//Written in the D programming language

/**
 * Implements functionality to read Comma Separated Values and its variants
 * from an input range of $(D dchar).
 *
 * Comma Separated Values provide a simple means to transfer and store
 * tabular data. It has been common for programs to use their own
 * variant of the CSV format. This parser will loosely follow the
 * $(HTTP tools.ietf.org/html/rfc4180, RFC-4180). CSV input should adhere
 * to the following criteria (differences from RFC-4180 in parentheses):
 *
 * $(UL
 *     $(LI A record is separated by a new line (CRLF,LF,CR))
 *     $(LI A final record may end with a new line)
 *     $(LI A header may be provided as the first record in input)
 *     $(LI A record has fields separated by a comma (customizable))
 *     $(LI A field containing new lines, commas, or double quotes
 *          should be enclosed in double quotes (customizable))
 *     $(LI Double quotes in a field are escaped with a double quote)
 *     $(LI Each record should contain the same number of fields)
 *   )
 *
 * Example:
 *
 * -------
 * import std.algorithm;
 * import std.array;
 * import std.csv;
 * import std.stdio;
 * import std.typecons;
 *
 * void main()
 * {
 *     auto text = "Joe,Carpenter,300000\nFred,Blacksmith,400000\r\n";
 *
 *     foreach (record; csvReader!(Tuple!(string, string, int))(text))
 *     {
 *         writefln("%s works as a %s and earns $%d per year",
 *                  record[0], record[1], record[2]);
 *     }
 *
 *     // To read the same string from the file "filename.csv":
 *
 *     auto file = File("filename.csv", "r");
 *     foreach (record;
 *         file.byLine.joiner("\n").csvReader!(Tuple!(string, string, int)))
 *     {
 *         writefln("%s works as a %s and earns $%d per year",
 *                  record[0], record[1], record[2]);
 *     }
 }
 * }
 * -------
 *
 * When an input contains a header the $(D Contents) can be specified as an
 * associative array. Passing null to signify that a header is present.
 *
 * -------
 * auto text = "Name,Occupation,Salary\r"
 *     "Joe,Carpenter,300000\nFred,Blacksmith,400000\r\n";
 *
 * foreach (record; csvReader!(string[string])
 *         (text, null))
 * {
 *     writefln("%s works as a %s and earns $%s per year.",
 *              record["Name"], record["Occupation"],
 *              record["Salary"]);
 * }
 * -------
 *
 * This module allows content to be iterated by record stored in a struct,
 * class, associative array, or as a range of fields. Upon detection of an
 * error an CSVException is thrown (can be disabled). csvNextToken has been
 * made public to allow for attempted recovery.
 *
 * Disabling exceptions will lift many restrictions specified above. A quote
 * can appear in a field if the field was not quoted. If in a quoted field any
 * quote by itself, not at the end of a field, will end processing for that
 * field. The field is ended when there is no input, even if the quote was not
 * closed.
 *
 *   See_Also:
 *      $(HTTP en.wikipedia.org/wiki/Comma-separated_values, Wikipedia
 *      Comma-separated values)
 *
 *   Copyright: Copyright 2011
 *   License:   $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
 *   Authors:   Jesse Phillips
 *   Source:    $(PHOBOSSRC std/_csv.d)
 */
module std.csv;

import std.conv;
import std.exception;  // basicExceptionCtors
import std.range.primitives;
import std.traits;

/**
 * Exception containing the row and column for when an exception was thrown.
 *
 * Numbering of both row and col start at one and corresponds to the location
 * in the file rather than any specified header. Special consideration should
 * be made when there is failure to match the header see $(LREF
 * HeaderMismatchException) for details.
 *
 * When performing type conversions, $(REF ConvException, std,conv) is stored in
 * the $(D next) field.
 */
class CSVException : Exception
{
    ///
    size_t row, col;

    // FIXME: Use std.exception.basicExceptionCtors here once bug #11500 is fixed

    this(string msg, string file = __FILE__, size_t line = __LINE__,
         Throwable next = null) @nogc @safe pure nothrow
    {
        super(msg, file, line, next);
    }

    this(string msg, Throwable next, string file = __FILE__,
         size_t line = __LINE__) @nogc @safe pure nothrow
    {
        super(msg, file, line, next);
    }

    this(string msg, size_t row, size_t col, Throwable next = null,
         string file = __FILE__, size_t line = __LINE__) @nogc @safe pure nothrow
    {
        super(msg, next, file, line);
        this.row = row;
        this.col = col;
    }

    override string toString() @safe pure const
    {
        return "(Row: " ~ to!string(row) ~
              ", Col: " ~ to!string(col) ~ ") " ~ msg;
    }
}

@safe pure unittest
{
    import std.string;
    auto e1 = new Exception("Foobar");
    auto e2 = new CSVException("args", e1);
    assert(e2.next is e1);

    size_t r = 13;
    size_t c = 37;

    auto e3 = new CSVException("argv", r, c);
    assert(e3.row == r);
    assert(e3.col == c);

    auto em = e3.toString();
    assert(em.indexOf("13") != -1);
    assert(em.indexOf("37") != -1);
}

/**
 * Exception thrown when a Token is identified to not be completed: a quote is
 * found in an unquoted field, data continues after a closing quote, or the
 * quoted field was not closed before data was empty.
 */
class IncompleteCellException : CSVException
{
    /**
     * Data pulled from input before finding a problem
     *
     * This field is populated when using $(LREF csvReader)
     * but not by $(LREF csvNextToken) as this data will have
     * already been fed to the output range.
     */
    dstring partialData;

    mixin basicExceptionCtors;
}

@safe pure unittest
{
    auto e1 = new Exception("Foobar");
    auto e2 = new IncompleteCellException("args", e1);
    assert(e2.next is e1);
}

/**
 * Exception thrown under different conditions based on the type of $(D
 * Contents).
 *
 * Structure, Class, and Associative Array
 * $(UL
 *     $(LI When a header is provided but a matching column is not found)
 *  )
 *
 * Other
 * $(UL
 *     $(LI When a header is provided but a matching column is not found)
 *     $(LI Order did not match that found in the input)
 *  )
 *
 * Since a row and column is not meaningful when a column specified by the
 * header is not found in the data, both row and col will be zero. Otherwise
 * row is always one and col is the first instance found in header that
 * occurred before the previous starting at one.
 */
class HeaderMismatchException : CSVException
{
    mixin basicExceptionCtors;
}

@safe pure unittest
{
    auto e1 = new Exception("Foobar");
    auto e2 = new HeaderMismatchException("args", e1);
    assert(e2.next is e1);
}

/**
 * Determines the behavior for when an error is detected.
 *
 * Disabling exception will follow these rules:
 * $(UL
 *     $(LI A quote can appear in a field if the field was not quoted.)
 *     $(LI If in a quoted field any quote by itself, not at the end of a
 *     field, will end processing for that field.)
 *     $(LI The field is ended when there is no input, even if the quote was
 *     not closed.)
 *     $(LI If the given header does not match the order in the input, the
 *     content will return as it is found in the input.)
 *     $(LI If the given header contains columns not found in the input they
 *     will be ignored.)
 *  )
*/
enum Malformed
{
    ignore,           /// No exceptions are thrown due to incorrect CSV.
    throwException    /// Use exceptions when input has incorrect CSV.
}

/**
 * Returns an input range for iterating over records found in $(D
 * input).
 *
 * The $(D Contents) of the input can be provided if all the records are the
 * same type such as all integer data:
 *
 * -------
 * string str = `76,26,22`;
 * int[] ans = [76,26,22];
 * auto records = csvReader!int(str);
 *
 * foreach (record; records)
 * {
 *     assert(equal(record, ans));
 * }
 * -------
 *
 * Example using a struct with modified delimiter:
 *
 * -------
 * string str = "Hello;65;63.63\nWorld;123;3673.562";
 * struct Layout
 * {
 *     string name;
 *     int value;
 *     double other;
 * }
 *
 * auto records = csvReader!Layout(str,';');
 *
 * foreach (record; records)
 * {
 *     writeln(record.name);
 *     writeln(record.value);
 *     writeln(record.other);
 * }
 * -------
 *
 * Specifying $(D ErrorLevel) as Malformed.ignore will lift restrictions
 * on the format. This example shows that an exception is not thrown when
 * finding a quote in a field not quoted.
 *
 * -------
 * string str = "A \" is now part of the data";
 * auto records = csvReader!(string,Malformed.ignore)(str);
 * auto record = records.front;
 *
 * assert(record.front == str);
 * -------
 *
 * Returns:
 *        An input range R as defined by
 *        $(REF isInputRange, std,range,primitives). When $(D Contents) is a
 *        struct, class, or an associative array, the element type of R is
 *        $(D Contents), otherwise the element type of R is itself a range with
 *        element type $(D Contents).
 *
 * Throws:
 *       $(LREF CSVException) When a quote is found in an unquoted field,
 *       data continues after a closing quote, the quoted field was not
 *       closed before data was empty, a conversion failed, or when the row's
 *       length does not match the previous length.
 *
 *       $(LREF HeaderMismatchException)  when a header is provided but a
 *       matching column is not found or the order did not match that found in
 *       the input. Read the exception documentation for specific details of
 *       when the exception is thrown for different types of $(D Contents).
 */
auto csvReader(Contents = string,Malformed ErrorLevel = Malformed.throwException, Range, Separator = char)(Range input,
                 Separator delimiter = ',', Separator quote = '"')
if (isInputRange!Range && is(Unqual!(ElementType!Range) == dchar)
    && isSomeChar!(Separator)
    && !is(Contents T : T[U], U : string))
{
    return CsvReader!(Contents,ErrorLevel,Range,
                    Unqual!(ElementType!Range),string[])
        (input, delimiter, quote);
}

/**
 * An optional $(D header) can be provided. The first record will be read in
 * as the header. If $(D Contents) is a struct then the header provided is
 * expected to correspond to the fields in the struct. When $(D Contents) is
 * not a type which can contain the entire record, the $(D header) must be
 * provided in the same order as the input or an exception is thrown.
 *
 * Read only column "b":
 *
 * -------
 * string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562";
 * auto records = csvReader!int(str, ["b"]);
 *
 * auto ans = [[65],[123]];
 * foreach (record; records)
 * {
 *     assert(equal(record, ans.front));
 *     ans.popFront();
 * }
 * -------
 *
 * Read from header of different order:
 *
 * -------
 * string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562";
 * struct Layout
 * {
 *     int value;
 *     double other;
 *     string name;
 * }
 *
 * auto records = csvReader!Layout(str, ["b","c","a"]);
 * -------
 *
 * The header can also be left empty if the input contains a header but
 * all columns should be iterated. The header from the input can always
 * be accessed from the header field.
 *
 * -------
 * string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562";
 * auto records = csvReader(str, null);
 *
 * assert(records.header == ["a","b","c"]);
 * -------
 *
 * Returns:
 *        An input range R as defined by
 *        $(REF isInputRange, std,range,primitives). When $(D Contents) is a
 *        struct, class, or an associative array, the element type of R is
 *        $(D Contents), otherwise the element type of R is itself a range with
 *        element type $(D Contents).
 *
 *        The returned range provides a header field for accessing the header
 *        from the input in array form.
 *
 * -------
 * string str = "a,b,c\nHello,65,63.63";
 * auto records = csvReader(str, ["a"]);
 *
 * assert(records.header == ["a","b","c"]);
 * -------
 *
 * Throws:
 *       $(LREF CSVException) When a quote is found in an unquoted field,
 *       data continues after a closing quote, the quoted field was not
 *       closed before data was empty, a conversion failed, or when the row's
 *       length does not match the previous length.
 *
 *       $(LREF HeaderMismatchException)  when a header is provided but a
 *       matching column is not found or the order did not match that found in
 *       the input. Read the exception documentation for specific details of
 *       when the exception is thrown for different types of $(D Contents).
 */
auto csvReader(Contents = string,
               Malformed ErrorLevel = Malformed.throwException,
               Range, Header, Separator = char)
                (Range input, Header header,
                 Separator delimiter = ',', Separator quote = '"')
if (isInputRange!Range && is(Unqual!(ElementType!Range) == dchar)
    && isSomeChar!(Separator)
    && isForwardRange!Header
    && isSomeString!(ElementType!Header))
{
    return CsvReader!(Contents,ErrorLevel,Range,
                    Unqual!(ElementType!Range),Header)
        (input, header, delimiter, quote);
}

///
auto csvReader(Contents = string,
               Malformed ErrorLevel = Malformed.throwException,
               Range, Header, Separator = char)
                (Range input, Header header,
                 Separator delimiter = ',', Separator quote = '"')
if (isInputRange!Range && is(Unqual!(ElementType!Range) == dchar)
    && isSomeChar!(Separator)
    && is(Header : typeof(null)))
{
    return CsvReader!(Contents,ErrorLevel,Range,
                    Unqual!(ElementType!Range),string[])
        (input, cast(string[]) null, delimiter, quote);
}

// Test standard iteration over input.
@safe pure unittest
{
    string str = `one,"two ""quoted"""` ~ "\n\"three\nnew line\",\nfive,six";
    auto records = csvReader(str);

    int count;
    foreach (record; records)
    {
        foreach (cell; record)
        {
            count++;
        }
    }
    assert(count == 6);
}

// Test newline on last record
@safe pure unittest
{
    string str = "one,two\nthree,four\n";
    auto records = csvReader(str);
    records.popFront();
    records.popFront();
    assert(records.empty);
}

// Test shorter row length
@safe pure unittest
{
    wstring str = "one,1\ntwo\nthree"w;
    struct Layout
    {
        string name;
        int value;
    }

    Layout[3] ans;
    ans[0].name = "one";
    ans[0].value = 1;
    ans[1].name = "two";
    ans[1].value = 0;
    ans[2].name = "three";
    ans[2].value = 0;

    auto records = csvReader!(Layout,Malformed.ignore)(str);

    int count;
    foreach (record; records)
    {
        assert(ans[count].name == record.name);
        assert(ans[count].value == record.value);
        count++;
    }
}

// Test shorter row length exception
@safe pure unittest
{
    import std.exception;

    struct A
    {
        string a,b,c;
    }

    auto strs = ["one,1\ntwo",
                 "one\ntwo,2,二\nthree,3,三",
                 "one\ntwo,2\nthree,3",
                 "one,1\ntwo\nthree,3"];

    foreach (str; strs)
    {
        auto records = csvReader!A(str);
        assertThrown!CSVException((){foreach (record; records) { }}());
    }
}


// Test structure conversion interface with unicode.
@safe pure unittest
{
    import std.math : abs;

    wstring str = "\U00010143Hello,65,63.63\nWorld,123,3673.562"w;
    struct Layout
    {
        string name;
        int value;
        double other;
    }

    Layout[2] ans;
    ans[0].name = "\U00010143Hello";
    ans[0].value = 65;
    ans[0].other = 63.63;
    ans[1].name = "World";
    ans[1].value = 123;
    ans[1].other = 3673.562;

    auto records = csvReader!Layout(str);

    int count;
    foreach (record; records)
    {
        assert(ans[count].name == record.name);
        assert(ans[count].value == record.value);
        assert(abs(ans[count].other - record.other) < 0.00001);
        count++;
    }
    assert(count == ans.length);
}

// Test input conversion interface
@safe pure unittest
{
    import std.algorithm;
    string str = `76,26,22`;
    int[] ans = [76,26,22];
    auto records = csvReader!int(str);

    foreach (record; records)
    {
        assert(equal(record, ans));
    }
}

// Test struct & header interface and same unicode
@safe unittest
{
    import std.math : abs;

    string str = "a,b,c\nHello,65,63.63\n➊➋➂❹,123,3673.562";
    struct Layout
    {
        int value;
        double other;
        string name;
    }

    auto records = csvReader!Layout(str, ["b","c","a"]);

    Layout[2] ans;
    ans[0].name = "Hello";
    ans[0].value = 65;
    ans[0].other = 63.63;
    ans[1].name = "➊➋➂❹";
    ans[1].value = 123;
    ans[1].other = 3673.562;

    int count;
    foreach (record; records)
    {
        assert(ans[count].name == record.name);
        assert(ans[count].value == record.value);
        assert(abs(ans[count].other - record.other) < 0.00001);
        count++;
    }
    assert(count == ans.length);

}

// Test header interface
@safe unittest
{
    import std.algorithm;

    string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562";
    auto records = csvReader!int(str, ["b"]);

    auto ans = [[65],[123]];
    foreach (record; records)
    {
        assert(equal(record, ans.front));
        ans.popFront();
    }

    try
    {
        csvReader(str, ["c","b"]);
        assert(0);
    }
    catch (HeaderMismatchException e)
    {
        assert(e.col == 2);
    }
    auto records2 = csvReader!(string,Malformed.ignore)
       (str, ["b","a"], ',', '"');

    auto ans2 = [["Hello","65"],["World","123"]];
    foreach (record; records2)
    {
        assert(equal(record, ans2.front));
        ans2.popFront();
    }

    str = "a,c,e\nJoe,Carpenter,300000\nFred,Fly,4";
    records2 = csvReader!(string,Malformed.ignore)
       (str, ["a","b","c","d"], ',', '"');

    ans2 = [["Joe","Carpenter"],["Fred","Fly"]];
    foreach (record; records2)
    {
        assert(equal(record, ans2.front));
        ans2.popFront();
    }
}

// Test null header interface
@safe unittest
{
    string str = "a,b,c\nHello,65,63.63\nWorld,123,3673.562";
    auto records = csvReader(str, ["a"]);

    assert(records.header == ["a","b","c"]);
}

// Test unchecked read
@safe pure unittest
{
    string str = "one \"quoted\"";
    foreach (record; csvReader!(string,Malformed.ignore)(str))
    {
        foreach (cell; record)
        {
            assert(cell == "one \"quoted\"");
        }
    }

    str = "one \"quoted\",two \"quoted\" end";
    struct Ans
    {
        string a,b;
    }
    foreach (record; csvReader!(Ans,Malformed.ignore)(str))
    {
        assert(record.a == "one \"quoted\"");
        assert(record.b == "two \"quoted\" end");
    }
}

// Test partial data returned
@safe pure unittest
{
    string str = "\"one\nnew line";

    try
    {
        foreach (record; csvReader(str))
        {}
        assert(0);
    }
    catch (IncompleteCellException ice)
    {
        assert(ice.partialData == "one\nnew line");
    }
}

// Test Windows line break
@safe pure unittest
{
    string str = "one,two\r\nthree";

    auto records = csvReader(str);
    auto record = records.front;
    assert(record.front == "one");
    record.popFront();
    assert(record.front == "two");
    records.popFront();
    record = records.front;
    assert(record.front == "three");
}


// Test associative array support with unicode separator
@safe unittest
{
  string str = "1❁2❁3\n34❁65❁63\n34❁65❁63";

  auto records = csvReader!(string[string])(str,["3","1"],'❁');
  int count;
  foreach (record; records)
  {
      count++;
      assert(record["1"] == "34");
      assert(record["3"] == "63");
  }
  assert(count == 2);
}

// Test restricted range
@safe unittest
{
    import std.typecons;
    struct InputRange
    {
        dstring text;

        this(dstring txt)
        {
            text = txt;
        }

        @property auto empty()
        {
            return text.empty;
        }

        void popFront()
        {
            text.popFront();
        }

        @property dchar front()
        {
            return text[0];
        }
    }
    auto ir = InputRange("Name,Occupation,Salary\r"d~
          "Joe,Carpenter,300000\nFred,Blacksmith,400000\r\n"d);

    foreach (record; csvReader(ir, cast(string[]) null))
        foreach (cell; record) {}
    foreach (record; csvReader!(Tuple!(string, string, int))
            (ir,cast(string[]) null)) {}
    foreach (record; csvReader!(string[string])
            (ir,cast(string[]) null)) {}
}

@safe unittest // const/immutable dchars
{
    import std.algorithm.iteration : map;
    import std.array : array;
    const(dchar)[] c = "foo,bar\n";
    assert(csvReader(c).map!array.array == [["foo", "bar"]]);
    immutable(dchar)[] i = "foo,bar\n";
    assert(csvReader(i).map!array.array == [["foo", "bar"]]);
}

/*
 * This struct is stored on the heap for when the structures
 * are passed around.
 */
private pure struct Input(Range, Malformed ErrorLevel)
{
    Range range;
    size_t row, col;
    static if (ErrorLevel == Malformed.throwException)
        size_t rowLength;
}

/*
 * Range for iterating CSV records.
 *
 * This range is returned by the $(LREF csvReader) functions. It can be
 * created in a similar manner to allow $(D ErrorLevel) be set to $(LREF
 * Malformed).ignore if best guess processing should take place.
 */
private struct CsvReader(Contents, Malformed ErrorLevel, Range, Separator, Header)
if (isSomeChar!Separator && isInputRange!Range
    && is(Unqual!(ElementType!Range) == dchar)
    && isForwardRange!Header && isSomeString!(ElementType!Header))
{
private:
    Input!(Range, ErrorLevel)* _input;
    Separator _separator;
    Separator _quote;
    size_t[] indices;
    bool _empty;
    static if (is(Contents == struct) || is(Contents == class))
    {
        Contents recordContent;
        CsvRecord!(string, ErrorLevel, Range, Separator) recordRange;
    }
    else static if (is(Contents T : T[U], U : string))
    {
        Contents recordContent;
        CsvRecord!(T, ErrorLevel, Range, Separator) recordRange;
    }
    else
        CsvRecord!(Contents, ErrorLevel, Range, Separator) recordRange;
public:
    /**
     * Header from the input in array form.
     *
     * -------
     * string str = "a,b,c\nHello,65,63.63";
     * auto records = csvReader(str, ["a"]);
     *
     * assert(records.header == ["a","b","c"]);
     * -------
     */
    string[] header;

    /**
     * Constructor to initialize the input, delimiter and quote for input
     * without a header.
     *
     * -------
     * string str = `76;^26^;22`;
     * int[] ans = [76,26,22];
     * auto records = CsvReader!(int,Malformed.ignore,string,char,string[])
     *       (str, ';', '^');
     *
     * foreach (record; records)
     * {
     *    assert(equal(record, ans));
     * }
     * -------
     */
    this(Range input, Separator delimiter, Separator quote)
    {
        _input = new Input!(Range, ErrorLevel)(input);
        _separator = delimiter;
        _quote = quote;

        prime();
    }

    /**
     * Constructor to initialize the input, delimiter and quote for input
     * with a header.
     *
     * -------
     * string str = `high;mean;low\n76;^26^;22`;
     * auto records = CsvReader!(int,Malformed.ignore,string,char,string[])
     *       (str, ["high","low"], ';', '^');
     *
     * int[] ans = [76,22];
     * foreach (record; records)
     * {
     *    assert(equal(record, ans));
     * }
     * -------
     *
     * Throws:
     *       $(LREF HeaderMismatchException)  when a header is provided but a
     *       matching column is not found or the order did not match that found
     *       in the input (non-struct).
     */
    this(Range input, Header colHeaders, Separator delimiter, Separator quote)
    {
        _input = new Input!(Range, ErrorLevel)(input);
        _separator = delimiter;
        _quote = quote;

        size_t[string] colToIndex;
        foreach (h; colHeaders)
        {
            colToIndex[h] = size_t.max;
        }

        auto r = CsvRecord!(string, ErrorLevel, Range, Separator)
            (_input, _separator, _quote, indices);

        size_t colIndex;
        foreach (col; r)
        {
            header ~= col;
            auto ptr = col in colToIndex;
            if (ptr)
                *ptr = colIndex;
            colIndex++;
        }
        // The above loop empties the header row.
        recordRange._empty = true;

        indices.length = colToIndex.length;
        int i;
        foreach (h; colHeaders)
        {
            immutable index = colToIndex[h];
            static if (ErrorLevel != Malformed.ignore)
                if (index == size_t.max)
                    throw new HeaderMismatchException
                        ("Header not found: " ~ to!string(h));
            indices[i++] = index;
        }

        static if (!is(Contents == struct) && !is(Contents == class))
        {
            static if (is(Contents T : T[U], U : string))
            {
                import std.algorithm.sorting : sort;
                sort(indices);
            }
            else static if (ErrorLevel == Malformed.ignore)
            {
                import std.algorithm.sorting : sort;
                sort(indices);
            }
            else
            {
                import std.algorithm.searching : findAdjacent;
                import std.algorithm.sorting : isSorted;
                if (!isSorted(indices))
                {
                    auto ex = new HeaderMismatchException
                           ("Header in input does not match specified header.");
                    findAdjacent!"a > b"(indices);
                    ex.row = 1;
                    ex.col = indices.front;

                    throw ex;
                }
            }
        }

        popFront();
    }

    /**
     * Part of an input range as defined by
     * $(REF isInputRange, std,range,primitives).
     *
     * Returns:
     *      If $(D Contents) is a struct, will be filled with record data.
     *
     *      If $(D Contents) is a class, will be filled with record data.
     *
     *      If $(D Contents) is a associative array, will be filled
     *      with record data.
     *
     *      If $(D Contents) is non-struct, a $(LREF CsvRecord) will be
     *      returned.
     */
    @property auto front()
    {
        assert(!empty);
        static if (is(Contents == struct) || is(Contents == class))
        {
            return recordContent;
        }
        else static if (is(Contents T : T[U], U : string))
        {
            return recordContent;
        }
        else
        {
            return recordRange;
        }
    }

    /**
     * Part of an input range as defined by
     * $(REF isInputRange, std,range,primitives).
     */
    @property bool empty() @safe @nogc pure nothrow const
    {
        return _empty;
    }

    /**
     * Part of an input range as defined by
     * $(REF isInputRange, std,range,primitives).
     *
     * Throws:
     *       $(LREF CSVException) When a quote is found in an unquoted field,
     *       data continues after a closing quote, the quoted field was not
     *       closed before data was empty, a conversion failed, or when the
     *       row's length does not match the previous length.
     */
    void popFront()
    {
        while (!recordRange.empty)
        {
            recordRange.popFront();
        }

        static if (ErrorLevel == Malformed.throwException)
            if (_input.rowLength == 0)
                _input.rowLength = _input.col;

        _input.col = 0;

        if (!_input.range.empty)
        {
            if (_input.range.front == '\r')
            {
                _input.range.popFront();
                if (!_input.range.empty && _input.range.front == '\n')
                    _input.range.popFront();
            }
            else if (_input.range.front == '\n')
                _input.range.popFront();
        }

        if (_input.range.empty)
        {
            _empty = true;
            return;
        }

        prime();
    }

    private void prime()
    {
        if (_empty)
            return;
        _input.row++;
        static if (is(Contents == struct) || is(Contents == class))
        {
            recordRange = typeof(recordRange)
                                 (_input, _separator, _quote, null);
        }
        else
        {
            recordRange = typeof(recordRange)
                                 (_input, _separator, _quote, indices);
        }

        static if (is(Contents T : T[U], U : string))
        {
            T[U] aa;
            try
            {
                for (; !recordRange.empty; recordRange.popFront())
                {
                    aa[header[_input.col-1]] = recordRange.front;
                }
            }
            catch (ConvException e)
            {
                throw new CSVException(e.msg, _input.row, _input.col, e);
            }

            recordContent = aa;
        }
        else static if (is(Contents == struct) || is(Contents == class))
        {
            static if (is(Contents == class))
                recordContent = new typeof(recordContent)();
            else
                recordContent = typeof(recordContent).init;
            size_t colIndex;
            try
            {
                for (; !recordRange.empty;)
                {
                    auto colData = recordRange.front;
                    scope(exit) colIndex++;
                    if (indices.length > 0)
                    {
                        foreach (ti, ToType; Fields!(Contents))
                        {
                            if (indices[ti] == colIndex)
                            {
                                static if (!isSomeString!ToType) skipWS(colData);
                                recordContent.tupleof[ti] = to!ToType(colData);
                            }
                        }
                    }
                    else
                    {
                        foreach (ti, ToType; Fields!(Contents))
                        {
                            if (ti == colIndex)
                            {
                                static if (!isSomeString!ToType) skipWS(colData);
                                recordContent.tupleof[ti] = to!ToType(colData);
                            }
                        }
                    }
                    recordRange.popFront();
                }
            }
            catch (ConvException e)
            {
                throw new CSVException(e.msg, _input.row, colIndex, e);
            }
        }
    }
}

@safe pure unittest
{
    import std.algorithm.comparison : equal;

    string str = `76;^26^;22`;
    int[] ans = [76,26,22];
    auto records = CsvReader!(int,Malformed.ignore,string,char,string[])
          (str, ';', '^');

    foreach (record; records)
    {
        assert(equal(record, ans));
    }
}

// Bugzilla 15545
// @system due to the catch for Throwable
@system pure unittest
{
    import std.exception : assertNotThrown;
    enum failData =
    "name, surname, age
    Joe, Joker, 99\r";
    auto r = csvReader(failData);
    assertNotThrown((){foreach (entry; r){}}());
}

/*
 * This input range is accessible through $(LREF CsvReader) when the
 * requested $(D Contents) type is neither a structure or an associative array.
 */
private struct CsvRecord(Contents, Malformed ErrorLevel, Range, Separator)
if (!is(Contents == class) && !is(Contents == struct))
{
    import std.array : appender;
private:
    Input!(Range, ErrorLevel)* _input;
    Separator _separator;
    Separator _quote;
    Contents curContentsoken;
    typeof(appender!(dchar[])()) _front;
    bool _empty;
    size_t[] _popCount;
public:
    /*
     * Params:
     *      input = Pointer to a character input range
     *      delimiter = Separator for each column
     *      quote = Character used for quotation
     *      indices = An array containing which columns will be returned.
     *             If empty, all columns are returned. List must be in order.
     */
    this(Input!(Range, ErrorLevel)* input, Separator delimiter,
         Separator quote, size_t[] indices)
    {
        _input = input;
        _separator = delimiter;
        _quote = quote;
        _front = appender!(dchar[])();
        _popCount = indices.dup;

        // If a header was given, each call to popFront will need
        // to eliminate so many tokens. This calculates
        // how many will be skipped to get to the next header column
        size_t normalizer;
        foreach (ref c; _popCount)
        {
            static if (ErrorLevel == Malformed.ignore)
            {
                // If we are not throwing exceptions
                // a header may not exist, indices are sorted
                // and will be size_t.max if not found.
                if (c == size_t.max)
                    break;
            }
            c -= normalizer;
            normalizer += c + 1;
        }

        prime();
    }

    /**
     * Part of an input range as defined by
     * $(REF isInputRange, std,range,primitives).
     */
    @property Contents front() @safe pure
    {
        assert(!empty);
        return curContentsoken;
    }

    /**
     * Part of an input range as defined by
     * $(REF isInputRange, std,range,primitives).
     */
    @property bool empty() @safe pure nothrow @nogc const
    {
        return _empty;
    }

    /*
     * CsvRecord is complete when input
     * is empty or starts with record break
     */
    private bool recordEnd()
    {
        if (_input.range.empty
           || _input.range.front == '\n'
           || _input.range.front == '\r')
        {
            return true;
        }
        return false;
    }


    /**
     * Part of an input range as defined by
     * $(REF isInputRange, std,range,primitives).
     *
     * Throws:
     *       $(LREF CSVException) When a quote is found in an unquoted field,
     *       data continues after a closing quote, the quoted field was not
     *       closed before data was empty, a conversion failed, or when the
     *       row's length does not match the previous length.
     */
    void popFront()
    {
        static if (ErrorLevel == Malformed.throwException)
            import std.format : format;
        // Skip last of record when header is depleted.
        if (_popCount.ptr && _popCount.empty)
            while (!recordEnd())
            {
                prime(1);
            }

        if (recordEnd())
        {
            _empty = true;
            static if (ErrorLevel == Malformed.throwException)
                if (_input.rowLength != 0)
                    if (_input.col != _input.rowLength)
                        throw new CSVException(
                           format("Row %s's length %s does not match "~
                                  "previous length of %s.", _input.row,
                                  _input.col, _input.rowLength));
            return;
        }
        else
        {
            static if (ErrorLevel == Malformed.throwException)
                if (_input.rowLength != 0)
                    if (_input.col > _input.rowLength)
                        throw new CSVException(
                           format("Row %s's length %s does not match "~
                                  "previous length of %s.", _input.row,
                                  _input.col, _input.rowLength));
        }

        // Separator is left on the end of input from the last call.
        // This cannot be moved to after the call to csvNextToken as
        // there may be an empty record after it.
        if (_input.range.front == _separator)
            _input.range.popFront();

        _front.shrinkTo(0);

        prime();
    }

    /*
     * Handles moving to the next skipNum token.
     */
    private void prime(size_t skipNum)
    {
        foreach (i; 0 .. skipNum)
        {
            _input.col++;
            _front.shrinkTo(0);
            if (_input.range.front == _separator)
                _input.range.popFront();

            try
                csvNextToken!(Range, ErrorLevel, Separator)
                                   (_input.range, _front, _separator, _quote,false);
            catch (IncompleteCellException ice)
            {
                ice.row = _input.row;
                ice.col = _input.col;
                ice.partialData = _front.data.idup;
                throw ice;
            }
            catch (ConvException e)
            {
                throw new CSVException(e.msg, _input.row, _input.col, e);
            }
        }
    }

    private void prime()
    {
        try
        {
            _input.col++;
            csvNextToken!(Range, ErrorLevel, Separator)
                (_input.range, _front, _separator, _quote,false);
        }
        catch (IncompleteCellException ice)
        {
            ice.row = _input.row;
            ice.col = _input.col;
            ice.partialData = _front.data.idup;
            throw ice;
        }

        auto skipNum = _popCount.empty ? 0 : _popCount.front;
        if (!_popCount.empty)
            _popCount.popFront();

        if (skipNum == size_t.max)
        {
            while (!recordEnd())
                prime(1);
            _empty = true;
            return;
        }

        if (skipNum)
            prime(skipNum);

        auto data = _front.data;
        static if (!isSomeString!Contents) skipWS(data);
        try curContentsoken = to!Contents(data);
        catch (ConvException e)
        {
            throw new CSVException(e.msg, _input.row, _input.col, e);
        }
    }
}

/**
 * Lower level control over parsing CSV
 *
 * This function consumes the input. After each call the input will
 * start with either a delimiter or record break (\n, \r\n, \r) which
 * must be removed for subsequent calls.
 *
 * Params:
 *       input = Any CSV input
 *       ans   = The first field in the input
 *       sep   = The character to represent a comma in the specification
 *       quote = The character to represent a quote in the specification
 *       startQuoted = Whether the input should be considered to already be in
 * quotes
 *
 * Throws:
 *       $(LREF IncompleteCellException) When a quote is found in an unquoted
 *       field, data continues after a closing quote, or the quoted field was
 *       not closed before data was empty.
 */
void csvNextToken(Range, Malformed ErrorLevel = Malformed.throwException,
                           Separator, Output)
                          (ref Range input, ref Output ans,
                           Separator sep, Separator quote,
                           bool startQuoted = false)
if (isSomeChar!Separator && isInputRange!Range
    && is(Unqual!(ElementType!Range) == dchar)
    && isOutputRange!(Output, dchar))
{
    bool quoted = startQuoted;
    bool escQuote;
    if (input.empty)
        return;

    if (input.front == '\n')
        return;
    if (input.front == '\r')
        return;

    if (input.front == quote)
    {
        quoted = true;
        input.popFront();
    }

    while (!input.empty)
    {
        assert(!(quoted && escQuote));
        if (!quoted)
        {
            // When not quoted the token ends at sep
            if (input.front == sep)
                break;
            if (input.front == '\r')
                break;
            if (input.front == '\n')
                break;
        }
        if (!quoted && !escQuote)
        {
            if (input.front == quote)
            {
                // Not quoted, but quote found
                static if (ErrorLevel == Malformed.throwException)
                    throw new IncompleteCellException(
                          "Quote located in unquoted token");
                else static if (ErrorLevel == Malformed.ignore)
                    ans.put(quote);
            }
            else
            {
                // Not quoted, non-quote character
                ans.put(input.front);
            }
        }
        else
        {
            if (input.front == quote)
            {
                // Quoted, quote found
                // By turning off quoted and turning on escQuote
                // I can tell when to add a quote to the string
                // escQuote is turned to false when it escapes a
                // quote or is followed by a non-quote (see outside else).
                // They are mutually exclusive, but provide different
                // information.
                if (escQuote)
                {
                    escQuote = false;
                    quoted = true;
                    ans.put(quote);
                } else
                {
                    escQuote = true;
                    quoted = false;
                }
            }
            else
            {
                // Quoted, non-quote character
                if (escQuote)
                {
                    static if (ErrorLevel == Malformed.throwException)
                        throw new IncompleteCellException(
                          "Content continues after end quote, " ~
                          "or needs to be escaped.");
                    else static if (ErrorLevel == Malformed.ignore)
                        break;
                }
                ans.put(input.front);
            }
        }
        input.popFront();
    }

    static if (ErrorLevel == Malformed.throwException)
        if (quoted && (input.empty || input.front == '\n' || input.front == '\r'))
            throw new IncompleteCellException(
                  "Data continues on future lines or trailing quote");

}

///
@safe unittest
{
    import std.array : appender;
    import std.range.primitives : popFront;

    string str = "65,63\n123,3673";

    auto a = appender!(char[])();

    csvNextToken(str,a,',','"');
    assert(a.data == "65");
    assert(str == ",63\n123,3673");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "63");
    assert(str == "\n123,3673");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "123");
    assert(str == ",3673");
}

// Test csvNextToken on simplest form and correct format.
@safe pure unittest
{
    import std.array;

    string str = "\U00010143Hello,65,63.63\nWorld,123,3673.562";

    auto a = appender!(dchar[])();
    csvNextToken!string(str,a,',','"');
    assert(a.data == "\U00010143Hello");
    assert(str == ",65,63.63\nWorld,123,3673.562");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "65");
    assert(str == ",63.63\nWorld,123,3673.562");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "63.63");
    assert(str == "\nWorld,123,3673.562");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "World");
    assert(str == ",123,3673.562");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "123");
    assert(str == ",3673.562");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "3673.562");
    assert(str == "");
}

// Test quoted tokens
@safe pure unittest
{
    import std.array;

    string str = `one,two,"three ""quoted""","",` ~ "\"five\nnew line\"\nsix";

    auto a = appender!(dchar[])();
    csvNextToken!string(str,a,',','"');
    assert(a.data == "one");
    assert(str == `,two,"three ""quoted""","",` ~ "\"five\nnew line\"\nsix");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "two");
    assert(str == `,"three ""quoted""","",` ~ "\"five\nnew line\"\nsix");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "three \"quoted\"");
    assert(str == `,"",` ~ "\"five\nnew line\"\nsix");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "");
    assert(str == ",\"five\nnew line\"\nsix");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "five\nnew line");
    assert(str == "\nsix");

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "six");
    assert(str == "");
}

// Test empty data is pulled at end of record.
@safe pure unittest
{
    import std.array;

    string str = "one,";
    auto a = appender!(dchar[])();
    csvNextToken(str,a,',','"');
    assert(a.data == "one");
    assert(str == ",");

    a.shrinkTo(0);
    csvNextToken(str,a,',','"');
    assert(a.data == "");
}

// Test exceptions
@safe pure unittest
{
    import std.array;

    string str = "\"one\nnew line";

    typeof(appender!(dchar[])()) a;
    try
    {
        a = appender!(dchar[])();
        csvNextToken(str,a,',','"');
        assert(0);
    }
    catch (IncompleteCellException ice)
    {
        assert(a.data == "one\nnew line");
        assert(str == "");
    }

    str = "Hello world\"";

    try
    {
        a = appender!(dchar[])();
        csvNextToken(str,a,',','"');
        assert(0);
    }
    catch (IncompleteCellException ice)
    {
        assert(a.data == "Hello world");
        assert(str == "\"");
    }

    str = "one, two \"quoted\" end";

    a = appender!(dchar[])();
    csvNextToken!(string,Malformed.ignore)(str,a,',','"');
    assert(a.data == "one");
    str.popFront();
    a.shrinkTo(0);
    csvNextToken!(string,Malformed.ignore)(str,a,',','"');
    assert(a.data == " two \"quoted\" end");
}

// Test modifying token delimiter
@safe pure unittest
{
    import std.array;

    string str = `one|two|/three "quoted"/|//`;

    auto a = appender!(dchar[])();
    csvNextToken(str,a, '|','/');
    assert(a.data == "one"d);
    assert(str == `|two|/three "quoted"/|//`);

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a, '|','/');
    assert(a.data == "two"d);
    assert(str == `|/three "quoted"/|//`);

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a, '|','/');
    assert(a.data == `three "quoted"`);
    assert(str == `|//`);

    str.popFront();
    a.shrinkTo(0);
    csvNextToken(str,a, '|','/');
    assert(a.data == ""d);
}

// Bugzilla 8908
@safe pure unittest
{
    string csv = `  1.0, 2.0, 3.0
                    4.0, 5.0, 6.0`;

    static struct Data { real a, b, c; }
    size_t i = 0;
    foreach (data; csvReader!Data(csv)) with (data)
    {
        int[] row = [cast(int) a, cast(int) b, cast(int) c];
        if (i == 0)
            assert(row == [1, 2, 3]);
        else
            assert(row == [4, 5, 6]);
        ++i;
    }

    i = 0;
    foreach (data; csvReader!real(csv))
    {
        auto a = data.front;    data.popFront();
        auto b = data.front;    data.popFront();
        auto c = data.front;
        int[] row = [cast(int) a, cast(int) b, cast(int) c];
        if (i == 0)
            assert(row == [1, 2, 3]);
        else
            assert(row == [4, 5, 6]);
        ++i;
    }
}