gnc-csv-model.c

#include "gnc-csv-model.h"

#include "gnc-book.h"

#include <glib/gi18n.h>
#include <goffice/utils/go-glib-extras.h>

#include <string.h>
#include <sys/time.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <regex.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdlib.h>
#include <time.h>
#ifndef HAVE_LOCALTIME_R
#include "localtime_r.h"
#endif

static QofLogModule log_module = GNC_MOD_IMPORT;

const int num_date_formats = 5;

const gchar* date_format_user[] = {N_("y-m-d"),
                                   N_("d-m-y"),
                                   N_("m-d-y"),
                                   N_("d-m"),
                                   N_("m-d")};

/* This array contains all of the different strings for different column types. */
gchar* gnc_csv_column_type_strs[GNC_CSV_NUM_COL_TYPES] = {N_("None"),
                                                          N_("Date"),
                                                          N_("Description"),
                                                          N_("Balance"),
                                                          N_("Deposit"),
                                                          N_("Withdrawal"),
                                                          N_("Num")};

static StfParseOptions_t* default_parse_options(void)
{
  StfParseOptions_t* options = stf_parse_options_new();
  stf_parse_options_set_type(options, PARSE_TYPE_CSV);
  stf_parse_options_csv_set_separators(options, ",", NULL);
  return options;
}

static time_t parse_date_with_year(const char* date_str, int format)
{
  time_t rawtime; /* The integer time */
  struct tm retvalue, test_retvalue; /* The time in a broken-down structure */
  
  int i, j, mem_length, orig_year = -1, orig_month = -1, orig_day = -1;

  /* Buffer for containing individual parts (e.g. year, month, day) of a date */
  char date_segment[5];

  /* The compiled regular expression */
  regex_t preg = {0};

  /* An array containing indices specifying the matched substrings in date_str */
  regmatch_t pmatch[4] = { {0}, {0}, {0}, {0} };

  /* The regular expression for parsing dates */
  const char* regex = "^ *([0-9]+) *[-/.'] *([0-9]+) *[-/.'] *([0-9]+).*$|^ *([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]).*$";

  /* We get our matches using the regular expression. */
  regcomp(&preg, regex, REG_EXTENDED);
  regexec(&preg, date_str, 4, pmatch, 0);
  regfree(&preg);

  /* If there wasn't a match, there was an error. */
  if(pmatch[0].rm_eo == 0)
    return -1;

  /* If this is a string without separators ... */
  if(pmatch[1].rm_so == -1)
  {
    /* ... we will fill in the indices based on the user's selection. */
    int k = 0; /* k traverses date_str by keeping track of where separators "should" be. */
    j = 1; /* j traverses pmatch. */
    for(i = 0; date_format_user[format][i]; i++)
    {
      char segment_type = date_format_user[format][i];
      /* Only do something if this is a meaningful character */
      if(segment_type == 'y' || segment_type == 'm' || segment_type == 'd')
      {
        pmatch[j].rm_so = k;
        switch(segment_type)
        {
        case 'm':
        case 'd':
          k += 2;
          break;

        case 'y':
          k += 4;
          break;
        }

        pmatch[j].rm_eo = k;
        j++;
      }
    }    
  }

  /* Put some sane values in retvalue by using the current time for
   * the non-year-month-day parts of the date. */
  time(&rawtime);
  localtime_r(&rawtime, &retvalue);

  /* j traverses pmatch (index 0 contains the entire string, so we
   * start at index 1 for the first meaningful match). */
  j = 1;
  /* Go through the date format and interpret the matches in order of
   * the sections in the date format. */
  for(i = 0; date_format_user[format][i]; i++)
  {
    char segment_type = date_format_user[format][i];
    /* Only do something if this is a meaningful character */
    if(segment_type == 'y' || segment_type == 'm' || segment_type == 'd')
    {
      /* Copy the matching substring into date_segment so that we can
       * convert it into an integer. */
      mem_length = pmatch[j].rm_eo - pmatch[j].rm_so;
      memcpy(date_segment, date_str + pmatch[j].rm_so, mem_length);
      date_segment[mem_length] = '\0';

      /* Set the appropriate member of retvalue. Save the original
       * values so that we can check if the change when we use mktime
       * below. */
      switch(segment_type)
      {
      case 'y':
        retvalue.tm_year = atoi(date_segment);

        /* Handle two-digit years. */
        if(retvalue.tm_year < 100)
        {
          /* We allow two-digit years in the range 1969 - 2068. */
          if(retvalue.tm_year < 69)
            retvalue.tm_year += 100;
        }
        else
          retvalue.tm_year -= 1900;
        orig_year = retvalue.tm_year;
        break;
        
      case 'm':
        orig_month = retvalue.tm_mon = atoi(date_segment) - 1;
        break;
        
      case 'd':
        orig_day = retvalue.tm_mday = atoi(date_segment);
        break;
      }
      j++;
    }
  }
  /* Convert back to an integer. If mktime leaves retvalue unchanged,
   * everything is okay; otherwise, an error has occurred. */
  /* We have to use a "test" date value to account for changes in
   * daylight savings time, which can cause a date change with mktime
   * near midnight, causing the code to incorrectly think a date is
   * incorrect. */
  test_retvalue = retvalue;
  mktime(&test_retvalue);
  retvalue.tm_isdst = test_retvalue.tm_isdst;
  rawtime = mktime(&retvalue);
  if(retvalue.tm_mday == orig_day &&
     retvalue.tm_mon == orig_month &&
     retvalue.tm_year == orig_year)
  {
    return rawtime;
  }
  else
  {
    return -1;
  }
}

static time_t parse_date_without_year(const char* date_str, int format)
{
  time_t rawtime; /* The integer time */
  struct tm retvalue, test_retvalue; /* The time in a broken-down structure */
  
  int i, j, mem_length, orig_year = -1, orig_month = -1, orig_day = -1;

  /* Buffer for containing individual parts (e.g. year, month, day) of a date */
  gchar* date_segment;

  /* The compiled regular expression */
  regex_t preg = {0};

  /* An array containing indices specifying the matched substrings in date_str */
  regmatch_t pmatch[3] = { {0}, {0}, {0} };

  /* The regular expression for parsing dates */
  const char* regex = "^ *([0-9]+) *[-/.'] *([0-9]+).*$";

  /* We get our matches using the regular expression. */
  regcomp(&preg, regex, REG_EXTENDED);
  regexec(&preg, date_str, 3, pmatch, 0);
  regfree(&preg);

  /* If there wasn't a match, there was an error. */
  if(pmatch[0].rm_eo == 0)
    return -1;

  /* Put some sane values in retvalue by using the current time for
   * the non-year-month-day parts of the date. */
  time(&rawtime);
  localtime_r(&rawtime, &retvalue);
  orig_year = retvalue.tm_year;

  /* j traverses pmatch (index 0 contains the entire string, so we
   * start at index 1 for the first meaningful match). */
  j = 1;
  /* Go through the date format and interpret the matches in order of
   * the sections in the date format. */
  for(i = 0; date_format_user[format][i]; i++)
  {
    char segment_type = date_format_user[format][i];
    /* Only do something if this is a meaningful character */
    if(segment_type == 'm' || segment_type == 'd')
    {
      /* Copy the matching substring into date_segment so that we can
       * convert it into an integer. */
      mem_length = pmatch[j].rm_eo - pmatch[j].rm_so;
      date_segment = g_new(gchar, mem_length);
      memcpy(date_segment, date_str + pmatch[j].rm_so, mem_length);
      date_segment[mem_length] = '\0';

      /* Set the appropriate member of retvalue. Save the original
       * values so that we can check if the change when we use mktime
       * below. */
      switch(segment_type)
      {
      case 'm':
        orig_month = retvalue.tm_mon = atoi(date_segment) - 1;
        break;
        
      case 'd':
        orig_day = retvalue.tm_mday = atoi(date_segment);
        break;
      }
      g_free(date_segment);
      j++;
    }
  }
  /* Convert back to an integer. If mktime leaves retvalue unchanged,
   * everything is okay; otherwise, an error has occurred. */
  /* We have to use a "test" date value to account for changes in
   * daylight savings time, which can cause a date change with mktime
   * near midnight, causing the code to incorrectly think a date is
   * incorrect. */
  test_retvalue = retvalue;
  mktime(&test_retvalue);
  retvalue.tm_isdst = test_retvalue.tm_isdst;
  rawtime = mktime(&retvalue);
  if(retvalue.tm_mday == orig_day &&
     retvalue.tm_mon == orig_month &&
     retvalue.tm_year == orig_year)
  {
    return rawtime;
  }
  else
  {
    return -1;
  }
}

static time_t parse_date(const char* date_str, int format)
{
  if(strchr(date_format_user[format], 'y'))
    return parse_date_with_year(date_str, format);
  else
    return parse_date_without_year(date_str, format);
}

GncCsvParseData* gnc_csv_new_parse_data(void)
{
  GncCsvParseData* parse_data = g_new(GncCsvParseData, 1);
  parse_data->encoding = "UTF-8";
  /* All of the data pointers are initially NULL. This is so that, if
   * gnc_csv_parse_data_free is called before all of the data is
   * initialized, only the data that needs to be freed is freed. */
  parse_data->raw_str.begin = parse_data->raw_str.end
    = parse_data->file_str.begin = parse_data->file_str.end = NULL;
  parse_data->orig_lines = NULL;
  parse_data->orig_row_lengths = NULL;
  parse_data->column_types = NULL;
  parse_data->error_lines = parse_data->transactions = NULL;
  parse_data->options = default_parse_options();
  parse_data->date_format = -1;
  parse_data->chunk = g_string_chunk_new(100 * 1024);
  return parse_data;
}

void gnc_csv_parse_data_free(GncCsvParseData* parse_data)
{
  /* All non-NULL pointers have been initialized and must be freed. */

  if(parse_data->raw_mapping != NULL)
    g_mapped_file_free(parse_data->raw_mapping);

  if(parse_data->file_str.begin != NULL)
    g_free(parse_data->file_str.begin);

  if(parse_data->orig_lines != NULL)
    stf_parse_general_free(parse_data->orig_lines);

  if(parse_data->orig_row_lengths != NULL)
    g_array_free(parse_data->orig_row_lengths, FALSE);

  if(parse_data->options != NULL)
    stf_parse_options_free(parse_data->options);

  if(parse_data->column_types != NULL)
    g_array_free(parse_data->column_types, TRUE);

  if(parse_data->error_lines != NULL)
    g_list_free(parse_data->error_lines);

  if(parse_data->transactions != NULL)
  {
    GList* transactions = parse_data->transactions;
    /* We have to free the GncCsvTransLine's that are at each node in
     * the list before freeing the entire list. */
    do
    {
      g_free(transactions->data);
      transactions = g_list_next(transactions);
    } while(transactions != NULL);
    g_list_free(parse_data->transactions);
  }

  g_free(parse_data->chunk);
  g_free(parse_data);
}

int gnc_csv_convert_encoding(GncCsvParseData* parse_data, const char* encoding,
                             GError** error)
{
  gsize bytes_read, bytes_written;

  /* If parse_data->file_str has already been initialized it must be
   * freed first. (This should always be the case, since
   * gnc_csv_load_file should always be called before this
   * function.) */
  if(parse_data->file_str.begin != NULL)
    g_free(parse_data->file_str.begin);

  /* Do the actual translation to UTF-8. */
  parse_data->file_str.begin = g_convert(parse_data->raw_str.begin,
                                         parse_data->raw_str.end - parse_data->raw_str.begin,
                                         "UTF-8", encoding, &bytes_read, &bytes_written,
                                         error);
  /* Handle errors that occur. */
  if(parse_data->file_str.begin == NULL)
    return 1;

  /* On success, save the ending pointer of the translated data and
   * the encoding type and return 0. */
  parse_data->file_str.end = parse_data->file_str.begin + bytes_written;
  parse_data->encoding = (gchar*)encoding;
  return 0;
}

int gnc_csv_load_file(GncCsvParseData* parse_data, const char* filename,
                      GError** error)
{
  const char* guess_enc;

  /* Get the raw data first and handle an error if one occurs. */
  parse_data->raw_mapping = g_mapped_file_new(filename, FALSE, error);
  if(parse_data->raw_mapping == NULL)
  {
    /* TODO Handle file opening errors more specifically,
     * e.g. inexistent file versus no read permission. */
    parse_data->raw_str.begin = NULL;
    g_set_error(error, 0, GNC_CSV_FILE_OPEN_ERR, _("File opening failed."));
    return 1;
  }

  /* Copy the mapping's contents into parse-data->raw_str. */
  parse_data->raw_str.begin = g_mapped_file_get_contents(parse_data->raw_mapping);
  parse_data->raw_str.end = parse_data->raw_str.begin + g_mapped_file_get_length(parse_data->raw_mapping);

  /* Make a guess at the encoding of the data. */
  guess_enc = go_guess_encoding((const char*)(parse_data->raw_str.begin),
                                (size_t)(parse_data->raw_str.end - parse_data->raw_str.begin),
                                "UTF-8", NULL);
  if(guess_enc == NULL)
  {
    g_set_error(error, 0, GNC_CSV_ENCODING_ERR, _("Unknown encoding."));
    return 1;
  }

  /* Convert using the guessed encoding into parse_data->file_str and
   * handle any errors that occur. */
  gnc_csv_convert_encoding(parse_data, guess_enc, error);
  if(parse_data->file_str.begin == NULL)
  {
    g_set_error(error, 0, GNC_CSV_ENCODING_ERR, _("Unknown encoding."));
    return 1;
  }
  else
    return 0;
}

int gnc_csv_parse(GncCsvParseData* parse_data, gboolean guessColTypes, GError** error)
{
  /* max_cols is the number of columns in the row with the most columns. */
  int i, max_cols = 0;

  if(parse_data->orig_lines != NULL)
  {
    stf_parse_general_free(parse_data->orig_lines);
  }

  /* If everything is fine ... */
  if(parse_data->file_str.begin != NULL)
  {
    /* Do the actual parsing. */
    parse_data->orig_lines = stf_parse_general(parse_data->options, parse_data->chunk,
                                               parse_data->file_str.begin,
                                               parse_data->file_str.end);
  }
  /* If we couldn't get the encoding right, we just want an empty array. */
  else
  {
    parse_data->orig_lines = g_ptr_array_new();
  }

  /* Record the original row lengths of parse_data->orig_lines. */
  if(parse_data->orig_row_lengths != NULL)
    g_array_free(parse_data->orig_row_lengths, FALSE);

  parse_data->orig_row_lengths =
    g_array_sized_new(FALSE, FALSE, sizeof(int), parse_data->orig_lines->len);
  g_array_set_size(parse_data->orig_row_lengths, parse_data->orig_lines->len);
  parse_data->orig_max_row = 0;
  for(i = 0; i < parse_data->orig_lines->len; i++)
  {
    int length = ((GPtrArray*)parse_data->orig_lines->pdata[i])->len;
    parse_data->orig_row_lengths->data[i] = length;
    if(length > parse_data->orig_max_row)
      parse_data->orig_max_row = length;
  }

  /* If it failed, generate an error. */
  if(parse_data->orig_lines == NULL)
  {
    g_set_error(error, 0, 0, "Parsing failed.");
    return 1;
  }

  /* Now that we have data, let's set max_cols. */
  for(i = 0; i < parse_data->orig_lines->len; i++)
  {
    if(max_cols < ((GPtrArray*)(parse_data->orig_lines->pdata[i]))->len)
      max_cols = ((GPtrArray*)(parse_data->orig_lines->pdata[i]))->len;
  }

  if(guessColTypes)
  {
    /* Free parse_data->column_types if it's already been created. */
    if(parse_data->column_types != NULL)
      g_array_free(parse_data->column_types, TRUE);

    /* Create parse_data->column_types and fill it with guesses based
     * on the contents of each column. */
    parse_data->column_types = g_array_sized_new(FALSE, FALSE, sizeof(int),
                                                 max_cols);
    g_array_set_size(parse_data->column_types, max_cols);
    /* TODO Make it actually guess. */
    for(i = 0; i < parse_data->column_types->len; i++)
    {
      parse_data->column_types->data[i] = GNC_CSV_NONE;
    }
  }
  else
  {
    /* If we don't need to guess column types, we will simply set any
     * new columns that are created that didn't exist before to "None"
     * since we don't want gibberish to appear. Note:
     * parse_data->column_types should have already been
     * initialized, so we don't check for it being NULL. */
    int i = parse_data->column_types->len;
    g_array_set_size(parse_data->column_types, max_cols);
    for(; i < parse_data->column_types->len; i++)
    {
      parse_data->column_types->data[i] = GNC_CSV_NONE;
    }
  }

  return 0;
}

typedef struct
{
  int date_format; 
  Account* account; 
  GList* properties; 
} TransPropertyList;

typedef struct
{
  int type; 
  void* value; 
  TransPropertyList* list; 
} TransProperty;

static TransProperty* trans_property_new(int type, TransPropertyList* list)
{
  TransProperty* prop = g_new(TransProperty, 1);
  prop->type = type;
  prop->list = list;
  prop->value = NULL;
  return prop;
}

static void trans_property_free(TransProperty* prop)
{
  switch(prop->type)
  {
    /* The types for "Date" and "Balance" (time_t and gnc_numeric,
     * respectively) are typically not pointed to, we have to free
     * them, unlike types like char* ("Description"). */
  case GNC_CSV_DATE:
  case GNC_CSV_BALANCE:
  case GNC_CSV_DEPOSIT:
  case GNC_CSV_WITHDRAWAL:
    if(prop->value != NULL)
      g_free(prop->value);
    break;
  }
  g_free(prop);
}

static gboolean trans_property_set(TransProperty* prop, char* str)
{
  char *endptr, *possible_currency_symbol, *str_dupe;
  double value;
  switch(prop->type)
  {
  case GNC_CSV_DATE:
    prop->value = g_new(time_t, 1);
    *((time_t*)(prop->value)) = parse_date(str, prop->list->date_format);
    return *((time_t*)(prop->value)) != -1;

  case GNC_CSV_DESCRIPTION:
  case GNC_CSV_NUM:
    prop->value = g_strdup(str);
    return TRUE;

  case GNC_CSV_BALANCE:
  case GNC_CSV_DEPOSIT:
  case GNC_CSV_WITHDRAWAL:
    str_dupe = g_strdup(str); /* First, we make a copy so we can't mess up real data. */

    /* Go through str_dupe looking for currency symbols. */
    for(possible_currency_symbol = str_dupe; *possible_currency_symbol;
        possible_currency_symbol = g_utf8_next_char(possible_currency_symbol))
    {
      if(g_unichar_type(g_utf8_get_char(possible_currency_symbol)) == G_UNICODE_CURRENCY_SYMBOL)
      {
        /* If we find a currency symbol, save the position just ahead
         * of the currency symbol (next_symbol), and find the null
         * terminator of the string (last_symbol). */
        char *next_symbol = g_utf8_next_char(possible_currency_symbol), *last_symbol = next_symbol;
        while(*last_symbol)
          last_symbol = g_utf8_next_char(last_symbol);

        /* Move all of the string (including the null byte, which is
         * why we have +1 in the size parameter) following the
         * currency symbol back one character, thereby overwriting the
         * currency symbol. */
        memmove(possible_currency_symbol, next_symbol, last_symbol - next_symbol + 1);
        break;
      }
    }

    /* Translate the string (now clean of currency symbols) into a number. */
    value = strtod(str_dupe, &endptr);

    /* If this isn't a valid numeric string, this is an error. */
    if(endptr != str_dupe + strlen(str_dupe))
    {
      g_free(str_dupe);
      return FALSE;
    }

    g_free(str_dupe);

    if(abs(value) > 0.00001)
    {
      prop->value = g_new(gnc_numeric, 1);
      *((gnc_numeric*)(prop->value)) =
        double_to_gnc_numeric(value, xaccAccountGetCommoditySCU(prop->list->account),
                              GNC_RND_ROUND);
    }
    return TRUE;
  }
  return FALSE; /* We should never actually get here. */
}

static TransPropertyList* trans_property_list_new(Account* account, int date_format)
{
  TransPropertyList* list = g_new(TransPropertyList, 1);
  list->account = account;
  list->date_format = date_format;
  list->properties = NULL;
  return list;
}

static void trans_property_list_free(TransPropertyList* list)
{
  /* Free all of the properties in this list before freeeing the list itself. */
  GList* properties_begin = list->properties;
  while(list->properties != NULL)
  {
    trans_property_free((TransProperty*)(list->properties->data));
    list->properties = g_list_next(list->properties);
  }
  g_list_free(properties_begin);
  g_free(list);
}

static void trans_property_list_add(TransProperty* property)
{
  property->list->properties = g_list_append(property->list->properties, property);
}

static void trans_add_split(Transaction* trans, Account* account, GNCBook* book,
                            gnc_numeric amount)
{
  Split* split = xaccMallocSplit(book);
  xaccSplitSetAccount(split, account);
  xaccSplitSetParent(split, trans);
  xaccSplitSetAmount(split, amount);
  xaccSplitSetValue(split, amount);
  xaccSplitSetAction(split, "Deposit");
}

static gboolean trans_property_list_verify_essentials(TransPropertyList* list, gchar** error)
{
  int i;
  /* possible_errors lists the ways in which a list can fail this test. */
  enum PossibleErrorTypes {NO_DATE, NO_AMOUNT, NUM_OF_POSSIBLE_ERRORS};
  gchar* possible_errors[NUM_OF_POSSIBLE_ERRORS] = {N_("No date column."),
                                                    N_("No balance, deposit, or withdrawal column.")};
  int possible_error_lengths[NUM_OF_POSSIBLE_ERRORS] = {0};
  GList *properties_begin = list->properties, *errors_list = NULL;

  /* Go through each of the properties and erase possible errors. */
  while(list->properties)
  {
    switch(((TransProperty*)(list->properties->data))->type)
    {
    case GNC_CSV_DATE:
      possible_errors[NO_DATE] = NULL;
      break;

    case GNC_CSV_BALANCE:
    case GNC_CSV_DEPOSIT:
    case GNC_CSV_WITHDRAWAL:
      possible_errors[NO_AMOUNT] = NULL;
      break;
    }
    list->properties = g_list_next(list->properties);
  }
  list->properties = properties_begin;

  /* Accumulate a list of the actual errors. */
  for(i = 0; i < NUM_OF_POSSIBLE_ERRORS; i++)
  {
    if(possible_errors[i] != NULL)
    {
      errors_list = g_list_append(errors_list, GINT_TO_POINTER(i));
      /* Since we added an error, we want to also store its length for
       * when we construct the full error string. */
      possible_error_lengths[i] = strlen(_(possible_errors[i]));
    }
  }

  /* If there are no errors, we can quit now. */
  if(errors_list == NULL)
    return TRUE;
  else
  {
    /* full_error_size is the full length of the error message. */
    int full_error_size = 0, string_length = 0;
    GList* errors_list_begin = errors_list;
    gchar *error_message, *error_message_begin;

    /* Find the value for full_error_size. */
    while(errors_list)
    {
      /* We add an extra 1 to account for spaces in between messages. */
      full_error_size += possible_error_lengths[GPOINTER_TO_INT(errors_list->data)] + 1;
      errors_list = g_list_next(errors_list);
    }
    errors_list = errors_list_begin;

    /* Append the error messages one after another. */
    error_message = error_message_begin = g_new(gchar, full_error_size);
    while(errors_list)
    {
      i = GPOINTER_TO_INT(errors_list->data);
      string_length = possible_error_lengths[i];
      
      /* Copy the error message and put a space after it. */
      strncpy(error_message, _(possible_errors[i]), string_length);
      error_message += string_length;
      *error_message = ' ';
      error_message++;
      
      errors_list = g_list_next(errors_list);
    }
    *error_message = '\0'; /* Replace the last space with the null byte. */
    g_list_free(errors_list_begin);

    *error = error_message_begin;
    return FALSE;
  }
}

static GncCsvTransLine* trans_property_list_to_trans(TransPropertyList* list, gchar** error)
{
  GncCsvTransLine* trans_line = g_new(GncCsvTransLine, 1);
  GList* properties_begin = list->properties;
  GNCBook* book = gnc_account_get_book(list->account);
  gnc_commodity* currency = xaccAccountGetCommodity(list->account);
  gnc_numeric amount = double_to_gnc_numeric(0.0, xaccAccountGetCommoditySCU(list->account),
                                             GNC_RND_ROUND);

  /* This flag is set to TRUE if we can use the "Deposit" or "Withdrawal" column. */
  gboolean amount_set = FALSE;

  /* The balance is 0 by default. */
  trans_line->balance_set = FALSE;
  trans_line->balance = amount;

  /* We make the line_no -1 just to mark that it hasn't been set. We
   * may get rid of line_no soon anyway, so it's not particularly
   * important. */
  trans_line->line_no = -1;

  /* Make sure this is a transaction with all the columns we need. */
  if(!trans_property_list_verify_essentials(list, error))
  {
    g_free(trans_line);
    return NULL;
  }

  trans_line->trans = xaccMallocTransaction(book);
  xaccTransBeginEdit(trans_line->trans);
  xaccTransSetCurrency(trans_line->trans, currency);

  /* Go through each of the properties and edit the transaction accordingly. */
  list->properties = properties_begin;
  while(list->properties != NULL)
  {
    TransProperty* prop = (TransProperty*)(list->properties->data);
    switch(prop->type)
    {
    case GNC_CSV_DATE:
      xaccTransSetDatePostedSecs(trans_line->trans, *((time_t*)(prop->value)));
      break;

    case GNC_CSV_DESCRIPTION:
      xaccTransSetDescription(trans_line->trans, (char*)(prop->value));
      break;

    case GNC_CSV_NUM:
      xaccTransSetNum(trans_line->trans, (char*)(prop->value));
      break;

    case GNC_CSV_DEPOSIT: /* Add deposits to the existing amount. */
      if(prop->value != NULL)
      {
        amount = gnc_numeric_add(*((gnc_numeric*)(prop->value)),
                                 amount,
                                 xaccAccountGetCommoditySCU(list->account),
                                 GNC_RND_ROUND);
        amount_set = TRUE;
        /* We will use the "Deposit" and "Withdrawal" columns in preference to "Balance". */
        trans_line->balance_set = FALSE;
      }
      break;

    case GNC_CSV_WITHDRAWAL: /* Withdrawals are just negative deposits. */
      if(prop->value != NULL)
      {
        amount = gnc_numeric_add(gnc_numeric_neg(*((gnc_numeric*)(prop->value))),
                                 amount,
                                 xaccAccountGetCommoditySCU(list->account),
                                 GNC_RND_ROUND);
        amount_set = TRUE;
        /* We will use the "Deposit" and "Withdrawal" columns in preference to "Balance". */
        trans_line->balance_set = FALSE;
      }
      break;

    case GNC_CSV_BALANCE: /* The balance gets stored in a separate field in trans_line. */
      /* We will use the "Deposit" and "Withdrawal" columns in preference to "Balance". */
      if(!amount_set && prop->value != NULL)
      {
        /* This gets put into the actual transaction at the end of gnc_csv_parse_to_trans. */
        trans_line->balance = *((gnc_numeric*)(prop->value));
        trans_line->balance_set = TRUE;
      }
      break;
    }
    list->properties = g_list_next(list->properties);
  }

  /* Add a split with the cumulative amount value. */
  trans_add_split(trans_line->trans, list->account, book, amount);

  return trans_line;
}

int gnc_csv_parse_to_trans(GncCsvParseData* parse_data, Account* account,
                           gboolean redo_errors)
{
  gboolean hasBalanceColumn;
  int i, j, max_cols = 0;
  GArray* column_types = parse_data->column_types;
  GList *error_lines = NULL, *begin_error_lines = NULL;

  /* last_transaction points to the last element in
   * parse_data->transactions, or NULL if it's empty. */
  GList* last_transaction = NULL;

  /* Free parse_data->error_lines and parse_data->transactions if they
   * already exist. */
  if(redo_errors) /* If we're redoing errors, we save freeing until the end. */
  {
    begin_error_lines = error_lines = parse_data->error_lines;
  }
  else
  {
    if(parse_data->error_lines != NULL)
    {
      g_list_free(parse_data->error_lines);
    }
    if(parse_data->transactions != NULL)
    {
      g_list_free(parse_data->transactions);
    }
  }
  parse_data->error_lines = NULL;

  if(redo_errors) /* If we're looking only at error data ... */
  {
    if(parse_data->transactions == NULL)
    {
      last_transaction = NULL;
    }
    else
    {
      /* Move last_transaction to the end. */
      last_transaction = parse_data->transactions;
      while(g_list_next(last_transaction) != NULL)
      {
        last_transaction = g_list_next(last_transaction);
      }
    }
    /* ... we use only the lines in error_lines. */
    if(error_lines == NULL)
      i = parse_data->orig_lines->len; /* Don't go into the for loop. */
    else
      i = GPOINTER_TO_INT(error_lines->data);
  }
  else /* Otherwise, we look at all the data. */
  {
    /* The following while-loop effectively behaves like the following for-loop:
     * for(i = 0; i < parse_data->orig_lines->len; i++). */
    i = 0;
    last_transaction = NULL;
  }
  while(i < parse_data->orig_lines->len)
  {
    GPtrArray* line = parse_data->orig_lines->pdata[i];
    /* This flag is TRUE if there are any errors in this row. */
    gboolean errors = FALSE;
    gchar* error_message = NULL;
    TransPropertyList* list = trans_property_list_new(account, parse_data->date_format);
    GncCsvTransLine* trans_line = NULL;

    for(j = 0; j < line->len; j++)
    {
      /* We do nothing in "None" columns. */
      if(column_types->data[j] != GNC_CSV_NONE)
      {
        /* Affect the transaction appropriately. */
        TransProperty* property = trans_property_new(column_types->data[j], list);
        gboolean succeeded = trans_property_set(property, line->pdata[j]);
        /* TODO Maybe move error handling to within TransPropertyList functions? */
        if(succeeded)
        {
          trans_property_list_add(property);
        }
        else
        {
          errors = TRUE;
          error_message = g_strdup_printf(_("%s column could not be understood."),
                                          _(gnc_csv_column_type_strs[property->type]));
          trans_property_free(property);
          break;
        }
      }
    }

    /* If we had success, add the transaction to parse_data->transaction. */
    if(!errors)
    {
      trans_line = trans_property_list_to_trans(list, &error_message);
      errors = trans_line == NULL;
    }

    trans_property_list_free(list);

    /* If there were errors, add this line to parse_data->error_lines. */
    if(errors)
    {
      parse_data->error_lines = g_list_append(parse_data->error_lines,
                                              GINT_TO_POINTER(i));
      /* If there's already an error message, we need to replace it. */
      if(line->len > (int)(parse_data->orig_row_lengths->data[i]))
      {
        g_free(line->pdata[line->len - 1]);
        line->pdata[line->len - 1] = error_message;
      }
      else
      {
        /* Put the error message at the end of the line. */
        g_ptr_array_add(line, error_message);
      }
    }
    else
    {
      /* If all went well, add this transaction to the list. */
      trans_line->line_no = i;

      /* We keep the transactions sorted by date. We start at the end
       * of the list and go backward, simply because the file itself
       * is probably also sorted by date (but we need to handle the
       * exception anyway). */

      /* If we can just put it at the end, do so and increment last_transaction. */
      if(last_transaction == NULL ||
         xaccTransGetDate(((GncCsvTransLine*)(last_transaction->data))->trans) <= xaccTransGetDate(trans_line->trans))
      {
        parse_data->transactions = g_list_append(parse_data->transactions, trans_line);
        /* If this is the first transaction, we need to get last_transaction on track. */
        if(last_transaction == NULL)
          last_transaction = parse_data->transactions;
        else /* Otherwise, we can just continue. */
          last_transaction = g_list_next(last_transaction);
      }
      /* Otherwise, search backward for the correct spot. */
      else
      {
        GList* insertion_spot = last_transaction;
        while(insertion_spot != NULL &&
              xaccTransGetDate(((GncCsvTransLine*)(insertion_spot->data))->trans) > xaccTransGetDate(trans_line->trans))
        {
          insertion_spot = g_list_previous(insertion_spot);
        }
        /* Move insertion_spot one location forward since we have to
         * use the g_list_insert_before function. */
        if(insertion_spot == NULL) /* We need to handle the case of inserting at the beginning of the list. */
          insertion_spot = parse_data->transactions;
        else
          insertion_spot = g_list_next(insertion_spot);
        
        parse_data->transactions = g_list_insert_before(parse_data->transactions, insertion_spot, trans_line);
      }
    }

    /* Increment to the next row. */
    if(redo_errors)
    {
      /* Move to the next error line in the list. */
      error_lines = g_list_next(error_lines);
      if(error_lines == NULL)
        i = parse_data->orig_lines->len; /* Don't continue the for loop. */
      else
        i = GPOINTER_TO_INT(error_lines->data);
    }
    else
    {
      i++;
    }
  }

  /* If we have a balance column, set the appropriate amounts on the transactions. */
  hasBalanceColumn = FALSE;
  for(i = 0; i < parse_data->column_types->len; i++)
  {
    if(parse_data->column_types->data[i] == GNC_CSV_BALANCE)
    {
      hasBalanceColumn = TRUE;
      break;
    }
  }

  if(hasBalanceColumn)
  {
    GList* transactions = parse_data->transactions;

    /* balance_offset is how much the balance currently in the account
     * differs from what it will be after the transactions are
     * imported. This will be sum of all the previous transactions for
     * any given transaction. */
    gnc_numeric balance_offset = double_to_gnc_numeric(0.0,
                                                       xaccAccountGetCommoditySCU(account),
                                                       GNC_RND_ROUND);
    while(transactions != NULL)
    {
      GncCsvTransLine* trans_line = (GncCsvTransLine*)transactions->data;
      if(trans_line->balance_set)
      {
        time_t date = xaccTransGetDate(trans_line->trans);
        /* Find what the balance should be by adding the offset to the actual balance. */
        gnc_numeric existing_balance = gnc_numeric_add(balance_offset,
                                                       xaccAccountGetBalanceAsOfDate(account, date),
                                                       xaccAccountGetCommoditySCU(account),
                                                       GNC_RND_ROUND);

        /* The amount of the transaction is the difference between the new and existing balance. */
        gnc_numeric amount = gnc_numeric_sub(trans_line->balance,
                                             existing_balance,
                                             xaccAccountGetCommoditySCU(account),
                                             GNC_RND_ROUND);

        SplitList* splits = xaccTransGetSplitList(trans_line->trans);
        while(splits)
        {
          SplitList* next_splits = g_list_next(splits);
          xaccSplitDestroy((Split*)splits->data);
          splits = next_splits;
        }

        trans_add_split(trans_line->trans, account, gnc_account_get_book(account), amount);
        
        /* This new transaction needs to be added to the balance offset. */
        balance_offset = gnc_numeric_add(balance_offset,
                                         amount,
                                         xaccAccountGetCommoditySCU(account),
                                         GNC_RND_ROUND);
      }
      transactions = g_list_next(transactions);
    }
  }

  if(redo_errors) /* Now that we're at the end, we do the freeing. */
  {
    g_list_free(begin_error_lines);
  }

  /* We need to resize parse_data->column_types since errors may have added columns. */
  for(i = 0; i < parse_data->orig_lines->len; i++)
  {
    if(max_cols < ((GPtrArray*)(parse_data->orig_lines->pdata[i]))->len)
      max_cols = ((GPtrArray*)(parse_data->orig_lines->pdata[i]))->len;
  }
  i = parse_data->column_types->len;
  parse_data->column_types = g_array_set_size(parse_data->column_types, max_cols);
  for(; i < max_cols; i++)
  {
    parse_data->column_types->data[i] = GNC_CSV_NONE;
  }

  return 0;
}

---