public CheckBuildVariableTable ( string tableName ) : bool | ||
tableName | string | |
Результат | bool |
public bool CheckBuildVariableTable(string tableName)
{
try
{
using (var conn = DbClient.GetConnection())
{
if (DataClient.HasTable(conn, DbClient, tableName))
{
if (!ReusePreviousJobTable)
{
DbClient.GetCommand(string.Format("DROP TABLE IF EXISTS \"{0}\";", tableName), conn).ExecuteNonQuery();
}
else
{
_log.DebugFormat("Table {0} was already built", tableName);
return true;
}
}
//gets a list of the LRUs we want
HashSet<string> requestedLRNs = GetFilteredLRUs(conn);
//gets all the variable mapping information we need
DataTable reqVariablesDT = GetRequestedVariables(conn);
if ((reqVariablesDT == null) || (reqVariablesDT.Rows.Count == 0))
{
_log.Info("Zero variables found: I couldn't understand those variables, can you check them and try again?");
//_log.Warn("I didn't understand those variables, can you check them and try again?");
return false;
}
DataTable newTable = new DataTable();
newTable.Columns.Add("LOGRECNO", typeof(string));
Dictionary<string, DataRow> rowsByLRN = new Dictionary<string, DataRow>(requestedLRNs.Count);
foreach (var id in requestedLRNs)
{
var row = newTable.NewRow();
row[0] = id;
newTable.Rows.Add(row);
rowsByLRN[id] = row;
}
bool shouldNotParseErrorValues = PreserveJam;
_log.Debug("Importing Columns");
int varNum = 0;
foreach (DataRow variableRow in reqVariablesDT.Rows)
{
if (this.IsCancelled()) { return false; }
varNum++;
var sequenceNo = Utilities.GetAs<int>(variableRow["SEQNO"] as string, -1);
var seqFile = Directory.GetFiles(this.GetAggregateDataPath(), "e*" + sequenceNo.ToString("0000") + "000.txt"); //0001000
if ((seqFile == null) || (seqFile.Length == 0))
{
_log.DebugFormat("Couldn't find sequence file {0}", sequenceNo);
continue;
}
var errorFile = Directory.GetFiles(this.GetAggregateDataPath(), "m*" + sequenceNo.ToString("0000") + "000.txt"); //0001000
if ((errorFile == null) || (errorFile.Length == 0))
{
_log.DebugFormat("Couldn't find error margin file {0}", sequenceNo);
}
//These next two if statement checks should probably be removed once
//#19869 is resolved
//Until that case is resolved, we can't guarantee rows in reqVariablesDT will be
//unique so they should stay.
//TODO: alternate column naming?
string newColumnName = variableRow["COLNAME"] as string;
if (newTable.Columns.Contains(newColumnName))
{
newColumnName = newColumnName + varNum;
}
newTable.Columns.Add(newColumnName, typeof(double));
//this really ought to be unique.
string newErrorMarginColumnName = "m" + newColumnName;
if (newTable.Columns.Contains(newErrorMarginColumnName))
{
newErrorMarginColumnName = newErrorMarginColumnName + varNum;
}
if (shouldNotParseErrorValues)
{
newTable.Columns.Add(newErrorMarginColumnName, typeof(string));
}
else
{
newTable.Columns.Add(newErrorMarginColumnName, typeof(double));
}
_log.DebugFormat("Importing {0}...", newColumnName);
int columnIDX = Utilities.GetAs<int>(variableRow["COLNO"] as string, -1);
CommaSeparatedValueReader reader = new CommaSeparatedValueReader(seqFile[0], false);
foreach (List<string> values in reader)
{
string lrn = values[5];
if (!requestedLRNs.Contains(lrn))
continue;
if (columnIDX < values.Count)
{
double val = Utilities.GetAs<double>(values[columnIDX], double.NaN);
if (!double.IsNaN(val))
{
rowsByLRN[lrn][newColumnName] = val;
}
}
if (this.IsCancelled()) { break; }
}
reader.Close();
//these error files better have the exact same format!
reader = new CommaSeparatedValueReader(errorFile[0], false);
foreach (List<string> values in reader)
{
string lrn = values[5];
if (!requestedLRNs.Contains(lrn))
continue;
if (columnIDX < values.Count)
{
if (shouldNotParseErrorValues)
{
rowsByLRN[lrn][newErrorMarginColumnName] = values[columnIDX];
}
else
{
double val = Utilities.GetAs<double>(values[columnIDX], double.NaN);
if (!double.IsNaN(val))
{
rowsByLRN[lrn][newErrorMarginColumnName] = val;
}
rowsByLRN[lrn][newErrorMarginColumnName] = val;
}
}
if (this.IsCancelled()) { break; }
}
reader.Close();
}
if (this.IsCancelled()) { return false; }
_log.DebugFormat("Creating Table {0}", tableName);
string createTableSQL = SqliteDataClient.GenerateTableSQLFromTable(tableName, newTable, "LOGRECNO");
DbClient.GetCommand(createTableSQL, conn).ExecuteNonQuery();
if (this.IsCancelled()) { return false; }
_log.DebugFormat("Saving Table {0}...", tableName);
var dba = DataClient.GetMagicAdapter(conn, DbClient, string.Format("SELECT * FROM \"{0}\"", tableName));
dba.Update(newTable);
_log.Debug("Done!");
_log.Debug("Import complete!");
//TODO: save the table to the database
}
return true;
}
catch (Exception ex)
{
_log.Error("Error while building table", ex);
}
return false;
}
/// <summary> /// Performs our main 'lifecycle' /// </summary> /// <returns></returns> public bool ExecuteJob() { _manager = null; _cancelled = false; this.UpdateProgress(0); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } DateTime startTime = DateTime.Now; try { if (!string.IsNullOrEmpty(DisplaySummaryLevels)) { Utilities.DisplayEnum<BoundaryLevels>("Summary Levels:", "{1} - {0}", new HashSet<string>(new string[] { "None" }) ); return true; } if (!string.IsNullOrEmpty(DisplayStateCodes)) { Utilities.DisplayEnumKeysOnly("State Codes:", typeof(AcsState), //Do not display internal control value new HashSet<string>(new string[] { "None" }) ); return true; } if (!string.IsNullOrEmpty(ListYears)) { var years = Settings.LoadYearConfigs(); //_log.DebugFormat("I found {0} year config files ", years.Count); _log.InfoFormat("I found {0} years available:", years.Count); foreach (var key in years.Keys) { //_log.DebugFormat("{0} - {1}", key, years[key].GetFilename()); _log.InfoFormat(" * {0} ", key); } _log.InfoFormat(Environment.NewLine + "Done!"); return true; } if (this.State == AcsState.None) { _log.Error("Invalid State selected, please select a state from the list and try again."); return false; } if ((string.IsNullOrEmpty(this.JobName)) || (this.JobName == true.ToString())) { this.JobName = string.Format("{0}_{1}_{2}", this.Year, this.State, DateTime.Now.ToShortDateString().Replace('/', '_')); _log.DebugFormat("Jobname was empty, using {0}", this.JobName); } WorkingFolder = FileUtilities.CleanPath(WorkingFolder); var manager = new AcsDataManager(this.State, WorkingFolder, this.Year); this._manager = manager; manager.WorkOffline = this.WorkOffline; //TODO: check for bad combinations of inputs manager.SummaryLevel = this.SummaryLevel; manager.ExportFilterFilename = this.ExportFilterShapefile; manager.DesiredVariablesFilename = IncludedVariableFile; manager.ReusePreviousJobTable = (!string.IsNullOrEmpty(this.ReusePreviousJobTable)); manager.OutputProjectionFilename = this.OutputProjection; manager.PreserveJam = (!string.IsNullOrEmpty(this.PreserveJam)); manager.AddStrippedGEOIDcolumn = (!string.IsNullOrEmpty(this.AddStrippedGEOIDcolumn)); manager.AddGeometryAttributesToOutput = (!string.IsNullOrEmpty(this.AddGeometryAttributesToOutput)); manager.OutputFolder = FileUtilities.CleanPath(OutputFolder); manager.IncludeEmptyGridCells = (!string.IsNullOrEmpty(this.IncludeEmptyGridCells)); if (FileUtilities.SafePathEnsure(OutputFolder) != OutputFolder) { _log.ErrorFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder); _log.FatalFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder); return false; } this.UpdateProgress(25); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (string.IsNullOrEmpty(this.OutputProjection)) { _log.Warn(Constants.Warning_MissingProjection); } _log.Debug("\r\n/*************************************/"); _log.Info(" Loading Prerequisites..."); _log.Debug("/*************************************/"); bool hasPrerequesites = true; hasPrerequesites &= !IsCancelled() && manager.CheckColumnMappingsFile(); hasPrerequesites &= !IsCancelled() && manager.CheckCensusAggregatedDataFile(); hasPrerequesites &= !IsCancelled() && manager.CheckDatabase(); hasPrerequesites &= !IsCancelled() && manager.CheckShapefiles(); if (!hasPrerequesites) { _log.Info("Loading Prerequisites... Failed!"); _log.Error("Import cannot continue, one or more prerequisites failed!"); return false; } _log.Debug("Loading Prerequisites... Done!\r\n"); this.UpdateProgress(35); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (!string.IsNullOrEmpty(IncludedVariableFile) && !string.IsNullOrEmpty(this.JobName)) { _log.Info("\r\n/*************************************/"); _log.Info(" Importing requested variables... "); _log.Info("/*************************************/"); if (!manager.CheckBuildVariableTable(this.JobName)) { _log.Error("Importing requested variables... Failed! A problem was detected, exiting."); return false; } else { _log.Debug("Importing requested variables... Done!"); } } this.UpdateProgress(50); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (!string.IsNullOrEmpty(ExportToShapefile)) { _log.Debug("\r\n/*************************************/"); _log.Info(" Exporting to shapefile... "); _log.Debug("/*************************************/"); if (!manager.ExportShapefile(this.JobName)) { _log.Error("There was an error while exporting the shapefile"); _log.Debug("\r\nExporting to shapefile... Failed!"); } else { _log.Debug("Exporting to shapefile... Done!"); } } else { _log.Debug("\r\n/**** No shapefile export requested ****/\r\n"); } this.UpdateProgress(75); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (!string.IsNullOrEmpty(ExportToGrid)) { _log.Debug("\r\n/*************************************/"); _log.Info(" Exporting to gridded shapefile..."); _log.Debug("/*************************************/"); manager.GridEnvelopeFilename = GridEnvelope; manager.SetGridParam(ExportToGrid); _log.DebugFormat("Exporting all requested variables to fishnet shapefile with grid cell size {0} ", ExportToGrid); manager.ExportGriddedShapefile(this.JobName); _log.Debug("Exporting to gridded shapefile... Done!"); } else { _log.Debug("\r\n/**** No gridded shapefile export requested ****/\r\n"); } this.UpdateProgress(100); _log.Info("Done!"); return true; } catch (Exception ex) { _log.Error("Error thrown during import job ", ex); _log.Fatal("Please see the errors log file for details"); } finally { TimeSpan elapsed = DateTime.Now - startTime; if (IsCancelled()) { _log.DebugFormat("Job was cancelled and stopped at {0} seconds", elapsed.TotalSeconds); } else { _log.DebugFormat("Job completed in {0} seconds", elapsed.TotalSeconds); } } return false; }