public ExportGriddedShapefile ( string tableName ) : bool | ||
tableName | string | |
return | bool |
public bool ExportGriddedShapefile(string tableName)
{
try
{
_log.DebugFormat("Retrieving contents of job {0}", tableName);
//don't filter out geometries, we'll do that at the cell level
var exportFeatures = GetShapeFeaturesToExport(tableName, false);
if ((exportFeatures == null) || (exportFeatures.Count == 0))
{
_log.ErrorFormat("Export of Job \"{0}\" failed, no features to export!", tableName);
return false;
}
ICoordinateSystem outputCrs = GeographicCoordinateSystem.WGS84;
if (!string.IsNullOrEmpty(this.OutputProjectionFilename))
{
outputCrs = Utilities.GetCoordinateSystemByWKTFile(this.OutputProjectionFilename);
}
//if we need to reproject:
List<IGeometry> filteringGeoms = GetFilteringGeometries(this.ExportFilterFilename, outputCrs);
//put everything into a basic spatial index
Envelope env = new Envelope();
var index = new GisSharpBlog.NetTopologySuite.Index.Strtree.STRtree();
for (int i = 0; i < exportFeatures.Count; i++)
{
Feature f = exportFeatures[i];
index.Insert(f.Geometry.EnvelopeInternal, f);
env.ExpandToInclude(f.Geometry.EnvelopeInternal);
}
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
index.Build();
//adjust envelope to only scan area inside filtering geometries
if (!string.IsNullOrEmpty(this.GridEnvelopeFilename))
{
//a specified envelope file overrides the envelope of the filtering geometries
env = GetGridEnvelope();
}
else if ((filteringGeoms != null) && (filteringGeoms.Count > 0))
{
//in the absence ... //TODO: finish--
env = new Envelope();
for (int i = 0; i < filteringGeoms.Count; i++)
{
env.ExpandToInclude(filteringGeoms[i].EnvelopeInternal);
}
}
//progress tracking
DateTime start = DateTime.Now, lastCheck = DateTime.Now;
int lastProgress = 0;
var features = new List<Feature>(exportFeatures.Count);
double cellWidth = GridCellWidth;
double cellHeight = GridCellHeight;
bool discardEmptyGridCells = !IncludeEmptyGridCells;
int numRows = (int)Math.Ceiling(env.Height / cellHeight);
int numCols = (int)Math.Ceiling(env.Width / cellWidth);
int expectedCells = numRows * numCols;
if (expectedCells > 1000000)
{
_log.Warn("**********************");
_log.Warn("Your selected area will produce a shapefile with over a million cells, is that a good idea?");
_log.WarnFormat("Area of {0}, Expected Cell Count of {1}", env.Area, expectedCells);
_log.Warn("**********************");
}
DbaseFileHeader header = null;
using (var conn = DbClient.GetConnection())
{
//Dictionary<string, DataRow> shapeDict = GetShapeRowsByLOGRECNO(conn);
var variablesDT = DataClient.GetMagicTable(conn, DbClient, string.Format("SELECT * FROM \"{0}\" where 0 = 1 ", tableName));
header = ShapefileHelper.SetupHeader(variablesDT);
}
ShapefileHelper.AddColumn(header, "CELLID", typeof(string));
ShapefileHelper.AddColumn(header, "GEOID", typeof(string));
if (this.AddStrippedGEOIDcolumn)
{
ShapefileHelper.AddColumn(header, "GEOID_STRP", typeof(string));
}
//lets not add these to the fishnet exports just yet
//if (this.AddGeometryAttributesToOutput)
//{
// ShapefileHelper.AddColumn(header, "AREA", typeof(double));
// ShapefileHelper.AddColumn(header, "PERIMETER", typeof(double));
// ShapefileHelper.AddColumn(header, "CENTROID", typeof(double));
//}
int cellCount = 0;
int xidx = 0;
for (double x = env.MinX; x < env.MaxX; x += cellWidth)
{
xidx++;
int yidx = 0;
for (double y = env.MinY; y < env.MaxY; y += cellHeight)
{
yidx++;
cellCount++;
string cellID = string.Format("{0}_{1}", xidx, yidx);
Envelope cellEnv = new Envelope(x, x + cellWidth, y, y + cellHeight);
IGeometry cellCenter = new Point(cellEnv.Centre);
IGeometry cellGeom = Utilities.IEnvToIGeometry(cellEnv);
Feature found = null;
IList mightMatch = index.Query(cellGeom.EnvelopeInternal);
foreach (Feature f in mightMatch)
{
if (f.Geometry.Contains(cellCenter))
{
found = f;
break;
}
}
if ((found == null) && (discardEmptyGridCells))
{
//_log.DebugFormat("No feature found for cell {0}", cellID);
continue;
}
//if we have filtering geometries, skip a cell if it isn't included
if (!IsIncluded(cellGeom, filteringGeoms))
{
continue;
}
if ((cellCount % 1000) == 0)
{
int step = (int)((((double)cellCount) / ((double)expectedCells)) * 100.0);
TimeSpan elapsed = (DateTime.Now - lastCheck);
if ((step != lastProgress) && (elapsed.TotalSeconds > 1))
{
_log.DebugFormat("{0:###.##}% complete, {1:#0.0#} seconds, {2} built, {3} checked, {4} left",
step, (DateTime.Now - start).TotalSeconds,
features.Count,
cellCount,
expectedCells - cellCount
);
lastCheck = DateTime.Now;
lastProgress = step;
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
}
}
//this is a lot of work just to add an id...
AttributesTable attribs = new AttributesTable();
if (found != null)
{
//if (!found.Attributes.GetNames().Contains("GEOID"))
// throw new Exception("GEOID NOT FOUND!!!!");
foreach (string name in found.Attributes.GetNames())
{
attribs.AddAttribute(name, found.Attributes[name]);
}
attribs.AddAttribute("CELLID", cellID);
}
else
{
foreach (var field in header.Fields)
{
attribs.AddAttribute(field.Name, null);
}
attribs["CELLID"] = cellID;
}
features.Add(new Feature(cellGeom, attribs));
}
}
_log.Debug("Done building cells, Saving Shapefile...");
header.NumRecords = features.Count;
if (features.Count == 0)
{
_log.Error("No features found, exiting!");
return false;
}
string newShapefilename = Path.Combine(Environment.CurrentDirectory, tableName);
if (!string.IsNullOrEmpty(OutputFolder))
{
newShapefilename = Path.Combine(OutputFolder, tableName);
}
if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; }
var writer = new ShapefileDataWriter(newShapefilename, ShapefileHelper.GetGeomFactory());
writer.Header = header;
writer.Write(features);
if (!string.IsNullOrEmpty(this.OutputProjectionFilename))
{
//Reproject everything in this file to the requested projection...
ShapefileHelper.MakeOutputProjFile(this.OutputProjectionFilename, newShapefilename);
}
else
{
ShapefileHelper.MakeCensusProjFile(newShapefilename);
}
_log.Debug("Done! Shapefile exported successfully");
return true;
}
catch (FileNotFoundException notFound)
{
string msg = "A needed file couldn't be found: " + notFound.FileName;
_log.Error(msg);
_log.Fatal("The export cannot continue. Exiting...");
throw new ApplicationException(msg);
}
catch (Exception ex)
{
_log.Error("Error while exporting shapefile", ex);
}
return false;
}
/// <summary> /// Performs our main 'lifecycle' /// </summary> /// <returns></returns> public bool ExecuteJob() { _manager = null; _cancelled = false; this.UpdateProgress(0); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } DateTime startTime = DateTime.Now; try { if (!string.IsNullOrEmpty(DisplaySummaryLevels)) { Utilities.DisplayEnum<BoundaryLevels>("Summary Levels:", "{1} - {0}", new HashSet<string>(new string[] { "None" }) ); return true; } if (!string.IsNullOrEmpty(DisplayStateCodes)) { Utilities.DisplayEnumKeysOnly("State Codes:", typeof(AcsState), //Do not display internal control value new HashSet<string>(new string[] { "None" }) ); return true; } if (!string.IsNullOrEmpty(ListYears)) { var years = Settings.LoadYearConfigs(); //_log.DebugFormat("I found {0} year config files ", years.Count); _log.InfoFormat("I found {0} years available:", years.Count); foreach (var key in years.Keys) { //_log.DebugFormat("{0} - {1}", key, years[key].GetFilename()); _log.InfoFormat(" * {0} ", key); } _log.InfoFormat(Environment.NewLine + "Done!"); return true; } if (this.State == AcsState.None) { _log.Error("Invalid State selected, please select a state from the list and try again."); return false; } if ((string.IsNullOrEmpty(this.JobName)) || (this.JobName == true.ToString())) { this.JobName = string.Format("{0}_{1}_{2}", this.Year, this.State, DateTime.Now.ToShortDateString().Replace('/', '_')); _log.DebugFormat("Jobname was empty, using {0}", this.JobName); } WorkingFolder = FileUtilities.CleanPath(WorkingFolder); var manager = new AcsDataManager(this.State, WorkingFolder, this.Year); this._manager = manager; manager.WorkOffline = this.WorkOffline; //TODO: check for bad combinations of inputs manager.SummaryLevel = this.SummaryLevel; manager.ExportFilterFilename = this.ExportFilterShapefile; manager.DesiredVariablesFilename = IncludedVariableFile; manager.ReusePreviousJobTable = (!string.IsNullOrEmpty(this.ReusePreviousJobTable)); manager.OutputProjectionFilename = this.OutputProjection; manager.PreserveJam = (!string.IsNullOrEmpty(this.PreserveJam)); manager.AddStrippedGEOIDcolumn = (!string.IsNullOrEmpty(this.AddStrippedGEOIDcolumn)); manager.AddGeometryAttributesToOutput = (!string.IsNullOrEmpty(this.AddGeometryAttributesToOutput)); manager.OutputFolder = FileUtilities.CleanPath(OutputFolder); manager.IncludeEmptyGridCells = (!string.IsNullOrEmpty(this.IncludeEmptyGridCells)); if (FileUtilities.SafePathEnsure(OutputFolder) != OutputFolder) { _log.ErrorFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder); _log.FatalFormat("Unable to set or create output folder, ( {0} ) exiting", OutputFolder); return false; } this.UpdateProgress(25); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (string.IsNullOrEmpty(this.OutputProjection)) { _log.Warn(Constants.Warning_MissingProjection); } _log.Debug("\r\n/*************************************/"); _log.Info(" Loading Prerequisites..."); _log.Debug("/*************************************/"); bool hasPrerequesites = true; hasPrerequesites &= !IsCancelled() && manager.CheckColumnMappingsFile(); hasPrerequesites &= !IsCancelled() && manager.CheckCensusAggregatedDataFile(); hasPrerequesites &= !IsCancelled() && manager.CheckDatabase(); hasPrerequesites &= !IsCancelled() && manager.CheckShapefiles(); if (!hasPrerequesites) { _log.Info("Loading Prerequisites... Failed!"); _log.Error("Import cannot continue, one or more prerequisites failed!"); return false; } _log.Debug("Loading Prerequisites... Done!\r\n"); this.UpdateProgress(35); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (!string.IsNullOrEmpty(IncludedVariableFile) && !string.IsNullOrEmpty(this.JobName)) { _log.Info("\r\n/*************************************/"); _log.Info(" Importing requested variables... "); _log.Info("/*************************************/"); if (!manager.CheckBuildVariableTable(this.JobName)) { _log.Error("Importing requested variables... Failed! A problem was detected, exiting."); return false; } else { _log.Debug("Importing requested variables... Done!"); } } this.UpdateProgress(50); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (!string.IsNullOrEmpty(ExportToShapefile)) { _log.Debug("\r\n/*************************************/"); _log.Info(" Exporting to shapefile... "); _log.Debug("/*************************************/"); if (!manager.ExportShapefile(this.JobName)) { _log.Error("There was an error while exporting the shapefile"); _log.Debug("\r\nExporting to shapefile... Failed!"); } else { _log.Debug("Exporting to shapefile... Done!"); } } else { _log.Debug("\r\n/**** No shapefile export requested ****/\r\n"); } this.UpdateProgress(75); if (IsCancelled()) { _log.Debug("Job Cancelled..."); return false; } if (!string.IsNullOrEmpty(ExportToGrid)) { _log.Debug("\r\n/*************************************/"); _log.Info(" Exporting to gridded shapefile..."); _log.Debug("/*************************************/"); manager.GridEnvelopeFilename = GridEnvelope; manager.SetGridParam(ExportToGrid); _log.DebugFormat("Exporting all requested variables to fishnet shapefile with grid cell size {0} ", ExportToGrid); manager.ExportGriddedShapefile(this.JobName); _log.Debug("Exporting to gridded shapefile... Done!"); } else { _log.Debug("\r\n/**** No gridded shapefile export requested ****/\r\n"); } this.UpdateProgress(100); _log.Info("Done!"); return true; } catch (Exception ex) { _log.Error("Error thrown during import job ", ex); _log.Fatal("Please see the errors log file for details"); } finally { TimeSpan elapsed = DateTime.Now - startTime; if (IsCancelled()) { _log.DebugFormat("Job was cancelled and stopped at {0} seconds", elapsed.TotalSeconds); } else { _log.DebugFormat("Job completed in {0} seconds", elapsed.TotalSeconds); } } return false; }