static int Export(AzureDataCatalog td, StreamWriter sw, string searchString)
{
const int countPerPage = 100;
bool firstTime = true;
int startPage = 1;
int totalResultsCount = 0;
int totalExportedSuccessfully = 0;
sw.Write("{\"catalog\":[");
do
{
string results = td.Search(searchString, startPage, countPerPage);
if (results == null)
{
return 0;
}
if (firstTime)
{
totalResultsCount = (int)JObject.Parse(results)["totalResults"];
}
var assetList = JObject.Parse(results)["results"].Children();
foreach (JObject asset in assetList["content"])
{
if (firstTime)
{
firstTime = false;
}
else
{
sw.Write(",");
}
(asset["properties"] as JObject).Remove("containerId");
var annotationsNode = asset.SelectToken("annotations") as JObject;
if (annotationsNode != null)
{
bool previewExists = annotationsNode.Remove("previews");
bool columnsDataProfilesExists = annotationsNode.Remove("columnsDataProfiles");
bool tableDataProfilesExist = annotationsNode.Remove("tableDataProfiles");
if (previewExists || columnsDataProfilesExists || tableDataProfilesExist)
{
var fullAsset = JObject.Parse(td.Get(asset["id"].ToString()));
if (previewExists)
{
annotationsNode.Add("previews", fullAsset["annotations"]["previews"]);
}
if (columnsDataProfilesExists)
{
annotationsNode.Add("columnsDataProfiles", fullAsset["annotations"]["columnsDataProfiles"]);
}
if (tableDataProfilesExist)
{
annotationsNode.Add("tableDataProfiles", fullAsset["annotations"]["tableDataProfiles"]);
}
}
}
//Set contributor equal to "Everyone" on all nodes. This allows them to be updated by others later. Ideally we would preserve the contributor but that requires
//a special platform to enable it.
JToken contributor = JObject.Parse("{'role': 'Contributor','members': [{'objectId': '00000000-0000-0000-0000-000000000201'}]}");
var roles = new JArray();
roles.Add(contributor);
foreach (var rolesNode in asset.SelectTokens("$..roles").ToList())
{
rolesNode.Replace(roles);
}
RemoveSystemProperties(asset);
sw.Write(JsonConvert.SerializeObject(asset));
totalExportedSuccessfully++;
if (totalExportedSuccessfully % 10 == 0)
{
Console.Write(".");
}
}
startPage++;
} while ((startPage - 1) * countPerPage < totalResultsCount);
sw.Write("]}");
Console.WriteLine("");
return totalExportedSuccessfully;
}