BlobSync.AzureOps.DefragBlob C# (CSharp) Method

DefragBlob() public method

Merge smaller blocks into something at least fragmentMergeSize bytes long. Only upload at most maxUploadLimit (0 == no limit). Should this be in CommonOps? Lame... really? DEFRAG? Then again I suppose the term IS appropriate.
public DefragBlob ( SizeBasedCompleteSignature blobSig, long maxUploadLimitMB = 2 ) : void
blobSig BlobSync.Datatypes.SizeBasedCompleteSignature
maxUploadLimitMB long
return void
        public void DefragBlob(SizeBasedCompleteSignature blobSig, long maxUploadLimitMB = 2)
        {
            var allBlobSigs = blobSig.Signatures.Values.SelectMany(x => x.SignatureList).OrderBy(a => a.Offset).ToList();

            var targetSigSize = ConfigHelper.SignatureSize;

            // loop through sigs, merge what we can but dont exceed maxUploadLimit
            long bytesToUpload = 0;
            var byteRangesToUpload = new List<RemainingBytes>();
            var defragNodeList = new List<DefragNode>();
            for (var i = 0; i < allBlobSigs.Count; i++)
            {
                uint sigSize = 0;
                var j = i;

                while (j < allBlobSigs.Count)
                {
                    var sig = allBlobSigs[j];
                    j++;

                    // break if we get too big.
                    if (sigSize + sig.Size > targetSigSize)
                    {
                        break;
                    }

                    sigSize += sig.Size;

                }

                defragNodeList.Add(new DefragNode { Offset = allBlobSigs[i].Offset, Size = sigSize, SigPos = i, NoSigs = j - i - 1 });
            }

            // defragNodeList is a list of sigs, and size. These ones will be merged.
            var sortedList = defragNodeList.OrderByDescending(n => n.NoSigs).ToList();

            // the entries in defragNodeList that has the max number of sigs in it (ie most fragmentation) will be the ones to get merged.
            foreach( var sig in sortedList)
            {
                DefragSigGroup(blobSig, sig);
                bytesToUpload += sig.Size;

                if (bytesToUpload > maxUploadLimitMB)
                {
                    break;
                }
            }
        }

Same methods

AzureOps::DefragBlob ( string filePath, long maxUploadLimitMB = 2 ) : void
AzureOps::DefragBlob ( string containerName, string blobName, long maxUploadLimitMB = 2 ) : void

Usage Example

Example #1
0
        static void Main(string[] args)
        {
            string command;
            string fileName;

            int parallelFactor = ConfigHelper.ParallelFactor;

            var sw = new Stopwatch();
            sw.Start();

            if (args.Length == 4)
            {
                command = args[0];
                fileName = args[1];
                var containerName = args[2];
                var blobName = args[3];
                var azureOps = new AzureOps();

                switch (command)
                {
                    case "upload":
                        var bytesUploaded = azureOps.UploadFile(containerName, blobName, fileName, parallelFactor);
                        Console.WriteLine("Uploaded {0} bytes", bytesUploaded);
                        break;
                    case "download":
                        var bytesDownloaded = azureOps.DownloadBlob(containerName, blobName, fileName, parallelFactor);
                        Console.WriteLine("Downloaded {0} bytes", bytesDownloaded);
                        break;
                    case "estimate":
                        var estimate = azureOps.CalculateDeltaSize(containerName, blobName, fileName);
                        Console.WriteLine( string.Format("Estimate to upload {0} bytes", estimate));
                        break;
                    default:
                        ShowExamples();
                        break;
                }
            }
            else if (args.Length == 3)
            {
                command = args[0];
                fileName = args[1];
                var localSigPath = args[2];
                var azureOps = new AzureOps();

                switch (command)
                {
                    case "estimatelocal":
                        var estimatelocal = azureOps.CalculateDeltaSizeFromLocalSig(localSigPath, fileName);
                        Console.WriteLine(string.Format("Estimate to upload {0} bytes", estimatelocal));
                        break;
                    case "createdeltasig":
                        var sig = azureOps.GenerateDeltaSigFromLocalResources(localSigPath, fileName);

                        var sigFileName = fileName + ".sig";
                        using (Stream s = new FileStream(sigFileName, FileMode.Create))
                        {
                            SerializationHelper.WriteBinarySizedBasedSignature(sig, s);
                        }
                        break;
                    case "showblocklist":

                        azureOps.GetBlockListInfo(args[1], args[2]);
                        break;
                    case "defrag":
                        azureOps.DefragBlob(args[1], args[2]);
                        break;

                    default:
                        ShowExamples();
                        break;
                }
            }
            else
            if (args.Length == 2)
            {
                command = args[0];
                fileName = args[1];
                var azureOps = new AzureOps();
                switch (command)
                {
                    case "createsig":
                        var sig = CommonOps.CreateSignatureForLocalFile(fileName);

                        var sigFileName = fileName + ".sig";
                        using (Stream s = new FileStream(sigFileName, FileMode.Create))
                        {
                            SerializationHelper.WriteBinarySizedBasedSignature(sig, s);
                        }

                        break;
                    case "showsig":
                        using (var fs = new FileStream(fileName, FileMode.Open))
                        {
                            var loadedSig = SerializationHelper.ReadSizeBasedBinarySignature(fs);

                            foreach( var sigSize in loadedSig.Signatures)
                            {
                                foreach( var s in sigSize.Value.SignatureList.OrderBy( s => s.Offset))
                                {
                                    Console.WriteLine(string.Format("{0}:{1}", s.Offset, s.Size));

                                }
                            }
                        }
                        break;
                    case "defraglocal":
                        azureOps.DefragBlob(args[1]);
                        break;
                    default:
                        ShowExamples();
                          break;
                }

            }
            else
            {
                ShowExamples();

            }

            sw.Stop();
            Console.WriteLine("Took {0}s", (double)sw.ElapsedMilliseconds / 1000.0);
        }