NetHadoop.FSClient.MutUpload C# (CSharp) Method

MutUpload() public method

public MutUpload ( BackgroundWorker worker, List localPath, string remoteRootPath, string localRootPath ) : void
worker System.ComponentModel.BackgroundWorker
localPath List
remoteRootPath string
localRootPath string
return void
        public void MutUpload(BackgroundWorker worker, List<string>localPath, string remoteRootPath,string localRootPath)
        {
            //相同操作
               bool sameOp = false;
               //是否覆盖
               bool IsOver = false;
               //准备创建连接
               Thrift.Transport.TBufferedTransport btsport = null;
               ThriftHadoopFileSystem.Client thriftClient = Connect(out btsport);
               List<string> NoSuccessList = new List<string>();
               List<string> skipList = new List<string>();
               if (thriftClient != null)//连接成功
               {
               int totalCount = localPath.Count;
               int currentCount = 0;
               //开始上传
               worker.ReportProgress(0, new ProgressState() { ListBoxMsg = "开始上传!",totalCount=totalCount, CurrentCount=0 });
               //循环
               foreach (string localFilePath in localPath)
               {
                   string fileName = Path.GetFileName(localFilePath);
                   if (!string.IsNullOrEmpty(localRootPath))//如果是文件夹,则包含原有路径
                       fileName = localFilePath.Replace(localRootPath, "");

                   string remoteFilePath = remoteRootPath + "/" + fileName;
                   currentCount++;
                   int pgPresent = (int)((double)currentCount / totalCount * 100);
                   //显示总进度
                   worker.ReportProgress(pgPresent, new ProgressState() { CurrentTitle = fileName, CurrentCount = currentCount });

                   #region 是否存在
                   if (!sameOp)
                   {
                       bool exsitFile = thriftClient.exists(new Pathname() { pathname = remoteFilePath });
                       if (exsitFile)
                       {
                           SureDialog myDialog = new SureDialog();
                           MyShowDialogResult myDR = new MyShowDialogResult();
                           myDialog.ShowDialog(fileName, myDR);
                           sameOp = myDR.IsCheck;
                           IsOver = myDR.Result;
                           if (!myDR.Result)
                           {
                               worker.ReportProgress(pgPresent, new ProgressState() { ListBoxMsg =  fileName +" 跳过 " });
                               skipList.Add(fileName);
                               continue;
                           }
                       }
                   }
                   else
                   {
                        bool exsitFile = thriftClient.exists(new Pathname() { pathname = remoteFilePath });
                        if (exsitFile)
                        {
                            if (!IsOver)
                            {
                                worker.ReportProgress(pgPresent, new ProgressState() { ListBoxMsg =fileName + "跳过 " });
                                skipList.Add(fileName);
                                continue;
                            }
                        }
                   }
                   #endregion

                    #region 上传单个文件
                   bool singleResult = false;
                   ThriftHandle th = null;
                   FileStream fs = null;
                   try
                   {
                       Pathname myNewFile=new Pathname() { pathname = remoteFilePath };

                       //创建一个文件
                       th = thriftClient.createFile(myNewFile, 1, true, 1024*1024*10, ConfigHelper.HDFSREPLICATION, 1024 * 1024 * 512);

                       UTF8Encoding utf8 = new UTF8Encoding(false, true);

                       fs = new FileStream(localFilePath, FileMode.Open, FileAccess.Read);

                       byte[] fileBuffer = new byte[1024 * 1024*10];	// 每次传1MB
                       int bytesRead;
                       long bytesTotal = 0;

                       while ((bytesRead = fs.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
                       {
                           bytesTotal += bytesRead;
                           byte[] realBuffer = new byte[bytesRead];
                           Array.Copy(fileBuffer, realBuffer, bytesRead);
                           //将utf8转为可存储编码
                           realBuffer = Encoding.Convert(Encoding.GetEncoding("iso-8859-1"), utf8, realBuffer);
                           //发送
                           thriftClient.write(th, realBuffer);
                           //清仓缓存
                           Array.Clear(fileBuffer, 0, fileBuffer.Length);
                           realBuffer = null;
                           //显示单个上传进度
                           int mypresent = (int)((double)bytesTotal / fs.Length * 100);
                           worker.ReportProgress(pgPresent, new ProgressState() { CurrentTitle = mypresent + "% " + fileName });
                       }
                       singleResult = true;
                   }
                   catch (Exception ee)
                   {
                       //显示上传错误
                       worker.ReportProgress(pgPresent, new ProgressState() { ListBoxMsg = ee.Message });
                   }
                   finally
                   {
                       if (th != null)
                           thriftClient.close(th);
                       if (fs != null)
                           fs.Close();
                   }
                    #endregion
                   //显示单个上传结果
                   string msg = string.Format("{0} 上传{1}", fileName, singleResult ? "成功" : "失败");
                   worker.ReportProgress(pgPresent, new ProgressState() { ListBoxMsg = msg });
                   if (!singleResult)
                   {
                       NoSuccessList.Add(fileName);
                   }
               }
               }

               //释放连接
               if (btsport != null)
               {
               btsport.Close();
               }
               //输出没有上传成功的
               if (NoSuccessList.Count > 0)
               {
               try
               {
                   File.WriteAllText("c:/UploadNoSuccess.txt", string.Join("\r\n", NoSuccessList.ToArray()));
                   worker.ReportProgress(100, new ProgressState() { ListBoxMsg = NoSuccessList.Count + "个上传错误!请查看c:/UploadNoSuccess.txt" });
               }
               catch (Exception ee)
               {
                   worker.ReportProgress(100, new ProgressState() { ListBoxMsg =ee.Message  });
               }
               }
               //输出跳过的
               if (skipList.Count > 0)
               {
               try
               {
                   File.WriteAllText("c:/UploadSkip.txt", string.Join("\r\n", skipList.ToArray()));
                   worker.ReportProgress(100, new ProgressState() { ListBoxMsg = "跳过" + skipList.Count + "个文件!请查看c:/UploadSkip.txt" });
               }
               catch (Exception ee)
               {
                   worker.ReportProgress(100, new ProgressState() { ListBoxMsg = ee.Message });
               }
               }
        }

Usage Example

示例#1
0
        //上传动作
        private void DoUploadWork(object sender, DoWorkEventArgs e)
        {
            UploadArg fileNames = e.Argument as UploadArg;

            FSClient fs = new FSClient();

            fs.MutUpload(worker, fileNames.FilePathList, ConfigHelper.HdfsRoot + CurrentPath, fileNames.LocalFileRoot);
        }
All Usage Examples Of NetHadoop.FSClient::MutUpload