Introduction
I recently needed to give an existing application the ability to download files via HTTP from Amazons S3 service for storing files. I found a lot of different sources for doing different bits of the job, but it took some time to collect everything I needed into one working class.
I thought it might be a good idea to share this library with others.
Background
One of the main things missing from code samples showing how to download files was any form of verification that the download was successful and no data was lost. I used separate sources to find out how to get the file length from the header of the URL and, if available, how to get the MD5 checksum of the file from the ETag. It took me a bit of time to throw it all together, so I though I might save someone some time if I provide the library for them.
Using the code
You can download a file by simply calling the boolean function DownloadFile() and passing it the URL, the directory to download the file to, the file name with extension and specifying the boolean whether you want to check the hash of the downloaded file, or if not available, rely on the file length for confirming successful download and doing any other checking after. Some example code is:
HTTPLib.HTTP _httpLib = new HTTPLib.HTTP();
if (_httpLib.DownloadFile(
"https://s3-downloadURL/", @"C:\DownloadDest\",
"File name with extension", true))
{
return true;
}
The file is downloaded with a .PART added after the file extension, and on verifying successful download, the .Part is removed.
You can find the CS file in the attached zip file. Also here is the code in full:
using System;
using System.IO;
using System.Net;
using System.Security.Cryptography;
namespace HTTPLib
{
class HTTP
{
public bool DownloadFile(string url, string filePath, string fileName, bool checkHash)
{
try
{
var partFilePath = String.Format(filePath + "{0}.PART", fileName);
var downloadedFilePath = String.Format("{0}{1}", filePath, fileName);
int fileSize = GetFileSize(String.Format("{0}{1}", url, fileName));
var etag = "";
var downloaded = 0;
var percent = 0;
var last = 0;
const int BUFFER_SIZE = 16 * 1020;
if (checkHash)
{
etag = GetETag(String.Format("{0}{1}", url, fileName));
}
if (File.Exists(partFilePath))
{
downloaded = (int)new FileInfo(partFilePath).Length;
percent = (downloaded * 100) / fileSize;
}
if (downloaded == 0 || downloaded >= fileSize)
{
downloaded = 0;
using (var outputFileStream = File.Create(partFilePath, BUFFER_SIZE))
{
var req = WebRequest.Create(new Uri(String.Format("{0}{1}", url, fileName)));
using (var response = req.GetResponse())
{
using (var responseStream = response.GetResponseStream())
{
var buffer = new byte[BUFFER_SIZE];
int bytesRead;
do
{
bytesRead = responseStream.Read(buffer, 0, BUFFER_SIZE);
outputFileStream.Write(buffer, 0, bytesRead);
downloaded += bytesRead;
percent = (downloaded * 100) / fileSize;
if (percent > last)
{
last = percent;
}
}
while (bytesRead > 0);
}
}
}
}
else
{
using (var _writeStream =
new FileStream(partFilePath, FileMode.Append, FileAccess.Write))
{
var _readStream = OpenReadStream(downloaded, fileSize,
String.Format("{0}{1}", url, fileName), _writeStream);
var buffer = new byte[BUFFER_SIZE];
int count = _readStream.Read(buffer, 0, buffer.Length);
do
{
_writeStream.Write(buffer, 0, count);
_writeStream.Flush();
count = _readStream.Read(buffer, 0, buffer.Length);
downloaded += count;
percent = (downloaded * 100) / fileSize;
if (percent > last)
{
last = percent;
}
}
while (count > 0);
}
}
if (File.Exists(partFilePath))
{
var downloadSuccessful = false;
if ((int)new FileInfo(partFilePath).Length >= fileSize)
{
if (checkHash)
{
var hash = GetMD5Hash(partFilePath);
if (hash.ToLower().Equals(etag.ToLower()))
{
downloadSuccessful = true;
}
}
else
{
downloadSuccessful = true;
}
if (downloadSuccessful)
{
File.Copy(partFilePath, downloadedFilePath, true);
File.SetAttributes(downloadedFilePath,
File.GetAttributes(downloadedFilePath) & ~FileAttributes.ReadOnly);
RemovePartFile(partFilePath);
return true;
}
else
{
File.Delete(partFilePath);
}
}
}
else
{
File.Delete(partFilePath);
}
return false;
}
catch (Exception ex)
{
throw new ApplicationException(
"Thrown from DownloadFile() -> " + ex.Message, ex);
}
}
public int GetFileSize(string url)
{
try
{
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(new Uri(url));
req.Method = "HEAD";
HttpWebResponse resp = (HttpWebResponse)(req.GetResponse());
int len = (int)resp.ContentLength;
resp.Close();
return len;
}
catch (Exception ex)
{
throw new ApplicationException(
"Thrown from GetFileSize() -> " + ex.Message, ex);
}
}
public string GetETag(string url)
{
try
{
var etag = "";
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(new Uri(url));
req.Method = "HEAD";
HttpWebResponse resp = (HttpWebResponse)(req.GetResponse());
etag = resp.Headers.Get("ETag").Replace("\"", "");
resp.Close();
return etag;
}
catch (Exception ex)
{
throw new ApplicationException("Thrown from GetETag() -> " + ex.Message, ex);
}
}
public Stream OpenReadStream(long start, long length, string url, FileStream _writeStream)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(new Uri(url));
request.AddRange((int)start, (int)length);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.ContentLength == length)
{
_writeStream.Seek(0, SeekOrigin.Begin);
}
Stream _readStream;
_readStream = response.GetResponseStream();
return _readStream;
}
catch (Exception ex)
{
throw new ApplicationException(
"Thrown from OpenReadStream() -> " + ex.Message, ex);
}
}
public static FileStream GetFileStream(string pathName)
{
return (new FileStream(pathName, FileMode.Open,
FileAccess.Read, FileShare.ReadWrite));
}
public static string GetMD5Hash(string pathName)
{
try
{
string strResult = "";
string strHashData = "";
byte[] arrbytHashValue;
FileStream oFileStream = null;
MD5CryptoServiceProvider oMD5Hasher =
new MD5CryptoServiceProvider();
oFileStream = GetFileStream(pathName);
arrbytHashValue = oMD5Hasher.ComputeHash(oFileStream);
oFileStream.Close();
strHashData = BitConverter.ToString(arrbytHashValue);
strHashData = strHashData.Replace("-", "");
strResult = strHashData;
return (strResult);
}
catch (Exception ex)
{
throw new ApplicationException(
"Thrown from GetMD5Hash() -> " + ex.Message, ex);
}
}
private void RemovePartFile(string filePath)
{
try
{
if (File.Exists(filePath))
{
FileInfo fileInfoMP4 = new FileInfo(filePath);
long bytesOfMP4 = fileInfoMP4.Length;
if (bytesOfMP4 > 0)
{
File.Delete((filePath));
}
}
}
catch (Exception ex)
{
throw new ApplicationException(
"Thrown from RemovePartFile() -> " + ex.Message, ex);
}
}
}
} You can also use the functions for getting the file size, ETag and hash sum of an existing file separately.
