// Server.cs
//
// Copyright (C) 2008,2009 Bartek Jasicki
//
// This file is part of Grubng.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
/**@file Server.cs
* Provide main class for server.*/
using System;
using System.Net;
using System.IO;
using System.Text;
namespace grubupload
{
/**Main class for server.*/
internal sealed class Server
{
/**HttpListener class object.*/
private HttpListener listener;
/**Full path to work directory.*/
private string workdir;
/**Full path to temporary directory.*/
private string tempdir;
/**Hash password for check correctness of .arc file.*/
private string hashpass;
/**True if log file enabled, otherwise false.*/
private bool enablelog;
/**Server version.*/
private string version;
/**True if server API enabled, otherwise false.*/
private bool enableapi;
/**True if database enabled, otherwise false.*/
private bool enabledb;
/**Free disk space - true if enough.*/
private bool freespace;
/**Minimal amount of free space on disk.*/
private long minfreespace;
/**Server time timer*/
private System.Timers.Timer servertime;
/**True is upload .arc files to hadoop is enabled, otherwise false.*/
private bool enableupload;
/**Amount of MB free on disk below which, upload to Hadoop starts.*/
private long hadoopfreespace;
///
/// Upload thread
///
System.Threading.Thread uploadthread;
/**True if server uploading .arc files to database, otherwise false.*/
private bool uploading;
/**Database class object.*/
private Database db;
/**File stream for log file.*/
private FileStream logfile;
/**Amount of connected clients.*/
private short connected;
/**Standard class contructor.*/
public Server()
{
System.Reflection.Assembly asm = System.Reflection.Assembly.GetExecutingAssembly();
this.version = asm.GetName().Version.Major.ToString() + "." + asm.GetName().Version.Minor.ToString()
+ "." + asm.GetName().Version.Build.ToString();
this.workdir = Config.ReadConfig("/configuration/workdir");
this.tempdir = Config.ReadConfig("/configuration/tempdir");
this.hashpass = Config.ReadConfig("/configuration/hashpass");
if (Config.ReadConfig("/configuration/enablelog") == "Y")
{
this.enablelog = true;
this.logfile = File.Open(Config.ReadConfig("/configuration/logpath"), FileMode.Append);
}
if (Config.ReadConfig("/configuration/enableapi") == "Y")
{
this.enableapi = true;
}
if (Config.ReadConfig("/configuration/enabledb") == "Y")
{
this.enabledb = true;
this.db = new Database(Config.ReadConfig("/configuration/dbpath"));
}
this.minfreespace = Convert.ToInt64(Config.ReadConfig("/configuration/minfreespace"));
if (Config.ReadConfig("/configuration/enableupload") == "Y")
{
this.enableupload = true;
this.hadoopfreespace = Convert.ToInt64(Config.ReadConfig("/configuration/hadoopfreespace"));
}
this.listener = new HttpListener();
}
///
/// Main server function, start server and serve requests from clients.
///
public void Run()
{
string address = Config.ReadConfig("/configuration/address");
string port = Config.ReadConfig("/configuration/port");
long freespace = this.CheckFreeSpace();
if (freespace < (this.hadoopfreespace * 1048576))
{
this.uploadthread = new System.Threading.Thread(this.UploadThread);
this.uploadthread.Name = "uploader";
this.uploadthread.Start(true);
}
this.servertime = new System.Timers.Timer();
this.servertime.Elapsed += new System.Timers.ElapsedEventHandler(OnTimeElapsed);
double minutes = Convert.ToDouble(Config.ReadConfig("/configuration/checkevery"));
this.servertime.Interval = (minutes * 60000);
this.servertime.Start();
this.listener.Prefixes.Add("http://" + address + ":" + port + "/");
this.listener.Start();
this.listener.BeginGetContext(new AsyncCallback(this.WebRequestCallback), this.listener);
while (true)
{
System.Threading.Thread.Sleep(100);
}
}
///
/// Function asynchronous serve reguests from clients.
///
///
/// A Asynchronous result of request.
///
private void WebRequestCallback(IAsyncResult result)
{
HttpListenerContext context = this.listener.EndGetContext(result);
this.listener.BeginGetContext(new AsyncCallback(this.WebRequestCallback), this.listener);
HttpListenerRequest request = context.Request;
//Obtain a response object
HttpListenerResponse response = context.Response;
string responseString = String.Empty;
try
{
//Construct a response
if ((request.HttpMethod == "OPTION") || (request.HttpMethod == "DELETE") ||
(request.HttpMethod == "TRACE") || (request.HttpMethod == "CONNECT"))
{
response.StatusCode = 405;
}
//Show directories and files on server
if (request.HttpMethod == "GET")
{
this.GetRequest(ref response, ref request);
return;
}
//Upload .arc file
if (request.HttpMethod == "PUT")
{
responseString = this.PutRequest(ref response, ref request);
if (responseString.Length == 0)
{
try
{
response.Close();
}
catch (System.Net.Sockets.SocketException)
{
}
return;
}
}
if (request.HttpMethod == "POST")
{
responseString = this.PostRequest(ref request, ref response);
}
this.Response(ref response, "text/plain; charset=utf-8", ref responseString);
}
catch (Exception e)
{
string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss");
FileStream errorlog = File.Open("error.log", FileMode.Append);
StreamWriter logstream = new StreamWriter(errorlog);
logstream.WriteLine(tdate + " Caught: " + e.GetType().ToString() + " " + e.Message);
logstream.WriteLine(" Source: " + e.Source);
logstream.WriteLine(" StackTrace: " + e.StackTrace);
logstream.WriteLine(" TargetSite: " + e.TargetSite);
logstream.Close();
response.StatusCode = 500;
responseString = "Internal server error.\n";
this.Response(ref response, "text/plain; charset=utf-8", ref responseString);
return;
}
}
///
/// Function return response to client.
///
///
/// A
///
///
/// A Content-Type header for reponse
///
///
/// A Body of response
///
void Response(ref HttpListenerResponse response, string contentType, ref string responseString)
{
response.Headers.Add(HttpResponseHeader.Server, "GrubNG " + this.version);
byte[] buffer2 = Encoding.UTF8.GetBytes(responseString);
response.ContentLength64 = buffer2.Length;
if (buffer2.Length > 0)
{
response.ContentType = contentType;
response.Headers.Add(HttpResponseHeader.ContentLanguage, "en");
}
try
{
response.OutputStream.Write(buffer2, 0, buffer2.Length);
response.OutputStream.Close();
response.Close();
}
catch (System.Net.Sockets.SocketException)
{
}
catch (IOException)
{
}
buffer2 = null;
}
///
/// Function show content of directories on server and send files to client (GET method)
///
///
/// A Response for client
///
///
/// A Request made by client
///
void GetRequest(ref HttpListenerResponse response, ref HttpListenerRequest request)
{
string readdir = workdir + System.Web.HttpUtility.UrlDecode(request.Url.AbsolutePath);
string responseString;
//Show directory
if (!File.Exists(readdir))
{
StringBuilder tmpString = new StringBuilder("
");
if (Directory.Exists(readdir))
{
response.StatusCode = 200;
tmpString.Append("| Name | Last Modified | Size |
");
string showdir;
System.Text.StringBuilder urldir = new StringBuilder();
string writetime;
if (request.Url.AbsolutePath != "/")
{
tmpString.Append("| Parent direcory | | |
");
}
string[] directories = Directory.GetDirectories(readdir);
foreach (string directory in directories)
{
writetime = Directory.GetLastWriteTimeUtc(directory).ToString("yyyy-MM-dd HH:mm:ss");
showdir = directory.Remove(0, readdir.Length);
urldir.Append(System.Web.HttpUtility.UrlEncode(showdir));
urldir.Append("/");
urldir = urldir.Replace("%2f", String.Empty);
tmpString.Append("| ");
tmpString.Append(showdir);
tmpString.Append("/ | ");
tmpString.Append(writetime);
tmpString.Append(" | |
");
urldir.Remove(0, urldir.Length);
}
directories = Directory.GetFiles(readdir);
FileInfo filein;
StringBuilder filelen = new StringBuilder();
double length;
foreach (string file in directories)
{
filein = new FileInfo(file);
writetime = File.GetLastWriteTimeUtc(file).ToString("yyyy-MM-dd HH:mm:ss");
filelen.Remove(0, filelen.Length);
if (filein.Length < 1024)
{
filelen.Append(filein.Length);
}
if ((filein.Length > 1023) && (filein.Length < 1048576))
{
filelen.Append((filein.Length / 1024));
filelen.Append("k");
}
if (filein.Length > 1048575)
{
length = Math.Round((Convert.ToDouble(filein.Length) / 1048576), 1);
filelen.Append(length);
filelen.Append("M");
}
showdir = file.Remove(0, readdir.Length);
urldir.Append(System.Web.HttpUtility.UrlEncode(showdir));
urldir = urldir.Replace("%2f", String.Empty);
tmpString.Append("| ");
tmpString.Append(showdir);
tmpString.Append(" | ");
tmpString.Append(writetime);
tmpString.Append(" | ");
tmpString.Append(filelen);
tmpString.Append(" |
");
urldir.Remove(0, urldir.Length);
}
tmpString.Append("
");
}
else
{
response.StatusCode = 404;
tmpString.Append("404 Not Found
");
}
tmpString.Append("");
responseString = tmpString.ToString();
tmpString = null;
this.Response(ref response, "text/html; charset=utf-8", ref responseString);
return;
}
else
{
//Upload selected file
using (FileStream file = new FileStream(readdir, FileMode.Open, FileAccess.Read))
{
response.ContentLength64 = file.Length;
response.Headers.Add(HttpResponseHeader.Server, "GrubNG " + this.version);
try
{
byte[] buffer = new byte[1024];
int bytesRead = 0;
while ((bytesRead = file.Read(buffer, 0, buffer.Length)) != 0)
{
response.OutputStream.Write(buffer, 0, bytesRead);
}
response.OutputStream.Close();
response.Close();
}
catch (IOException)
{
}
catch (System.Net.Sockets.SocketException)
{
}
}
return;
}
}
///
/// Function upload .arc file on server, decompress it and check correctness of file (PUT method)
///
///
/// A Response for client
///
///
/// A Request made by client
///
///
/// A HTTP body with informations which been send to client
///
string PutRequest(ref HttpListenerResponse response, ref HttpListenerRequest request)
{
string responseString = String.Empty;
if (!this.freespace)
{
response.StatusCode = 500;
return "No enough free space on disk\n";
}
//Download .arc file and put it on server
this.connected ++;
string[] parts = request.Url.AbsolutePath.Split('/');
string filename = tempdir + Path.DirectorySeparatorChar +
parts[(parts.Length - 1)];
filename = System.Web.HttpUtility.UrlDecode(filename);
byte[] buffer = new byte[1024];
int bytesRead = 0;
FileStream file;
try
{
file = File.Create(filename);
}
catch (IOException)
{
response.StatusCode = 500;
this.connected --;
return "Other crawler send this file now, please wait before you try again\n";
}
try
{
while ((bytesRead = request.InputStream.Read(buffer, 0, buffer.Length)) != 0)
{
file.Write(buffer, 0, bytesRead);
file.Flush();
}
}
catch (IOException)
{
file.Close();
file.Dispose();
File.Delete(filename);
this.connected --;
return String.Empty;
}
file.Close();
file.Dispose();
request.InputStream.Close();
request.InputStream.Dispose();
bool valid = true;
string tmpfilename = filename.Replace(this.tempdir + Path.DirectorySeparatorChar, "");
string[] extensions = tmpfilename.Split('.');
//1. Check for extensions
if (extensions.Length < 4)
{
response.StatusCode = 401;
File.Delete(filename);
this.connected --;
return "Invalid file sent\n";
}
if (extensions[(extensions.Length - 2)] != "arc")
{
response.StatusCode = 401;
File.Delete(filename);
this.connected --;
return "This is not .arc file\n";
}
if (!File.Exists(filename))
{
response.StatusCode = 500;
this.connected --;
return "Cannot find uploaded file, please send it again\n";
}
//2.Decompress file to memory
MemoryStream memstream = new MemoryStream();
file = File.OpenRead(filename);
if (extensions[(extensions.Length - 1)] == "gz")
{
using (System.IO.Compression.GZipStream gzstream = new System.IO.Compression.GZipStream(file,
System.IO.Compression.CompressionMode.Decompress, true))
{
try
{
while ((bytesRead = gzstream.Read(buffer, 0, buffer.Length)) != 0)
{
memstream.Write(buffer, 0, bytesRead);
memstream.Flush();
}
}
catch (InvalidDataException)
{
valid = false;
responseString = "Invalid data sent by client.";
}
}
}
else
{
if (extensions[(extensions.Length - 1)] == "lzma")
{
byte[] properties = new byte[5];
file.Read(properties, 0, 5);
long outSize = 0;
for (int i = 0; i < 8; i++)
{
bytesRead = file.ReadByte();
outSize |= ((long)(byte)bytesRead) << (8 * i);
}
using (Decoder decoder = new Decoder())
{
decoder.SetDecoderProperties(properties);
decoder.Code(file, memstream, outSize);
}
}
else
{
valid = false;
responseString = "Invalid data sent by client";
}
}
file.Close();
file.Dispose();
//3.Validate .arc file
string[] resultvalue = new string[3] {String.Empty, String.Empty, String.Empty};
//Check for correctness of url's in .arc file
if (valid)
{
//Check hash for file
resultvalue = ARCCheck.CheckFile(memstream, String.Join(".", extensions, 0, (extensions.Length - 3)),
extensions[(extensions.Length - 3)], this.hashpass);
if (resultvalue[0] == "valid")
{
valid = true;
}
else
{
valid = false;
responseString = resultvalue[1];
}
if (valid)
{
//Write files on server
string date = DateTime.UtcNow.ToString("yyyy" +
Path.DirectorySeparatorChar +
"MM" +
Path.DirectorySeparatorChar +
"dd" +
Path.DirectorySeparatorChar +
"HH");
string directory = workdir +
Path.DirectorySeparatorChar +
date;
if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
if (File.Exists(filename))
{
File.Copy(filename, directory + Path.DirectorySeparatorChar + tmpfilename, true);
}
else
{
response.StatusCode = 500;
this.connected --;
return "Cannot find uploaded file, please send it again\n";
}
}
}
memstream.Close();
memstream.Dispose();
if (File.Exists(filename))
{
File.Delete(filename);
}
//If .arc file is valid, write it to database
if (valid)
{
response.StatusCode = 200;
responseString = "File uploaded";
if (this.enabledb)
{
this.db.WriteDB(String.Join(".", extensions, 0, (extensions.Length - 3)), resultvalue[2]);
}
}
else
{
response.StatusCode = 401;
response.StatusDescription = "Invalid .arc file";
}
responseString += "\n";
//If log file enabled, write information about send .arc.gz file
if (this.enablelog)
{
if (valid)
{
buffer = System.Text.Encoding.UTF8.GetBytes(DateTime.UtcNow.ToString("yyyy/MM/dd/HH:mm:ss") + " " +
request.RemoteEndPoint.Address + " " + tmpfilename + " status: OK" + Environment.NewLine);
}
else
{
buffer = System.Text.Encoding.UTF8.GetBytes(DateTime.UtcNow.ToString("yyyy/MM/dd/HH:mm:ss") + " " +
request.RemoteEndPoint.Address + " " + tmpfilename + " status: Invalid" + Environment.NewLine);
}
this.logfile.Write(buffer, 0, buffer.Length);
this.logfile.Flush();
}
this.connected --;
return responseString;
}
///
/// Function create response for client from server API (POST method)
///
///
/// A request send by client
///
///
/// A response for client
///
///
/// A HTTP body for response
///
string PostRequest(ref HttpListenerRequest request, ref HttpListenerResponse response)
{
if (this.enableapi)
{
//Get request body
byte[] buffer = new byte[100];
int bytesSend = request.InputStream.Read(buffer, 0, buffer.Length);
string requestString = Encoding.UTF8.GetString(buffer, 0, bytesSend);
request.InputStream.Close();
request.InputStream.Dispose();
//Get server status
if (requestString == "status")
{
return API.Status(this.freespace, this.uploading, this.connected, ref this.version, ref this.workdir);
}
else
{
//Read informations from database
if (this.enabledb)
{
string[] requestparts = requestString.Split(',', '=');
string year = String.Empty;
string month = String.Empty;
int tempint;
bool valid;
if ((requestparts[0] == "user") || (requestparts[0] == "list"))
{
if (requestparts.Length < 5)
{
if (requestparts[0] == "list")
{
year = DateTime.UtcNow.Year.ToString();
}
}
else
{
valid = int.TryParse(requestparts[5], out tempint);
if (valid)
{
year = tempint.ToString();
}
else
{
year = DateTime.UtcNow.Year.ToString();
}
}
if (requestparts.Length < 7)
{
if (requestparts[0] == "list")
{
month = DateTime.UtcNow.Month.ToString();
}
}
else
{
valid = int.TryParse(requestparts[7], out tempint);
if (valid)
{
month = tempint.ToString();
}
else
{
month = DateTime.UtcNow.Month.ToString();
}
}
}
//Get stats for selected user
if (requestparts[0] == "user")
{
if (requestparts.Length > 3)
{
this.ValidateUsername(ref requestparts[1]);
switch (requestparts[3])
{
case "weekly":
return API.UserWeekly(requestparts[1], ref this.db);
case "monthly":
if (String.IsNullOrEmpty(month))
{
year = String.Empty;
}
if (String.IsNullOrEmpty(year))
{
month = String.Empty;
}
return API.UserStats(requestparts[1], year, month, "monthly", ref this.db);
case "yearly":
return API.UserStats(requestparts[1], year, month, "yearly", ref this.db);
default:
response.StatusCode = 400;
return String.Empty;
}
}
else
{
response.StatusCode = 400;
return String.Empty;
}
}
//Get stats for list of users
if (requestparts[0] == "list")
{
tempint = -1;
valid = int.TryParse(requestparts[1], out tempint);
if (!valid)
{
response.StatusCode = 400;
}
else
{
switch (requestparts[3])
{
case "weekly":
return API.ListUsers(requestparts[1], year, month, "weekly", ref this.db);
case "monthly":
return API.ListUsers(requestparts[1], year, month, "monthly", ref this.db);
case "yearly":
return API.ListUsers(requestparts[1], year, month,"yearly", ref this.db);
default:
response.StatusCode = 400;
return String.Empty;
}
}
}
//Get overall statistics
if (requestparts[0] == "overall")
{
//Get overall statistics for selected user
if (requestparts.Length == 3)
{
this.ValidateUsername(ref requestparts[2]);
return API.OverallStats(requestparts[2], ref this.db);
}
else
{
return API.OverallStats(String.Empty, ref this.db);
}
}
response.StatusCode = 400;
return String.Empty;
}
else
{
return "Database disabled";
}
}
}
return "Server API disabled";
}
///
/// Timer for check free disk space, cleanup old incomplete files and send .arc files to Hadoop.
///
///
/// A Unused
///
///
/// A Unused
///
void OnTimeElapsed(object sender, System.Timers.ElapsedEventArgs e)
{
if (sender != null)
{
sender = null;
}
if (e != null)
{
e = null;
}
//Check for available free space on disk
long freespace = this.CheckFreeSpace();
//Clean unfinished files
this.CleanFiles();
//Run garbage collector for clean memory
GC.GetTotalMemory(true);
//Upload .arc files to Hadoop
if (this.enableupload)
{
if (DateTime.UtcNow.Hour == 0)
{
if (!this.uploading)
{
this.uploadthread = new System.Threading.Thread(this.UploadThread);
this.uploadthread.Name = "uploader";
this.uploadthread.Start(false);
}
}
if (freespace < (this.hadoopfreespace * 1048576))
{
if (!this.uploading)
{
this.uploadthread = new System.Threading.Thread(this.UploadThread);
this.uploadthread.Name = "uploader";
this.uploadthread.Start(true);
}
}
}
}
///
/// Function delete old, incomplete .arc files from temporary directory.
///
void CleanFiles()
{
//Get list of all files
string[] files = Directory.GetFiles(this.tempdir);
//Process files
System.Text.StringBuilder extension = new StringBuilder();
long difftime;
this.connected = 0;
foreach (string file in files)
{
//If file is .arc.gz or .gz, check it last modification date
extension.Remove(0, extension.Length);
extension = extension.Append(file.Substring((file.Length - 2)));
if ((extension.ToString() == "gz") || (extension.ToString() == "ma"))
{
difftime = DateTime.UtcNow.Ticks - File.GetLastWriteTimeUtc(file).Ticks;
//If file was modified more than 15 minutes ago, delete it
if (difftime > (TimeSpan.TicksPerMinute * 15))
{
File.Delete(file);
}
else
{
this.connected ++;
}
}
}
}
///
/// Thread for upload .arc files to Hadoop
///
///
/// A If true, upload files from today, otherwise from yesterday
///
void UploadThread(object data)
{
this.uploading = true;
string param = String.Empty;
if ((bool)data)
{
param = "today";
}
System.Diagnostics.ProcessStartInfo procinfo = new System.Diagnostics.ProcessStartInfo();
procinfo.FileName = "mono";
procinfo.Arguments = "grub-uploader.exe " + param;
procinfo.UseShellExecute = false;
System.Diagnostics.Process proc = System.Diagnostics.Process.Start(procinfo);
proc.WaitForExit();
proc.Close();
proc.Dispose();
this.uploading = false;
}
///
/// Function check available free space on server
///
///
/// A Amount of free bytes
///
long CheckFreeSpace()
{
Mono.Unix.UnixDriveInfo drive = new Mono.Unix.UnixDriveInfo(this.workdir);
long freedisk = drive.AvailableFreeSpace;
drive = null;
if (freedisk < (this.minfreespace * 1048576))
{
this.freespace = false;
}
else
{
this.freespace = true;
}
return freedisk;
}
///
/// Function delete all bad characters from username
///
///
/// A Username to validate
///
static void ValidateUsername(ref string username)
{
username = username.Replace("'", String.Empty);
username = username.Replace("\"", String.Empty);
username = username.Replace("<", String.Empty);
username = username.ToLower();
}
}
}