// Server.cs // // Copyright (C) 2008,2009 Bartek Jasicki // // This file is part of Grubng. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // /**@file Server.cs * Provide main class for server.*/ using System; using System.Net; using System.IO; using System.Text; namespace grubupload { /**Main class for server.*/ internal sealed class Server { /**HttpListener class object.*/ private HttpListener listener; /**Full path to work directory.*/ private string workdir; /**Full path to temporary directory.*/ private string tempdir; /**Hash password for check correctness of .arc file.*/ private string hashpass; /**True if log file enabled, otherwise false.*/ private bool enablelog; /**Server version.*/ private string version; /**True if server API enabled, otherwise false.*/ private bool enableapi; /**True if database enabled, otherwise false.*/ private bool enabledb; /**Free disk space - true if enough.*/ private bool freespace; /**Minimal amount of free space on disk.*/ private long minfreespace; /**Server time timer*/ private System.Timers.Timer servertime; /**True is upload .arc files to hadoop is enabled, otherwise false.*/ private bool enableupload; /**Amount of MB free on disk below which, upload to Hadoop starts.*/ private long hadoopfreespace; /// /// Upload thread /// System.Threading.Thread uploadthread; /**True if server uploading .arc files to database, otherwise false.*/ private bool uploading; /**Database class object.*/ private Database db; /**File stream for log file.*/ private FileStream logfile; /**Amount of connected clients.*/ private short connected; /**Standard class contructor.*/ public Server() { System.Reflection.Assembly asm = System.Reflection.Assembly.GetExecutingAssembly(); this.version = asm.GetName().Version.Major.ToString() + "." + asm.GetName().Version.Minor.ToString() + "." + asm.GetName().Version.Build.ToString(); this.workdir = Config.ReadConfig("/configuration/workdir"); this.tempdir = Config.ReadConfig("/configuration/tempdir"); this.hashpass = Config.ReadConfig("/configuration/hashpass"); if (Config.ReadConfig("/configuration/enablelog") == "Y") { this.enablelog = true; this.logfile = File.Open(Config.ReadConfig("/configuration/logpath"), FileMode.Append); } if (Config.ReadConfig("/configuration/enableapi") == "Y") { this.enableapi = true; } if (Config.ReadConfig("/configuration/enabledb") == "Y") { this.enabledb = true; this.db = new Database(Config.ReadConfig("/configuration/dbpath")); } this.minfreespace = Convert.ToInt64(Config.ReadConfig("/configuration/minfreespace")); if (Config.ReadConfig("/configuration/enableupload") == "Y") { this.enableupload = true; this.hadoopfreespace = Convert.ToInt64(Config.ReadConfig("/configuration/hadoopfreespace")); } this.listener = new HttpListener(); } /// /// Main server function, start server and serve requests from clients. /// public void Run() { string address = Config.ReadConfig("/configuration/address"); string port = Config.ReadConfig("/configuration/port"); long freespace = this.CheckFreeSpace(); if (freespace < (this.hadoopfreespace * 1048576)) { this.uploadthread = new System.Threading.Thread(this.UploadThread); this.uploadthread.Name = "uploader"; this.uploadthread.Start(true); } this.servertime = new System.Timers.Timer(); this.servertime.Elapsed += new System.Timers.ElapsedEventHandler(OnTimeElapsed); double minutes = Convert.ToDouble(Config.ReadConfig("/configuration/checkevery")); this.servertime.Interval = (minutes * 60000); this.servertime.Start(); this.listener.Prefixes.Add("http://" + address + ":" + port + "/"); this.listener.Start(); this.listener.BeginGetContext(new AsyncCallback(this.WebRequestCallback), this.listener); while (true) { System.Threading.Thread.Sleep(100); } } /// /// Function asynchronous serve reguests from clients. /// /// /// A Asynchronous result of request. /// private void WebRequestCallback(IAsyncResult result) { HttpListenerContext context = this.listener.EndGetContext(result); this.listener.BeginGetContext(new AsyncCallback(this.WebRequestCallback), this.listener); HttpListenerRequest request = context.Request; //Obtain a response object HttpListenerResponse response = context.Response; string responseString = String.Empty; try { //Construct a response if ((request.HttpMethod == "OPTION") || (request.HttpMethod == "DELETE") || (request.HttpMethod == "TRACE") || (request.HttpMethod == "CONNECT")) { response.StatusCode = 405; } //Show directories and files on server if (request.HttpMethod == "GET") { this.GetRequest(ref response, ref request); return; } //Upload .arc file if (request.HttpMethod == "PUT") { responseString = this.PutRequest(ref response, ref request); if (responseString.Length == 0) { try { response.Close(); } catch (System.Net.Sockets.SocketException) { } return; } } if (request.HttpMethod == "POST") { responseString = this.PostRequest(ref request, ref response); } this.Response(ref response, "text/plain; charset=utf-8", ref responseString); } catch (Exception e) { string tdate = DateTime.Now.ToString("yyyy/MM/dd/HH:mm:ss"); FileStream errorlog = File.Open("error.log", FileMode.Append); StreamWriter logstream = new StreamWriter(errorlog); logstream.WriteLine(tdate + " Caught: " + e.GetType().ToString() + " " + e.Message); logstream.WriteLine(" Source: " + e.Source); logstream.WriteLine(" StackTrace: " + e.StackTrace); logstream.WriteLine(" TargetSite: " + e.TargetSite); logstream.Close(); response.StatusCode = 500; responseString = "Internal server error.\n"; this.Response(ref response, "text/plain; charset=utf-8", ref responseString); return; } } /// /// Function return response to client. /// /// /// A /// /// /// A Content-Type header for reponse /// /// /// A Body of response /// void Response(ref HttpListenerResponse response, string contentType, ref string responseString) { response.Headers.Add(HttpResponseHeader.Server, "GrubNG " + this.version); byte[] buffer2 = Encoding.UTF8.GetBytes(responseString); response.ContentLength64 = buffer2.Length; if (buffer2.Length > 0) { response.ContentType = contentType; response.Headers.Add(HttpResponseHeader.ContentLanguage, "en"); } try { response.OutputStream.Write(buffer2, 0, buffer2.Length); response.OutputStream.Close(); response.Close(); } catch (System.Net.Sockets.SocketException) { } catch (IOException) { } buffer2 = null; } /// /// Function show content of directories on server and send files to client (GET method) /// /// /// A Response for client /// /// /// A Request made by client /// void GetRequest(ref HttpListenerResponse response, ref HttpListenerRequest request) { string readdir = workdir + System.Web.HttpUtility.UrlDecode(request.Url.AbsolutePath); string responseString; //Show directory if (!File.Exists(readdir)) { StringBuilder tmpString = new StringBuilder(""); if (Directory.Exists(readdir)) { response.StatusCode = 200; tmpString.Append(""); string showdir; System.Text.StringBuilder urldir = new StringBuilder(); string writetime; if (request.Url.AbsolutePath != "/") { tmpString.Append(""); } string[] directories = Directory.GetDirectories(readdir); foreach (string directory in directories) { writetime = Directory.GetLastWriteTimeUtc(directory).ToString("yyyy-MM-dd HH:mm:ss"); showdir = directory.Remove(0, readdir.Length); urldir.Append(System.Web.HttpUtility.UrlEncode(showdir)); urldir.Append("/"); urldir = urldir.Replace("%2f", String.Empty); tmpString.Append(""); urldir.Remove(0, urldir.Length); } directories = Directory.GetFiles(readdir); FileInfo filein; StringBuilder filelen = new StringBuilder(); double length; foreach (string file in directories) { filein = new FileInfo(file); writetime = File.GetLastWriteTimeUtc(file).ToString("yyyy-MM-dd HH:mm:ss"); filelen.Remove(0, filelen.Length); if (filein.Length < 1024) { filelen.Append(filein.Length); } if ((filein.Length > 1023) && (filein.Length < 1048576)) { filelen.Append((filein.Length / 1024)); filelen.Append("k"); } if (filein.Length > 1048575) { length = Math.Round((Convert.ToDouble(filein.Length) / 1048576), 1); filelen.Append(length); filelen.Append("M"); } showdir = file.Remove(0, readdir.Length); urldir.Append(System.Web.HttpUtility.UrlEncode(showdir)); urldir = urldir.Replace("%2f", String.Empty); tmpString.Append(""); urldir.Remove(0, urldir.Length); } tmpString.Append("
NameLast ModifiedSize
Parent direcory
"); tmpString.Append(showdir); tmpString.Append("/"); tmpString.Append(writetime); tmpString.Append("
"); tmpString.Append(showdir); tmpString.Append(""); tmpString.Append(writetime); tmpString.Append(""); tmpString.Append(filelen); tmpString.Append("
"); } else { response.StatusCode = 404; tmpString.Append("

404 Not Found

"); } tmpString.Append(""); responseString = tmpString.ToString(); tmpString = null; this.Response(ref response, "text/html; charset=utf-8", ref responseString); return; } else { //Upload selected file using (FileStream file = new FileStream(readdir, FileMode.Open, FileAccess.Read)) { response.ContentLength64 = file.Length; response.Headers.Add(HttpResponseHeader.Server, "GrubNG " + this.version); try { byte[] buffer = new byte[1024]; int bytesRead = 0; while ((bytesRead = file.Read(buffer, 0, buffer.Length)) != 0) { response.OutputStream.Write(buffer, 0, bytesRead); } response.OutputStream.Close(); response.Close(); } catch (IOException) { } catch (System.Net.Sockets.SocketException) { } } return; } } /// /// Function upload .arc file on server, decompress it and check correctness of file (PUT method) /// /// /// A Response for client /// /// /// A Request made by client /// /// /// A HTTP body with informations which been send to client /// string PutRequest(ref HttpListenerResponse response, ref HttpListenerRequest request) { string responseString = String.Empty; if (!this.freespace) { response.StatusCode = 500; return "No enough free space on disk\n"; } //Download .arc file and put it on server this.connected ++; string[] parts = request.Url.AbsolutePath.Split('/'); string filename = tempdir + Path.DirectorySeparatorChar + parts[(parts.Length - 1)]; filename = System.Web.HttpUtility.UrlDecode(filename); byte[] buffer = new byte[1024]; int bytesRead = 0; FileStream file; try { file = File.Create(filename); } catch (IOException) { response.StatusCode = 500; this.connected --; return "Other crawler send this file now, please wait before you try again\n"; } try { while ((bytesRead = request.InputStream.Read(buffer, 0, buffer.Length)) != 0) { file.Write(buffer, 0, bytesRead); file.Flush(); } } catch (IOException) { file.Close(); file.Dispose(); File.Delete(filename); this.connected --; return String.Empty; } file.Close(); file.Dispose(); request.InputStream.Close(); request.InputStream.Dispose(); bool valid = true; string tmpfilename = filename.Replace(this.tempdir + Path.DirectorySeparatorChar, ""); string[] extensions = tmpfilename.Split('.'); //1. Check for extensions if (extensions.Length < 4) { response.StatusCode = 401; File.Delete(filename); this.connected --; return "Invalid file sent\n"; } if (extensions[(extensions.Length - 2)] != "arc") { response.StatusCode = 401; File.Delete(filename); this.connected --; return "This is not .arc file\n"; } if (!File.Exists(filename)) { response.StatusCode = 500; this.connected --; return "Cannot find uploaded file, please send it again\n"; } //2.Decompress file to memory MemoryStream memstream = new MemoryStream(); file = File.OpenRead(filename); if (extensions[(extensions.Length - 1)] == "gz") { using (System.IO.Compression.GZipStream gzstream = new System.IO.Compression.GZipStream(file, System.IO.Compression.CompressionMode.Decompress, true)) { try { while ((bytesRead = gzstream.Read(buffer, 0, buffer.Length)) != 0) { memstream.Write(buffer, 0, bytesRead); memstream.Flush(); } } catch (InvalidDataException) { valid = false; responseString = "Invalid data sent by client."; } } } else { if (extensions[(extensions.Length - 1)] == "lzma") { byte[] properties = new byte[5]; file.Read(properties, 0, 5); long outSize = 0; for (int i = 0; i < 8; i++) { bytesRead = file.ReadByte(); outSize |= ((long)(byte)bytesRead) << (8 * i); } using (Decoder decoder = new Decoder()) { decoder.SetDecoderProperties(properties); decoder.Code(file, memstream, outSize); } } else { valid = false; responseString = "Invalid data sent by client"; } } file.Close(); file.Dispose(); //3.Validate .arc file string[] resultvalue = new string[3] {String.Empty, String.Empty, String.Empty}; //Check for correctness of url's in .arc file if (valid) { //Check hash for file resultvalue = ARCCheck.CheckFile(memstream, String.Join(".", extensions, 0, (extensions.Length - 3)), extensions[(extensions.Length - 3)], this.hashpass); if (resultvalue[0] == "valid") { valid = true; } else { valid = false; responseString = resultvalue[1]; } if (valid) { //Write files on server string date = DateTime.UtcNow.ToString("yyyy" + Path.DirectorySeparatorChar + "MM" + Path.DirectorySeparatorChar + "dd" + Path.DirectorySeparatorChar + "HH"); string directory = workdir + Path.DirectorySeparatorChar + date; if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } if (File.Exists(filename)) { File.Copy(filename, directory + Path.DirectorySeparatorChar + tmpfilename, true); } else { response.StatusCode = 500; this.connected --; return "Cannot find uploaded file, please send it again\n"; } } } memstream.Close(); memstream.Dispose(); if (File.Exists(filename)) { File.Delete(filename); } //If .arc file is valid, write it to database if (valid) { response.StatusCode = 200; responseString = "File uploaded"; if (this.enabledb) { this.db.WriteDB(String.Join(".", extensions, 0, (extensions.Length - 3)), resultvalue[2]); } } else { response.StatusCode = 401; response.StatusDescription = "Invalid .arc file"; } responseString += "\n"; //If log file enabled, write information about send .arc.gz file if (this.enablelog) { if (valid) { buffer = System.Text.Encoding.UTF8.GetBytes(DateTime.UtcNow.ToString("yyyy/MM/dd/HH:mm:ss") + " " + request.RemoteEndPoint.Address + " " + tmpfilename + " status: OK" + Environment.NewLine); } else { buffer = System.Text.Encoding.UTF8.GetBytes(DateTime.UtcNow.ToString("yyyy/MM/dd/HH:mm:ss") + " " + request.RemoteEndPoint.Address + " " + tmpfilename + " status: Invalid" + Environment.NewLine); } this.logfile.Write(buffer, 0, buffer.Length); this.logfile.Flush(); } this.connected --; return responseString; } /// /// Function create response for client from server API (POST method) /// /// /// A request send by client /// /// /// A response for client /// /// /// A HTTP body for response /// string PostRequest(ref HttpListenerRequest request, ref HttpListenerResponse response) { if (this.enableapi) { //Get request body byte[] buffer = new byte[100]; int bytesSend = request.InputStream.Read(buffer, 0, buffer.Length); string requestString = Encoding.UTF8.GetString(buffer, 0, bytesSend); request.InputStream.Close(); request.InputStream.Dispose(); //Get server status if (requestString == "status") { return API.Status(this.freespace, this.uploading, this.connected, ref this.version, ref this.workdir); } else { //Read informations from database if (this.enabledb) { string[] requestparts = requestString.Split(',', '='); string year = String.Empty; string month = String.Empty; int tempint; bool valid; if ((requestparts[0] == "user") || (requestparts[0] == "list")) { if (requestparts.Length < 5) { if (requestparts[0] == "list") { year = DateTime.UtcNow.Year.ToString(); } } else { valid = int.TryParse(requestparts[5], out tempint); if (valid) { year = tempint.ToString(); } else { year = DateTime.UtcNow.Year.ToString(); } } if (requestparts.Length < 7) { if (requestparts[0] == "list") { month = DateTime.UtcNow.Month.ToString(); } } else { valid = int.TryParse(requestparts[7], out tempint); if (valid) { month = tempint.ToString(); } else { month = DateTime.UtcNow.Month.ToString(); } } } //Get stats for selected user if (requestparts[0] == "user") { if (requestparts.Length > 3) { this.ValidateUsername(ref requestparts[1]); switch (requestparts[3]) { case "weekly": return API.UserWeekly(requestparts[1], ref this.db); case "monthly": if (String.IsNullOrEmpty(month)) { year = String.Empty; } if (String.IsNullOrEmpty(year)) { month = String.Empty; } return API.UserStats(requestparts[1], year, month, "monthly", ref this.db); case "yearly": return API.UserStats(requestparts[1], year, month, "yearly", ref this.db); default: response.StatusCode = 400; return String.Empty; } } else { response.StatusCode = 400; return String.Empty; } } //Get stats for list of users if (requestparts[0] == "list") { tempint = -1; valid = int.TryParse(requestparts[1], out tempint); if (!valid) { response.StatusCode = 400; } else { switch (requestparts[3]) { case "weekly": return API.ListUsers(requestparts[1], year, month, "weekly", ref this.db); case "monthly": return API.ListUsers(requestparts[1], year, month, "monthly", ref this.db); case "yearly": return API.ListUsers(requestparts[1], year, month,"yearly", ref this.db); default: response.StatusCode = 400; return String.Empty; } } } //Get overall statistics if (requestparts[0] == "overall") { //Get overall statistics for selected user if (requestparts.Length == 3) { this.ValidateUsername(ref requestparts[2]); return API.OverallStats(requestparts[2], ref this.db); } else { return API.OverallStats(String.Empty, ref this.db); } } response.StatusCode = 400; return String.Empty; } else { return "Database disabled"; } } } return "Server API disabled"; } /// /// Timer for check free disk space, cleanup old incomplete files and send .arc files to Hadoop. /// /// /// A Unused /// /// /// A Unused /// void OnTimeElapsed(object sender, System.Timers.ElapsedEventArgs e) { if (sender != null) { sender = null; } if (e != null) { e = null; } //Check for available free space on disk long freespace = this.CheckFreeSpace(); //Clean unfinished files this.CleanFiles(); //Run garbage collector for clean memory GC.GetTotalMemory(true); //Upload .arc files to Hadoop if (this.enableupload) { if (DateTime.UtcNow.Hour == 0) { if (!this.uploading) { this.uploadthread = new System.Threading.Thread(this.UploadThread); this.uploadthread.Name = "uploader"; this.uploadthread.Start(false); } } if (freespace < (this.hadoopfreespace * 1048576)) { if (!this.uploading) { this.uploadthread = new System.Threading.Thread(this.UploadThread); this.uploadthread.Name = "uploader"; this.uploadthread.Start(true); } } } } /// /// Function delete old, incomplete .arc files from temporary directory. /// void CleanFiles() { //Get list of all files string[] files = Directory.GetFiles(this.tempdir); //Process files System.Text.StringBuilder extension = new StringBuilder(); long difftime; this.connected = 0; foreach (string file in files) { //If file is .arc.gz or .gz, check it last modification date extension.Remove(0, extension.Length); extension = extension.Append(file.Substring((file.Length - 2))); if ((extension.ToString() == "gz") || (extension.ToString() == "ma")) { difftime = DateTime.UtcNow.Ticks - File.GetLastWriteTimeUtc(file).Ticks; //If file was modified more than 15 minutes ago, delete it if (difftime > (TimeSpan.TicksPerMinute * 15)) { File.Delete(file); } else { this.connected ++; } } } } /// /// Thread for upload .arc files to Hadoop /// /// /// A If true, upload files from today, otherwise from yesterday /// void UploadThread(object data) { this.uploading = true; string param = String.Empty; if ((bool)data) { param = "today"; } System.Diagnostics.ProcessStartInfo procinfo = new System.Diagnostics.ProcessStartInfo(); procinfo.FileName = "mono"; procinfo.Arguments = "grub-uploader.exe " + param; procinfo.UseShellExecute = false; System.Diagnostics.Process proc = System.Diagnostics.Process.Start(procinfo); proc.WaitForExit(); proc.Close(); proc.Dispose(); this.uploading = false; } /// /// Function check available free space on server /// /// /// A Amount of free bytes /// long CheckFreeSpace() { Mono.Unix.UnixDriveInfo drive = new Mono.Unix.UnixDriveInfo(this.workdir); long freedisk = drive.AvailableFreeSpace; drive = null; if (freedisk < (this.minfreespace * 1048576)) { this.freespace = false; } else { this.freespace = true; } return freedisk; } /// /// Function delete all bad characters from username /// /// /// A Username to validate /// static void ValidateUsername(ref string username) { username = username.Replace("'", String.Empty); username = username.Replace("\"", String.Empty); username = username.Replace("<", String.Empty); username = username.ToLower(); } } }