Package lib :: Package cuckoo :: Package core :: Module resultserver
[hide private]
[frames] | no frames]

Source Code for Module lib.cuckoo.core.resultserver

  1  # Copyright (C) 2010-2015 Cuckoo Foundation. 
  2  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org 
  3  # See the file 'docs/LICENSE' for copying permission. 
  4   
  5  import os 
  6  import socket 
  7  import select 
  8  import logging 
  9  import datetime 
 10  import SocketServer 
 11  from threading import Event, Thread 
 12   
 13  from lib.cuckoo.common.config import Config 
 14  from lib.cuckoo.common.constants import CUCKOO_ROOT 
 15  from lib.cuckoo.common.exceptions import CuckooOperationalError 
 16  from lib.cuckoo.common.exceptions import CuckooCriticalError 
 17  from lib.cuckoo.common.exceptions import CuckooResultError 
 18  from lib.cuckoo.common.netlog import NetlogParser, BsonParser 
 19  from lib.cuckoo.common.utils import create_folder, Singleton, logtime 
 20   
 21  log = logging.getLogger(__name__) 
 22   
 23  BUFSIZE = 16 * 1024 
 24  EXTENSIONS = { 
 25      NetlogParser: ".raw", 
 26      BsonParser: ".bson", 
 27  } 
 28   
29 -class Disconnect(Exception):
30 pass
31 32
33 -class ResultServer(SocketServer.ThreadingTCPServer, object):
34 """Result server. Singleton! 35 36 This class handles results coming back from the analysis machines. 37 """ 38 39 __metaclass__ = Singleton 40 41 allow_reuse_address = True 42 daemon_threads = True 43
44 - def __init__(self, *args, **kwargs):
45 self.cfg = Config() 46 self.analysistasks = {} 47 self.analysishandlers = {} 48 49 ip = self.cfg.resultserver.ip 50 self.port = int(self.cfg.resultserver.port) 51 while True: 52 try: 53 server_addr = ip, self.port 54 SocketServer.ThreadingTCPServer.__init__(self, 55 server_addr, 56 ResultHandler, 57 *args, 58 **kwargs) 59 except Exception as e: 60 # In Linux /usr/include/asm-generic/errno-base.h. 61 # EADDRINUSE 98 (Address already in use) 62 # In Mac OS X or FreeBSD: 63 # EADDRINUSE 48 (Address already in use) 64 if e.errno == 98 or e.errno == 48: 65 log.warning("Cannot bind ResultServer on port {0}, " 66 "trying another port.".format(self.port)) 67 self.port += 1 68 else: 69 raise CuckooCriticalError("Unable to bind ResultServer on " 70 "{0}:{1}: {2}".format( 71 ip, self.port, str(e))) 72 else: 73 log.debug("ResultServer running on {0}:{1}.".format(ip, self.port)) 74 self.servethread = Thread(target=self.serve_forever) 75 self.servethread.setDaemon(True) 76 self.servethread.start() 77 break
78
79 - def add_task(self, task, machine):
80 """Register a task/machine with the ResultServer.""" 81 self.analysistasks[machine.ip] = task, machine 82 self.analysishandlers[task.id] = []
83
84 - def del_task(self, task, machine):
85 """Delete ResultServer state and wait for pending RequestHandlers.""" 86 x = self.analysistasks.pop(machine.ip, None) 87 if not x: 88 log.warning("ResultServer did not have {0} in its task " 89 "info.".format(machine.ip)) 90 handlers = self.analysishandlers.pop(task.id, None) 91 for h in handlers: 92 h.end_request.set() 93 h.done_event.wait()
94
95 - def register_handler(self, handler):
96 """Register a RequestHandler so that we can later wait for it.""" 97 task, machine = self.get_ctx_for_ip(handler.client_address[0]) 98 if not task or not machine: 99 return False 100 101 self.analysishandlers[task.id].append(handler)
102
103 - def get_ctx_for_ip(self, ip):
104 """Return state for this IP's task.""" 105 x = self.analysistasks.get(ip) 106 if not x: 107 log.critical("ResultServer unable to map ip to " 108 "context: {0}.".format(ip)) 109 return None, None 110 111 return x
112
113 - def build_storage_path(self, ip):
114 """Initialize analysis storage folder.""" 115 task, machine = self.get_ctx_for_ip(ip) 116 if not task or not machine: 117 return 118 119 return os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task.id))
120 121
122 -class ResultHandler(SocketServer.BaseRequestHandler):
123 """Result handler. 124 125 This handler speaks our analysis log network protocol. 126 """ 127
128 - def setup(self):
129 self.logfd = None 130 self.rawlogfd = None 131 self.protocol = None 132 self.startbuf = "" 133 self.end_request = Event() 134 self.done_event = Event() 135 self.pid, self.ppid, self.procname = None, None, None 136 self.server.register_handler(self)
137
138 - def finish(self):
139 self.done_event.set()
140
141 - def wait_sock_or_end(self):
142 while True: 143 if self.end_request.isSet(): 144 return False 145 rs, _, _ = select.select([self.request], [], [], 1) 146 if rs: 147 return True
148
149 - def read(self, length):
150 buf = "" 151 while len(buf) < length: 152 if not self.wait_sock_or_end(): 153 raise Disconnect() 154 tmp = self.request.recv(length-len(buf)) 155 if not tmp: 156 raise Disconnect() 157 buf += tmp 158 159 if isinstance(self.protocol, (NetlogParser, BsonParser)): 160 if self.rawlogfd: 161 self.rawlogfd.write(buf) 162 else: 163 self.startbuf += buf 164 return buf
165
166 - def read_any(self):
167 if not self.wait_sock_or_end(): 168 raise Disconnect() 169 tmp = self.request.recv(BUFSIZE) 170 if not tmp: 171 raise Disconnect() 172 return tmp
173
174 - def read_newline(self):
175 buf = "" 176 while "\n" not in buf: 177 buf += self.read(1) 178 return buf
179
180 - def negotiate_protocol(self):
181 # Read until newline. 182 buf = self.read_newline() 183 184 if "NETLOG" in buf: 185 self.protocol = NetlogParser(self) 186 elif "BSON" in buf: 187 self.protocol = BsonParser(self) 188 elif "FILE" in buf: 189 self.protocol = FileUpload(self) 190 elif "LOG" in buf: 191 self.protocol = LogHandler(self) 192 else: 193 raise CuckooOperationalError("Netlog failure, unknown " 194 "protocol requested.")
195
196 - def handle(self):
197 ip, port = self.client_address 198 self.connect_time = datetime.datetime.now() 199 log.debug("New connection from: {0}:{1}".format(ip, port)) 200 201 self.storagepath = self.server.build_storage_path(ip) 202 if not self.storagepath: 203 return 204 205 # Create all missing folders for this analysis. 206 self.create_folders() 207 208 try: 209 # Initialize the protocol handler class for this connection. 210 self.negotiate_protocol() 211 212 while self.protocol.read_next_message(): 213 pass 214 except CuckooResultError as e: 215 log.warning("ResultServer connection stopping because of " 216 "CuckooResultError: %s.", str(e)) 217 except (Disconnect, socket.error): 218 pass 219 except: 220 log.exception("FIXME - exception in resultserver connection %s", 221 str(self.client_address)) 222 223 self.protocol.close() 224 225 if self.logfd: 226 self.logfd.close() 227 if self.rawlogfd: 228 self.rawlogfd.close() 229 230 log.debug("Connection closed: {0}:{1}".format(ip, port))
231
232 - def log_process(self, ctx, timestring, pid, ppid, modulepath, procname):
233 if self.pid is not None: 234 log.debug("ResultServer got a new process message but already " 235 "has pid %d ppid %s procname %s.", 236 pid, str(ppid), procname) 237 raise CuckooResultError("ResultServer connection state " 238 "inconsistent.") 239 240 log.debug("New process (pid={0}, ppid={1}, name={2}, " 241 "path={3})".format(pid, ppid, procname, modulepath)) 242 243 # CSV format files are optional. 244 if self.server.cfg.resultserver.store_csvs: 245 path = os.path.join(self.storagepath, "logs", "%d.csv" % pid) 246 self.logfd = open(path, "wb") 247 248 # Raw Bson or Netlog extension. 249 ext = EXTENSIONS.get(type(self.protocol), ".raw") 250 path = os.path.join(self.storagepath, "logs", str(pid) + ext) 251 self.rawlogfd = open(path, "wb") 252 self.rawlogfd.write(self.startbuf) 253 254 self.pid, self.ppid, self.procname = pid, ppid, procname
255
256 - def log_thread(self, context, pid):
257 log.debug("New thread (tid={0}, pid={1})".format(context[3], pid))
258
259 - def log_anomaly(self, subcategory, tid, funcname, msg):
260 log.debug("Anomaly (tid=%s, category=%s, funcname=%s): %s", 261 tid, subcategory, funcname, msg)
262
263 - def log_call(self, context, apiname, modulename, arguments):
264 if not self.rawlogfd: 265 raise CuckooOperationalError("Netlog failure, call " 266 "before process.") 267 268 apiindex, status, returnval, tid, timediff = context 269 270 # log.debug("log_call> tid:{0} apiname:{1}".format(tid, apiname)) 271 272 current_time = \ 273 self.connect_time + datetime.timedelta(0, 0, timediff*1000) 274 timestring = logtime(current_time) 275 276 argumentstrings = ["{0}->{1}".format(argname, repr(str(r))[1:-1]) 277 for argname, r in arguments] 278 279 if self.logfd: 280 print >>self.logfd, ",".join("\"{0}\"".format(i) for i in [ 281 timestring, self.pid, self.procname, tid, self.ppid, 282 modulename, apiname, status, returnval] + argumentstrings)
283
284 - def log_error(self, emsg):
285 log.warning("ResultServer error condition on connection %s " 286 "(pid %s procname %s): %s", str(self.client_address), 287 str(self.pid), str(self.procname), emsg)
288
289 - def create_folders(self):
290 folders = "shots", "files", "logs" 291 292 for folder in folders: 293 try: 294 create_folder(self.storagepath, folder=folder) 295 except CuckooOperationalError: 296 log.error("Unable to create folder %s" % folder) 297 return False
298 299
300 -class FileUpload(object):
301 RESTRICTED_DIRECTORIES = "reports/", 302
303 - def __init__(self, handler):
304 self.handler = handler 305 self.upload_max_size = \ 306 self.handler.server.cfg.resultserver.upload_max_size 307 self.storagepath = self.handler.storagepath 308 self.fd = None
309
310 - def read_next_message(self):
311 # Read until newline for file path, e.g., 312 # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin 313 314 buf = self.handler.read_newline().strip().replace("\\", "/") 315 log.debug("File upload request for {0}".format(buf)) 316 317 dir_part, filename = os.path.split(buf) 318 319 if "./" in buf or not dir_part or buf.startswith("/"): 320 raise CuckooOperationalError("FileUpload failure, banned path.") 321 322 for restricted in self.RESTRICTED_DIRECTORIES: 323 if restricted in dir_part: 324 raise CuckooOperationalError("FileUpload failure, banned path.") 325 326 try: 327 create_folder(self.storagepath, dir_part) 328 except CuckooOperationalError: 329 log.error("Unable to create folder %s" % dir_part) 330 return False 331 332 file_path = os.path.join(self.storagepath, buf.strip()) 333 334 if not file_path.startswith(self.storagepath): 335 raise CuckooOperationalError("FileUpload failure, path sanitization failed.") 336 337 if os.path.exists(file_path): 338 log.warning("Analyzer tried to overwrite an existing file, closing connection.") 339 return False 340 341 self.fd = open(file_path, "wb") 342 chunk = self.handler.read_any() 343 while chunk: 344 self.fd.write(chunk) 345 346 if self.fd.tell() >= self.upload_max_size: 347 log.warning("Uploaded file length larger than upload_max_size, stopping upload.") 348 self.fd.write("... (truncated)") 349 break 350 351 try: 352 chunk = self.handler.read_any() 353 except: 354 break 355 356 log.debug("Uploaded file length: {0}".format(self.fd.tell()))
357
358 - def close(self):
359 if self.fd: 360 self.fd.close()
361 362
363 -class LogHandler(object):
364 - def __init__(self, handler):
365 self.handler = handler 366 self.logpath = os.path.join(handler.storagepath, "analysis.log") 367 self.fd = self._open() 368 log.debug("LogHandler for live analysis.log initialized.")
369
370 - def read_next_message(self):
371 if not self.fd: 372 return False 373 374 buf = self.handler.read_newline() 375 if not buf: 376 return False 377 self.fd.write(buf) 378 self.fd.flush() 379 return True
380
381 - def close(self):
382 if self.fd: 383 self.fd.close()
384
385 - def _open(self):
386 if not os.path.exists(self.logpath): 387 return open(self.logpath, "wb") 388 389 log.debug("Log analysis.log already existing, appending data.") 390 fd = open(self.logpath, "ab") 391 392 # add a fake log entry, saying this had to be re-opened 393 # use the same format as the default logger, in case anyone wants to parse this 394 # 2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC injection. 395 now = datetime.datetime.now() 396 print >>fd, "\n%s,%03.0f [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended." % ( 397 now.strftime("%Y-%m-%d %H:%M:%S"), now.microsecond / 1000.0 398 ) 399 400 return fd
401