1
2
3
4
5 import os
6 import socket
7 import select
8 import logging
9 import datetime
10 import SocketServer
11 from threading import Event, Thread
12
13 from lib.cuckoo.common.config import Config
14 from lib.cuckoo.common.constants import CUCKOO_ROOT
15 from lib.cuckoo.common.exceptions import CuckooOperationalError
16 from lib.cuckoo.common.exceptions import CuckooCriticalError
17 from lib.cuckoo.common.exceptions import CuckooResultError
18 from lib.cuckoo.common.netlog import NetlogParser, BsonParser
19 from lib.cuckoo.common.utils import create_folder, Singleton, logtime
20
21 log = logging.getLogger(__name__)
22
23 BUFSIZE = 16 * 1024
24 EXTENSIONS = {
25 NetlogParser: ".raw",
26 BsonParser: ".bson",
27 }
28
31
32
34 """Result server. Singleton!
35
36 This class handles results coming back from the analysis machines.
37 """
38
39 __metaclass__ = Singleton
40
41 allow_reuse_address = True
42 daemon_threads = True
43
45 self.cfg = Config()
46 self.analysistasks = {}
47 self.analysishandlers = {}
48
49 ip = self.cfg.resultserver.ip
50 self.port = int(self.cfg.resultserver.port)
51 while True:
52 try:
53 server_addr = ip, self.port
54 SocketServer.ThreadingTCPServer.__init__(self,
55 server_addr,
56 ResultHandler,
57 *args,
58 **kwargs)
59 except Exception as e:
60
61
62
63
64 if e.errno == 98 or e.errno == 48:
65 log.warning("Cannot bind ResultServer on port {0}, "
66 "trying another port.".format(self.port))
67 self.port += 1
68 else:
69 raise CuckooCriticalError("Unable to bind ResultServer on "
70 "{0}:{1}: {2}".format(
71 ip, self.port, str(e)))
72 else:
73 log.debug("ResultServer running on {0}:{1}.".format(ip, self.port))
74 self.servethread = Thread(target=self.serve_forever)
75 self.servethread.setDaemon(True)
76 self.servethread.start()
77 break
78
80 """Register a task/machine with the ResultServer."""
81 self.analysistasks[machine.ip] = task, machine
82 self.analysishandlers[task.id] = []
83
85 """Delete ResultServer state and wait for pending RequestHandlers."""
86 x = self.analysistasks.pop(machine.ip, None)
87 if not x:
88 log.warning("ResultServer did not have {0} in its task "
89 "info.".format(machine.ip))
90 handlers = self.analysishandlers.pop(task.id, None)
91 for h in handlers:
92 h.end_request.set()
93 h.done_event.wait()
94
96 """Register a RequestHandler so that we can later wait for it."""
97 task, machine = self.get_ctx_for_ip(handler.client_address[0])
98 if not task or not machine:
99 return False
100
101 self.analysishandlers[task.id].append(handler)
102
104 """Return state for this IP's task."""
105 x = self.analysistasks.get(ip)
106 if not x:
107 log.critical("ResultServer unable to map ip to "
108 "context: {0}.".format(ip))
109 return None, None
110
111 return x
112
120
121
123 """Result handler.
124
125 This handler speaks our analysis log network protocol.
126 """
127
129 self.logfd = None
130 self.rawlogfd = None
131 self.protocol = None
132 self.startbuf = ""
133 self.end_request = Event()
134 self.done_event = Event()
135 self.pid, self.ppid, self.procname = None, None, None
136 self.server.register_handler(self)
137
139 self.done_event.set()
140
142 while True:
143 if self.end_request.isSet():
144 return False
145 rs, _, _ = select.select([self.request], [], [], 1)
146 if rs:
147 return True
148
149 - def read(self, length):
150 buf = ""
151 while len(buf) < length:
152 if not self.wait_sock_or_end():
153 raise Disconnect()
154 tmp = self.request.recv(length-len(buf))
155 if not tmp:
156 raise Disconnect()
157 buf += tmp
158
159 if isinstance(self.protocol, (NetlogParser, BsonParser)):
160 if self.rawlogfd:
161 self.rawlogfd.write(buf)
162 else:
163 self.startbuf += buf
164 return buf
165
173
175 buf = ""
176 while "\n" not in buf:
177 buf += self.read(1)
178 return buf
179
195
197 ip, port = self.client_address
198 self.connect_time = datetime.datetime.now()
199 log.debug("New connection from: {0}:{1}".format(ip, port))
200
201 self.storagepath = self.server.build_storage_path(ip)
202 if not self.storagepath:
203 return
204
205
206 self.create_folders()
207
208 try:
209
210 self.negotiate_protocol()
211
212 while self.protocol.read_next_message():
213 pass
214 except CuckooResultError as e:
215 log.warning("ResultServer connection stopping because of "
216 "CuckooResultError: %s.", str(e))
217 except (Disconnect, socket.error):
218 pass
219 except:
220 log.exception("FIXME - exception in resultserver connection %s",
221 str(self.client_address))
222
223 self.protocol.close()
224
225 if self.logfd:
226 self.logfd.close()
227 if self.rawlogfd:
228 self.rawlogfd.close()
229
230 log.debug("Connection closed: {0}:{1}".format(ip, port))
231
232 - def log_process(self, ctx, timestring, pid, ppid, modulepath, procname):
233 if self.pid is not None:
234 log.debug("ResultServer got a new process message but already "
235 "has pid %d ppid %s procname %s.",
236 pid, str(ppid), procname)
237 raise CuckooResultError("ResultServer connection state "
238 "inconsistent.")
239
240 log.debug("New process (pid={0}, ppid={1}, name={2}, "
241 "path={3})".format(pid, ppid, procname, modulepath))
242
243
244 if self.server.cfg.resultserver.store_csvs:
245 path = os.path.join(self.storagepath, "logs", "%d.csv" % pid)
246 self.logfd = open(path, "wb")
247
248
249 ext = EXTENSIONS.get(type(self.protocol), ".raw")
250 path = os.path.join(self.storagepath, "logs", str(pid) + ext)
251 self.rawlogfd = open(path, "wb")
252 self.rawlogfd.write(self.startbuf)
253
254 self.pid, self.ppid, self.procname = pid, ppid, procname
255
257 log.debug("New thread (tid={0}, pid={1})".format(context[3], pid))
258
259 - def log_anomaly(self, subcategory, tid, funcname, msg):
260 log.debug("Anomaly (tid=%s, category=%s, funcname=%s): %s",
261 tid, subcategory, funcname, msg)
262
263 - def log_call(self, context, apiname, modulename, arguments):
264 if not self.rawlogfd:
265 raise CuckooOperationalError("Netlog failure, call "
266 "before process.")
267
268 apiindex, status, returnval, tid, timediff = context
269
270
271
272 current_time = \
273 self.connect_time + datetime.timedelta(0, 0, timediff*1000)
274 timestring = logtime(current_time)
275
276 argumentstrings = ["{0}->{1}".format(argname, repr(str(r))[1:-1])
277 for argname, r in arguments]
278
279 if self.logfd:
280 print >>self.logfd, ",".join("\"{0}\"".format(i) for i in [
281 timestring, self.pid, self.procname, tid, self.ppid,
282 modulename, apiname, status, returnval] + argumentstrings)
283
285 log.warning("ResultServer error condition on connection %s "
286 "(pid %s procname %s): %s", str(self.client_address),
287 str(self.pid), str(self.procname), emsg)
288
290 folders = "shots", "files", "logs"
291
292 for folder in folders:
293 try:
294 create_folder(self.storagepath, folder=folder)
295 except CuckooOperationalError:
296 log.error("Unable to create folder %s" % folder)
297 return False
298
299
301 RESTRICTED_DIRECTORIES = "reports/",
302
304 self.handler = handler
305 self.upload_max_size = \
306 self.handler.server.cfg.resultserver.upload_max_size
307 self.storagepath = self.handler.storagepath
308 self.fd = None
309
311
312
313
314 buf = self.handler.read_newline().strip().replace("\\", "/")
315 log.debug("File upload request for {0}".format(buf))
316
317 dir_part, filename = os.path.split(buf)
318
319 if "./" in buf or not dir_part or buf.startswith("/"):
320 raise CuckooOperationalError("FileUpload failure, banned path.")
321
322 for restricted in self.RESTRICTED_DIRECTORIES:
323 if restricted in dir_part:
324 raise CuckooOperationalError("FileUpload failure, banned path.")
325
326 try:
327 create_folder(self.storagepath, dir_part)
328 except CuckooOperationalError:
329 log.error("Unable to create folder %s" % dir_part)
330 return False
331
332 file_path = os.path.join(self.storagepath, buf.strip())
333
334 if not file_path.startswith(self.storagepath):
335 raise CuckooOperationalError("FileUpload failure, path sanitization failed.")
336
337 if os.path.exists(file_path):
338 log.warning("Analyzer tried to overwrite an existing file, closing connection.")
339 return False
340
341 self.fd = open(file_path, "wb")
342 chunk = self.handler.read_any()
343 while chunk:
344 self.fd.write(chunk)
345
346 if self.fd.tell() >= self.upload_max_size:
347 log.warning("Uploaded file length larger than upload_max_size, stopping upload.")
348 self.fd.write("... (truncated)")
349 break
350
351 try:
352 chunk = self.handler.read_any()
353 except:
354 break
355
356 log.debug("Uploaded file length: {0}".format(self.fd.tell()))
357
359 if self.fd:
360 self.fd.close()
361
362
365 self.handler = handler
366 self.logpath = os.path.join(handler.storagepath, "analysis.log")
367 self.fd = self._open()
368 log.debug("LogHandler for live analysis.log initialized.")
369
371 if not self.fd:
372 return False
373
374 buf = self.handler.read_newline()
375 if not buf:
376 return False
377 self.fd.write(buf)
378 self.fd.flush()
379 return True
380
382 if self.fd:
383 self.fd.close()
384
386 if not os.path.exists(self.logpath):
387 return open(self.logpath, "wb")
388
389 log.debug("Log analysis.log already existing, appending data.")
390 fd = open(self.logpath, "ab")
391
392
393
394
395 now = datetime.datetime.now()
396 print >>fd, "\n%s,%03.0f [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended." % (
397 now.strftime("%Y-%m-%d %H:%M:%S"), now.microsecond / 1000.0
398 )
399
400 return fd
401