1
2
3
4
5 import os
6 import time
7 import shutil
8 import logging
9 import Queue
10 from threading import Thread, Lock
11
12 from lib.cuckoo.common.config import Config
13 from lib.cuckoo.common.constants import CUCKOO_ROOT
14 from lib.cuckoo.common.exceptions import CuckooMachineError, CuckooGuestError
15 from lib.cuckoo.common.exceptions import CuckooOperationalError
16 from lib.cuckoo.common.exceptions import CuckooCriticalError
17 from lib.cuckoo.common.objects import File
18 from lib.cuckoo.common.utils import create_folder
19 from lib.cuckoo.core.database import Database, TASK_COMPLETED, TASK_REPORTED
20 from lib.cuckoo.core.guest import GuestManager
21 from lib.cuckoo.core.plugins import list_plugins, RunAuxiliary, RunProcessing
22 from lib.cuckoo.core.plugins import RunSignatures, RunReporting
23 from lib.cuckoo.core.resultserver import ResultServer
24
25 log = logging.getLogger(__name__)
26
27 machinery = None
28 machine_lock = Lock()
29 latest_symlink_lock = Lock()
30
31 active_analysis_count = 0
32
33
35 """Exception thrown when a machine turns dead.
36
37 When this exception has been thrown, the analysis task will start again,
38 and will try to use another machine, when available.
39 """
40 pass
41
42
44 """Analysis Manager.
45
46 This class handles the full analysis process for a given task. It takes
47 care of selecting the analysis machine, preparing the configuration and
48 interacting with the guest agent and analyzer components to launch and
49 complete the analysis and store, process and report its results.
50 """
51
53 """@param task: task object containing the details for the analysis."""
54 Thread.__init__(self)
55 Thread.daemon = True
56
57 self.task = task
58 self.errors = error_queue
59 self.cfg = Config()
60 self.storage = ""
61 self.binary = ""
62 self.machine = None
63
65 """Initialize analysis storage folder."""
66 self.storage = os.path.join(CUCKOO_ROOT,
67 "storage",
68 "analyses",
69 str(self.task.id))
70
71
72
73 if os.path.exists(self.storage):
74 log.error("Analysis results folder already exists at path \"%s\","
75 " analysis aborted", self.storage)
76 return False
77
78
79
80 try:
81 create_folder(folder=self.storage)
82 except CuckooOperationalError:
83 log.error("Unable to create analysis folder %s", self.storage)
84 return False
85
86 return True
87
98
100 """Store a copy of the file being analyzed."""
101 if not os.path.exists(self.task.target):
102 log.error("The file to analyze does not exist at path \"%s\", "
103 "analysis aborted", self.task.target)
104 return False
105
106 sha256 = File(self.task.target).get_sha256()
107 self.binary = os.path.join(CUCKOO_ROOT, "storage", "binaries", sha256)
108
109 if os.path.exists(self.binary):
110 log.info("File already exists at \"%s\"", self.binary)
111 else:
112
113
114 try:
115 shutil.copy(self.task.target, self.binary)
116 except (IOError, shutil.Error) as e:
117 log.error("Unable to store file from \"%s\" to \"%s\", "
118 "analysis aborted", self.task.target, self.binary)
119 return False
120
121 try:
122 new_binary_path = os.path.join(self.storage, "binary")
123
124 if hasattr(os, "symlink"):
125 os.symlink(self.binary, new_binary_path)
126 else:
127 shutil.copy(self.binary, new_binary_path)
128 except (AttributeError, OSError) as e:
129 log.error("Unable to create symlink/copy from \"%s\" to "
130 "\"%s\": %s", self.binary, self.storage, e)
131
132 return True
133
170
198
200 """Start analysis."""
201 succeeded = False
202 dead_machine = False
203
204 log.info("Starting analysis of %s \"%s\" (task=%d)",
205 self.task.category.upper(), self.task.target, self.task.id)
206
207
208 if not self.init_storage():
209 return False
210
211 if self.task.category == "file":
212
213
214 if not self.check_file():
215 return False
216
217
218 if not self.store_file():
219 return False
220
221
222 try:
223 self.acquire_machine()
224 except CuckooOperationalError as e:
225 log.error("Cannot acquire machine: {0}".format(e))
226 return False
227
228
229 options = self.build_options()
230
231
232 try:
233 ResultServer().add_task(self.task, self.machine)
234 except Exception as e:
235 machinery.release(self.machine.label)
236 self.errors.put(e)
237
238 aux = RunAuxiliary(task=self.task, machine=self.machine)
239 aux.start()
240
241 try:
242
243 guest_log = Database().guest_start(self.task.id,
244 self.machine.name,
245 self.machine.label,
246 machinery.__class__.__name__)
247
248 machinery.start(self.machine.label)
249
250
251 guest = GuestManager(self.machine.name, self.machine.ip,
252 self.machine.platform)
253
254
255 guest.start_analysis(options)
256
257 guest.wait_for_completion()
258 succeeded = True
259 except CuckooMachineError as e:
260 log.error(str(e), extra={"task_id": self.task.id})
261 dead_machine = True
262 except CuckooGuestError as e:
263 log.error(str(e), extra={"task_id": self.task.id})
264 finally:
265
266 aux.stop()
267
268
269 if self.cfg.cuckoo.memory_dump or self.task.memory:
270 try:
271 dump_path = os.path.join(self.storage, "memory.dmp")
272 machinery.dump_memory(self.machine.label, dump_path)
273 except NotImplementedError:
274 log.error("The memory dump functionality is not available "
275 "for the current machine manager.")
276 except CuckooMachineError as e:
277 log.error(e)
278
279 try:
280
281 machinery.stop(self.machine.label)
282 except CuckooMachineError as e:
283 log.warning("Unable to stop machine %s: %s",
284 self.machine.label, e)
285
286
287
288
289 Database().guest_stop(guest_log)
290
291
292
293 ResultServer().del_task(self.task, self.machine)
294
295 if dead_machine:
296
297
298
299 Database().guest_remove(guest_log)
300
301
302
303 shutil.rmtree(self.storage)
304
305
306
307
308 raise CuckooDeadMachine()
309
310 try:
311
312
313 machinery.release(self.machine.label)
314 except CuckooMachineError as e:
315 log.error("Unable to release machine %s, reason %s. "
316 "You might need to restore it manually.",
317 self.machine.label, e)
318
319 return succeeded
320
322 """Process the analysis results and generate the enabled reports."""
323 results = RunProcessing(task_id=self.task.id).run()
324 RunSignatures(results=results).run()
325 RunReporting(task_id=self.task.id, results=results).run()
326
327
328
329 if self.task.category == "file" and self.cfg.cuckoo.delete_original:
330 if not os.path.exists(self.task.target):
331 log.warning("Original file does not exist anymore: \"%s\": "
332 "File not found.", self.task.target)
333 else:
334 try:
335 os.remove(self.task.target)
336 except OSError as e:
337 log.error("Unable to delete original file at path "
338 "\"%s\": %s", self.task.target, e)
339
340
341
342 if self.task.category == "file" and self.cfg.cuckoo.delete_bin_copy:
343 if not os.path.exists(self.binary):
344 log.warning("Copy of the original file does not exist anymore: \"%s\": File not found", self.binary)
345 else:
346 try:
347 os.remove(self.binary)
348 except OSError as e:
349 log.error("Unable to delete the copy of the original file at path \"%s\": %s", self.binary, e)
350
351 log.info("Task #%d: reports generation completed (path=%s)",
352 self.task.id, self.storage)
353
354 return True
355
357 """Run manager thread."""
358 global active_analysis_count
359 active_analysis_count += 1
360 try:
361 while True:
362 try:
363 success = self.launch_analysis()
364 except CuckooDeadMachine:
365 continue
366
367 break
368
369 Database().set_status(self.task.id, TASK_COMPLETED)
370
371 log.debug("Released database task #%d with status %s",
372 self.task.id, success)
373
374 if self.cfg.cuckoo.process_results:
375 self.process_results()
376 Database().set_status(self.task.id, TASK_REPORTED)
377
378
379
380
381 if hasattr(os, "symlink"):
382 latest = os.path.join(CUCKOO_ROOT, "storage",
383 "analyses", "latest")
384
385
386
387
388 latest_symlink_lock.acquire()
389 try:
390 if os.path.exists(latest):
391 os.remove(latest)
392
393 os.symlink(self.storage, latest)
394 except OSError as e:
395 log.warning("Error pointing latest analysis symlink: %s" % e)
396 finally:
397 latest_symlink_lock.release()
398
399 log.info("Task #%d: analysis procedure completed", self.task.id)
400 except:
401 log.exception("Failure in AnalysisManager.run")
402
403 active_analysis_count -= 1
404
406 """Tasks Scheduler.
407
408 This class is responsible for the main execution loop of the tool. It
409 prepares the analysis machines and keep waiting and loading for new
410 analysis tasks.
411 Whenever a new task is available, it launches AnalysisManager which will
412 take care of running the full analysis process and operating with the
413 assigned analysis machine.
414 """
416 self.running = True
417 self.cfg = Config()
418 self.db = Database()
419 self.maxcount = maxcount
420 self.total_analysis_count = 0
421
423 """Initialize the machine manager."""
424 global machinery
425
426 machinery_name = self.cfg.cuckoo.machinery
427
428 log.info("Using \"%s\" machine manager", machinery_name)
429
430
431
432 plugin = list_plugins("machinery")[0]
433
434 machinery = plugin()
435
436
437 conf = os.path.join(CUCKOO_ROOT, "conf", "%s.conf" % machinery_name)
438
439 if not os.path.exists(conf):
440 raise CuckooCriticalError("The configuration file for machine "
441 "manager \"{0}\" does not exist at path:"
442 " {1}".format(machinery_name, conf))
443
444
445
446 machinery.set_options(Config(machinery_name))
447
448
449 try:
450 machinery.initialize(machinery_name)
451 except CuckooMachineError as e:
452 raise CuckooCriticalError("Error initializing machines: %s" % e)
453
454
455
456
457 if not len(machinery.machines()):
458 raise CuckooCriticalError("No machines available.")
459 else:
460 log.info("Loaded %s machine/s", len(machinery.machines()))
461
462 if len(machinery.machines()) > 1 and self.db.engine.name == "sqlite":
463 log.warning("As you've configured Cuckoo to execute parallel "
464 "analyses, we recommend you to switch to a MySQL "
465 "a PostgreSQL database as SQLite might cause some "
466 "issues.")
467
468 if len(machinery.machines()) > 4 and self.cfg.cuckoo.process_results:
469 log.warning("When running many virtual machines it is recommended "
470 "to process the results in a separate process.py to "
471 "increase throughput and stability. Please read the "
472 "documentation about the `Processing Utility`.")
473
479
481 """Start scheduler."""
482 self.initialize()
483
484 log.info("Waiting for analysis tasks.")
485
486
487 errors = Queue.Queue()
488
489
490 if self.maxcount is None:
491 self.maxcount = self.cfg.cuckoo.max_analysis_count
492
493
494 while self.running:
495 time.sleep(1)
496
497
498
499
500 if self.cfg.cuckoo.freespace:
501
502
503 dir_path = os.path.join(CUCKOO_ROOT, "storage", "analyses")
504
505
506 if hasattr(os, "statvfs"):
507 dir_stats = os.statvfs(dir_path)
508
509
510 space_available = dir_stats.f_bavail * dir_stats.f_frsize
511 space_available /= 1024 * 1024
512
513 if space_available < self.cfg.cuckoo.freespace:
514 log.error("Not enough free disk space! (Only %d MB!)",
515 space_available)
516 continue
517
518
519 if self.cfg.cuckoo.max_machines_count > 0:
520
521 if len(machinery.running()) >= self.cfg.cuckoo.max_machines_count:
522 continue
523
524
525
526 if not machinery.availables():
527 continue
528
529
530
531 if self.maxcount and self.total_analysis_count >= self.maxcount:
532 if active_analysis_count <= 0:
533 self.stop()
534 else:
535
536
537 for machine in self.db.get_available_machines():
538
539 task = self.db.fetch(machine=machine.name)
540 if task:
541 log.debug("Processing task #%s", task.id)
542 self.total_analysis_count += 1
543
544
545 analysis = AnalysisManager(task, errors)
546 analysis.start()
547
548
549 try:
550 raise errors.get(block=False)
551 except Queue.Empty:
552 pass
553