1
2
3
4
5 import os
6 import logging
7 import datetime
8
9 from lib.cuckoo.common.abstracts import Processing
10 from lib.cuckoo.common.config import Config
11 from lib.cuckoo.common.netlog import NetlogParser, BsonParser
12 from lib.cuckoo.common.utils import convert_to_printable, logtime
13 from lib.cuckoo.common.utils import cleanup_value
14
15 log = logging.getLogger(__name__)
16
18 """Fix a registry key to have it normalized.
19 @param key: raw key
20 @returns: normalized key
21 """
22 res = key
23 if key.lower().startswith("registry\\machine\\"):
24 res = "HKEY_LOCAL_MACHINE\\" + key[17:]
25 elif key.lower().startswith("registry\\user\\"):
26 res = "HKEY_USERS\\" + key[14:]
27 elif key.lower().startswith("\\registry\\machine\\"):
28 res = "HKEY_LOCAL_MACHINE\\" + key[18:]
29 elif key.lower().startswith("\\registry\\user\\"):
30 res = "HKEY_USERS\\" + key[15:]
31
32 return res
33
35 """Parses process log file."""
36
38 """@param log_path: log file path."""
39 self._log_path = log_path
40 self.fd = None
41 self.parser = None
42
43 self.process_id = None
44 self.process_name = None
45 self.parent_id = None
46 self.first_seen = None
47 self.calls = self
48 self.lastcall = None
49 self.call_id = 0
50
51 if os.path.exists(log_path) and os.stat(log_path).st_size > 0:
52 self.parse_first_and_reset()
53
55 self.fd = open(self._log_path, "rb")
56
57 if self._log_path.endswith(".bson"):
58 self.parser = BsonParser(self)
59 elif self._log_path.endswith(".raw"):
60 self.parser = NetlogParser(self)
61 else:
62 self.fd.close()
63 self.fd = None
64 return
65
66
67
68 while not self.process_id:
69 self.parser.read_next_message()
70
71 self.fd.seek(0)
72
73 - def read(self, length):
74 if not length:
75 return ''
76 buf = self.fd.read(length)
77 if not buf or len(buf) != length:
78 raise EOFError()
79 return buf
80
85
87 return "<ParseProcessLog log-path: %r>" % self._log_path
88
91
93 self.fd.seek(0)
94 self.lastcall = None
95 self.call_id = 0
96
98 """Compare two calls for equality. Same implementation as before netlog.
99 @param a: call a
100 @param b: call b
101 @return: True if a == b else False
102 """
103 if a["api"] == b["api"] and \
104 a["status"] == b["status"] and \
105 a["arguments"] == b["arguments"] and \
106 a["return"] == b["return"]:
107 return True
108 return False
109
111 while not self.lastcall:
112 try:
113 if not self.parser.read_next_message():
114 return False
115 except EOFError:
116 return False
117
118 return True
119
121 if not self.fd:
122 raise StopIteration()
123
124 if not self.wait_for_lastcall():
125 self.reset()
126 raise StopIteration()
127
128 nextcall, self.lastcall = self.lastcall, None
129
130 self.wait_for_lastcall()
131 while self.lastcall and self.compare_calls(nextcall, self.lastcall):
132 nextcall["repeated"] += 1
133 self.lastcall = None
134 self.wait_for_lastcall()
135
136 nextcall["id"] = self.call_id
137 self.call_id += 1
138
139 return nextcall
140
141 - def log_process(self, context, timestring, pid, ppid, modulepath, procname):
142 self.process_id, self.parent_id, self.process_name = pid, ppid, procname
143 self.first_seen = timestring
144
147
148 - def log_anomaly(self, subcategory, tid, funcname, msg):
149 self.lastcall = dict(thread_id=tid, category="anomaly", api="",
150 subcategory=subcategory, funcname=funcname,
151 msg=msg)
152
153 - def log_call(self, context, apiname, category, arguments):
154 apiindex, status, returnval, tid, timediff = context
155
156 current_time = self.first_seen + datetime.timedelta(0, 0, timediff*1000)
157 timestring = logtime(current_time)
158
159 self.lastcall = self._parse([timestring,
160 tid,
161 category,
162 apiname,
163 status,
164 returnval] + arguments)
165
167 log.warning("ParseProcessLog error condition on log %s: %s", str(self._log_path), emsg)
168
170 """Parse log row.
171 @param row: row data.
172 @return: parsed information dict.
173 """
174 call = {}
175 arguments = []
176
177 try:
178 timestamp = row[0]
179 thread_id = row[1]
180 category = row[2]
181 api_name = row[3]
182 status_value = row[4]
183 return_value = row[5]
184 except IndexError as e:
185 log.debug("Unable to parse process log row: %s", e)
186 return None
187
188
189
190 for index in range(6, len(row)):
191 argument = {}
192
193
194 try:
195 arg_name, arg_value = row[index]
196 except ValueError as e:
197 log.debug("Unable to parse analysis row argument (row=%s): %s", row[index], e)
198 continue
199
200 argument["name"] = arg_name
201
202 argument["value"] = convert_to_printable(cleanup_value(arg_value))
203 arguments.append(argument)
204
205 call["timestamp"] = timestamp
206 call["thread_id"] = str(thread_id)
207 call["category"] = category
208 call["api"] = api_name
209 call["status"] = bool(int(status_value))
210
211 if isinstance(return_value, int):
212 call["return"] = "0x%.08x" % return_value
213 else:
214 call["return"] = convert_to_printable(cleanup_value(return_value))
215
216 call["arguments"] = arguments
217 call["repeated"] = 0
218
219 return call
220
222 """Processes analyzer."""
223
225 """@param logs_path: logs path."""
226 self._logs_path = logs_path
227 self.cfg = Config()
228
230 """Run analysis.
231 @return: processes infomartion list.
232 """
233 results = []
234
235 if not os.path.exists(self._logs_path):
236 log.warning("Analysis results folder does not exist at path \"%s\".", self._logs_path)
237 return results
238
239
240
241 if len(os.listdir(self._logs_path)) == 0:
242 log.info("Analysis results folder does not contain any file or injection was disabled.")
243 return results
244
245 for file_name in os.listdir(self._logs_path):
246 file_path = os.path.join(self._logs_path, file_name)
247
248 if os.path.isdir(file_path):
249 continue
250
251
252 if os.stat(file_path).st_size > self.cfg.processing.analysis_size_limit:
253 log.warning("Behavioral log {0} too big to be processed, skipped.".format(file_name))
254 continue
255
256
257 current_log = ParseProcessLog(file_path)
258 if current_log.process_id is None:
259 continue
260
261
262
263 results.append({
264 "process_id": current_log.process_id,
265 "process_name": current_log.process_name,
266 "parent_id": current_log.parent_id,
267 "first_seen": logtime(current_log.first_seen),
268 "calls": current_log.calls,
269 })
270
271
272
273 results.sort(key=lambda process: process["first_seen"])
274
275 return results
276
278 """Generates summary information."""
279
280 key = "summary"
281
283 self.keys = []
284 self.mutexes = []
285 self.files = []
286 self.handles = []
287
289 for known_handle in self.handles:
290 if handle != 0 and handle == known_handle["handle"]:
291 return None
292
293 name = ""
294
295 if registry == 0x80000000:
296 name = "HKEY_CLASSES_ROOT\\"
297 elif registry == 0x80000001:
298 name = "HKEY_CURRENT_USER\\"
299 elif registry == 0x80000002:
300 name = "HKEY_LOCAL_MACHINE\\"
301 elif registry == 0x80000003:
302 name = "HKEY_USERS\\"
303 elif registry == 0x80000004:
304 name = "HKEY_PERFORMANCE_DATA\\"
305 elif registry == 0x80000005:
306 name = "HKEY_CURRENT_CONFIG\\"
307 elif registry == 0x80000006:
308 name = "HKEY_DYN_DATA\\"
309 else:
310 for known_handle in self.handles:
311 if registry == known_handle["handle"]:
312 name = known_handle["name"] + "\\"
313
314 key = fix_key(name + subkey)
315 self.handles.append({"handle": handle, "name": key})
316 return key
317
319 """Generate processes list from streamed calls/processes.
320 @return: None.
321 """
322
323 if call["api"].startswith("RegOpenKeyEx") or call["api"].startswith("RegCreateKeyEx"):
324 registry = 0
325 subkey = ""
326 handle = 0
327
328 for argument in call["arguments"]:
329 if argument["name"] == "Registry":
330 registry = int(argument["value"], 16)
331 elif argument["name"] == "SubKey":
332 subkey = argument["value"]
333 elif argument["name"] == "Handle":
334 handle = int(argument["value"], 16)
335
336 name = self._check_registry(registry, subkey, handle)
337 if name and name not in self.keys:
338 self.keys.append(name)
339 elif call["api"].startswith("NtOpenKey"):
340 registry = -1
341 subkey = ""
342 handle = 0
343
344 for argument in call["arguments"]:
345 if argument["name"] == "ObjectAttributes":
346 subkey = argument["value"]
347 elif argument["name"] == "KeyHandle":
348 handle = int(argument["value"], 16)
349
350 name = self._check_registry(registry, subkey, handle)
351 if name and name not in self.keys:
352 self.keys.append(name)
353 elif call["api"].startswith("NtDeleteValueKey"):
354 registry = -1
355 subkey = ""
356 handle = 0
357
358 for argument in call["arguments"]:
359 if argument["name"] == "ValueName":
360 subkey = argument["value"]
361 elif argument["name"] == "KeyHandle":
362 handle = int(argument["value"], 16)
363
364 name = self._check_registry(registry, subkey, handle)
365 if name and name not in self.keys:
366 self.keys.append(name)
367 elif call["api"].startswith("RegCloseKey"):
368 handle = 0
369
370 for argument in call["arguments"]:
371 if argument["name"] == "Handle":
372 handle = int(argument["value"], 16)
373
374 if handle != 0:
375 for a in self.handles:
376 if a["handle"] == handle:
377 try:
378 self.handles.remove(a)
379 except ValueError:
380 pass
381
382 elif call["category"] == "filesystem":
383 for argument in call["arguments"]:
384 if argument["name"] == "FileName":
385 value = argument["value"].strip()
386 if not value:
387 continue
388
389 if value not in self.files:
390 self.files.append(value)
391
392 elif call["category"] == "synchronization":
393 for argument in call["arguments"]:
394 if argument["name"] == "MutexName":
395 value = argument["value"].strip()
396 if not value:
397 continue
398
399 if value not in self.mutexes:
400 self.mutexes.append(value)
401
403 """Get registry keys, mutexes and files.
404 @return: Summary of keys, mutexes and files.
405 """
406 return {"files": self.files, "keys": self.keys, "mutexes": self.mutexes}
407
409 """Generates a more extensive high-level representation than Summary."""
410
411 key = "enhanced"
412
414 """
415 @param details: Also add some (not so relevant) Details to the log
416 """
417 self.currentdir = "C: "
418 self.eid = 0
419 self.details = details
420 self.filehandles = {}
421 self.servicehandles = {}
422 self.keyhandles = {
423 "0x80000000": "HKEY_CLASSES_ROOT\\",
424 "0x80000001": "HKEY_CURRENT_USER\\",
425 "0x80000002": "HKEY_LOCAL_MACHINE\\",
426 "0x80000003": "HKEY_USERS\\",
427 "0x80000004": "HKEY_PERFORMANCE_DATA\\",
428 "0x80000005": "HKEY_CURRENT_CONFIG\\",
429 "0x80000006": "HKEY_DYN_DATA\\"
430 }
431 self.modules = {}
432 self.procedures = {}
433 self.events = []
434
436 """
437 Add a procedure address
438 """
439 self.procedures[base] = "{0}:{1}".format(self._get_loaded_module(mbase), name)
440
442 """
443 Add a loaded module to the internal database
444 """
445 self.modules[base] = name
446
448 """
449 Get the name of a loaded module from the internal db
450 """
451 return self.modules.get(base, "")
452
453
455 """
456 @registry: returned, new handle
457 @handle: handle to base key
458 @subkey: subkey to add
459 """
460 if handle != 0 and handle in self.keyhandles:
461 return self.keyhandles[handle]
462
463 name = ""
464 if registry and registry != "0x00000000" and \
465 registry in self.keyhandles:
466 name = self.keyhandles[registry]
467
468 nkey = name + subkey
469 nkey = fix_key(nkey)
470
471 self.keyhandles[handle] = nkey
472
473 return nkey
474
482
485
487 """ Gets files calls
488 @return: information list
489 """
490 def _load_args(call):
491 """
492 Load arguments from call
493 """
494 res = {}
495 for argument in call["arguments"]:
496 res[argument["name"]] = argument["value"]
497
498 return res
499
500 def _generic_handle_details(self, call, item):
501 """
502 Generic handling of api calls
503 @call: the call dict
504 @item: Generic item to process
505 """
506 event = None
507 if call["api"] in item["apis"]:
508 args = _load_args(call)
509 self.eid += 1
510
511 event = {
512 "event": item["event"],
513 "object": item["object"],
514 "timestamp": call["timestamp"],
515 "eid": self.eid,
516 "data": {}
517 }
518
519 for logname, dataname in item["args"]:
520 event["data"][logname] = args.get(dataname)
521 return event
522
523 def _generic_handle(self, data, call):
524 """Generic handling of api calls."""
525 for item in data:
526 event = _generic_handle_details(self, call, item)
527 if event:
528 return event
529
530 return None
531
532
533 def _add_handle(handles, handle, filename):
534 handles[handle] = filename
535
536 def _remove_handle(handles, handle):
537 if handle in handles:
538 handles.pop(handle)
539
540 def _get_handle(handles, handle):
541 return handles.get(handle)
542
543 def _get_service_action(control_code):
544 """@see: http://msdn.microsoft.com/en-us/library/windows/desktop/ms682108%28v=vs.85%29.aspx"""
545 codes = {1: "stop",
546 2: "pause",
547 3: "continue",
548 4: "info"}
549
550 default = "user" if control_code >= 128 else "notify"
551 return codes.get(control_code, default)
552
553 event = None
554
555 gendat = [
556 {
557 "event": "move",
558 "object": "file",
559 "apis": [
560 "MoveFileWithProgressW",
561 "MoveFileExA",
562 "MoveFileExW"
563 ],
564 "args": [
565 ("from", "ExistingFileName"),
566 ("to", "NewFileName")
567 ]
568 },
569 {
570 "event": "copy",
571 "object": "file",
572 "apis": [
573 "CopyFileA",
574 "CopyFileW",
575 "CopyFileExW",
576 "CopyFileExA"
577 ],
578 "args": [
579 ("from", "ExistingFileName"),
580 ("to", "NewFileName")
581 ]
582 },
583 {
584 "event": "delete",
585 "object": "file",
586 "apis": [
587 "DeleteFileA",
588 "DeleteFileW",
589 "NtDeleteFile"
590 ],
591 "args": [("file", "FileName")]
592 },
593 {
594 "event": "delete",
595 "object": "dir",
596 "apis": [
597 "RemoveDirectoryA",
598 "RemoveDirectoryW"
599 ],
600 "args": [("file", "DirectoryName")]
601 },
602 {
603 "event": "create",
604 "object": "dir",
605 "apis": [
606 "CreateDirectoryW",
607 "CreateDirectoryExW"
608 ],
609 "args": [("file", "DirectoryName")]
610 },
611 {
612 "event": "write",
613 "object": "file",
614 "apis": [
615 "URLDownloadToFileW",
616 "URLDownloadToFileA"
617 ],
618 "args": [("file", "FileName")]
619 },
620 {
621 "event": "execute",
622 "object": "file",
623 "apis": [
624 "CreateProcessAsUserA",
625 "CreateProcessAsUserW",
626 "CreateProcessA",
627 "CreateProcessW",
628 "NtCreateProcess",
629 "NtCreateProcessEx"
630 ],
631 "args": [("file", "FileName")]
632 },
633 {
634 "event": "execute",
635 "object": "file",
636 "apis": [
637 "CreateProcessInternalW",
638 ],
639 "args": [("file", "CommandLine")]
640 },
641 {
642 "event": "execute",
643 "object": "file",
644 "apis": [
645 "ShellExecuteExA",
646 "ShellExecuteExW",
647 ],
648 "args": [("file", "FilePath")]
649 },
650 {
651 "event": "load",
652 "object": "library",
653 "apis": [
654 "LoadLibraryA",
655 "LoadLibraryW",
656 "LoadLibraryExA",
657 "LoadLibraryExW",
658 "LdrLoadDll",
659 "LdrGetDllHandle"
660 ],
661 "args": [
662 ("file", "FileName"),
663 ("pathtofile", "PathToFile"),
664 ("moduleaddress", "BaseAddress")
665 ]
666 },
667 {
668 "event": "findwindow",
669 "object": "windowname",
670 "apis": [
671 "FindWindowA",
672 "FindWindowW",
673 "FindWindowExA",
674 "FindWindowExW"
675 ],
676 "args": [
677 ("classname", "ClassName"),
678 ("windowname", "WindowName")
679 ]
680 },
681 {
682 "event": "read",
683 "object": "file",
684 "apis": [
685 "NtReadFile",
686 "ReadFile"
687 ],
688 "args": []
689 },
690 {
691 "event": "write",
692 "object": "file",
693 "apis": ["NtWriteFile"],
694 "args": []
695 },
696 {
697 "event": "delete",
698 "object": "registry",
699 "apis": [
700 "RegDeleteKeyA",
701 "RegDeleteKeyW"
702 ],
703 "args": []
704 },
705 {
706 "event": "write",
707 "object": "registry",
708 "apis": [
709 "RegSetValueExA",
710 "RegSetValueExW"
711 ],
712 "args": [
713 ("content", "Buffer"),
714 ("object", "object")
715 ]
716 },
717 {
718 "event": "read",
719 "object": "registry",
720 "apis": [
721 "RegQueryValueExA",
722 "RegQueryValueExW",
723 "NtQueryValueKey"
724 ],
725 "args": []
726 },
727 {
728 "event": "delete",
729 "object": "registry",
730 "apis": [
731 "RegDeleteValueA",
732 "RegDeleteValueW",
733 "NtDeleteValueKey"
734 ],
735 "args": []
736 },
737 {
738 "event": "create",
739 "object": "windowshook",
740 "apis": ["SetWindowsHookExA"],
741 "args": [
742 ("id", "HookIdentifier"),
743 ("moduleaddress", "ModuleAddress"),
744 ("procedureaddress", "ProcedureAddress")
745 ]
746 },
747 {
748 "event": "modify",
749 "object": "service",
750 "apis": ["ControlService"],
751 "args": [("controlcode", "ControlCode")]
752 },
753 {
754 "event": "delete",
755 "object": "service",
756 "apis": ["DeleteService"],
757 "args": [],
758 },
759 ]
760
761
762
763
764
765
766
767
768
769
770 event = _generic_handle(self, gendat, call)
771 args = _load_args(call)
772
773 if event:
774 if call["api"] in ["NtReadFile", "ReadFile", "NtWriteFile"]:
775 event["data"]["file"] = _get_handle(self.filehandles, args["FileHandle"])
776
777 elif call["api"] in ["RegDeleteKeyA", "RegDeleteKeyW"]:
778 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "")), args.get("SubKey", ""))
779
780 elif call["api"] in ["RegSetValueExA", "RegSetValueExW"]:
781 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "")), args.get("ValueName", ""))
782
783 elif call["api"] in ["RegQueryValueExA", "RegQueryValueExW", "RegDeleteValueA", "RegDeleteValueW"]:
784 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("Handle", "UNKNOWN")), args.get("ValueName", ""))
785
786 elif call["api"] in ["NtQueryValueKey", "NtDeleteValueKey"]:
787 event["data"]["regkey"] = "{0}{1}".format(self._get_keyhandle(args.get("KeyHandle", "UNKNOWN")), args.get("ValueName", ""))
788
789 elif call["api"] in ["LoadLibraryA", "LoadLibraryW", "LoadLibraryExA", "LoadLibraryExW", "LdrGetDllHandle"] and call["status"]:
790 self._add_loaded_module(args.get("FileName", ""), args.get("ModuleHandle", ""))
791
792 elif call["api"] in ["LdrLoadDll"] and call["status"]:
793 self._add_loaded_module(args.get("FileName", ""), args.get("BaseAddress", ""))
794
795 elif call["api"] in ["LdrGetProcedureAddress"] and call["status"]:
796 self._add_procedure(args.get("ModuleHandle", ""), args.get("FunctionName", ""), args.get("FunctionAddress", ""))
797 event["data"]["module"] = self._get_loaded_module(args.get("ModuleHandle", ""))
798
799 elif call["api"] in ["SetWindowsHookExA"]:
800 event["data"]["module"] = self._get_loaded_module(args.get("ModuleAddress", ""))
801
802 if call["api"] in ["ControlService", "DeleteService"]:
803 event["data"]["service"] = _get_handle(self.servicehandles, args["ServiceHandle"])
804
805 if call["api"] in ["ControlService"]:
806 event["data"]["action"] = _get_service_action(args["ControlCode"])
807
808 return event
809
810 elif call["api"] in ["SetCurrentDirectoryA", "SetCurrentDirectoryW"]:
811 self.currentdir = args["Path"]
812
813
814 elif call["api"] in ["NtCreateFile", "NtOpenFile"]:
815 _add_handle(self.filehandles, args["FileHandle"], args["FileName"])
816
817 elif call["api"] in ["CreateFileW"]:
818 _add_handle(self.filehandles, call["return"], args["FileName"])
819
820 elif call["api"] in ["NtClose", "CloseHandle"]:
821 _remove_handle(self.filehandles, args["Handle"])
822
823
824 elif call["api"] in ["OpenServiceW"]:
825 _add_handle(self.servicehandles, call["return"], args["ServiceName"])
826
827
828 elif call["api"] in ["RegOpenKeyExA", "RegOpenKeyExW", "RegCreateKeyExA", "RegCreateKeyExW"]:
829 self._add_keyhandle(args.get("Registry", ""), args.get("SubKey", ""), args.get("Handle", ""))
830
831 elif call["api"] in ["NtOpenKey"]:
832 self._add_keyhandle(None, args.get("ObjectAttributes", ""), args.get("KeyHandle", ""))
833
834 elif call["api"] in ["RegCloseKey"]:
835 self._remove_keyhandle(args.get("Handle", ""))
836
837 return event
838
840 """Generate processes list from streamed calls/processes.
841 @return: None.
842 """
843 event = self._process_call(call)
844 if event:
845 self.events.append(event)
846
848 """Get registry keys, mutexes and files.
849 @return: Summary of keys, mutexes and files.
850 """
851 return self.events
852
853
855 """Anomaly detected during analysis.
856 For example: a malware tried to remove Cuckoo's hooks.
857 """
858
859 key = "anomaly"
860
863
865 """Process API calls.
866 @param call: API call object
867 @param process: process object
868 """
869 if call["category"] != "anomaly":
870 return
871
872 category, funcname, message = None, None, None
873 for row in call["arguments"]:
874 if row["name"] == "Subcategory":
875 category = row["value"]
876 if row["name"] == "FunctionName":
877 funcname = row["value"]
878 if row["name"] == "Message":
879 message = row["value"]
880
881 self.anomalies.append(dict(
882 name=process["process_name"],
883 pid=process["process_id"],
884 category=category,
885 funcname=funcname,
886 message=message,
887 ))
888
890 """Fetch all anomalies."""
891 return self.anomalies
892
893
895 """Generates process tree."""
896
897 key = "processtree"
898
900 self.processes = []
901 self.tree = []
902
904 """Add a node to a process tree.
905 @param node: node to add.
906 @param tree: processes tree.
907 @return: boolean with operation success status.
908 """
909
910 for process in tree:
911
912
913 if process["pid"] == node["parent_id"]:
914 process["children"].append(node)
915
916 else:
917 self.add_node(node, process["children"])
918
920 for entry in self.processes:
921 if entry["pid"] == process["process_id"]:
922 return
923
924 self.processes.append(dict(
925 name=process["process_name"],
926 pid=process["process_id"],
927 parent_id=process["parent_id"],
928 children=[]
929 ))
930
932 children = []
933
934
935 for process in self.processes:
936 has_parent = False
937
938 for process_again in self.processes:
939
940
941 if process_again["pid"] == process["parent_id"]:
942 has_parent = True
943
944
945 if has_parent:
946 children.append(process)
947
948 else:
949 self.tree.append(process)
950
951
952 for process in children:
953 self.add_node(process, self.tree)
954
955 return self.tree
956
958 """Behavior Analyzer."""
959
960 key = "behavior"
961
996