;;; This is (C) Ingvar Mattsson, 2004 ;;; <ingvar@hexapodia.net> ;;; This code uses the file format specification outlined in store.h ;;; in the netflow logger daemon downloadable from ;;; http://www.mindrot.org/flowd.html ;;; ;;; This code is available under the BSD license, please preserve the ;;; relevant copyright notices. ;;; ;;; The store.h file is: ;;; /* ;;; * Copyright (c) 2004 Damien Miller <djm@mindrot.org> ;;; * ;;; * Permission to use, copy, modify, and distribute this software for any ;;; * purpose with or without fee is hereby granted, provided that the above ;;; * copyright notice and this permission notice appear in all copies. ;;; * ;;; * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES ;;; * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF ;;; * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ;;; * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES ;;; * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ;;; * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF ;;; * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ;;; */ (defpackage #:flowd (:use #:cl) (:shadow #:tag #:stream) (:export #:+store-magic+ #:+store-version+ #:store-field-tag #:store-field-recv-time #:store-field-proto-flags-tos #:store-field-agent-addr4 #:store-field-agent-addr6 #:store-field-src-addr4 #:store-field-src-addr6 #:store-field-dst-addr4 #:store-field-dst-addr6 #:store-field-gateway-addr4 #:store-field-gateway-addr6 #:store-field-srcdst-port #:store-field-packets #:store-field-octets #:store-field-if-indices #:store-field-agent-info #:store-field-flow-times #:store-field-as-info #:store-field-flow-engine-info #:store-field-crc32 #:store-field-all #:open-log #:read-flow #:close-log #:fields #:tag #:recv-time #:proto-flags-tos #:agent-addr #:src-addr #:dst-addr #:gateway-addr #:src-port #:dst-port #:packets #:if-index-in #:if-index-out #:sys-uptime-ms #:time-sec #:time-nanosec #:netflow-version #:flow-start #:flow-finish #:src-as #:dst-as #:src-mask #:dst-mask #:engine-type #:engine-id #:flow-sequence #:src-net #:dst-net #:octets #:packets #:start-time #:format-ipv4)) (in-package #:flowd) (defvar *ipv4-netmasks* (apply #'vector (loop for n from 0 to 32 for mask = 0 then (logior #x80000000 (ash mask -1)) collect mask))) (defconstant +store-magic+ #x012cf047) (defconstant +store-version+ 2) (defconstant +store-field-tag+ (ash 1 0)) (defconstant +store-field-recv-time+ (ash 1 1)) (defconstant +store-field-proto-flags-tos+ (ash 1 2)) (defconstant +store-field-agent-addr4+ (ash 1 3)) (defconstant +store-field-agent-addr6+ (ash 1 4)) (defconstant +store-field-src-addr4+ (ash 1 5)) (defconstant +store-field-src-addr6+ (ash 1 6)) (defconstant +store-field-dst-addr4+ (ash 1 7)) (defconstant +store-field-dst-addr6+ (ash 1 8)) (defconstant +store-field-gateway-addr4+ (ash 1 9)) (defconstant +store-field-gateway-addr6+ (ash 1 10)) (defconstant +store-field-srcdst-port+ (ash 1 11)) (defconstant +store-field-packets+ (ash 1 12)) (defconstant +store-field-octets+ (ash 1 13)) (defconstant +store-field-if-indices+ (ash 1 14)) (defconstant +store-field-agent-info+ (ash 1 15)) (defconstant +store-field-flow-times+ (ash 1 16)) (defconstant +store-field-as-info+ (ash 1 17)) (defconstant +store-field-flow-engine-info+ (ash 1 18)) (defconstant +store-field-crc32+ (ash 1 30)) (defconstant +store-field-all+ (1- (ash 1 19))) (defclass store-header () ((magic :reader magic :initarg :magic) (version :reader version :initarg :version) (start-time :reader start-time :initarg :start-time) (flags :reader flags :initarg :flags) (stream :reader stream :initarg :stream) )) (defclass flow () ((fields :accessor fields :initarg :fields :initform nil) (tag :accessor tag :initarg :tag :initform nil) (recv-time :accessor recv-time :initarg :recv-time :initform nil) (tcp-flags :accessor tcp-flags :initarg :tcp-flags :initform nil) (protocol :accessor protocol :initarg :protocol :initform nil) (tos :accessor tos :initarg :tos :initform nil) (agent-addr :accessor agent-addr :initarg :agent-addr :initform nil) (src-addr :accessor src-addr :initarg :src-addr :initform nil) (dst-addr :accessor dst-addr :initarg :dst-addr :initform nil) (gateway-addr :accessor gateway-addr :initarg :gateway-addr :initform nil) (src-port :accessor src-port :initarg :src-port :initform nil) (dst-port :accessor dst-port :initarg :dst-port :initform nil) (packets :accessor packets :initarg :packets :initform nil) (octets :accessor octets :initarg :octets :initform nil) (if-index-in :accessor if-index-in :initarg :if-index-in :initform nil) (if-index-out :accessor if-index-out :initarg :if-index-out :initform nil) (sys-uptime-ms :accessor sys-uptime-ms :initarg :sys-uptime-ms :initform nil) (time-sec :accessor time-sec :initarg :time-sec :initform nil) (time-nanosec :accessor time-nanosec :initarg :time-nanosec :initform nil) (netflow-version :accessor netflow-version :initarg :netflow-version :initform nil) (flow-start :accessor flow-start :initarg :flow-start :initform nil) (flow-finish :accessor flow-finish :initarg :flow-finish :initform nil) (src-as :accessor src-as :initarg :src-as :initform nil) (dst-as :accessor dst-as :initarg :dst-as :initform nil) (src-mask :accessor src-mask :initarg :src-mask :initform nil) (dst-mask :accessor dst-mask :initarg :dst-mask :initform nil) (engine-type :accessor engine-type :initarg :engine-type :initform nil) (engine-id :accessor engine-id :initarg :engine-id :initform nil) (flow-sequence :accessor flow-sequence :initarg :flow-sequence :initform nil) )) (defclass ipaddr () ((address :accessor address :initarg :address))) (defclass ipv4 (ipaddr) ()) (defclass ipv6 (ipaddr) ()) (defun make-ipv4 (addr) "This function is currently a no-op" ;;(make-instance 'ipv4 :address addr) (identity addr) ) (defun make-ipv6 (addr) "This function is currently a no-op" ;;(make-instance 'ipv6 :address addr) (identity addr) ) (defmacro when-flagged (flag &body body) "Checks if a given flag is set. The flag field is expected to be named FIELDS and is for use inside READ-FLOW only!" `(when (not (zerop (logand fields ,flag))) ,@body)) (defun read-n-bytes (stream n) "Read from STREAM a total of N bytes, mung them together as a single integer. Expects 8-bit bytes." (let ((acc 0)) (loop for r from 1 to n do (setf acc (logior (ash acc 8) (read-byte stream)))) acc)) (defun read-flow (flow-header &optional flow-obj) "(read-flow <flow-header> &optional flow-object) This function reads one flow entry from a log file (return value from OPEN-LOG) and returns it. If a flow object is passed in as an optional parameter, this flow object is re-used for storage instead of allocating a new instance." (let ((stream (stream flow-header))) (let ((fields (read-n-bytes stream 4))) (let ((flow (if flow-obj (progn (setf (fields flow-obj) fields) flow-obj) (make-instance 'flow :fields fields))) pad) (when-flagged +store-field-tag+ (setf (tag flow) (read-n-bytes stream 4))) (when-flagged +store-field-recv-time+ (setf (recv-time flow) (read-n-bytes stream 4))) (when-flagged +store-field-proto-flags-tos+ (setf (tcp-flags flow) (read-n-bytes stream 1)) (setf (protocol flow) (read-n-bytes stream 1)) (setf (tos flow) (read-n-bytes stream 1)) (setf pad (read-n-bytes stream 1))) (when-flagged +store-field-agent-addr4+ (setf (agent-addr flow) (make-ipv4 (read-n-bytes stream 4)))) (when-flagged +store-field-agent-addr6+ (setf (agent-addr flow) (make-ipv6 (read-n-bytes stream 16)))) (when-flagged +store-field-src-addr4+ (setf (src-addr flow) (make-ipv4 (read-n-bytes stream 4)))) (when-flagged +store-field-src-addr6+ (setf (src-addr flow) (make-ipv6 (read-n-bytes stream 16)))) (when-flagged +store-field-dst-addr4+ (setf (dst-addr flow) (make-ipv4 (read-n-bytes stream 4)))) (when-flagged +store-field-dst-addr6+ (setf (dst-addr flow) (make-ipv6 (read-n-bytes stream 16)))) (when-flagged +store-field-gateway-addr4+ (setf (gateway-addr flow) (make-ipv4 (read-n-bytes stream 4)))) (when-flagged +store-field-gateway-addr6+ (setf (gateway-addr flow) (make-ipv6 (read-n-bytes stream 16)))) (when-flagged +store-field-srcdst-port+ (setf (src-port flow) (read-n-bytes stream 2)) (setf (dst-port flow) (read-n-bytes stream 2))) (when-flagged +store-field-packets+ (setf (packets flow) (read-n-bytes stream 8))) (when-flagged +store-field-octets+ (setf (octets flow) (read-n-bytes stream 8))) (when-flagged +store-field-if-indices+ (setf (if-index-in flow) (read-n-bytes stream 2)) (setf (if-index-out flow) (read-n-bytes stream 2))) (when-flagged +store-field-agent-info+ (setf (sys-uptime-ms flow) (read-n-bytes stream 4)) (setf (time-sec flow) (read-n-bytes stream 4)) (setf (time-nanosec flow) (read-n-bytes stream 4)) (setf (netflow-version flow) (read-n-bytes stream 2)) (setf pad (read-n-bytes stream 2))) (when-flagged +store-field-flow-times+ (setf (flow-start flow) (read-n-bytes stream 4)) (setf (flow-finish flow) (read-n-bytes stream 4))) (when-flagged +store-field-as-info+ (setf (src-as flow) (read-n-bytes stream 2)) (setf (dst-as flow) (read-n-bytes stream 2)) (setf (src-mask flow) (read-n-bytes stream 1)) (setf (dst-mask flow) (read-n-bytes stream 1)) (setf pad (read-n-bytes stream 2))) (when-flagged +store-field-flow-engine-info+ (setf (engine-type flow) (read-n-bytes stream 1)) (setf (engine-id flow) (read-n-bytes stream 1)) (setf pad (read-n-bytes stream 2)) (setf (flow-sequence flow) (read-n-bytes stream 4))) (when-flagged +store-field-crc32+ (setf pad (read-n-bytes stream 4))) flow)))) (defun open-log (file) "(open-log <file name> This function opens a new log file and returns a header structure containing the relevant file header information." (let ((stream (open file :element-type '(unsigned-byte 8) :direction :input))) (let ((magic (read-n-bytes stream 4))) (let ((version (read-n-bytes stream 4))) (let ((start-time (read-n-bytes stream 4))) (let ((flags (read-n-bytes stream 4))) (make-instance 'store-header :magic magic :version version :start-time start-time :flags flags :stream stream))))))) (defun close-log (flow) "(close-log flow) This function closes the log file associated with a storage header." (close (stream flow))) (defmacro formatted-addr (flow-obj slot) (let ((flags (case slot (dst-addr (list +store-field-dst-addr4+ +store-field-dst-addr6+)) (src-addr (list +store-field-src-addr4+ +store-field-src-addr6+)) (gateway-addr (list +store-field-gateway-addr4+ +store-field-gateway-addr6+)) (agent-addr (list +store-field-agent-addr4+ +store-field-agent-addr6+)))) (flow flow-obj)) `(format-addr ,flow ',flags (,slot ,flow)))) (defun format-addr (flow-obj flags chunk) (let ((flag4 (car flags)) (flag6 (cadr flags)) (fields (fields flow-obj))) (or (when (= flag4 (logand flag4 fields)) (format-ipv4 chunk)) (when (= flag6 (logand flag6 fields)) (format nil "~X" chunk))))) (defun format-ipv4 (chunk &optional stream mask) "(format-ipv4 binary-chunk &optiona stream mask) This function outputs an IPv4 address as a dotted quad to STREAM. If a netmask is passed in, it's outputted with the dotted quad in CIDR notation." (if mask (format stream "~D.~D.~D.~D/~D" (ldb (byte 8 24) chunk) (ldb (byte 8 16) chunk) (ldb (byte 8 8) chunk) (ldb (byte 8 0) chunk) mask) (format stream "~D.~D.~D.~D" (ldb (byte 8 24) chunk) (ldb (byte 8 16) chunk) (ldb (byte 8 8) chunk) (ldb (byte 8 0) chunk)))) (defun src-net (flow-obj &optional (stream nil) (formatted nil)) "(src-net flow-obj &optional stream formatted-p) This function extracts the source network and masks it against the relevant IPv4 netmask and returns the network part. If given a STREAM and FORMATTED-P is not null, the resulting netblock is emitted using FORMAT-IPV4 to the indicated stream." (let ((bitmask (logior +store-field-as-info+ +store-field-src-addr4+))) (when (= bitmask (logand (fields flow-obj) bitmask)) (let ((masklen (src-mask flow-obj))) (let ((netmask (aref *ipv4-netmasks* masklen))) (let ((netblock (logand (src-addr flow-obj) netmask))) (if formatted (format-ipv4 netblock stream masklen) netblock))))))) (defun dst-net (flow-obj &optional (stream nil) (formatted nil)) "(dst-net flow-obj &optional stream formatted-p) This function extracts the destination network and masks it against the relevant IPv4 netmask and returns the network part. If given a STREAM and FORMATTED-P is not null, the resulting netblock is emitted using FORMAT-IPV4 to the indicated stream." (let ((bitmask (logior +store-field-as-info+ +store-field-dst-addr4+))) (when (= bitmask (logand (fields flow-obj) bitmask)) (let ((masklen (dst-mask flow-obj))) (let ((netmask (aref *ipv4-netmasks* masklen))) (let ((netblock (logand (dst-addr flow-obj) netmask))) (if formatted (format-ipv4 netblock stream masklen) netblock))))))) -- //Ingvar