yafcore.h
Go to the documentation of this file.
1 /*
2  *
3  ** @file yafcore.h
4  ** YAF core I/O routines
5  **
6  ** ------------------------------------------------------------------------
7  ** Copyright (C) 2006-2015 Carnegie Mellon University. All Rights Reserved.
8  ** ------------------------------------------------------------------------
9  ** Authors: Brian Trammell
10  ** ------------------------------------------------------------------------
11  ** Use of the YAF system and related source code is subject to the terms
12  ** of the following licenses:
13  **
14  ** GNU Public License (GPL) Rights pursuant to Version 2, June 1991
15  ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
16  **
17  ** NO WARRANTY
18  **
19  ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER
20  ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY
21  ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN
22  ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY
23  ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT
24  ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE,
25  ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE
26  ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT,
27  ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY
28  ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF
29  ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES.
30  ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF
31  ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON
32  ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE
33  ** DELIVERABLES UNDER THIS LICENSE.
34  **
35  ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
36  ** Mellon University, its trustees, officers, employees, and agents from
37  ** all claims or demands made against them (and any related losses,
38  ** expenses, or attorney's fees) arising out of, or relating to Licensee's
39  ** and/or its sub licensees' negligent use or willful misuse of or
40  ** negligent conduct or willful misconduct regarding the Software,
41  ** facilities, or other rights or assistance granted by Carnegie Mellon
42  ** University under this License, including, but not limited to, any
43  ** claims of product liability, personal injury, death, damage to
44  ** property, or violation of any laws or regulations.
45  **
46  ** Carnegie Mellon University Software Engineering Institute authored
47  ** documents are sponsored by the U.S. Department of Defense under
48  ** Contract FA8721-05-C-0003. Carnegie Mellon University retains
49  ** copyrights in all material produced under this contract. The U.S.
50  ** Government retains a non-exclusive, royalty-free license to publish or
51  ** reproduce these documents, or allow others to do so, for U.S.
52  ** Government purposes only pursuant to the copyright license under the
53  ** contract clause at 252.227.7013.
54  **
55  ** ------------------------------------------------------------------------
56  */
57 
98 #ifndef _YAF_CORE_H_
99 #define _YAF_CORE_H_
100 
101 #include <yaf/autoinc.h>
102 #include <fixbuf/public.h>
103 #include <stdlib.h>
104 #include <math.h>
110 #define CERT_PEN 6871
111 
117 #define YAF_ERROR_DOMAIN (g_quark_from_string("certYAFError"))
118 
119 #define YAF_ERROR_HEADER 1
120 
121 #define YAF_ERROR_ARGUMENT 2
122 
123 #define YAF_ERROR_IO 3
124 
125 #define YAF_ERROR_IPFIX 4
126 
127 #define YAF_ERROR_IMPL 5
128 
129 #define YAF_ERROR_INTERNAL 6
130 
131 #define YAF_ERROR_LIMIT 7
132 
133 #define YAF_ERROR_EOF 8
134 
135 #define YAF_ERROR_ALIGNMENT 9
136 
137 #define YAF_ERROR_PACKET_PAYLOAD 10
138 
139 
140 
145 #define YAF_FLOW_ACTIVE 0
146 
147 #define YAF_END_IDLE 1
148 
149 #define YAF_END_ACTIVE 2
150 
151 #define YAF_END_CLOSED 3
152 
153 #define YAF_END_FORCED 4
154 
155 #define YAF_END_RESOURCE 5
156 
157 #define YAF_END_UDPFORCE 0x1F
158 
159 #define YAF_SAME_SIZE 0x01
160 
161 #define YAF_OUT_OF_SEQUENCE 0x02
162 
163 #define YAF_MP_CAPABLE 0x04
164 
165 #define YAF_FRAGMENTS 0x08
166 
167 #define YAF_PARTIAL_FRAGS 0x10
168 
169 #define YAF_FRAG_ACTIVE 0x03
170 
171 #define YAF_FRAG_PASSIVE 0x04
172 
173 #define YAF_END_MASK 0x7F
174 
176 #define YAF_ENDF_ISCONT 0x80
177 
179 #define YAF_IP_ICMP 1
180 
181 #define YAF_IP_TCP 6
182 
183 #define YAF_IP_UDP 17
184 
188 #define YFP_IPTCPHEADER_SIZE 128
189 
190 #define ETHERNET_MAC_ADDR_LENGTH 6
191 
192 #define YAF_MAX_HOOKS 4
193 
196 #define YAF_HOOKS_MAX_EXPORT 1500
197 
198 #define YAF_MAX_PKT_BOUNDARY 25
199 
200 #define YAF_PCAP_MAX 5000000
201 
202 #define YAF_MAX_MPLS_LABELS 3
203 
208 typedef struct yfFlowKey_st {
210  uint16_t sp;
212  uint16_t dp;
214  uint8_t proto;
216  uint8_t version;
218  uint16_t vlanId;
220  uint8_t tos;
224 #if YAF_ENABLE_DAG_SEPARATE_INTERFACES || YAF_ENABLE_SEPARATE_INTERFACES
225  uint8_t netIf;
226 #endif
227 
228  union {
229  struct {
231  uint32_t sip;
233  uint32_t dip;
234  } v4;
235  struct {
237  uint8_t sip[16];
239  uint8_t dip[16];
240  } v6;
241  } addr;
242 } yfFlowKey_t;
243 
247 typedef struct yfFlowStats_st {
249  uint64_t iaarray[10];
251  uint16_t pktsize[10];
253  uint64_t payoct;
255  uint64_t ltime;
257  uint32_t tcpurgct;
259  uint32_t smallpktct;
261  uint32_t nonemptypktct;
263  uint32_t largepktct;
265  uint32_t aitime;
267  uint16_t firstpktsize;
269  uint16_t maxpktsize;
270 } yfFlowStats_t;
271 
276 typedef struct yfFlowVal_st {
278  uint64_t oct;
280  uint64_t pkt;
281 # if YAF_ENABLE_PAYLOAD
282 
283  uint32_t paylen;
285  uint8_t *payload;
287  size_t *paybounds;
288 # endif
289 
290  uint32_t isn;
292  uint32_t lsn;
294  uint16_t first_pkt_size;
296  uint16_t attributes;
298  uint8_t iflags;
300  uint8_t uflags;
302  uint8_t appkt;
304  uint16_t vlan;
305 # if YAF_ENABLE_SEPARATE_INTERFACES
306  uint8_t netIf;
307 # endif
308 # if YAF_ENABLE_ENTROPY
309 
310  uint8_t entropy;
312  uint8_t entpad[7];
313 # endif
314 # if YAF_ENABLE_P0F
315 
316  const char *osname;
318  const char *osver;
320  uint8_t fuzzyMatch;
322  uint8_t fuzzyPad[7];
324  char *osFingerPrint;
325 # endif
326 # if YAF_ENABLE_FPEXPORT
327 
328  uint32_t firstPacketLen;
330  uint32_t secondPacketLen;
332  uint8_t *firstPacket;
334  uint8_t *secondPacket;
335 # endif
336 
338 } yfFlowVal_t;
339 
340 #if YAF_MPLS
341 typedef struct yfMPLSNode_st {
343  GHashTable *tab;
345  uint32_t mpls_label[YAF_MAX_MPLS_LABELS];
347  int tab_count;
348 } yfMPLSNode_t;
349 #endif
350 
351 typedef struct yfMPTCPFlow_st {
353  uint64_t idsn;
355  uint32_t token;
357  uint16_t mss;
359  uint8_t addrid;
361  uint8_t flags;
362 } yfMPTCPFlow_t;
363 
364 
371 typedef struct yfFlow_st {
373  uint64_t stime;
375  uint64_t etime;
376 #ifdef YAF_ENABLE_HOOKS
377 
381  void *hfctx[YAF_MAX_HOOKS];
382 #endif
383  /*
384  * Reverse flow delta start time in milliseconds. Equivalent to initial
385  * packet round-trip time; useful for decomposing biflows into uniflows.
386  */
387  int32_t rdtime;
388 #if YAF_ENABLE_APPLABEL
389 
390  uint16_t appLabel;
391 #endif
392 #if YAF_ENABLE_NDPI
393  uint16_t ndpi_master;
394  uint16_t ndpi_sub;
395 #endif
396 
397  uint8_t reason;
399  uint8_t pcap_serial;
401  uint8_t sourceMacAddr[ETHERNET_MAC_ADDR_LENGTH];
403  uint8_t destinationMacAddr[ETHERNET_MAC_ADDR_LENGTH];
405  uint8_t pcap_file_no;
407  uint8_t pktdir;
409  uint8_t rtos;
411  pcap_dumper_t *pcap;
412 #if YAF_MPLS
413 
414  yfMPLSNode_t *mpls;
415 #endif
416 
424 } yfFlow_t;
425 
434 void yfAlignmentCheck(void);
435 
436 
445 void yfFlowPrepare(
446  yfFlow_t *flow);
447 
455 void yfFlowCleanup(
456  yfFlow_t *flow);
457 
470 fBuf_t *yfWriterForFile(
471  const char *path,
472  uint32_t domain,
473  gboolean export_meta,
474  GError **err);
475 
490 fBuf_t *yfWriterForFP(
491  FILE *fp,
492  uint32_t domain,
493  gboolean export_meta,
494  GError **err);
495 
506 fBuf_t *yfWriterForSpec(
507  fbConnSpec_t *spec,
508  uint32_t domain,
509  gboolean export_meta,
510  GError **err);
511 
512 
513 #ifdef HAVE_SPREAD
514 
527 fBuf_t *yfWriterForSpread(
528  fbSpreadParams_t *params,
529  uint32_t domain,
530  uint16_t *spreadGroupIndex,
531  gboolean export_meta,
532  GError **err);
533 
534 #endif /* HAVE_SPREAD */
535 
550 gboolean yfWriteStatsFlow(
551  void *yfContext,
552  uint32_t pcap_drop,
553  GTimer *timer,
554  GError **err);
555 
568 gboolean yfWriteFlow(
569  void *yfContext,
570  yfFlow_t *flow,
571  GError **err);
572 
585 gboolean yfWriterClose(
586  fBuf_t *fbuf,
587  gboolean flush,
588  GError **err);
589 
595  int max_payload);
596 
602  gboolean map_mode);
603 
616 fBuf_t *yfReaderForFP(
617  fBuf_t *fbuf,
618  FILE *fp,
619  GError **err);
620 
637 fbListener_t *yfListenerForSpec(
638  fbConnSpec_t *spec,
639  fbListenerAppInit_fn appinit,
640  fbListenerAppFree_fn appfree,
641  GError **err);
642 
658 gboolean yfReadFlow(
659  fBuf_t *fbuf,
660  yfFlow_t *flow,
661  GError **err);
662 
681 gboolean yfReadFlowExtended(
682  fBuf_t *fbuf,
683  yfFlow_t *flow,
684  GError **err);
685 
693 void yfPrintString(
694  GString *rstr,
695  yfFlow_t *flow);
696 
706  GString *rstr,
707  yfFlow_t *flow,
708  gboolean yaft_mac);
709 
719 gboolean yfPrint(
720  FILE *out,
721  yfFlow_t *flow,
722  GError **err);
723 
734 gboolean yfPrintDelimited(
735  FILE *out,
736  yfFlow_t *flow,
737  gboolean yaft_mac,
738  GError **err);
739 
750  FILE *out,
751  gboolean yaft_mac,
752  GError **err);
753 
754 #if YAF_ENABLE_HOOKS
755 
760 fbInfoModel_t *yfDPIInfoModel();
761 #endif
762 
763 
764 
765 #endif
uint32_t nonemptypktct
total number of non empty pkts
Definition: yafcore.h:261
uint32_t sip
Source IPv4 address.
Definition: yafcore.h:231
uint8_t pktdir
non empty packet directions, 1, or 0
Definition: yafcore.h:407
uint16_t vlan
VLAN TAG (also in key, but want to record both sides)
Definition: yafcore.h:304
uint64_t oct
Octet count.
Definition: yafcore.h:278
uint16_t maxpktsize
largest pkt size
Definition: yafcore.h:269
uint32_t aitime
average interarrival time in milliseconds
Definition: yafcore.h:265
uint8_t uflags
Union of remaining TCP flags.
Definition: yafcore.h:300
struct yfFlowKey_st yfFlowKey_t
A YAF flow key.
uint32_t token
receiver token
Definition: yafcore.h:355
uint16_t first_pkt_size
First Packet Size - to determine whether to turn on fixed size flag.
Definition: yafcore.h:294
yfMPTCPFlow_t mptcp
MPTCP Flow.
Definition: yafcore.h:417
uint8_t pcap_file_no
Pcap File "ID" so we know when to make entries in metadata file.
Definition: yafcore.h:405
uint8_t appkt
packets with payload - don&#39;t care if this wraps.
Definition: yafcore.h:302
yfFlowVal_t val
Forward value.
Definition: yafcore.h:419
uint8_t pcap_serial
Keep track of number of pcap files for this flow.
Definition: yafcore.h:399
Definition: yafcore.h:351
uint64_t idsn
initial data seq no.
Definition: yafcore.h:353
gboolean yfPrint(FILE *out, yfFlow_t *flow, GError **err)
Print a YAF flow to a file.
fBuf_t * yfReaderForFP(fBuf_t *fbuf, FILE *fp, GError **err)
Get an IPFIX message buffer for reading YAF flows from an open file pointer.
uint64_t etime
Flow end time in epoch milliseconds.
Definition: yafcore.h:375
void yfPrintDelimitedString(GString *rstr, yfFlow_t *flow, gboolean yaft_mac)
Print a YAF flow to a GString in pipe-delimited (tabular) format.
uint32_t isn
Initial TCP sequence number.
Definition: yafcore.h:290
yfFlowVal_t rval
Reverse value.
Definition: yafcore.h:421
void yfAlignmentCheck(void)
yfAlignmentCheck
void yfWriterExportMappedV6(gboolean map_mode)
FIXME doc.
gboolean yfReadFlow(fBuf_t *fbuf, yfFlow_t *flow, GError **err)
Read a single flow from an IPFIX message buffer.
gboolean yfReadFlowExtended(fBuf_t *fbuf, yfFlow_t *flow, GError **err)
Read a single flow from an IPFIX message buffer.
uint16_t attributes
flowAttributes
Definition: yafcore.h:296
uint32_t smallpktct
Number of packets with 60 bytes or less of data.
Definition: yafcore.h:259
A YAF flow.
Definition: yafcore.h:371
fBuf_t * yfWriterForFP(FILE *fp, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to an open file pointer.
uint64_t ltime
used to calculate interarrival time
Definition: yafcore.h:255
void yfPrintColumnHeaders(FILE *out, gboolean yaft_mac, GError **err)
Print column headers for the pipe-delimited (tabular) format.
#define YAF_MAX_MPLS_LABELS
Maximum number of labels we&#39;re going to keep around.
Definition: yafcore.h:202
fBuf_t * yfWriterForSpec(fbConnSpec_t *spec, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to a socket.
void yfFlowCleanup(yfFlow_t *flow)
Clean up after a static flow buffer prepared by yfFlowPrepare.
A YAF uniflow value.
Definition: yafcore.h:276
void yfPrintString(GString *rstr, yfFlow_t *flow)
Print a YAF flow to a GString.
uint16_t vlanId
VLAN Tag - only fwd.
Definition: yafcore.h:218
yfFlowStats_t * stats
yaf flow statistics
Definition: yafcore.h:337
uint8_t iflags
Initial TCP flags.
Definition: yafcore.h:298
void yfFlowPrepare(yfFlow_t *flow)
Prepare a static flow buffer for use with yaf_flow_read().
uint64_t stime
Flow start time in epoch milliseconds.
Definition: yafcore.h:373
uint8_t tos
Type of Service/Traffic Class.
Definition: yafcore.h:220
uint8_t version
IP Version.
Definition: yafcore.h:216
fBuf_t * yfWriterForFile(const char *path, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to a named file.
void yfWriterExportPayload(int max_payload)
FIXME doc.
fbListener_t * yfListenerForSpec(fbConnSpec_t *spec, fbListenerAppInit_fn appinit, fbListenerAppFree_fn appfree, GError **err)
Get an IPFIX connection listener for collecting YAF flows via IPFIX from the network.
uint16_t dp
Destination transport port.
Definition: yafcore.h:212
uint32_t lsn
Last TCP sequence number.
Definition: yafcore.h:292
#define YAF_MAX_HOOKS
maximum number of hooks (plugins) allowed at one time
Definition: yafcore.h:192
uint8_t reason
Flow termination reason (YAF_END_ macros, per IPFIX standard)
Definition: yafcore.h:397
uint8_t proto
IP protocol.
Definition: yafcore.h:214
A YAF flow key.
Definition: yafcore.h:208
pcap_dumper_t * pcap
Pcap File Ptr.
Definition: yafcore.h:411
union yfFlowKey_st::@0 addr
for DAG cards need to record the interface, may only be seeing unidirectional flows on each interface...
gboolean yfPrintDelimited(FILE *out, yfFlow_t *flow, gboolean yaft_mac, GError **err)
Print a YAF flow to a file in pipe-delimited (tabular) format.
uint64_t pkt
Packet count.
Definition: yafcore.h:280
#define ETHERNET_MAC_ADDR_LENGTH
length of Ethernet MAC Address
Definition: yafcore.h:190
gboolean yfWriterClose(fBuf_t *fbuf, gboolean flush, GError **err)
Close the connection underlying an IPFIX message buffer created by yfWriterForFP() or yfWriterForSpec...
uint32_t dip
Destination IPv4 address.
Definition: yafcore.h:233
uint16_t mss
max segment size
Definition: yafcore.h:357
uint8_t rtos
reverse ToS (fwd in flowKey)
Definition: yafcore.h:409
struct yfFlowStats_st yfFlowStats_t
yaf flow statistics
gboolean yfWriteStatsFlow(void *yfContext, uint32_t pcap_drop, GTimer *timer, GError **err)
Write an options data record to an IPFIX Message buffer.
uint32_t tcpurgct
Number of urgent packets.
Definition: yafcore.h:257
uint8_t addrid
addr id
Definition: yafcore.h:359
uint8_t flags
hash_flags
Definition: yafcore.h:361
struct yfFlowVal_st yfFlowVal_t
A YAF uniflow value.
uint64_t payoct
total amount of payload data
Definition: yafcore.h:253
gboolean yfWriteFlow(void *yfContext, yfFlow_t *flow, GError **err)
Write a single flow to an IPFIX message buffer.
uint16_t sp
Source transport port.
Definition: yafcore.h:210
struct yfFlow_st yfFlow_t
A YAF flow.
uint16_t firstpktsize
payload length of first non-empty pkt
Definition: yafcore.h:267
yaf flow statistics
Definition: yafcore.h:247
uint32_t largepktct
total number of packets with 225 bytes or more
Definition: yafcore.h:263
yfFlowKey_t key
Flow key.
Definition: yafcore.h:423