validate.cpp

Go to the documentation of this file.

/* 
   
* FIMI output validator/sorter program

Contains parts of:


 * Cryptographic API.
 *
 * MD5 Message Digest Algorithm (RFC1321).
 *
 * Derived from cryptoapi implementation, originally based on the
 * public domain implementation written by Colin Plumb in 1993.
 *
 * Copyright (c) Cryptoapi developers.
 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
 * 
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option) 
 * any later version.
 *
 */


#include <stdio.h>
#include <getopt.h>

#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <inttypes.h>

typedef uint8_t u8;
typedef uint32_t u32;
typedef uint64_t u64;



// ---------------------- MD5 CHECKSUM ROUTINE ------------------

/*

WORKS ONLY ON LITTLE ENDIAN CPUS

*/

#define __le32_to_cpus(buf)
#define __cpu_to_le32s(buf)



#define MD5_DIGEST_SIZE         16
#define MD5_HMAC_BLOCK_SIZE     64
#define MD5_BLOCK_WORDS         16
#define MD5_HASH_WORDS          4

#define F1(x, y, z)     (z ^ (x & (y ^ z)))
#define F2(x, y, z)     F1(z, x, y)
#define F3(x, y, z)     (x ^ y ^ z)
#define F4(x, y, z)     (y ^ (x | ~z))

#define MD5STEP(f, w, x, y, z, in, s) \
        (w += f(x, y, z) + in, w = (w<<s | w>>(32-s)) + x)

struct md5_ctx {
        u32 hash[MD5_HASH_WORDS];
        u32 block[MD5_BLOCK_WORDS];
        u64 byte_count;
};

static void md5_transform(u32 *hash, u32 const *in)
{
        u32 a, b, c, d;

        a = hash[0];
        b = hash[1];
        c = hash[2];
        d = hash[3];

        MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
        MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
        MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
        MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);
        MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);
        MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);
        MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);
        MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);
        MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);
        MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);
        MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
        MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
        MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
        MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
        MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
        MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);

        MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
        MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
        MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
        MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);
        MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);
        MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
        MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
        MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);
        MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);
        MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
        MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);
        MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);
        MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
        MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
        MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
        MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);

        MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
        MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
        MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
        MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
        MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);
        MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);
        MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);
        MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
        MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
        MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);
        MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);
        MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);
        MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);
        MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
        MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
        MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);

        MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
        MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
        MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
        MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);
        MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
        MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);
        MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
        MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);
        MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);
        MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
        MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);
        MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
        MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);
        MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
        MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
        MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);

        hash[0] += a;
        hash[1] += b;
        hash[2] += c;
        hash[3] += d;
}

/* XXX: this stuff can be optimized */
static inline void le32_to_cpu_array(u32 *buf, unsigned int words)
{
        while (words--) {
                __le32_to_cpus(buf);
                buf++;
        }
}

static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
{
        while (words--) {
                __cpu_to_le32s(buf);
                buf++;
        }
}

static inline void md5_transform_helper(struct md5_ctx *ctx)
{
        le32_to_cpu_array(ctx->block, sizeof(ctx->block) / sizeof(u32));
        md5_transform(ctx->hash, ctx->block);
}

static void md5_init(void *ctx)
{
        struct md5_ctx *mctx = (struct md5_ctx *)ctx;

        mctx->hash[0] = 0x67452301;
        mctx->hash[1] = 0xefcdab89;
        mctx->hash[2] = 0x98badcfe;
        mctx->hash[3] = 0x10325476;
        mctx->byte_count = 0;
}

static void md5_update(void *ctx, const u8 *data, unsigned int len)
{
        struct md5_ctx *mctx = (struct md5_ctx *)ctx;
        const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f);

        mctx->byte_count += len;

        if (avail > len) {
                memcpy((char *)mctx->block + (sizeof(mctx->block) - avail),
                       data, len);
                return;
        }

        memcpy((char *)mctx->block + (sizeof(mctx->block) - avail),
               data, avail);

        md5_transform_helper(mctx);
        data += avail;
        len -= avail;

        while (len >= sizeof(mctx->block)) {
                memcpy(mctx->block, data, sizeof(mctx->block));
                md5_transform_helper(mctx);
                data += sizeof(mctx->block);
                len -= sizeof(mctx->block);
        }

        memcpy(mctx->block, data, len);
}

static void md5_final(void *ctx, u8 *out)
{
        struct md5_ctx *mctx = (struct md5_ctx *)ctx;
        const unsigned int offset = mctx->byte_count & 0x3f;
        char *p = (char *)mctx->block + offset;
        int padding = 56 - (offset + 1);

        *p++ = 0x80;
        if (padding < 0) {
                memset(p, 0x00, padding + sizeof (u64));
                md5_transform_helper(mctx);
                p = (char *)mctx->block;
                padding = 56;
        }

        memset(p, 0, padding);
        mctx->block[14] = mctx->byte_count << 3;
        mctx->block[15] = mctx->byte_count >> 29;
        le32_to_cpu_array(mctx->block, (sizeof(mctx->block) -
                          sizeof(u64)) / sizeof(u32));
        md5_transform(mctx->hash, mctx->block);
        cpu_to_le32_array(mctx->hash, sizeof(mctx->hash) / sizeof(u32));
        memcpy(out, mctx->hash, sizeof(mctx->hash));
        memset(mctx, 0, sizeof(*mctx));
}


//--------------- END OF MD5 CHECKSUM ROUTINE ------------------


#include <vector>
using namespace std;
#include <algorithm>

void usage(char *s) {
  fprintf(stderr,"Usage: %s [-h] infilename\n",s);
  fprintf(stderr,"Computes a checksum from the FIMI output file infilename\n");
  fprintf(stderr,"-h prints this help text\n");
  exit(1);
}

char *infilename;

void parseparams(int argc, char *argv[]) {
  char ch;
  while ((ch=getopt(argc, argv, "h"))>=0) {
    switch(ch) {
    case 'h':
      usage(argv[0]);
      break;
    case '?':
      fprintf(stderr,"unknown option %c\n",optopt);
      usage(argv[0]);
      break;
    default:
      fprintf(stderr,"unknown option %c\n",ch);
      usage(argv[0]);
      break;
    }      
  }
  if (optind>=argc) {
    usage(argv[0]);
  }
}


bool parse_fimi_line(int & supp, vector<int> &itemset, char *linebuf) {
  char *bptr;
  itemset.clear();
  bptr=linebuf;
  while ((*bptr) && (*bptr!='(') && (*bptr != '\n')) {
    int lastnum=0;
    if (!((*bptr >='0')&&(*bptr<='9'))) {
      fprintf(stderr,"Input file format error: Expecting number, got %c Erroneous line:\n%s",*bptr,linebuf);
      return false;
    }
    while ((*bptr >='0')&&(*bptr<='9')) {
      lastnum*=10;
      lastnum+=*bptr-'0';
      bptr++;
    }
    itemset.push_back(lastnum);
    while (*bptr==' ') {
      bptr++;
    }
  }
  if (*bptr!='(') {
    fprintf(stderr,"Input file format error. Expecting (, got %c. Erroneous line:\n%s",*bptr,linebuf);
    return false;
  }
  bptr++;
  int lastnum=0;
  while ((*bptr >='0')&&(*bptr<='9')) {
    lastnum*=10;
    lastnum+=*bptr-'0';
    bptr++;
  }
  if (*bptr!=')') {
    fprintf(stderr,"Input file format error. Expecting ), got %c. Erroneous line:\n%s",*bptr,linebuf);
    return false;
  }
  bptr++;
  if (*bptr!='\n') {
    fprintf(stderr,"Input file format error. Expecting newline, got %c. Erroneous line:\n%s",*bptr,linebuf);
    return false;
  }
  supp=lastnum;
  return true;
}

void printsum(uint32_t* sum) {
  static char* hexdigits="0123456789abcdef";
  for(size_t i=0;i<MD5_HASH_WORDS;++i) {
    for (int q=7;q>=0;--q) {
      int c=sum[i];
      c>>=(q*4);
      c&=15;
      putchar(hexdigits[c]);
    }
  }
}

bool checksumfile(char *name) {
  char buffer[128*1024];
  FILE *fin = fopen(name,"r");
  if (!fin) {
    perror("open");
    return false;
  }
  setbuffer(fin,buffer,sizeof(buffer));
  struct md5_ctx ctx;
  uint32_t finalsum1[MD5_HASH_WORDS];
  uint32_t finalsum2[MD5_HASH_WORDS];
  bzero(finalsum1,MD5_HASH_WORDS*sizeof(int));
  bzero(finalsum2,MD5_HASH_WORDS*sizeof(int));
  char linebuf[10000];
  vector<int> itemset;
  int supp;
  int lines=0;
  while(!feof(fin)) {
    uint32_t currsum[MD5_HASH_WORDS];
    if (!fgets(linebuf,sizeof(linebuf),fin)) {
      break;
    }
    parse_fimi_line(supp,itemset,linebuf);
    md5_init(&ctx);
    md5_update(&ctx,(u8*)&supp,sizeof(supp));
    sort(itemset.begin(),itemset.end());
    md5_update(&ctx,(u8*)&(itemset[0]),itemset.size()*sizeof(int));
    md5_final(&ctx,(u8*)currsum);
    for(size_t i=0;i<MD5_HASH_WORDS;++i) {
      finalsum1[i]^=currsum[i];
      finalsum2[i]+=currsum[i];
    }
    lines++;
  }
  for (int i=0;i<MD5_HASH_WORDS;++i) {
    finalsum1[i]^=finalsum2[i];
  }
  printsum(finalsum1);
  printf(" (%d lines) %s\n",lines,name);
  return true;
}


int main(int argc, char *argv[]) {
  parseparams(argc,argv);
  while(optind<argc){
    if (!checksumfile(argv[optind++])) {
      exit(1);
    }
  }
  return 0;
}

---