ZenHAX

Free Game Research Forum | Official QuickBMS support | twitter @zenhax | SSL HTTPS://zenhax.com
It is currently Thu Nov 23, 2017 4:46 pm

All times are UTC




Post new topic  Reply to topic  [ 17 posts ] 
Author Message
PostPosted: Tue Sep 06, 2016 9:04 am 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
Well the heading says it all,i need quick bms to inject back to the decompressed bpe file,
here is the bpe source code,except in this case,it has been modified to work with Yukes games for Sony PS2..

Code:
#include <stdio.h>
#include <stdlib.h>

#define BLOCKSIZE 10000    /* maximum block size */
#define HASHSIZE 8192       /* size of hash table */
#define MAXCHARS 220       /* char set per block */
#define THRESHOLD 3       /* minimum pair count */

unsigned char buffer[BLOCKSIZE];      /* data block */
unsigned char leftcode[256];      /* pair table */
unsigned char rightcode[256];      /* pair table */
unsigned char left[HASHSIZE];      /* hash table */
unsigned char right[HASHSIZE];      /* hash table */
unsigned char count[HASHSIZE];      /* pair count */
int size;      /* size of current data block */

/* function prototypes */
int lookup (unsigned char, unsigned char, int );
int fileread (FILE *, int, int, int);
void filewrite (FILE *);
void compress (FILE *, FILE *, int, int, int, int);

/* return index of character pair in hash table */
/* deleted nodes have a count of 1 for hashing */
int lookup (unsigned char a, unsigned char b, int hs)
{
  int index;   /* ?  - will add question marks until I understand each variable */


  /* compute hash key from both characters */
  index = (a ^ (b << 5)) & (hs-1);   /* ? */
/* if b = 10110101 then '(b << 5)' --> b = 10100000. */
/* ie shift the bits in b left by five positions and fill holes with zeros */

  /* search for pair or first empty slot */
  while ((left[index] != a || right[index] != b) && count[index] != 0)
  {
    index = (index + 1) & (hs - 1);
  }

  left[index] = a;
  right[index] = b;
  return index;
}

/* read next block from input file into buffer */
int fileread (FILE *input, int bs, int hs, int mc)
{
  int c, index, used=0;

  /* reset hash table and pair table */
  for (c = 0; c < hs; c++)
    count[c] = 0;
  for (c = 0; c < 256; c++)
  {
    leftcode[c] = c;
    rightcode[c] = 0;
  }
  size = 0;

  /* read data until full or few unused chars */
  while (size < bs && used < mc && (c = getc(input)) != EOF)
  {
    if (size > 0)
    {
      index = lookup(buffer[size-1], c, hs);
      if (count[index] < 255)
      {
        ++count[index];
      }
    }
    buffer[size++] = c;

    /* use right code to flag data chars found */
    if (!rightcode[c])
    {
      rightcode[c] = 1;
      used++;
    }
  }
  return c == EOF;
}

/* write each pair table and data block to output */
void filewrite( FILE *output )
{
  int i, len, c = 0;

  /* for each character 0..255 */
  while ( c < 256 )
  {
    /* if not a pair code, count run of literals */
    if ( c == leftcode[c] )
    {
      len = 1; c++;
      while ( len < 127 && c < 256 && c == leftcode[c])
      {
        len++; c++;
      }
      putc( len + 127, output );
      len = 0;
      if ( c == 256 ) break;
    }
   
    /* else count run of pair codes */
    else
    {
      len = 0;
      c++;


/* original, will add extra brackets per compiler suggestions:      while ( len < 127 && c < 256 && c != leftcode[c] || len < 125 && c < 254 && c+1 != leftcode[c+1]) */
      while (( len < 127 && c < 256 && c != leftcode[c]) || (len < 125
&& c < 254 && c+1 != leftcode[c+1]))
      {
        len++;
        c++;
      }
      putc(len, output);
      c -= len+1;
    }

    /* write range of pairs to output */
    for ( i = 0; i <= len; i++ )
    {
      putc(leftcode[c], output);
      if ( c != leftcode[c] )
      { putc(rightcode[c], output); }
      c++;
    }
  }
  /* write size bytes and compressed data block */
  putc(size%256, output);
  putc(size/256, output);
  fwrite(buffer, size, 1, output);
}

/* compress from input file to output file */
void compress( FILE *infile, FILE *outfile,
               int bs, int hs, int mc, int th )
{
  int leftch, rightch, code, oldsize;
  int index, r, w, best, done = 0;

  /* compress each data block until end of file */
  while ( !done )
  {
    done = fileread(infile, bs, hs, mc);
    code = 256;
   
    /* compress this block */
    for(;;)
    {
      /* get next unused chr for pair code */
      for ( code--; code >= 0; code-- )
      {
        if ( code == leftcode[code] && !rightcode[code] )
        {
          break;
        }
      }
 
      /* must quit if no unused chars left */
      if ( code < 0 )
      {
        break;
      }

      /* find most frequent pair of chars */
      for ( best = 2, index = 0; index < hs; index++ )
      {
        if (count[index] > best)
        {
          best = count[index];
          leftch = left[index];
          rightch = right[index];
        }
      }
   
      /* done if no more compression possible */
      if ( best < th )
      {
        break;
      }
   
      /* Replace pairs in data, adjust pair counts */
      oldsize = size - 1;
      for ( w = 0, r = 0; r < oldsize ; r++ )
      {
        if (buffer[r] == leftch && buffer[r+1] == rightch)
        {
          if ( r > 0 )
          {
            index = lookup(buffer[w-1], leftch, hs);
            if ( count[index] > 1 )
            {
              --count[index];
            }
            index = lookup( buffer[w-1], code, hs );
            if ( count[index] < 255 )
            {
              ++count[index];
            }
          }
          if ( r < oldsize - 1 )
          {
            index = lookup( rightch, buffer[r+2] , hs);
            if ( count[index] > 1 )
            {
              --count[index];
            }
            index = lookup( code, buffer[r+2], hs );
            if ( count[index] < 255 )
            {
              ++count[index];
            }
          }
          buffer[w++] = code;
          r++;
          size--;
        }
        else
        {
          buffer[w++] = buffer[r];
        }
      }
      buffer[w] = buffer[r];

      /* add to pair substitution table */
      leftcode[code] = leftch;
      rightcode[code] = rightch;
   
      /* delete pair from hash table */
      index = lookup( leftch, rightch, hs );
      count[index] = 1;
    }
    filewrite( outfile );
  }
}

void main (int argc, char *argv[] )
{
  FILE *infile, *outfile;
/* argc = 7              */
/*   argv[0] = command   */
/*   argv[1] = infile    */
/*   argv[2] = outfile   */
/*   argv[3] = BLOCKSIZE */
/*   argv[4] = HASHSIZE  */
/*   argv[5] = MAXCHARS  */
/*   argv[6] = THRESHOLD */
  int bs = 20000; /* maxval */
  int hs = 16384; /* maxval */
  int mc = 200;   /* default value */
  int th = 3;     /* default min */


  if (argc != 7)
  {
    printf("Usage: bpe infile outfile blocksize hashsize maxchars threshold\n");
    printf("typical: bpe infile outfile 5000 4096 200 3\n");
  }
  else
  {
    if (( infile = fopen( argv[1], "rb" )) == NULL )
    {
      printf("Error opening input %s\n",argv[1]);
    }
    else
    {
      if (( outfile = fopen( argv[2], "wb" )) == NULL )
      {
        printf("Error opening output %s\n",argv[2]);
      }
      else
      {
        bs = atoi( argv[3] );
        hs = atoi( argv[4] );
        mc = atoi( argv[5] );
        th = atoi( argv[6] );
        /* because these inputs come from the command line generated
           by clustor, I have included very little error checking here
        */   

        compress( infile, outfile, bs, hs, mc, th );
        fclose( outfile );
        fclose( infile );
      }
    }
  }
}

/* end of file */




Top
   
PostPosted: Tue Sep 06, 2016 10:32 am 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
That code you posted is the SAME of the one here:
https://github.com/wwylele/KEYTranslati ... /gfa/bpe.c

And I don't see references there about working with yuke/WWE, so, are you 100% sure?


Top
   
PostPosted: Tue Sep 06, 2016 10:34 am 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
And it's also the same identical implementation already available in quickbms!


Top
   
PostPosted: Tue Sep 06, 2016 12:42 pm 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
aluigi wrote:
And it's also the same identical implementation already available in quickbms!

Yes i made modifications for it to work with WWE Yukes...

Modified two lines of code to make it work...could you add a inject feature for this code?


Top
   
PostPosted: Tue Sep 06, 2016 1:00 pm 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
What are these lines? The code is identical.


Top
   
PostPosted: Tue Sep 06, 2016 1:08 pm 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
Do you mean the endianess at line 139?


Top
   
PostPosted: Tue Sep 06, 2016 1:15 pm 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
yes i modified the endianness at line 139, i switched places of these codes.
In the place of size/256 i replaced it with size%256 and in the place of size%256,i replaced it with size/256


Last edited by eri619 on Tue Sep 06, 2016 1:21 pm, edited 1 time in total.

Top
   
PostPosted: Tue Sep 06, 2016 1:21 pm 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
Ok, I can add a yuke_compress in the next quickbms.


Top
   
PostPosted: Tue Sep 06, 2016 1:22 pm 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
aluigi wrote:
Ok, I can add a yuke_compress in the next quickbms.


Thank You,it would be really helpful.It can inject back with the same file size as original right?


Top
   
PostPosted: Tue Sep 06, 2016 2:38 pm 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
If you use the reimport feature you are bound to the size limits.
I'm not 100% sure if there are problems of compressed size like happens with lz4 and other algorithms, it depends by the decompression implementation.
What I mean is that the game may not accept the edited archive.


Top
   
PostPosted: Tue Sep 06, 2016 5:35 pm 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
aluigi wrote:
If you use the reimport feature you are bound to the size limits.
I'm not 100% sure if there are problems of compressed size like happens with lz4 and other algorithms, it depends by the decompression implementation.
What I mean is that the game may not accept the edited archive.


Because thats what is wanted,SVR games accept compressed bpe files of any size,it has been tested.
But PS1 game Smackdown 2 requires the bpe file to be of its original size.The game reads only bpe file which is identical in size.


Top
   
PostPosted: Thu Sep 08, 2016 2:11 am 

Joined: Sun Jul 17, 2016 5:23 am
Posts: 9
If you're going to add yuke compress support,remember to add the BPE Header,which is those 16 bytes below
The first 4 bytes: "BPE "
The next 4 bytes: "\x00\x01\x00\x00"
The next 4 bytes are compressed size in hex,the last 4 bytes are uncompressed size in hex.Size is in bytes


Top
   
PostPosted: Thu Sep 08, 2016 4:30 am 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
What you mention is part of the format and not part of the compression algorithm.


Top
   
PostPosted: Sun Sep 11, 2016 7:21 am 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
aluigi wrote:
What you mention is part of the format and not part of the compression algorithm.

yes you are right


Top
   
PostPosted: Sun Sep 11, 2016 7:22 am 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
eatrawmeat391 wrote:
If you're going to add yuke compress support,remember to add the BPE Header,which is those 16 bytes below
The first 4 bytes: "BPE "
The next 4 bytes: "\x00\x01\x00\x00"
The next 4 bytes are compressed size in hex,the last 4 bytes are uncompressed size in hex.Size is in bytes

There is no need to add a bpe header as quickbms injects the new file back into the old file provided the file sizes are the same


Top
   
PostPosted: Wed Dec 07, 2016 9:58 am 

Joined: Sun Nov 29, 2015 11:06 pm
Posts: 20
has this been implemented in the new version yet?


Top
   
PostPosted: Wed Dec 07, 2016 2:54 pm 
Site Admin
User avatar

Joined: Wed Jul 30, 2014 9:32 pm
Posts: 7142
yuke_compress is available in quickbms 0.7.7, it uses the code of bpe_compress swapping the 16 bits values it writes.


Top
   
Display posts from previous:  Sort by  
Post new topic  Reply to topic  [ 17 posts ] 

All times are UTC


You cannot post new topics in this forum
You cannot reply to topics in this forum
You cannot edit your posts in this forum
You cannot delete your posts in this forum
You cannot post attachments in this forum

Search for:
Powered by phpBB® Forum Software © phpBB Limited