fork/slippc

git clone https://git.julia.blue/fork/slippc

Patrick Crainfixed rare memory leak in parser; updated changelog and readmes for version 0.8.0b79f91c

0.8.0
50.1 KiB1272 linesraw
1#include "compressor.h"
2
3namespace slip {
4
5  Compressor::Compressor(int debug_level) {
6    _debug = debug_level;
7    _bp    = 0;
8  }
9
10  Compressor::~Compressor() {
11    if (_rb != nullptr)               { delete[] _rb; }
12    if (_wb != nullptr)               { delete[] _wb; }
13    if (_outfilename != nullptr)      { delete   _outfilename; }
14    if (_outgeckofilename != nullptr) { delete   _outgeckofilename; }
15  }
16
17  bool Compressor::loadFromFile(const char* replayfilename) {
18    DOUT1("  Loading " << replayfilename);
19    std::ifstream myfile;
20    myfile.open(replayfilename,std::ios::binary | std::ios::in);
21    if (myfile.fail()) {
22      FAIL("    File " << replayfilename << " could not be opened or does not exist");
23      return false;
24    }
25
26    myfile.seekg(0, myfile.end);
27    _file_size = myfile.tellg();
28    if (_file_size < MIN_REPLAY_LENGTH) {
29      FAIL("    File " << replayfilename << " is too short to be a valid Slippi replay");
30      return false;
31    }
32    myfile.seekg(0, myfile.beg);
33
34    _rb = new char[_file_size];
35    myfile.read(_rb,_file_size);
36    myfile.close();
37
38    // Check if we have a compressed stream
39    bool is_compressed = same4(&_rb[0],LZMA_HEADER);
40    if (is_compressed) {
41      DOUT1("    File Size: " << +_file_size << ", compressed");
42      // Decompress the read buffer
43      std::string decomp = decompressWithLzma(_rb, _file_size);
44      // Get the new file size
45      _file_size    = decomp.size();
46      // Delete the old read buffer
47      delete [] _rb;
48      // Reallocate it with more spce
49      _rb = new char[_file_size];
50      // Copy buffer from the decompressed string
51      memcpy(_rb,decomp.c_str(),_file_size);
52    } else {
53      DOUT1("    File Size: " << +_file_size);
54    }
55
56    _infilename = replayfilename;
57
58    _wb           = new char[_file_size];
59    memcpy(_wb,_rb,sizeof(char)*_file_size);
60
61    return this->_parse();
62  }
63
64  void Compressor::saveToFile(bool rawencode) {
65    if (fileExists(*_outfilename)) {
66      FAIL("  File " << *_outfilename << " exists, refusing to overwrite");
67      return;
68    }
69
70    std::ofstream ofile;
71    ofile.open(*_outfilename, std::ios::binary | std::ios::out);
72    // If this is the unencoded version, compress it first
73    if (!(_encode_ver || rawencode)) {
74      // Compress the write buffer
75      std::string comp = compressWithLzma(_wb, _file_size);
76      DOUT1("  Compression Ratio = " << float(_file_size-comp.size())/_file_size);
77      // Write compressed buffer to file
78      ofile.write(comp.c_str(),sizeof(char)*comp.size());
79    } else {
80      // Write normal buffer to file
81      ofile.write(_wb,sizeof(char)*_file_size);
82    }
83
84    ofile.close();
85
86  }
87
88  bool Compressor::setOutputFilename(const char* fname) {
89    if (fileExists(fname)) {
90      return false;
91    }
92    if (_outfilename != nullptr) {
93      delete   _outfilename;
94    }
95    _outfilename = new std::string(fname);
96    return true;
97  }
98
99  bool Compressor::setGeckoOutputFilename(const char* fname) {
100    if (_outgeckofilename != nullptr) {
101      delete   _outgeckofilename;
102    }
103    _outgeckofilename = new std::string(fname);
104    _outgeckofilename->append(".dat");
105    if (fileExists(*_outgeckofilename)) {
106      return false;
107    }
108    return true;
109  }
110
111  unsigned Compressor::saveToBuff(char** buffer) {
112    // buffer = new char[_file_size];
113    *buffer = new char[_file_size];
114    memcpy(*buffer,_wb,sizeof(char)*_file_size);
115    return _file_size;
116  }
117
118  bool Compressor::loadFromBuff(char** buffer, unsigned size) {
119    _file_size = size;
120    _wb        = new char[_file_size];
121    _rb        = new char[_file_size];
122    memcpy(_wb,*buffer,sizeof(char)*_file_size);
123    memcpy(_rb,*buffer,sizeof(char)*_file_size);
124    return this->_parse();
125  }
126
127  bool Compressor::validate() {
128    if (_encode_ver) {
129      return true;
130    }
131
132    char *enc_buff = nullptr, *dec_buff = nullptr;
133    unsigned size  = this->saveToBuff(&enc_buff);
134    Compressor *d  = new slip::Compressor(_debug);
135    d->loadFromBuff(&enc_buff,size);
136    d->saveToBuff(&dec_buff);
137
138    bool success = (memcmp(_rb,dec_buff,size) == 0);
139    if (!success) {
140      unsigned diff = 0;
141      for(unsigned i = 0; i < size; ++i) {
142        if(_rb[i] != dec_buff[i]) {
143          if ((++diff) <= 500) {
144            DOUT2("    Byte " << i << " differs! orig "
145            << int(_rb[i]) << " != " << int(dec_buff[i]) << " new");
146          }
147        }
148      }
149      DOUT1("Differs in " << diff << "/" << size << " bytes");
150    }
151
152    delete[] dec_buff;
153    delete[] enc_buff;
154    delete d;
155
156    return success;
157  }
158
159  bool Compressor::_parse() {
160    _bp = 0; //Start reading from byte 0
161    if (not this->_parseHeader()) {
162      FAIL("  Failed to parse header");
163      return false;
164    }
165    if (not this->_parseEventDescriptions()) {
166      FAIL("  Failed to parse event descriptions");
167      return false;
168    }
169    if (not this->_parseEvents()) {
170      FAIL("  Failed to parse events proper");
171      return false;
172    }
173    DOUT1("  Successfully parsed replay!");
174    return true;
175  }
176
177  bool Compressor::_parseHeader() {
178    DOUT1("  Parsing header");
179
180    //First 15 bytes contain header information
181    if (same8(&_rb[_bp],SLP_HEADER)) {
182      DOUT1("    Slippi Header Matched");
183    } else {
184      FAIL_CORRUPT("    Header did not match expected Slippi file header");
185      return false;
186    }
187    _length_raw_start = readBE4U(&_rb[_bp+11]);
188    if(_length_raw_start == 0) {  //TODO: this is /technically/ recoverable
189      FAIL_CORRUPT("    0-byte raw data detected");
190      return false;
191    }
192    DOUT1("    Raw portion = " << _length_raw_start << " bytes");
193    if (_length_raw_start > _file_size) {
194      FAIL_CORRUPT("      Raw data size exceeds file size of " << _file_size << " bytes");
195      return false;
196    }
197    _length_raw = _length_raw_start;
198    _bp += 15;
199    return true;
200  }
201
202  bool Compressor::_parseEventDescriptions() {
203    DOUT1("  Parsing event descriptions");
204
205    //Next 2 bytes should be 0x35
206    if (_rb[_bp] != Event::EV_PAYLOADS) {
207      FAIL_CORRUPT("    Expected Event 0x" << std::hex
208        << Event::EV_PAYLOADS << std::dec << " (Event Payloads)");
209      return false;
210    }
211    uint8_t ev_bytes = _rb[_bp+1]-1; //Subtract 1 because the last byte we read counted as part of the payload
212    _payload_sizes[Event::EV_PAYLOADS] = int32_t(ev_bytes+1);
213    DOUT1("    Event description length = " << int32_t(ev_bytes+1) << " bytes");
214    _bp += 2;
215
216    //Next ev_bytes bytes describe events
217    for(unsigned i = 0; i < ev_bytes; i+=3) {
218      unsigned ev_code = uint8_t(_rb[_bp+i]);
219      if (_payload_sizes[ev_code] > 0) {
220        if (ev_code >= Event::EV_PAYLOADS && ev_code <= Event::GAME_END) {
221          FAIL_CORRUPT("    Event " << Event::name[ev_code-Event::EV_PAYLOADS] << " payload size set multiple times");
222        } else {
223          FAIL_CORRUPT("    Event " << hex(ev_code) << std::dec << " payload size set multiple times");
224        }
225        return false;
226      }
227      _payload_sizes[ev_code] = readBE2U(&_rb[_bp+i+1])+1; //Add one because of the event code itself
228      DOUT1("    Payload size for event "
229        << hex(ev_code) << std::dec << ": " << _payload_sizes[ev_code]
230        << " bytes");
231    }
232
233    //Sanity checks to verify we at least have Payload Sizes, Game Start, Pre Frame, Post Frame, and Game End Events
234    for(unsigned i = Event::EV_PAYLOADS; i <= Event::GAME_END; ++i) {
235      if (_payload_sizes[i] == 0) {
236        FAIL_CORRUPT("    Event " << Event::name[i-Event::EV_PAYLOADS] << " payload size not set");
237        return false;
238      }
239    }
240
241    //Update the remaining length of the raw data to sift through
242    _bp         += ev_bytes;
243    _length_raw -= (2+ev_bytes);
244    return true;
245  }
246
247  bool Compressor::_parseEvents() {
248    DOUT1("  Parsing events proper");
249
250    bool success = true;
251    for( ; _length_raw > 0; ) {
252      unsigned ev_code = uint8_t(_rb[_bp]);
253
254      unsigned shift   = _payload_sizes[ev_code];
255      if (shift > _length_raw) {
256        FAIL_CORRUPT("    Event byte offset exceeds raw data length");
257        return false;
258      }
259      DOUT2("    EV code " << hex(ev_code) << " encountered");
260      switch(ev_code) { //Determine the event code
261        case Event::GAME_START:  success = _parseGameStart(); break;
262        case Event::SPLIT_MSG:   success = _parseGeckoCodes(); break;
263        case Event::FRAME_START: success = _parseFrameStart(); break;
264        case Event::PRE_FRAME:   success = _parsePreFrame();   break;
265        case Event::ITEM_UPDATE: success = _parseItemUpdate(); break;
266        case Event::POST_FRAME:  success = _parsePostFrame();  break;
267        case Event::BOOKEND:     success = _parseBookend();    break;
268        case Event::GAME_END:
269            _game_end_found = true;
270            _game_loop_end = _bp;
271            if (! _encode_ver) {
272                _shuffleEvents();
273            }
274            success        = true;
275            break;
276        default:
277          DOUT1("    Warning: unknown event code " << hex(ev_code) << " encountered; skipping");
278          break;
279      }
280      if (not success) {
281        return false;
282      }
283      if (_payload_sizes[ev_code] == 0) {
284        FAIL_CORRUPT("    Uninitialized event " << hex(ev_code) << " encountered");
285        return false;
286      }
287      _length_raw    -= shift;
288      _bp            += shift;
289      DOUT2("    Raw bytes remaining: " << +_length_raw);
290    }
291    if(!_game_end_found) {
292      FAIL_CORRUPT("    No game end event found");
293      return false;
294    }
295    return true;
296  }
297
298  bool Compressor::_parseGameStart() {
299    DOUT1("  Parsing game start event at byte " << +_bp);
300
301    //Get Slippi version
302    _slippi_maj = uint8_t(_rb[_bp+O_SLP_MAJ]); //Major version
303    _slippi_min = uint8_t(_rb[_bp+O_SLP_MIN]); //Minor version
304    _slippi_rev = uint8_t(_rb[_bp+O_SLP_REV]); //Build version (4th char unused)
305    _encode_ver = uint8_t(_rb[_bp+O_SLP_ENC]); //Whether the file is encoded
306    if (MIN_VERSION(3,13,0)) {
307      FAIL("    Version is " << GET_VERSION() << ". Replays from Slippi 3.13.0 and higher are not supported");
308      return false;
309    }
310
311    //Set encoding status for output buffer
312    if(_wb[_bp+O_SLP_ENC]) {
313      _wb[_bp+O_SLP_ENC] = 0;  //Input is encoded, so we will decode
314    } else {
315      _wb[_bp+O_SLP_ENC] = COMPRETZ_VERSION;  //Input is not encoded, use current version
316    }
317
318    // ensure we have a good output filename based on encode status
319    if (!ensureAppropriateFilename()) {
320      return false;
321    }
322
323    //Print slippi version
324    std::stringstream ss;
325    ss << +_slippi_maj << "." << +_slippi_min << "." << +_slippi_rev;
326    DOUT1("    Slippi Version: " << ss.str() << ", " << (_encode_ver ? "encoded" : "raw"));
327    if(_encode_ver) {
328      DOUT1("      Compressed with: slippc compression ver. " << int(_encode_ver));
329    }
330
331    //Get starting RNG state
332    _rng       = readBE4U(&_rb[_bp+O_RNG_GAME_START]);
333    _rng_start = _rng;
334    DOUT1("    Starting RNG: " << _rng);
335
336    return true;
337  }
338
339  bool Compressor::_parseGeckoCodes() {
340    if(ENCODE_VERSION_MIN(2)) {
341      // Debug output for actually dumping the gecko code messages to a file
342      if(_outgeckofilename) {
343        char* main_buf = (_encode_ver ? _wb : _rb);
344        std::fstream fin;
345        if(_message_count) {
346          fin.open(*_outgeckofilename, std::ios::out | std::ios::app | std::ios::binary);
347        } else {
348          fin.open(*_outgeckofilename, std::ios::out | std::ios::trunc | std::ios::binary);
349        }
350        fin.write(&(main_buf[_bp]), MESSAGE_SIZE);
351        fin.close();
352        ++_message_count;
353      }
354      //Assuming everyone has the same default Gecko codes doesn't work, so V2 doesn't do anything fancy
355      return true;
356    }
357
358    // V1 tried to XOR gecko codes but implemented it wrong anyway
359    //   because it used a function-local variable to count messages,
360    //   so it ended up XORing the same 512 bytes over and over again
361    //   Must keep this here as a reminder of my shame ):
362
363    // Load default Gecko codes
364    const char* codes = readLegacyGeckoCodes();
365
366    // XOR encode everything but the command byte
367    for(unsigned i = 1; i < MESSAGE_SIZE; ++i) {
368      _wb[_bp+i] = _rb[_bp+i] ^ codes[i];
369    }
370
371    return true;
372  }
373
374  bool Compressor::_parseItemUpdate() {
375    //Encodings so far
376      //0x00 - 0x00 | Command Byte         | No encoding
377      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1)
378      //0x05 - 0x06 | Type ID              | XORed with last item-slot value
379      //0x07 - 0x07 | State                | XORed with last item-slot value
380      //0x08 - 0x0B | Facing Direction     | XORed with last item-slot value
381      //0x0C - 0x0F | X Velocity           | Predicted from X position on this / last frame
382      //0x10 - 0x13 | Y Velocity           | Predicted from Y position on this / last frame
383      //0x14 - 0x17 | X Position           | Predictive encoding (2-frame delta)
384      //0x18 - 0x1B | Y Position           | Predictive encoding (2-frame delta)
385      //0x1C - 0x1D | Damage Taken         | XORed with last item-slot value
386      //0x1E - 0x21 | Expiration Timer     | Predictive encoding (2-frame delta)
387      //0x22 - 0x25 | Spawn ID             | No encoding
388      //0x26 - 0x29 | Misc Values          | XORed with last item-slot value
389      //0x2A - 0x2A | Owner                | XORed with last item-slot value
390
391    DOUT2("  Compressing item event at byte " << +_bp);
392    char* main_buf = (_encode_ver ? _wb : _rb);
393
394    //Encode frame number, predicting the last frame number +1
395    int cur_item_frame  = readBE4S(&_rb[_bp+O_FRAME]);
396    int pred_item_frame = predictFrame(cur_item_frame, lastitemstartframe, &_wb[_bp+O_FRAME]);
397    lastitemstartframe = _encode_ver ? pred_item_frame : cur_item_frame;
398    DOUT2("    Item frame is " << +lastitemstartframe);
399
400    if (_debug >= 3) {
401        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
402        int inum = readBE4U(&_rb[_bp+O_ITEM_ID]);
403        DOUT3("    EVID " << +fnum << ".3" << +inum);
404    }
405
406    //Get a storage slot for the item
407    uint8_t slot = readBE4U(&_rb[_bp+O_ITEM_ID]) % ITEM_SLOTS;
408
409    //XOR all of the remaining data for the item
410    uint16_t itype;
411    // TODO: not sure if I even need this split
412    if (ENCODE_VERSION_MIN(2)) {
413      // we use ITEM_TYPE for other purposes, so don't XOR it
414      xorEncodeRange(O_ITEM_STATE,O_ITEM_XVEL,_x_item[slot]);
415      // Ignore the bits used for storing defer information
416      itype = decodeDeferInfo(readBE2U(&main_buf[_bp+O_ITEM_TYPE]));
417    } else {
418      xorEncodeRange(O_ITEM_TYPE,O_ITEM_XVEL,_x_item[slot]);
419      itype = readBE2U(&main_buf[_bp+O_ITEM_TYPE]);
420    }
421    xorEncodeRange(O_ITEM_DAMAGE,O_ITEM_EXPIRE,_x_item[slot]);
422
423    //Predict item positions based on velocity
424    predictVelocItem(slot,O_ITEM_XPOS);
425
426    //V2 and up of compressor predicts velocities y position based on item type
427    if (ENCODE_VERSION_MIN(2)) {
428      switch(itype) {
429        case 0xd2:  //Shy Guy
430          // Shy guys are jerks (yay puns)
431          predictJoltItem(slot,O_ITEM_YPOS); break;
432        default:  //Everything else just uses V1 behavior
433          predictAccelItem(slot,O_ITEM_YPOS); break;
434      }
435    } else { // V1 of compressor always used predictAccelItem()
436      predictAccelItem(slot,O_ITEM_YPOS);
437    }
438
439    // Use item positions to determine their velocties
440    predictAsDifference(O_ITEM_XVEL,O_ITEM_XPOS,_x_item_p[slot]);
441    predictAsDifference(O_ITEM_YVEL,O_ITEM_YPOS,_x_item_p[slot]);
442
443    //Predict item expiration based on velocity
444    predictVelocItem(slot,O_ITEM_EXPIRE);
445
446    if(MIN_VERSION(3,2,0)) {
447      xorEncodeRange(O_ITEM_MISC,O_ITEM_OWNER,_x_item[slot]);
448      if (MIN_VERSION(3,6,0)) {
449        xorEncodeRange(O_ITEM_OWNER,O_ITEM_END,_x_item[slot]);
450      }
451    }
452
453    if (_debug == 0) { return true; }
454
455    return true;
456  }
457
458  bool Compressor::_parseFrameStart() {
459    //Encodings so far
460      //0x00 - 0x00 | Command Byte         | No encoding
461      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1), second bit flipped if raw RNG
462      //0x05 - 0x08 | RNG Seed             | Predictive encoding (# of RNG rolls, or raw)
463
464    if (_game_loop_start == 0) {
465        _game_loop_start = _bp;
466        if (_encode_ver) {
467          _unshuffleEvents();
468        }
469    }
470
471    if (_debug >= 3) {
472        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
473        DOUT3("    EVID " << +fnum << ".10");
474    }
475
476    //Encode frame number, predicting the last frame number +1
477    int cur_frame  = readBE4S(&_rb[_bp+O_FRAME]);
478    int pred_frame = predictFrame(cur_frame, laststartframe, &_wb[_bp+O_FRAME]);
479
480    //Update laststartframe since we just crossed a new frame boundary
481    laststartframe = _encode_ver ? pred_frame : cur_frame;
482
483    //Predict RNG value from real (not delta encoded) frame number
484    predictRNG(O_FRAME,O_RNG_FS);
485
486    return true;
487  }
488
489  bool Compressor::_parseBookend() {
490    //Encodings so far
491      //0x00 - 0x00 | Command Byte         | No encoding
492      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1)
493      //0x05 - 0x08 | Rollback Frame       | Predictive encoding (last frame + 1)
494
495    if (_debug >= 3) {
496      if (MIN_VERSION(3,7,0)) {
497        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
498        int ffin = readBE4S(&_rb[_bp+O_ROLLBACK_FRAME]);
499        DOUT3("    EVID " << +fnum << ".50 (" << ffin << ")");
500      }
501    }
502
503    //Encode actual frame number, predicting and updating laststartframe
504    predictFrame(readBE4S(&_rb[_bp+O_FRAME]), laststartframe, &_wb[_bp+O_FRAME]);
505    if (MIN_VERSION(3,7,0)) {
506      //Encode rollback frame number, predicting laststartframe without update
507      predictFrame(readBE4S(&_rb[_bp+O_ROLLBACK_FRAME]), laststartframe, &_wb[_bp+O_ROLLBACK_FRAME]);
508    }
509
510    return true;
511  }
512
513  bool Compressor::_parsePreFrame() {
514    //Encodings so far
515      //0x00 - 0x00 | Command Byte         | No encoding
516      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1), second bit flipped if raw RNG
517      //0x05 - 0x05 | Player Index         | No encoding (never changes)
518      //0x06 - 0x06 | Is Follower          | No encoding (never changes)
519      //0x07 - 0x0A | RNG Seed             | Predictive encoding (# of RNG rolls, or raw)
520      //0x0B - 0x0C | Action State         | XORed with last post-frame value
521      //0x0D - 0x10 | X Position           | XORed with last post-frame value
522      //0x11 - 0x14 | Y Position           | XORed with last post-frame value
523      //0x15 - 0x18 | Facing Direction     | XORed with last post-frame value
524      //0x19 - 0x1C | Joystick X           | Value retrieved from float map
525      //0x1D - 0x20 | Joystick Y           | Value retrieved from float map
526      //0x21 - 0x24 | C-Stick X            | Value retrieved from float map
527      //0x25 - 0x28 | C-Stick Y            | Value retrieved from float map
528      //0x29 - 0x2C | Trigger              | Value retrieved from float map
529      //0x2D - 0x30 | Processed Buttons    | No encoding (already sparse)
530      //0x31 - 0x32 | Physical Buttons     | XORed with lower bits of Processed Buttons
531      //0x33 - 0x36 | Physical L           | Value retrieved from float map
532      //0x37 - 0x3A | Physical R           | Value retrieved from float map
533      //0x3B - 0x3B | UCF X analog         | No encoding
534      //0x3C - 0x3F | Damage               | No encoding (already sparse)
535
536    // Old code for trying to predict Joy X from Joy Y and UCF X
537    // if (!_encode_ver) {
538    //   float   joyx  = readBE4F(&_rb[_bp+O_JOY_X]);
539    //   float   joyy  = readBE4F(&_rb[_bp+O_JOY_Y]);
540    //   int8_t  ucfx  = _rb[_bp+O_UCF_ANALOG];
541    //   int8_t  intx  = float(joyx*80.0f);
542    //   int8_t  inty  = float(joyy*80.0f);
543
544    //   int8_t predx;
545    //   if (inty > 27 || inty < -58) {
546    //     predx = 0;
547    //   } else if (ucfx < -22) {
548    //     predx = ucfx;
549    //   } else if (ucfx > 22) {
550    //     predx = ucfx;
551    //   } else {
552    //     predx = 0;
553    //   }
554
555    //   if (predx > 79) {
556    //     predx = 79;
557    //   } else if (predx < -79) {
558    //     predx = -79;
559    //   }
560
561    //   std::cout
562    //     << " PRED "  << padString(predx,4)
563    //     << " INT X " << padString(intx,4)
564    //     << " UCF X " << padString(ucfx,4)
565    //     << " INT Y " << padString(inty,4)
566    //     << (((predx != 0) && (predx != intx)) ? "   MISS" : "")
567    //     << std::endl;
568    // }
569
570    DOUT2("  Compressing pre frame event at byte " << +_bp);
571    char* main_buf = (_encode_ver ? _wb : _rb);
572
573    //Get player index
574    uint8_t p = uint8_t(_rb[_bp+O_PLAYER])+4*uint8_t(_rb[_bp+O_FOLLOWER]); //Includes follower
575    if (p > 7) {
576      FAIL_CORRUPT("    Invalid player index " << +p);
577      return false;
578    }
579
580    if (_debug >= 3) {
581        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
582        DOUT3("    EVID " << +fnum << ".2" << +p);
583    }
584
585    //Encode frame number, predicting the last frame number +1
586    int cur_frame  = readBE4S(&_rb[_bp+O_FRAME]);
587    int pred_frame = predictFrame(cur_frame, lastpreframe[p], &_wb[_bp+O_FRAME]);
588    //Update lastpreframe[p] since we just crossed a new frame boundary
589    lastpreframe[p] = _encode_ver ? pred_frame : cur_frame;
590
591    DOUT2("    Pre frame is " << +lastpreframe[p]);
592
593    //Predict RNG value from real (not delta encoded) frame number
594    predictRNG(O_FRAME,O_RNG_PRE);
595
596    //Carry over action state from last post-frame
597    writeBE2U(readBE2U(&_rb[_bp+O_ACTION_PRE]) ^ readBE2U(&_x_post_frame[p][O_ACTION_POST]),&_wb[_bp+O_ACTION_PRE]);
598    memcpy(&_x_pre_frame[p][O_ACTION_PRE],&main_buf[_bp+O_ACTION_PRE],2);
599
600    //Carry over x position from last post-frame
601    writeBE4U(readBE4U(&_rb[_bp+O_XPOS_PRE]) ^ readBE4U(&_x_post_frame[p][O_XPOS_POST]),&_wb[_bp+O_XPOS_PRE]);
602    memcpy(&_x_pre_frame[p][O_XPOS_PRE],&main_buf[_bp+O_XPOS_PRE],4);
603
604    //Carry over y position from last post-frame
605    writeBE4U(readBE4U(&_rb[_bp+O_YPOS_PRE]) ^ readBE4U(&_x_post_frame[p][O_YPOS_POST]),&_wb[_bp+O_YPOS_PRE]);
606    memcpy(&_x_pre_frame[p][O_YPOS_PRE],&main_buf[_bp+O_YPOS_PRE],4);
607
608    //Carry over facing direction from last post-frame
609    writeBE4U(readBE4U(&_rb[_bp+O_FACING_PRE]) ^ readBE4U(&_x_post_frame[p][O_FACING_POST]),&_wb[_bp+O_FACING_PRE]);
610    memcpy(&_x_pre_frame[p][O_FACING_PRE],&main_buf[_bp+O_FACING_PRE],4);
611
612    // Encode analog stick and trigger values as integers
613    encodeAnalog(O_JOY_X,   80.0f);
614    encodeAnalog(O_JOY_Y,   80.0f);
615    encodeAnalog(O_CX,      80.0f);
616    encodeAnalog(O_CY,      80.0f);
617    encodeAnalog(O_TRIGGER,140.0f);
618    encodeAnalog(O_PHYS_L, 140.0f);
619    encodeAnalog(O_PHYS_R, 140.0f);
620
621    // Encode physical buttons from lower bits of processed buttons
622    uint16_t phys_buttons = readBE2U(&main_buf[_bp+O_BUTTONS]);
623    uint16_t proc_buttons = readBE2U(&main_buf[_bp+O_PROCESSED+2]);
624    writeBE2U(proc_buttons^phys_buttons,&_wb[_bp+O_BUTTONS]);
625
626    return true;
627  }
628
629  bool Compressor::_parsePostFrame() {
630    //Encodings so far
631      //0x00 - 0x00 | Command Byte         | No encoding
632      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1)
633      //0x05 - 0x05 | Player Index         | No encoding (never changes)
634      //0x06 - 0x06 | Is Follower          | No encoding (never changes)
635      //0x07 - 0x07 | Char ID              | No encoding (already sparse)
636      //0x08 - 0x09 | Action State         | No encoding, but used to predict pre-frame
637      //0x0A - 0x0D | X Position           | Predictive encoding (3-frame accel)
638      //0x0E - 0x11 | Y Position           | Predictive encoding (3-frame accel)
639      //0x12 - 0x15 | Facing Direction     | No encoding, but used to predict pre-frame
640      //0x16 - 0x19 | Damage               | No encoding, but used to predict pre-frame
641      //0x1A - 0x1D | Shield               | Predictive encoding (2-frame delta)
642      //0x1E - 0x1E | Last Hitting Attack  | XOR encoding
643      //0x1F - 0x1F | Combo Count          | XOR encoding
644      //0x20 - 0x20 | Last Hit By          | XOR encoding
645      //0x21 - 0x21 | Stocks Remaining     | XOR encoding
646      //0x22 - 0x25 | AS Frame Counter     | Predictive encoding (2-frame delta)
647      //0x26 - 0x2A | State Bit Flags      | XOR encoding
648      //0x2B - 0x2E | Hitstun Counter      | Predictive encoding (2-frame delta)
649      //0x2F - 0x2F | Ground / Air State   | No encoding (already sparse)
650      //0x30 - 0x31 | Ground ID            | No encoding (already sparse)
651      //0x32 - 0x32 | Jumps Remaining      | No encoding (already sparse)
652      //0x33 - 0x33 | L Cancel Status      | No encoding (already sparse)
653      //0x34 - 0x34 | Hurtbox Status       | No encoding (already sparse)
654      //0x35 - 0x38 | Self Air X-velocity  | Predictive encoding (2-frame delta)
655      //0x39 - 0x3C | Self Y-velocity      | Predictive encoding (2-frame delta)
656      //0x3D - 0x40 | Attack X-velocity    | Predictive encoding (2-frame delta)
657      //0x41 - 0x44 | Attack Y-velocity    | Predictive encoding (2-frame delta)
658      //0x45 - 0x48 | Self Ground X-vel.   | Predictive encoding (2-frame delta)
659
660    DOUT2("  Compressing post frame event at byte " << +_bp);
661    char* main_buf = (_encode_ver ? _wb : _rb);
662
663    union { float f; uint32_t u; } float_true, float_pred, float_temp;
664    int32_t fnum = readBE4S(&_rb[_bp+O_FRAME]);
665    int32_t f    = fnum-LOAD_FRAME;
666
667    //Get player index
668    uint8_t p = uint8_t(_rb[_bp+O_PLAYER])+4*uint8_t(_rb[_bp+O_FOLLOWER]); //Includes follower
669    if (p > 7) {
670      FAIL_CORRUPT("    Invalid player index " << +p);
671      return false;
672    }
673
674    if (_debug >= 3) {
675        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
676        DOUT3("    EVID " << +fnum << ".4" << +p);
677    }
678
679    //Encode frame number, predicting the last frame number +1
680    int cur_frame  = readBE4S(&_rb[_bp+O_FRAME]);
681    int pred_frame = predictFrame(cur_frame, lastpostframe[p], &_wb[_bp+O_FRAME]);
682    //Update lastpreframe[p] since we just crossed a new frame boundary
683    lastpostframe[p] = _encode_ver ? pred_frame : cur_frame;
684
685    DOUT2("    Post frame is " << +lastpostframe[p]);
686
687    //Predict x position based on velocity and acceleration
688    predictAccelPost(p,O_XPOS_POST);
689    //Predict y position based on velocity and acceleration
690    predictAccelPost(p,O_YPOS_POST);
691
692    //Copy action state to post-frame
693    memcpy(&_x_post_frame[p][O_ACTION_POST],&main_buf[_bp+O_ACTION_POST],2);
694    //Copy x position to post-frame
695    memcpy(&_x_post_frame[p][O_XPOS_POST],&main_buf[_bp+O_XPOS_POST],4);
696    //Copy y position to post-frame
697    memcpy(&_x_post_frame[p][O_YPOS_POST],&main_buf[_bp+O_YPOS_POST],4);
698    //Copy facing direction to post-frame
699    memcpy(&_x_post_frame[p][O_FACING_POST],&main_buf[_bp+O_FACING_POST],4);
700    //Copy damage to post-frame
701    memcpy(&_x_post_frame[p][O_DAMAGE_POST],&main_buf[_bp+O_DAMAGE_POST],4);
702
703    //Predict shield decay as velocity
704    predictVelocPost(p,O_SHIELD);
705
706    //Compress single byte values with XOR encoding
707    xorEncodeRange(O_LAST_HIT_ID,O_ACTION_FRAMES,_x_post_frame[p]);
708
709    //Predict this frame's action state counter from the last 2 frames' counters
710    if (MIN_VERSION(0,2,0)) {
711      predictVelocPost(p,O_ACTION_FRAMES);
712    }
713
714    if (MIN_VERSION(2,0,0)) {
715      //XOR encode state bit flags
716      xorEncodeRange(O_STATE_BITS_1,O_HITSTUN,_x_post_frame[p]);
717
718      //Predict this frame's hitstun counter from the last 2 frames' counters
719      predictVelocPost(p,O_HITSTUN);
720
721      if (MIN_VERSION(3,5,0)) {
722        //Predict delta of various speeds based on previous frames' velocities
723        predictVelocPost(p,O_SELF_AIR_Y);
724        predictVelocPost(p,O_ATTACK_X);
725        predictVelocPost(p,O_ATTACK_Y);
726        predictVelocPost(p,O_SELF_GROUND_X);
727        predictVelocPost(p,O_SELF_AIR_X);
728
729        if (MIN_VERSION(3,8,0)) {
730          //Predict this frame's hitlag counter from the last 2 frames' counters
731          predictVelocPost(p,O_HITLAG);
732        }
733      }
734    }
735
736    if (_debug == 0) { return true; }
737    //Below here is still in testing mode
738
739    return true;
740  }
741
742  bool Compressor::_unshuffleEvents() {
743    if (MAX_VERSION(3,0,0)) {
744        return true;
745    }
746    bool success = _shuffleEvents(true);
747    if (success) {
748        // Copy the relevant portion of _rb to _wb
749        memcpy(
750          &_wb[_game_loop_start],
751          &_rb[_game_loop_start],
752          sizeof(char)*(_game_loop_end-_game_loop_start)
753          );
754    } else {
755        FAIL("BIG PROBLEM UNSHUFFLING");
756    }
757    return success;
758  }
759
760  #define DOUNSHUFFLE(index,event) \
761    off         = _payload_sizes[event]; \
762    memcpy(&main_buf[b],&ev_buf[index][cpos[index]],off); \
763    cpos[index] += off; \
764    b           += off;
765
766  #define DECODEFRAMEMATCHES(index,last) \
767    (decodeFrame(readBE4S(&ev_buf[index][cpos[index]+O_FRAME]), last) == fnum)
768
769
770  bool Compressor::_shuffleEvents(bool unshuffle) {
771    const unsigned ETYPES = 64;  //Number of types of events
772    const unsigned EMAX   = 30;  //Max event to include as part of the game loop
773
774    // Can't do this for replays without frame start events yet
775    if (MAX_VERSION(3,0,0)) {
776        return true;
777    }
778
779    // Flag for if we fail anywhere
780    bool success = true;
781
782    // Determine if we're using the input or output buffer
783    char* main_buf = unshuffle ? _rb : _wb;
784    if (unshuffle) {
785      // We don't actually know where _game_loop_end is yet
786      _game_loop_end = _file_size;
787      // We also need to unshuffle columns
788      _unshuffleColumns(main_buf);
789    }
790
791    //Allocate space for storing shuffled events
792    int max_events          = ALLOC_EVENTS;  //initial number of events to allocate space for
793    unsigned offset[ETYPES] = {0};  //Size of individual event arrays
794    unsigned* ev_counts     = new unsigned[ETYPES];
795    unsigned* ev_max        = new unsigned[ETYPES];
796    unsigned* ev_size       = new unsigned[ETYPES];
797    ev_size[0]  = _payload_sizes[Event::FRAME_START];
798    ev_size[9]  = _payload_sizes[Event::ITEM_UPDATE];
799    ev_size[18] = _payload_sizes[Event::BOOKEND];
800    ev_size[19] = _payload_sizes[Event::SPLIT_MSG];
801    for (unsigned i = 0; i < 8; ++i) {
802      ev_size[1+i]  = _payload_sizes[Event::PRE_FRAME];
803      ev_size[10+i] = _payload_sizes[Event::POST_FRAME];
804    }
805    char** ev_buf           = new char*[ETYPES];
806    for (unsigned i = 0; i < 20; ++i) {
807      ev_counts[i] = 0;          //set event counter to 0
808      ev_max[i]    = max_events; //initially allocate space for arrays
809      ev_buf[i]    = new char[ev_max[i]*ev_size[i]];
810    }
811    int *frame_counter          = new int[max_events]{0};
812    unsigned *finalized_counter = new unsigned[max_events]{0};
813    unsigned start_fp           = 0;  //Frame pointer to next start frame
814    unsigned end_fp             = 0;  //Frame pointer to next end frame
815
816    //Initialize data for frame deferral counting
817    char* defer_pre[8];
818    char* defer_post[8];
819    for (unsigned i = 0; i < 8; ++i) {
820      defer_pre[i]  = new char[RB_SIZE]{0};
821      defer_post[i] = new char[RB_SIZE]{0};
822    }
823    char* dupe_frames = new char[RB_SIZE]{0};
824    int max_frame          = -125;
825    unsigned max_item_seen = 0;
826    int modframe           = 0;
827    uint8_t defer          = 0;
828
829    //Rearrange memory
830    uint8_t pid; //Temporary variable for player id
831    uint8_t oid; //Index into the offset array we're currently working with
832    int cur_frame      = -125;
833    unsigned oldshuffleframe = lastshuffleframe;
834    for (unsigned b = _game_loop_start; b < _game_loop_end; ) {
835      unsigned ev_code = uint8_t(main_buf[b]);
836      unsigned shift   = _payload_sizes[ev_code];
837      // std::cout << "ev code" << hex(ev_code) << " at byte " << b << "/" << _game_loop_end << std::endl;
838      switch(ev_code) {
839        case Event::FRAME_START:
840            oid = 0;
841            if (unshuffle) {
842                cur_frame        = decodeFrame(readBE4S(&main_buf[b+O_FRAME]), lastshuffleframe);
843                lastshuffleframe = cur_frame;
844            } else {
845                cur_frame        = readBE4S(&_rb[b+O_FRAME]);
846            }
847
848            // If we're shuffling, we need to keep track of deferred frames
849            //   once we unshuffle events back into order
850            if(!unshuffle) {
851              // Determine the number of times each recent frame has been duplicated
852              modframe = ((cur_frame+256)%RB_SIZE);
853              if (cur_frame > max_frame) {
854                max_frame = cur_frame;
855                dupe_frames[modframe] = 0;
856                // reset the defer counters for each player
857                for (unsigned i = 0; i < 8; ++i) {
858                  defer_pre[i][modframe]  = 0;
859                  defer_post[i][modframe] = 0;
860                }
861              } else {
862                dupe_frames[modframe] += 1;
863              }
864            }
865
866            frame_counter[start_fp] = cur_frame;
867            ++start_fp;
868
869            // check if we've reached the event limit, and reallocate memory for more
870            //   events if so
871            if((int)start_fp == max_events) {
872              DOUT1("    Maxed out frame count, resizing to " << max_events*2 << " events");
873              // double the size of the buffer for storing frame count information
874              int* newstartbuf = new int[max_events*2];
875              // copy data from the old buffer to the new one
876              memcpy(&newstartbuf[0],&frame_counter[0],max_events);
877              // free the old frame counter buffer
878              delete[] frame_counter;
879              // set the buffer pointer to the new buffer
880              frame_counter = newstartbuf;
881
882              // double the size of the buffer for storing finalized frame information
883              unsigned* newendbuf = new unsigned[max_events*2];
884              // copy data from the old buffer to the new one
885              memcpy(&newendbuf[0],&finalized_counter[0],max_events);
886              // free the old finalized counter buffer
887              delete[] finalized_counter;
888              // set the buffer pointer to the new buffer
889              finalized_counter = newendbuf;
890
891              // actually properly double the size of the event buffer
892              max_events *= 2;
893            }
894            // std::cout << "Started frame " << cur_frame << std::endl;
895            break;
896        case Event::PRE_FRAME: //Includes follower
897            pid = uint8_t(main_buf[b+O_PLAYER])+4*uint8_t(main_buf[b+O_FOLLOWER]);
898            oid = 1+pid;
899            if(!unshuffle) {
900              // check if we got rollback'd into existence
901              defer = (dupe_frames[modframe] - defer_pre[pid][modframe]);
902              if (defer > 0) {
903                // encode the number of frames we need to defer writing
904                //   this rollback'd preframe into the stream when decoding
905                unsigned asid = readBE2U(&main_buf[b+O_ACTION_PRE]);
906                asid = encodeDeferInfo(asid,defer);
907                writeBE2U(asid,&main_buf[b+O_ACTION_PRE]);
908              }
909              // std::cout << "    set defer to " << +(dupe_frames[modframe]) << " - " << +(defer_pre[pid][modframe]) << "+ 1 = " << +(defer_pre[pid][modframe]) << std::endl;
910              defer_pre[pid][modframe] = dupe_frames[modframe]+1;
911            }
912            break;
913        case Event::ITEM_UPDATE:
914            oid = 9;
915            // XOR first byte of item id with last finalized frame
916            if (! unshuffle) {
917              unsigned item_id = readBE4U(&main_buf[b+O_ITEM_ID]);
918
919              int ff = 0;
920              if (MIN_VERSION(3,7,0)) {
921                ff               = lookAheadToFinalizedFrame(&_rb[b]);
922
923                // if we've seen a new item, check if it was rollbacked into existence
924                if(item_id >= max_item_seen) {
925                  max_item_seen = item_id+1;
926                  if (dupe_frames[modframe] > 0) {
927                    // encode the number of frames we need to defer writing
928                    //   this rollback'd item into the stream when decoding
929                    unsigned item_type = readBE2U(&main_buf[b+O_ITEM_TYPE]);
930                    item_type = encodeDeferInfo(item_type,dupe_frames[modframe]);
931                    writeBE2U(item_type,&main_buf[b+O_ITEM_TYPE]);
932                  }
933                }
934
935                item_id          = encodeFrameIntoItemId(item_id,ff);
936                unsigned dec_id  = encodeFrameIntoItemId(item_id,ff);
937                if (encodeFrameIntoItemId(encodeFrameIntoItemId(dec_id,ff),ff) != dec_id) {
938                  return false;
939                }
940                writeBE4U(item_id,&main_buf[b+O_ITEM_ID]);
941              } else {
942                ff = start_fp;
943                int before = 0;
944                // check last 128 frames for duplicates
945                // start at 2 because frame_ptr is incremented
946                for(unsigned fi = 2; fi < MAX_ROLLBACK; ++fi) {
947                  if(frame_counter[start_fp-fi] == cur_frame) {
948                    before += 1;
949                  }
950                }
951                if (before > 0) {
952                  // std::cout << "Item " << item_id << " has " << +before << " duplicate frames before frame " << cur_frame << std::endl;
953                }
954                item_id          = encodeFrameIntoItemId(item_id,before);
955                unsigned dec_id  = encodeFrameIntoItemId(item_id,before);
956                if (encodeFrameIntoItemId(encodeFrameIntoItemId(dec_id,ff),ff) != dec_id) {
957                  return false;
958                }
959                writeBE4U(item_id,&main_buf[b+O_ITEM_ID]);
960              }
961              // std::cout << "ITEM ID = " << item_id << " (" << dec_id << ") FINAL " << ff << std::endl;
962            }
963            break;
964        case Event::POST_FRAME: //Includes follower
965            pid = uint8_t(main_buf[b+O_PLAYER])+4*uint8_t(main_buf[b+O_FOLLOWER]);
966            oid = 10+pid;
967            if(!unshuffle) {
968              // check if we got rollback'd into existence
969              defer = (dupe_frames[modframe] - defer_post[pid][modframe]);
970              if (defer > 0) {
971                // encode the number of frames we need to defer writing
972                //   this rollback'd preframe into the stream when decoding
973                unsigned asid = readBE2U(&main_buf[b+O_ACTION_POST]);
974                asid = encodeDeferInfo(asid,defer);
975                writeBE2U(asid,&main_buf[b+O_ACTION_POST]);
976              }
977              // std::cout << "    set defer to " << +(dupe_frames[modframe]) << " - " << +(defer_pre[pid][modframe]) << "+ 1 = " << +(defer_pre[pid][modframe]) << std::endl;
978              defer_post[pid][modframe] = dupe_frames[modframe]+1;
979            }
980            break;
981        case Event::BOOKEND:
982            oid = 18;
983            // Frame bookend rollback frames aren't defined before 3.7.0
984            if (MIN_VERSION(3,7,0)) {
985              int final_frame;
986              if (unshuffle) {
987                lastshuffleframe = frame_counter[end_fp];
988                final_frame = decodeFrame(readBE4S(&main_buf[b+O_ROLLBACK_FRAME]),lastshuffleframe);
989              } else {
990                final_frame = readBE4S(&_rb[b+O_ROLLBACK_FRAME]);
991              }
992              finalized_counter[end_fp] = (final_frame+256) % 256;
993              ++end_fp;
994            } else {
995              // TODO: MANUALLY COMPUTE FINALIZED FRAME
996            }
997            // std::cout << "Finalized frame " << cur_frame << std::endl;
998            break;
999        case Event::SPLIT_MSG: //Includes follower
1000            oid = EMAX+1; break;
1001        case Event::GAME_END:
1002            oid            = EMAX;
1003            _game_loop_end = b;
1004            // std::cout << "GAME LOOP END 0x"
1005            //     << std::hex << ev_code << std::dec
1006            //     << " at byte " << +b << std::endl;
1007            break;
1008        default:
1009            oid = ETYPES-1;
1010            // std::cout << "NOT GOOD 0x"
1011            //     << std::hex << ev_code << std::dec
1012            //     << " at byte " << +b << std::endl;
1013            break;
1014      }
1015      if (oid < EMAX) {
1016        // std::cout << "FINE 0x"
1017        //   << std::hex << ev_code << std::dec
1018        //   << " at byte " << +b << std::endl;
1019        memcpy(&ev_buf[oid][offset[oid]],&main_buf[b],sizeof(char)*shift);
1020        ++(ev_counts[oid]);
1021        if(ev_counts[oid] == ev_max[oid]) {
1022          DOUT1("    Maxed out events for " << +oid << ", resizing to " << ev_max[oid]*2 << " events");
1023          // double the size of the buffer for storing event information
1024          char* newbuf = new char[ev_max[oid]*ev_size[oid]*2];
1025          // copy data from the old buffer to the new one
1026          memcpy(&newbuf[0],&ev_buf[oid][0],ev_max[oid]*ev_size[oid]);
1027          // actually properly double the size of the event buffer
1028          ev_max[oid] *= 2;
1029          // free the old buffer
1030          delete[] ev_buf[oid];
1031          // set the buffer pointer to the new buffer
1032          ev_buf[oid] = newbuf;
1033        }
1034      }
1035      offset[oid] += shift;
1036      b           += shift;
1037    }
1038    lastshuffleframe = oldshuffleframe;
1039
1040    //Sanity checks to make sure the number of individual event bytes
1041    //  sums to the total number of bytes in the main game loop
1042    unsigned tsize = 0;
1043    for(unsigned i = 0; i < EMAX; ++ i) {  //Skip game end event
1044        tsize += offset[i];
1045        // std::cout << "SIZE of " << i << " is " << offset[i] << std::endl;
1046    }
1047    if (tsize != (_game_loop_end-_game_loop_start)) {
1048        FAIL("    SOMETHING WENT HORRENDOUSLY WRONG D:");
1049        FAIL("      tsize=" << tsize);
1050        FAIL("      _game_loop_end=" << _game_loop_end);
1051        FAIL("      _game_loop_start=" << _game_loop_start);
1052        success = false;
1053    }
1054
1055    // Copy memory over if we haven't failed horrendously yet
1056    if(success) {
1057        unsigned off = 0;
1058        unsigned b   = _game_loop_start;
1059        if (!unshuffle) { //Shuffle into main memory, excluding message event
1060            // Copy all events (except 19 == messages, makes things worse)
1061            for(unsigned i = 0; i <= 18; ++i) {
1062                memcpy(&main_buf[b],&ev_buf[i][0],offset[i]);
1063                b += offset[i];
1064            }
1065            // Shuffle columns
1066            _shuffleColumns(offset);
1067        } else { //Unshuffle into main memory, excluding message event
1068            unsigned cpos[EMAX] = {0};  //Buffer positions we're copying out of
1069            int* dec_frames = new int[_game_loop_end];
1070            for(unsigned frame_ptr = 0; b < _game_loop_end; ++frame_ptr) {
1071                // Sanity checks to make sure this is an actual frame start event
1072                if (cpos[0] >= offset[0]) {
1073                    FAIL("    THAT SUCKS D: at frame" << frame_ptr);
1074                    FAIL("      " << cpos[0] << " >= " << offset[0]);
1075                    success = false;
1076                    break;
1077                }
1078                if (ev_buf[0][cpos[0]] != Event::FRAME_START) {
1079                    FAIL("    OH SNAP D:");
1080                    success = false;
1081                    break;
1082                }
1083
1084                // Get the current frame from the next frame start event
1085                int enc_frame             = readBE4S(&ev_buf[0][cpos[0]+O_FRAME]);
1086                int fnum                  = decodeFrame(enc_frame, lastshuffleframe);
1087                lastshuffleframe          = fnum;
1088                dec_frames[frame_ptr]     = fnum;
1089
1090                // Determine the number of times each recent frame has been duplicated
1091                modframe = ((fnum+256)%RB_SIZE);
1092                if (fnum > max_frame) {
1093                  max_frame = fnum;
1094                  dupe_frames[modframe] = 0;
1095                } else {
1096                  dupe_frames[modframe] += 1;
1097                }
1098
1099                // std::cout << "Decoded frame at " << frame_ptr << " as " << fnum << std::endl;
1100                DOUT3("    " << (b) << " bytes read, " << (_game_loop_end-b) << " bytes left at frame " << fnum);
1101
1102                // TODO: this isn't working for Game_20210207T004448.slp
1103                if (fnum < -123 || fnum > (1 << 30)) {
1104                    FAIL("    SOMETHING WENT HORRENDOUSLY WRONG D:");
1105                    success = false;
1106                    break;
1107                }
1108
1109                // Copy the frame start event over to the main buffer
1110                DOUNSHUFFLE(0,Event::FRAME_START);
1111
1112                // Check if each player has a pre-frame this frame
1113                for(unsigned p = 0; p < 8; ++p) {
1114                    // True player order: 0,4,1,5,2,6,3,7
1115                    unsigned i = 1 + (4*(p%2)) + unsigned(p/2);
1116                    // If the player has no more pre-frame data, move on
1117                    if (cpos[i] >= offset[i]) {
1118                        continue;
1119                    }
1120                    // If the next frame isn't the one we're expecting, move on
1121                    if (!DECODEFRAMEMATCHES(i,lastshufflepreframe[p])) {
1122                        // std::cout << fnum << " NO MATCH pre-frame " << i+1 << " @ " << decodeFrame(readBE4S(&ev_buf[1+i][cpos[1+i]+O_FRAME]), lastshuffleframe) << std::endl;
1123                        continue;
1124                    }
1125                    // Verify we don't need to defer writing this preframe event
1126                    //   to the output stream due to rollback shenanigans
1127                    unsigned asid  = readBE2U(&ev_buf[i][cpos[i]+O_ACTION_PRE]);
1128                    unsigned defer = getDeferInfo(asid);
1129                    if (defer > 0) {
1130                      if ((int)defer != dupe_frames[modframe]) {
1131                        continue;
1132                      }
1133                      // zero out the first two bits of action state id holding defer information
1134                      writeBE2U(decodeDeferInfo(asid),&ev_buf[i][cpos[i]+O_ACTION_PRE]);
1135                    }
1136                    // Copy the pre frame event over to the main buffer
1137                    DOUNSHUFFLE(i,Event::PRE_FRAME);
1138                    lastshufflepreframe[p] = fnum;
1139                    // std::cout << fnum << " pre-frame " << i+1 << std::endl;
1140                }
1141
1142                // Copy over any items with matching frames
1143                int last_id = -1;
1144                for(;;) {
1145                    // If there are no more items, move on
1146                    if (cpos[9] >= offset[9]) { break; }
1147
1148                    // If the next item frame isn't the current frame, move on
1149                    if (!DECODEFRAMEMATCHES(9,lastitemshuffleframe)) {
1150                      break;
1151                    }
1152
1153                    // Double check we're on the correctly finalized frame
1154                    unsigned item_id = readBE4U(&ev_buf[9][cpos[9]+O_ITEM_ID]);
1155
1156                    if(MIN_VERSION(3,7,0)) {
1157                      unsigned ff = finalized_counter[frame_ptr];
1158                      // Decode item_id encoded with frame number
1159                      if (getFrameModFromItemId(item_id) != ff) {
1160                        break;
1161                      }
1162                      item_id  = encodeFrameIntoItemId(item_id,ff);
1163                    } else { // 3.6.x does not have a frame bookend rollback frame
1164                      unsigned dupes = 0;
1165                      // check last 128 frames for duplicates
1166                      // start at 1 because frame_ptr is not incremented
1167                      for(unsigned fi = 1; fi < MAX_ROLLBACK; ++fi) {
1168                        if (frame_ptr<fi) {
1169                          break;
1170                        }
1171                        if(dec_frames[frame_ptr-fi] == fnum) {
1172                          dupes += 1;
1173                        }
1174                      }
1175
1176                      // verify item appeared on this duplicate frame
1177                      if (getFrameModFromItemId(item_id) != dupes) {
1178                        break;
1179                      }
1180
1181                      item_id = encodeFrameIntoItemId(item_id,dupes);
1182                    }
1183
1184                    // Verify we aren't repeating items this frame
1185                    if (int(item_id) <= last_id) { break; }
1186
1187                    // Verify we don't need to defer writing this item event
1188                    //   to the output stream due to rollback shenanigans
1189                    unsigned item_type = readBE2U(&ev_buf[9][cpos[9]+O_ITEM_TYPE]);
1190                    unsigned defer     = getDeferInfo(item_type);
1191                    if (defer > 0) {
1192                      if ((int)defer != dupe_frames[modframe]) {
1193                        break;
1194                      }
1195                      // zero out the first two bits of item type holding defer information
1196                      writeBE2U(decodeDeferInfo(item_type),&ev_buf[9][cpos[9]+O_ITEM_TYPE]);
1197                    }
1198
1199                    // Restore the actual item ID to the buffer
1200                    writeBE4U(item_id,&ev_buf[9][cpos[9]+O_ITEM_ID]);
1201
1202                    // Update the last seen item id
1203                    last_id = item_id;
1204
1205                    // Update lastitemshuffleframe
1206                    lastitemshuffleframe = fnum;
1207
1208                    // Copy the item event over to the main buffer
1209                    DOUNSHUFFLE(9,Event::ITEM_UPDATE);
1210                }
1211
1212                // Check if each player has a post-frame this frame
1213                for(unsigned p = 0; p < 8; ++p) {
1214                    // True player order: 0,4,1,5,2,6,3,7
1215                    unsigned i = 10 + (4*(p%2)) + unsigned(p/2);
1216                    //If the player has no more post-frame data, move on
1217                    if (cpos[i] >= offset[i]) {
1218                        continue;
1219                    }
1220                    // If the next frame isn't the one we're expecting, move on
1221                    if (!DECODEFRAMEMATCHES(i,lastshufflepostframe[p])) {
1222                        continue;
1223                    }
1224                    // Verify we don't need to defer writing this postframe event
1225                    //   to the output stream due to rollback shenanigans
1226                    unsigned asid  = readBE2U(&ev_buf[i][cpos[i]+O_ACTION_POST]);
1227                    unsigned defer = getDeferInfo(asid);
1228                    if (defer > 0) {
1229                      if ((int)defer != dupe_frames[modframe]) {
1230                        continue;
1231                      }
1232                      // zero out the first two bits of action state id holding defer information
1233                      writeBE2U(decodeDeferInfo(asid),&ev_buf[i][cpos[i]+O_ACTION_POST]);
1234                    }
1235                    // Copy the post frame event over to the main buffer
1236                    DOUNSHUFFLE(i,Event::POST_FRAME);
1237                    lastshufflepostframe[p] = fnum;
1238                    // std::cout << fnum << " post-frame " << i+1 << std::endl;
1239                }
1240
1241                // Copy the frame end event over to the main buffer
1242                if (DECODEFRAMEMATCHES(18,lastshuffleframe)) {
1243                    DOUNSHUFFLE(18,Event::BOOKEND);
1244                }
1245            }
1246            delete[] dec_frames;
1247        }
1248    }
1249
1250    //Free memory
1251
1252    delete[] ev_counts;
1253    delete[] ev_max;
1254    delete[] ev_size;
1255
1256    for (unsigned i = 0; i < 8; ++i) {
1257      delete[] defer_pre[i];
1258      delete[] defer_post[i];
1259    }
1260    delete[] dupe_frames;
1261
1262    for (unsigned i = 0; i < 20; ++i) {
1263        delete[] ev_buf[i];
1264    }
1265    delete[] ev_buf;
1266    delete[] frame_counter;
1267    delete[] finalized_counter;
1268
1269    return success;
1270  }
1271
1272}