fork/slippc

git clone https://git.julia.blue/fork/slippc

Patrick CrainUpdated readme and changelog; added GPL license; tagging release as version 0.6.04cdedc6

0.6.0
39.0 KiB1005 linesraw
1#include "compressor.h"
2
3namespace slip {
4
5  Compressor::Compressor(int debug_level) {
6    _debug = debug_level;
7    FRAME_ENC_DELTA = _debug ? 1 : 1;
8    _bp    = 0;
9  }
10
11  Compressor::~Compressor() {
12    if (_rb != nullptr)          { delete[] _rb; }
13    if (_wb != nullptr)          { delete[] _wb; }
14    if (_outfilename != nullptr) { delete   _outfilename; }
15  }
16
17  bool Compressor::loadFromFile(const char* replayfilename) {
18    DOUT1("Loading " << replayfilename << std::endl);
19    std::ifstream myfile;
20    myfile.open(replayfilename,std::ios::binary | std::ios::in);
21    if (myfile.fail()) {
22      FAIL("File " << replayfilename << " could not be opened or does not exist");
23      return false;
24    }
25
26    myfile.seekg(0, myfile.end);
27    _file_size = myfile.tellg();
28    if (_file_size < MIN_REPLAY_LENGTH) {
29      FAIL("File " << replayfilename << " is too short to be a valid Slippi replay");
30      return false;
31    }
32    myfile.seekg(0, myfile.beg);
33
34    _rb = new char[_file_size];
35    myfile.read(_rb,_file_size);
36    myfile.close();
37
38    // Check if we have a compressed stream
39    bool is_compressed = same4(&_rb[0],LZMA_HEADER);
40    if (is_compressed) {
41      DOUT1("  File Size: " << +_file_size << ", compressed" << std::endl);
42      // Copy the buffer to a string
43      std::string rs(_rb, _file_size);
44      // Decompress the string
45      std::string decomp = decompressWithLzma(rs);
46      // Get the new file size
47      _file_size    = decomp.size();
48      // Delete the old read buffer
49      delete [] _rb;
50      // Reallocate it with more spce
51      _rb = new char[_file_size];
52      // Copy buffer from the decompressed string
53      memcpy(_rb,decomp.c_str(),_file_size);
54    } else {
55      DOUT1("  File Size: " << +_file_size << std::endl);
56    }
57    if (_outfilename == nullptr) {
58      std::string fname = std::string(replayfilename);
59      _outfilename = new std::string(
60        fname.substr(0,
61          fname.find_last_of("."))+
62          std::string(is_compressed ? ".slp" : ".zlp"));
63      if (fileExists(*_outfilename)) {
64        std::cerr << "File " << *_outfilename << " already exists or is invalid" << std::endl;
65        return false;
66      }
67    }
68
69    _wb           = new char[_file_size];
70    memcpy(_wb,_rb,sizeof(char)*_file_size);
71
72    return this->_parse();
73  }
74
75  void Compressor::saveToFile(bool rawencode) {
76    if (fileExists(*_outfilename)) {
77      std::cerr << "File " << *_outfilename << " exists, refusing to overwrite" << std::endl;
78      return;
79    }
80
81    std::ofstream ofile;
82    ofile.open(*_outfilename, std::ios::binary | std::ios::out);
83    // If this is the unencoded version, compress it first
84    if (!(_is_encoded || rawencode)) {
85      // Copy the buffer to a string
86      std::string ws(_wb, _file_size);
87      // Compress the string
88      std::string comp = compressWithLzma(ws);
89      DOUT1("Compression Ratio = " << float(_file_size-comp.size())/_file_size << std::endl);
90      // Write compressed buffer to file
91      ofile.write(comp.c_str(),sizeof(char)*comp.size());
92    } else {
93      // Write norma buffer to file
94      ofile.write(_wb,sizeof(char)*_file_size);
95    }
96
97    ofile.close();
98
99  }
100
101  bool Compressor::setOutputFilename(const char* fname) {
102    if (fileExists(fname)) {
103      return false;
104    }
105    if (_outfilename != nullptr) {
106      delete   _outfilename;
107    }
108    _outfilename = new std::string(fname);
109    return true;
110  }
111
112  unsigned Compressor::saveToBuff(char** buffer) {
113    // buffer = new char[_file_size];
114    *buffer = new char[_file_size];
115    memcpy(*buffer,_wb,sizeof(char)*_file_size);
116    return _file_size;
117  }
118
119  bool Compressor::loadFromBuff(char** buffer, unsigned size) {
120    _file_size = size;
121    _wb        = new char[_file_size];
122    _rb        = new char[_file_size];
123    memcpy(_wb,*buffer,sizeof(char)*_file_size);
124    memcpy(_rb,*buffer,sizeof(char)*_file_size);
125    return this->_parse();
126  }
127
128  bool Compressor::validate() {
129    if (_is_encoded) {
130      return true;
131    }
132
133    char *enc_buff = nullptr, *dec_buff = nullptr;
134    unsigned size  = this->saveToBuff(&enc_buff);
135    Compressor *d  = new slip::Compressor(_debug);
136    d->loadFromBuff(&enc_buff,size);
137    d->saveToBuff(&dec_buff);
138
139    bool success = (memcmp(_rb,dec_buff,size) == 0);
140
141    delete dec_buff;
142    delete enc_buff;
143    delete d;
144
145    return success;
146  }
147
148  bool Compressor::_parse() {
149    _bp = 0; //Start reading from byte 0
150    if (not this->_parseHeader()) {
151      std::cerr << "Failed to parse header" << std::endl;
152      return false;
153    }
154    if (not this->_parseEventDescriptions()) {
155      std::cerr << "Failed to parse event descriptions" << std::endl;
156      return false;
157    }
158    if (not this->_parseEvents()) {
159      std::cerr << "Failed to parse events proper" << std::endl;
160      return false;
161    }
162    DOUT1("Successfully parsed replay!" << std::endl);
163    return true;
164  }
165
166  bool Compressor::_parseHeader() {
167    DOUT1("Parsing header" << std::endl);
168
169    //First 15 bytes contain header information
170    if (same8(&_rb[_bp],SLP_HEADER)) {
171      DOUT1("  Slippi Header Matched" << std::endl);
172    } else {
173      FAIL_CORRUPT("Header did not match expected Slippi file header");
174      return false;
175    }
176    _length_raw_start = readBE4U(&_rb[_bp+11]);
177    if(_length_raw_start == 0) {  //TODO: this is /technically/ recoverable
178      FAIL_CORRUPT("0-byte raw data detected");
179      return false;
180    }
181    DOUT1("  Raw portion = " << _length_raw_start << " bytes" << std::endl);
182    if (_length_raw_start > _file_size) {
183      FAIL_CORRUPT("Raw data size exceeds file size of " << _file_size << " bytes");
184      return false;
185    }
186    _length_raw = _length_raw_start;
187    _bp += 15;
188    return true;
189  }
190
191  bool Compressor::_parseEventDescriptions() {
192    DOUT1("Parsing event descriptions" << std::endl);
193
194    //Next 2 bytes should be 0x35
195    if (_rb[_bp] != Event::EV_PAYLOADS) {
196      FAIL_CORRUPT("Expected Event 0x" << std::hex
197        << Event::EV_PAYLOADS << std::dec << " (Event Payloads)");
198      return false;
199    }
200    uint8_t ev_bytes = _rb[_bp+1]-1; //Subtract 1 because the last byte we read counted as part of the payload
201    _payload_sizes[Event::EV_PAYLOADS] = int32_t(ev_bytes+1);
202    DOUT1("  Event description length = " << int32_t(ev_bytes+1) << " bytes" << std::endl);
203    _bp += 2;
204
205    //Next ev_bytes bytes describe events
206    for(unsigned i = 0; i < ev_bytes; i+=3) {
207      unsigned ev_code = uint8_t(_rb[_bp+i]);
208      if (_payload_sizes[ev_code] > 0) {
209        if (ev_code >= Event::EV_PAYLOADS && ev_code <= Event::GAME_END) {
210          FAIL_CORRUPT("Event " << Event::name[ev_code-Event::EV_PAYLOADS] << " payload size set multiple times");
211        } else {
212          FAIL_CORRUPT("Event " << hex(ev_code) << std::dec << " payload size set multiple times");
213        }
214        return false;
215      }
216      _payload_sizes[ev_code] = readBE2U(&_rb[_bp+i+1])+1; //Add one because of the event code itself
217      DOUT1("  Payload size for event "
218        << hex(ev_code) << std::dec << ": " << _payload_sizes[ev_code]
219        << " bytes" << std::endl);
220    }
221
222    //Sanity checks to verify we at least have Payload Sizes, Game Start, Pre Frame, Post Frame, and Game End Events
223    for(unsigned i = Event::EV_PAYLOADS; i <= Event::GAME_END; ++i) {
224      if (_payload_sizes[i] == 0) {
225        FAIL_CORRUPT("Event " << Event::name[i-Event::EV_PAYLOADS] << " payload size not set");
226        return false;
227      }
228    }
229
230    //Update the remaining length of the raw data to sift through
231    _bp         += ev_bytes;
232    _length_raw -= (2+ev_bytes);
233    return true;
234  }
235
236  bool Compressor::_parseEvents() {
237    DOUT1("Parsing events proper" << std::endl);
238
239    bool success = true;
240    for( ; _length_raw > 0; ) {
241      unsigned ev_code = uint8_t(_rb[_bp]);
242
243      unsigned shift   = _payload_sizes[ev_code];
244      if (shift > _length_raw) {
245        FAIL_CORRUPT("Event byte offset exceeds raw data length");
246        return false;
247      }
248      switch(ev_code) { //Determine the event code
249        case Event::GAME_START:  success = _parseGameStart(); break;
250        case Event::SPLIT_MSG:   success = _parseGeckoCodes(); break;
251        case Event::FRAME_START: success = _parseFrameStart(); break;
252        case Event::PRE_FRAME:   success = _parsePreFrame();   break;
253        case Event::ITEM_UPDATE: success = _parseItemUpdate(); break;
254        case Event::POST_FRAME:  success = _parsePostFrame();  break;
255        case Event::BOOKEND:     success = _parseBookend();    break;
256        case Event::GAME_END:
257            _game_loop_end = _bp;
258            if (! _is_encoded) {
259                _shuffleEvents();
260            }
261            success        = true;
262            break;
263        default:
264          DOUT1("  Warning: unknown event code " << hex(ev_code) << " encountered; skipping" << std::endl);
265          break;
266      }
267      if (not success) {
268        return false;
269      }
270      if (_payload_sizes[ev_code] == 0) {
271        FAIL_CORRUPT("Uninitialized event " << hex(ev_code) << " encountered");
272        return false;
273      }
274      _length_raw    -= shift;
275      _bp            += shift;
276      DOUT2("  Raw bytes remaining: " << +_length_raw << std::endl);
277    }
278
279    return true;
280  }
281
282  bool Compressor::_parseGameStart() {
283    DOUT1("  Parsing game start event at byte " << +_bp << std::endl);
284
285    //Get Slippi version
286    _slippi_maj = uint8_t(_rb[_bp+O_SLP_MAJ]); //Major version
287    _slippi_min = uint8_t(_rb[_bp+O_SLP_MIN]); //Minor version
288    _slippi_rev = uint8_t(_rb[_bp+O_SLP_REV]); //Build version (4th char unused)
289    _is_encoded = uint8_t(_rb[_bp+O_SLP_ENC]); //Whether the file is encoded
290    if (MAX_VERSION(1,0,0)) {
291      FAIL("Replays from Slippi 0.x.x are not supported");
292      return false;
293    }
294    if (MIN_VERSION(3,8,0)) {
295      FAIL("Replays from Slippi 3.8.0 and higher are not supported");
296      return false;
297    }
298
299    //Flip encoding status for output buffer
300    _wb[_bp+O_SLP_ENC] ^= 1;
301
302    //Print slippi version
303    std::stringstream ss;
304    ss << +_slippi_maj << "." << +_slippi_min << "." << +_slippi_rev;
305    DOUT1("Slippi Version: " << ss.str() << ", " << (_is_encoded ? "encoded" : "raw") << std::endl);
306
307    //Get starting RNG state
308    _rng       = readBE4U(&_rb[_bp+O_RNG_GAME_START]);
309    _rng_start = _rng;
310    DOUT1("Starting RNG: " << _rng << std::endl);
311
312    return true;
313  }
314
315  bool Compressor::_parseGeckoCodes() {
316    unsigned message_count = 0;
317
318    // Load default Gecko codes
319    char* gecko_codes = nullptr;
320    readDefaultGeckoCodes(&gecko_codes);
321
322    // XOR encode everything but the command byte
323    unsigned offset = message_count*MESSAGE_SIZE;
324    if (offset < CODE_SIZE) {
325      for(unsigned i = 1; i < MESSAGE_SIZE; ++i) {
326        _wb[_bp+i] = _rb[_bp+i] ^ gecko_codes[offset + i];
327      }
328    }
329
330    delete gecko_codes;
331
332    // Increment the message count and return
333    ++message_count;
334    return true;
335  }
336
337  bool Compressor::_parseItemUpdate() {
338    //Encodings so far
339      //0x00 - 0x00 | Command Byte         | No encoding
340      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1)
341      //0x05 - 0x06 | Type ID              | XORed with last item-slot value
342      //0x07 - 0x07 | State                | XORed with last item-slot value
343      //0x08 - 0x0B | Facing Direction     | XORed with last item-slot value
344      //0x0C - 0x0F | X Velocity           | Predicted from X position on this / last frame
345      //0x10 - 0x13 | Y Velocity           | Predicted from Y position on this / last frame
346      //0x14 - 0x17 | X Position           | Predictive encoding (2-frame delta)
347      //0x18 - 0x1B | Y Position           | Predictive encoding (2-frame delta)
348      //0x1C - 0x1D | Damage Taken         | XORed with last item-slot value
349      //0x1E - 0x21 | Expiration Timer     | Predictive encoding (2-frame delta)
350      //0x22 - 0x25 | Spawn ID             | No encoding
351      //0x26 - 0x29 | Misc Values          | XORed with last item-slot value
352      //0x2A - 0x2A | Owner                | XORed with last item-slot value
353
354    DOUT2("  Compressing item event at byte " << +_bp << std::endl);
355
356    //Encode frame number, predicting the last frame number +1
357    int cur_item_frame  = readBE4S(&_rb[_bp+O_FRAME]);
358    int pred_item_frame = predictFrame(cur_item_frame, lastitemstartframe, &_wb[_bp+O_FRAME]);
359    lastitemstartframe = _is_encoded ? pred_item_frame : cur_item_frame;
360
361    if (_debug >= 3) {
362        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
363        int inum = readBE4U(&_rb[_bp+O_ITEM_ID]);
364        DOUT3("    EVID " << +fnum << ".3" << +inum << std::endl);
365    }
366
367    //Get a storage slot for the item
368    uint8_t slot = readBE4U(&_rb[_bp+O_ITEM_ID]) % ITEM_SLOTS;
369
370    //Predict item positions based on velocity
371    predictVelocItem(slot,O_ITEM_XPOS);
372    predictAccelItem(slot,O_ITEM_YPOS);
373
374    // Use item positions to determine their velocties
375    predictAsDifference(O_ITEM_XVEL,O_ITEM_XPOS,_x_item_p[slot]);
376    predictAsDifference(O_ITEM_YVEL,O_ITEM_YPOS,_x_item_p[slot]);
377
378    //Predict item expiration based on velocity
379    predictVelocItem(slot,O_ITEM_EXPIRE);
380
381    //XOR all of the remaining data for the item
382    xorEncodeRange(O_ITEM_TYPE,O_ITEM_XVEL,_x_item[slot]);
383    xorEncodeRange(O_ITEM_DAMAGE,O_ITEM_EXPIRE,_x_item[slot]);
384    xorEncodeRange(O_ITEM_MISC,O_ITEM_END,_x_item[slot]);
385
386    if (_debug == 0) { return true; }
387
388    return true;
389  }
390
391  bool Compressor::_parseFrameStart() {
392    //Encodings so far
393      //0x00 - 0x00 | Command Byte         | No encoding
394      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1), second bit flipped if raw RNG
395      //0x05 - 0x08 | RNG Seed             | Predictive encoding (# of RNG rolls, or raw)
396
397    if (_game_loop_start == 0) {
398        _game_loop_start = _bp;
399        if (_is_encoded) {
400          _unshuffleEvents();
401        }
402    }
403
404    if (_debug >= 3) {
405        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
406        DOUT3("    EVID " << +fnum << ".10" << std::endl);
407    }
408
409    //Encode frame number, predicting the last frame number +1
410    int cur_frame  = readBE4S(&_rb[_bp+O_FRAME]);
411    int pred_frame = predictFrame(cur_frame, laststartframe, &_wb[_bp+O_FRAME]);
412
413    //Update laststartframe since we just crossed a new frame boundary
414    laststartframe = _is_encoded ? pred_frame : cur_frame;
415
416    //Predict RNG value from real (not delta encoded) frame number
417    predictRNG(O_FRAME,O_RNG_FS);
418
419    return true;
420  }
421
422  bool Compressor::_parseBookend() {
423    //Encodings so far
424      //0x00 - 0x00 | Command Byte         | No encoding
425      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1)
426      //0x05 - 0x08 | Rollback Frame       | Predictive encoding (last frame + 1)
427
428    if (_debug >= 3) {
429        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
430        int ffin = readBE4S(&_rb[_bp+O_ROLLBACK_FRAME]);
431        DOUT3("    EVID " << +fnum << ".50 (" << ffin << ")" << std::endl);
432    }
433
434    //Encode actual frame number, predicting and updating laststartframe
435    predictFrame(readBE4S(&_rb[_bp+O_FRAME]), laststartframe, &_wb[_bp+O_FRAME]);
436    if (MIN_VERSION(3,7,0)) {
437      //Encode rollback frame number, predicting laststartframe without update
438      predictFrame(readBE4S(&_rb[_bp+O_ROLLBACK_FRAME]), laststartframe, &_wb[_bp+O_ROLLBACK_FRAME]);
439    }
440
441    return true;
442  }
443
444  bool Compressor::_parsePreFrame() {
445    //Encodings so far
446      //0x00 - 0x00 | Command Byte         | No encoding
447      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1), second bit flipped if raw RNG
448      //0x05 - 0x05 | Player Index         | No encoding (never changes)
449      //0x06 - 0x06 | Is Follower          | No encoding (never changes)
450      //0x07 - 0x0A | RNG Seed             | Predictive encoding (# of RNG rolls, or raw)
451      //0x0B - 0x0C | Action State         | XORed with last post-frame value
452      //0x0D - 0x10 | X Position           | XORed with last post-frame value
453      //0x11 - 0x14 | Y Position           | XORed with last post-frame value
454      //0x15 - 0x18 | Facing Direction     | XORed with last post-frame value
455      //0x19 - 0x1C | Joystick X           | Value retrieved from float map
456      //0x1D - 0x20 | Joystick Y           | Value retrieved from float map
457      //0x21 - 0x24 | C-Stick X            | Value retrieved from float map
458      //0x25 - 0x28 | C-Stick Y            | Value retrieved from float map
459      //0x29 - 0x2C | Trigger              | Value retrieved from float map
460      //0x2D - 0x30 | Processed Buttons    | No encoding (already sparse)
461      //0x31 - 0x32 | Physical Buttons     | XORed with lower bits of Processed Buttons
462      //0x33 - 0x36 | Physical L           | Value retrieved from float map
463      //0x37 - 0x3A | Physical R           | Value retrieved from float map
464      //0x3B - 0x3B | UCF X analog         | No encoding
465      //0x3C - 0x3F | Damage               | No encoding (already sparse)
466
467    // Old code for trying to predict Joy X from Joy Y and UCF X
468    // if (!_is_encoded) {
469    //   float   joyx  = readBE4F(&_rb[_bp+O_JOY_X]);
470    //   float   joyy  = readBE4F(&_rb[_bp+O_JOY_Y]);
471    //   int8_t  ucfx  = _rb[_bp+O_UCF_ANALOG];
472    //   int8_t  intx  = float(joyx*80.0f);
473    //   int8_t  inty  = float(joyy*80.0f);
474
475    //   int8_t predx;
476    //   if (inty > 27 || inty < -58) {
477    //     predx = 0;
478    //   } else if (ucfx < -22) {
479    //     predx = ucfx;
480    //   } else if (ucfx > 22) {
481    //     predx = ucfx;
482    //   } else {
483    //     predx = 0;
484    //   }
485
486    //   if (predx > 79) {
487    //     predx = 79;
488    //   } else if (predx < -79) {
489    //     predx = -79;
490    //   }
491
492    //   std::cout
493    //     << " PRED "  << padString(predx,4)
494    //     << " INT X " << padString(intx,4)
495    //     << " UCF X " << padString(ucfx,4)
496    //     << " INT Y " << padString(inty,4)
497    //     << (((predx != 0) && (predx != intx)) ? "   MISS" : "")
498    //     << std::endl;
499    // }
500
501    DOUT2("  Compressing pre frame event at byte " << +_bp << std::endl);
502    char* main_buf = (_is_encoded ? _wb : _rb);
503
504    //Get player index
505    uint8_t p = uint8_t(_rb[_bp+O_PLAYER])+4*uint8_t(_rb[_bp+O_FOLLOWER]); //Includes follower
506    if (p > 7) {
507      FAIL_CORRUPT("Invalid player index " << +p);
508      return false;
509    }
510
511    if (_debug >= 3) {
512        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
513        DOUT3("    EVID " << +fnum << ".2" << +p << std::endl);
514    }
515
516    //Encode frame number, predicting the last frame number +1
517    int cur_frame  = readBE4S(&_rb[_bp+O_FRAME]);
518    int pred_frame = predictFrame(cur_frame, lastpreframe[p], &_wb[_bp+O_FRAME]);
519    //Update lastpreframe[p] since we just crossed a new frame boundary
520    lastpreframe[p] = _is_encoded ? pred_frame : cur_frame;
521
522    //Predict RNG value from real (not delta encoded) frame number
523    predictRNG(O_FRAME,O_RNG_PRE);
524
525    //Carry over action state from last post-frame
526    writeBE2U(readBE2U(&_rb[_bp+O_ACTION_PRE]) ^ readBE2U(&_x_post_frame[p][O_ACTION_POST]),&_wb[_bp+O_ACTION_PRE]);
527    memcpy(&_x_pre_frame[p][O_ACTION_PRE],&main_buf[_bp+O_ACTION_PRE],2);
528
529    //Carry over x position from last post-frame
530    writeBE4U(readBE4U(&_rb[_bp+O_XPOS_PRE]) ^ readBE4U(&_x_post_frame[p][O_XPOS_POST]),&_wb[_bp+O_XPOS_PRE]);
531    memcpy(&_x_pre_frame[p][O_XPOS_PRE],&main_buf[_bp+O_XPOS_PRE],4);
532
533    //Carry over y position from last post-frame
534    writeBE4U(readBE4U(&_rb[_bp+O_YPOS_PRE]) ^ readBE4U(&_x_post_frame[p][O_YPOS_POST]),&_wb[_bp+O_YPOS_PRE]);
535    memcpy(&_x_pre_frame[p][O_YPOS_PRE],&main_buf[_bp+O_YPOS_PRE],4);
536
537    //Carry over facing direction from last post-frame
538    writeBE4U(readBE4U(&_rb[_bp+O_FACING_PRE]) ^ readBE4U(&_x_post_frame[p][O_FACING_POST]),&_wb[_bp+O_FACING_PRE]);
539    memcpy(&_x_pre_frame[p][O_FACING_PRE],&main_buf[_bp+O_FACING_PRE],4);
540
541    // Encode analog stick and trigger values as integers
542    encodeAnalog(O_JOY_X,   80.0f);
543    encodeAnalog(O_JOY_Y,   80.0f);
544    encodeAnalog(O_CX,      80.0f);
545    encodeAnalog(O_CY,      80.0f);
546    encodeAnalog(O_TRIGGER,140.0f);
547    encodeAnalog(O_PHYS_L, 140.0f);
548    encodeAnalog(O_PHYS_R, 140.0f);
549
550    // Encode physical buttons from lower bits of processed buttons
551    uint16_t phys_buttons = readBE2U(&main_buf[_bp+O_BUTTONS]);
552    uint16_t proc_buttons = readBE2U(&main_buf[_bp+O_PROCESSED+2]);
553    writeBE2U(proc_buttons^phys_buttons,&_wb[_bp+O_BUTTONS]);
554
555    return true;
556  }
557
558  bool Compressor::_parsePostFrame() {
559    //Encodings so far
560      //0x00 - 0x00 | Command Byte         | No encoding
561      //0x01 - 0x04 | Frame Number         | Predictive encoding (last frame + 1)
562      //0x05 - 0x05 | Player Index         | No encoding (never changes)
563      //0x06 - 0x06 | Is Follower          | No encoding (never changes)
564      //0x07 - 0x07 | Char ID              | No encoding (already sparse)
565      //0x08 - 0x09 | Action State         | No encoding, but used to predict pre-frame
566      //0x0A - 0x0D | X Position           | Predictive encoding (3-frame accel)
567      //0x0E - 0x11 | Y Position           | Predictive encoding (3-frame accel)
568      //0x12 - 0x15 | Facing Direction     | No encoding, but used to predict pre-frame
569      //0x16 - 0x19 | Damage               | No encoding, but used to predict pre-frame
570      //0x1A - 0x1D | Shield               | Predictive encoding (2-frame delta)
571      //0x1E - 0x1E | Last Hitting Attack  | XOR encoding
572      //0x1F - 0x1F | Combo Count          | XOR encoding
573      //0x20 - 0x20 | Last Hit By          | XOR encoding
574      //0x21 - 0x21 | Stocks Remaining     | XOR encoding
575      //0x22 - 0x25 | AS Frame Counter     | Predictive encoding (2-frame delta)
576      //0x26 - 0x2A | State Bit Flags      | XOR encoding
577      //0x2B - 0x2E | Hitstun Counter      | Predictive encoding (2-frame delta)
578      //0x2F - 0x2F | Ground / Air State   | No encoding (already sparse)
579      //0x30 - 0x31 | Ground ID            | No encoding (already sparse)
580      //0x32 - 0x32 | Jumps Remaining      | No encoding (already sparse)
581      //0x33 - 0x33 | L Cancel Status      | No encoding (already sparse)
582      //0x34 - 0x34 | Hurtbox Status       | No encoding (already sparse)
583      //0x35 - 0x38 | Self Air X-velocity  | Predictive encoding (2-frame delta)
584      //0x39 - 0x3C | Self Y-velocity      | Predictive encoding (2-frame delta)
585      //0x3D - 0x40 | Attack X-velocity    | Predictive encoding (2-frame delta)
586      //0x41 - 0x44 | Attack Y-velocity    | Predictive encoding (2-frame delta)
587      //0x45 - 0x48 | Self Ground X-vel.   | Predictive encoding (2-frame delta)
588
589    DOUT2("  Compressing post frame event at byte " << +_bp << std::endl);
590    char* main_buf = (_is_encoded ? _wb : _rb);
591
592    union { float f; uint32_t u; } float_true, float_pred, float_temp;
593    int32_t fnum = readBE4S(&_rb[_bp+O_FRAME]);
594    int32_t f    = fnum-LOAD_FRAME;
595
596    //Get player index
597    uint8_t p = uint8_t(_rb[_bp+O_PLAYER])+4*uint8_t(_rb[_bp+O_FOLLOWER]); //Includes follower
598    if (p > 7) {
599      FAIL_CORRUPT("Invalid player index " << +p);
600      return false;
601    }
602
603    if (_debug >= 3) {
604        int fnum = readBE4S(&_rb[_bp+O_FRAME]);
605        DOUT3("    EVID " << +fnum << ".4" << +p << std::endl);
606    }
607
608    //Encode frame number, predicting the last frame number +1
609    int cur_frame  = readBE4S(&_rb[_bp+O_FRAME]);
610    int pred_frame = predictFrame(cur_frame, lastpostframe[p], &_wb[_bp+O_FRAME]);
611    //Update lastpreframe[p] since we just crossed a new frame boundary
612    lastpostframe[p] = _is_encoded ? pred_frame : cur_frame;
613
614    //Predict x position based on velocity and acceleration
615    predictAccelPost(p,O_XPOS_POST);
616    //Predict y position based on velocity and acceleration
617    predictAccelPost(p,O_YPOS_POST);
618
619    //Copy action state to post-frame
620    memcpy(&_x_post_frame[p][O_ACTION_POST],&main_buf[_bp+O_ACTION_POST],2);
621    //Copy x position to post-frame
622    memcpy(&_x_post_frame[p][O_XPOS_POST],&main_buf[_bp+O_XPOS_POST],4);
623    //Copy y position to post-frame
624    memcpy(&_x_post_frame[p][O_YPOS_POST],&main_buf[_bp+O_YPOS_POST],4);
625    //Copy facing direction to post-frame
626    memcpy(&_x_post_frame[p][O_FACING_POST],&main_buf[_bp+O_FACING_POST],4);
627    //Copy damage to post-frame
628    memcpy(&_x_post_frame[p][O_DAMAGE_POST],&main_buf[_bp+O_DAMAGE_POST],4);
629
630    //Predict shield decay as velocity
631    predictVelocPost(p,O_SHIELD);
632
633    //Compress single byte values with XOR encoding
634    xorEncodeRange(O_LAST_HIT_ID,O_ACTION_FRAMES,_x_post_frame[p]);
635
636    //Predict this frame's action state counter from the last 2 frames' counters
637    predictVelocPost(p,O_ACTION_FRAMES);
638
639    if (MIN_VERSION(2,0,0)) {
640      //XOR encode state bit flags
641      xorEncodeRange(O_STATE_BITS_1,O_HITSTUN,_x_post_frame[p]);
642
643      //Predict this frame's hitstun counter from the last 2 frames' counters
644      predictVelocPost(p,O_HITSTUN);
645
646      if (MIN_VERSION(3,5,0)) {
647        //Predict delta of various speeds based on previous frames' velocities
648        predictVelocPost(p,O_SELF_AIR_Y);
649        predictVelocPost(p,O_ATTACK_X);
650        predictVelocPost(p,O_ATTACK_Y);
651        predictVelocPost(p,O_SELF_GROUND_X);
652        predictVelocPost(p,O_SELF_AIR_X);
653      }
654    }
655
656    if (_debug == 0) { return true; }
657    //Below here is still in testing mode
658
659    return true;
660  }
661
662  bool Compressor::_unshuffleEvents() {
663    if (MAX_VERSION(3,0,0)) {
664        return true;
665    }
666    bool success = _shuffleEvents(true);
667    if (success) {
668        // Copy the relevant portion of _rb to _wb
669        memcpy(
670          &_wb[_game_loop_start],
671          &_rb[_game_loop_start],
672          sizeof(char)*(_game_loop_end-_game_loop_start)
673          );
674    } else {
675        std::cerr << "BIG PROBLEM UNSHUFFLING" << std::endl;
676    }
677    return success;
678  }
679
680  bool Compressor::_shuffleEvents(bool unshuffle) {
681    const unsigned ETYPES = 64;  //Number of types of events
682    const unsigned EMAX   = 30;  //Max event to include as part of the game loop
683
684    // Can't do this for replays without frame start events yet
685    if (MAX_VERSION(3,0,0)) {
686        return true;
687    }
688
689    // Flag for if we fail anywhere
690    bool success = true;
691
692    // Determine if we're using the input or output buffer
693    char* main_buf = unshuffle ? _rb : _wb;
694    if (unshuffle) {
695      // We don't actually know where _game_loop_end is yet
696      _game_loop_end = 999999999;
697      // We also need to unshuffle columns
698      _unshuffleColumns(main_buf);
699    }
700
701    //Allocate space for storing shuffled events
702    //TODO: lazy space calculations, should be more robust later
703    const int MAX_EVENTS    = 100000;
704    unsigned offset[ETYPES] = {0};  //Size of individual event arrays
705    char** ev_buf           = new char*[ETYPES];
706    ev_buf[0]               = new char[MAX_EVENTS*(_payload_sizes[Event::FRAME_START])];
707    ev_buf[9]               = new char[MAX_EVENTS*(_payload_sizes[Event::ITEM_UPDATE])];
708    ev_buf[18]              = new char[MAX_EVENTS*(_payload_sizes[Event::BOOKEND])];
709    ev_buf[19]              = new char[MAX_EVENTS*(_payload_sizes[Event::SPLIT_MSG])];
710    for (unsigned i = 0; i < 8; ++i) {
711      ev_buf[1+i]  = new char[MAX_EVENTS*(_payload_sizes[Event::PRE_FRAME])];
712      ev_buf[10+i] = new char[MAX_EVENTS*(_payload_sizes[Event::POST_FRAME])];
713    }
714    int *frame_counter     = new int[MAX_EVENTS]{0};
715    int *finalized_counter = new int[MAX_EVENTS]{0};
716    unsigned start_fp = 0;  //Frame pointer to next start frame
717    unsigned end_fp = 0;    //Frame pointer to next end frame
718
719    //Rearrange memory
720    uint8_t oid; //Index into the offset array we're currently working with
721    int last_finalized = -125; //Last finalized frame
722    int cur_frame      = -125;
723    unsigned oldshuffleframe = lastshuffleframe;
724    for (unsigned b = _game_loop_start; b < _game_loop_end; ) {
725      unsigned ev_code = uint8_t(main_buf[b]);
726      unsigned shift   = _payload_sizes[ev_code];
727      switch(ev_code) {
728        case Event::FRAME_START:
729            oid = 0;
730            if (unshuffle) {
731                cur_frame = decodeFrame(readBE4S(&main_buf[b+O_FRAME]), lastshuffleframe);
732                lastshuffleframe = cur_frame;
733            } else {
734                cur_frame = readBE4S(&_rb[b+O_FRAME]);
735            }
736            frame_counter[start_fp] = cur_frame;
737            ++start_fp;
738            // std::cout << "Started frame " << cur_frame << std::endl;
739            break;
740        case Event::PRE_FRAME: //Includes follower
741            oid = 1+uint8_t(main_buf[b+O_PLAYER])+4*uint8_t(main_buf[b+O_FOLLOWER]); break;
742        case Event::ITEM_UPDATE:
743            oid = 9;
744            // XOR first byte of item id with last finalized frame
745            if (! unshuffle) {
746              unsigned item_id = readBE4U(&main_buf[b+O_ITEM_ID]);
747              // int ff           = finalized_counter[end_fp-1];
748              int ff           = lookAheadToFinalizedFrame(&_rb[b]);
749              item_id          = encodeFrameIntoItemId(item_id,ff);
750              unsigned dec_id  = encodeFrameIntoItemId(item_id,ff);
751              if (encodeFrameIntoItemId(encodeFrameIntoItemId(dec_id,ff),ff) != dec_id) {
752                return false;
753              }
754              writeBE4U(item_id,&main_buf[b+O_ITEM_ID]);
755              // std::cout << "ITEM ID = " << item_id << " (" << dec_id << ") FINAL " << ff << std::endl;
756            }
757            break;
758        case Event::POST_FRAME: //Includes follower
759            oid = 10+uint8_t(main_buf[b+O_PLAYER])+4*uint8_t(main_buf[b+O_FOLLOWER]); break;
760        case Event::BOOKEND:
761            oid = 18;
762            if (unshuffle) {
763                lastshuffleframe = frame_counter[end_fp];
764                cur_frame = decodeFrame(readBE4S(&main_buf[b+O_ROLLBACK_FRAME]),lastshuffleframe);
765            } else {
766                cur_frame = readBE4S(&_rb[b+O_ROLLBACK_FRAME]);
767            }
768            finalized_counter[end_fp] = (cur_frame+256) % 256;
769            ++end_fp;
770            // std::cout << "Finalized frame " << cur_frame << std::endl;
771            break;
772        case Event::SPLIT_MSG: //Includes follower
773            oid = EMAX+1; break;
774        case Event::GAME_END:
775            oid = EMAX;
776            _game_loop_end = b;
777            // std::cout << "GAME LOOP END 0x"
778            //     << std::hex << ev_code << std::dec
779            //     << " at byte " << +b << std::endl;
780            break;
781        default:
782            oid = ETYPES-1;
783            // std::cout << "NOT GOOD 0x"
784            //     << std::hex << ev_code << std::dec
785            //     << " at byte " << +b << std::endl;
786            break;
787      }
788      if (oid < EMAX) {
789        // std::cout << "FINE 0x"
790        //   << std::hex << ev_code << std::dec
791        //   << " at byte " << +b << std::endl;
792        memcpy(&ev_buf[oid][offset[oid]],&main_buf[b],sizeof(char)*shift);
793      }
794      offset[oid] += shift;
795      b           += shift;
796    }
797    lastshuffleframe = oldshuffleframe;
798
799    //Sanity checks to make sure the number of individual event bytes
800    //  sums to the total number of bytes in the main game loop
801    unsigned tsize = 0;
802    for(unsigned i = 0; i < EMAX; ++ i) {  //Skip game end event
803        tsize += offset[i];
804        // std::cout << "SIZE of " << i << " is " << offset[i] << std::endl;
805    }
806    if (tsize != (_game_loop_end-_game_loop_start)) {
807        std::cerr << "SOMETHING WENT HORRENDOUSLY WRONG D:" << std::endl;
808        std::cerr << "tsize=" << tsize << std::endl;
809        std::cerr << "_game_loop_end=" << _game_loop_end << std::endl;
810        std::cerr << "_game_loop_start=" << _game_loop_start << std::endl;
811        success = false;
812    }
813
814    // Copy memory over if we haven't failed horrendously yet
815    if(success) {
816        unsigned off = 0;
817        unsigned b   = _game_loop_start;
818        if (unshuffle) { //Unshuffle into main memory
819            unsigned cpos[EMAX] = {0};  //Buffer positions we're copying out of
820
821            // Unshuffle message events first (makes things worse)
822            // off = sizeof(char)*_payload_sizes[Event::SPLIT_MSG];
823            // while(cpos[19] < offset[19]) {
824            //     memcpy(&main_buf[b],&ev_buf[19][cpos[19]],off);
825            //     cpos[19] += off;
826            //     b        += off;
827            // }
828
829            for(unsigned frame_ptr = 0; b < _game_loop_end; ++frame_ptr) {
830                // Sanity checks to make sure this is an actual frame start event
831                if (cpos[0] >= offset[0]) {
832                    std::cerr << "THAT SUCKS D:" << std::endl;
833                    success = false;
834                    break;
835                }
836                if (ev_buf[0][cpos[0]] != Event::FRAME_START) {
837                    std::cerr << "OH SNAP D:" << std::endl;
838                    success = false;
839                    break;
840                }
841
842                // Get the current frame from the next frame start event
843                int enc_frame    = readBE4S(&ev_buf[0][cpos[0]+O_FRAME]);
844                int fnum         = decodeFrame(enc_frame, lastshuffleframe);
845                lastshuffleframe = fnum;
846                DOUT3((b) << " bytes read, " << (_game_loop_end-b) << " bytes left at frame " << fnum << std::endl);
847
848                // TODO: this isn't working for Game_20210207T004448.slp
849                if (fnum < -123 || fnum > pow(2,30)) {
850                    std::cerr << "SOMETHING WENT HORRENDOUSLY WRONG D:" << std::endl;
851                    success = false;
852                    break;
853                }
854
855                // Copy the frame start event over to the main buffer
856                off = sizeof(char)*_payload_sizes[Event::FRAME_START];
857                memcpy(&main_buf[b],&ev_buf[0][cpos[0]],off);
858                cpos[0] += off;
859                b       += off;
860
861                // Check if each player has a pre-frame this frame
862                for(unsigned p = 0; p < 8; ++p) {
863                    // True player order: 0,4,1,5,2,6,3,7
864                    unsigned i = (4*(p%2)) + unsigned(p/2);
865                    //If the player has no more pre-frame data, move on
866                    if (cpos[1+i] >= offset[1+i]) {
867                        continue;
868                    }
869                    // If the next frame isn't the one we're expecting, move on
870                    if (decodeFrame(readBE4S(&ev_buf[1+i][cpos[1+i]+O_FRAME]), lastshufflepreframe[p]) != fnum) {
871                        std::cout << fnum << " NO MATCH pre-frame " << i+1 << " @ " << decodeFrame(readBE4S(&ev_buf[1+i][cpos[1+i]+O_FRAME]), lastshuffleframe) << std::endl;
872                        continue;
873                    }
874                    // Copy the pre frame event over to the main buffer
875                    off = sizeof(char)*_payload_sizes[Event::PRE_FRAME];
876                    memcpy(&main_buf[b],&ev_buf[1+i][cpos[1+i]],off);
877                    cpos[1+i] += off;
878                    b         += off;
879                    lastshufflepreframe[p] = fnum;
880                    // std::cout << fnum << " pre-frame " << i+1 << std::endl;
881                }
882
883                // Copy over any items with matching frames
884                int last_id = -1;
885                for(;;) {
886                    // If there are no more items, move on
887                    if (cpos[9] >= offset[9]) { break; }
888
889                    // If the next item frame isn't the current frame, move on
890                    int iframe = readBE4S(&ev_buf[9][cpos[9]+O_FRAME]);
891                    iframe = decodeFrame(iframe,lastitemshuffleframe);
892                    if (iframe != fnum) { break; }
893
894                    // Double check we're on the correctly finalized frame
895                    int item_id = readBE4U(&ev_buf[9][cpos[9]+O_ITEM_ID]);
896                    int ff      = finalized_counter[frame_ptr];
897                    int fmod    = getFrameModFromItemId(item_id);
898                    if (fmod != ff) {
899                      std::cout << "ITEM fmod " << fmod << "!=" << ff << std::endl;
900                      break;
901                    }
902
903
904                    // Decode item_id encoded with frame number
905                    unsigned dec_id = encodeFrameIntoItemId(item_id,ff);
906                    item_id = dec_id;
907
908                    // Verify we aren't repeating items this frame
909                    if (item_id <= last_id) {
910                      break;
911                    }
912
913
914                    // Update lastitemshuffleframe
915                    lastitemshuffleframe = iframe;
916
917                    // Restore the actual item ID to the buffer
918                    writeBE4U(dec_id,&ev_buf[9][cpos[9]+O_ITEM_ID]);
919
920                    // Copy the item event over to the main buffer
921                    off = sizeof(char)*_payload_sizes[Event::ITEM_UPDATE];
922                    memcpy(&main_buf[b],&ev_buf[9][cpos[9]],off);
923                    cpos[9] += off;
924                    b       += off;
925                    last_id  = item_id;
926                    // std::cout << fnum << " item " << std::endl;
927                }
928
929                // Check if each player has a post-frame this frame
930                for(unsigned p = 0; p < 8; ++p) {
931                    // True player order: 0,4,1,5,2,6,3,7
932                    unsigned i = (4*(p%2)) + unsigned(p/2);
933                    //If the player has no more post-frame data, move on
934                    if (cpos[10+i] >= offset[10+i]) {
935                        continue;
936                    }
937                    // If the next frame isn't the one we're expecting, move on
938                    if (decodeFrame(readBE4S(&ev_buf[10+i][cpos[10+i]+O_FRAME]), lastshufflepostframe[p]) != fnum) {
939                        continue;
940                    }
941                    // Copy the post frame event over to the main buffer
942                    off = sizeof(char)*_payload_sizes[Event::POST_FRAME];
943                    memcpy(&main_buf[b],&ev_buf[10+i][cpos[10+i]],off);
944                    cpos[10+i] += off;
945                    b          += off;
946                    lastshufflepostframe[p] = fnum;
947                    // std::cout << fnum << " post-frame " << i+1 << std::endl;
948                }
949
950                // Copy the frame end event over to the main buffer
951                if (decodeFrame(readBE4S(&ev_buf[18][cpos[18]+O_FRAME]), lastshuffleframe) == fnum) {
952                    off = sizeof(char)*_payload_sizes[Event::BOOKEND];
953                    memcpy(&main_buf[b],&ev_buf[18][cpos[18]],off);
954                    cpos[18] += off;
955                    b        += off;
956                    // std::cout << fnum << " end " << std::endl;
957                }
958
959                // finalized = decodeFrame(readBE4S(&ev_buf[18][cpos[18]+0x5]), lastshuffleframe);
960            }
961        } else {  //Shuffle into main memory
962            // Copy message events (makes things worse)
963            // memcpy(&main_buf[b],&ev_buf[19][0],offset[19]);
964            // b += offset[19];
965
966            // Copy frame start events
967            memcpy(&main_buf[b],&ev_buf[0][0],offset[0]);
968            b += offset[0];
969
970            // Copy pre-frame events
971            for(unsigned i = 0; i < 8; ++i) {
972                memcpy(&main_buf[b],&ev_buf[1+i][0],offset[1+i]);
973                b += offset[1+i];
974            }
975
976            // Copy item events
977            memcpy(&main_buf[b],&ev_buf[9][0],offset[9]);
978            b += offset[9];
979
980            // Copy post-frame events
981            for(unsigned i = 0; i < 8; ++i) {
982                memcpy(&main_buf[b],&ev_buf[10+i][0],offset[10+i]);
983                b += offset[10+i];
984            }
985
986            // Copy frame end events
987            memcpy(&main_buf[b],&ev_buf[18][0],offset[18]);
988            b += offset[18];
989
990            // Shuffle columns
991            _shuffleColumns(offset);
992        }
993    }
994
995    //Free memory
996    for (unsigned i = 0; i < 19; ++i) {
997        delete ev_buf[i];
998    }
999    delete ev_buf;
1000    delete frame_counter;
1001
1002    return success;
1003  }
1004
1005}