Text files patching optimisation

.gitignore

@@ -1,5 +1,6 @@
 # cmake build directory
 cmake-build-debug 
+cmake-build-release
 # ide project directory
 .idea
 
@@ -23,4 +24,4 @@ install/*
 
 
 !bin/LotRO_dat_extractor.exe
-!bin/LotRO_dat_patcher.exe
+!bin/LotRO_dat_patcher.exe

include/LotroDat.h

@@ -7,4 +7,4 @@
 #include "SubfileData.h"
 
 #include "yaml-cpp/yaml.h"
-#include "zlib.h"
+#include "ZLib/zlib.h"

include/Subfiles/TextSubFile.h

@@ -6,10 +6,23 @@
 #define LOTRO_DAT_LIBRARY_TEXTSUBFILE_H
 
 #include "SubFile.h"
+#include "SubfileData.h"
+#include "BinaryData.h"
+
 #include <vector>
 #include <unordered_map>
 
 namespace LOTRO_DAT {
+    struct TextFragment {
+        long long fragment_id;
+        std::u16string text;
+        std::string args;
+
+        bool operator < (const TextFragment &other) const {
+            return fragment_id < other.fragment_id;
+        }
+    };
+
     class TextSubFile : public SubFile {
     public:
         TextSubFile() = delete;
@@ -30,22 +43,28 @@ namespace LOTRO_DAT {
         BinaryData MakeForImport(const BinaryData &old_data, const SubfileData &data) override;
 
     private:
-        std::vector<SubfileData> FileFragments;
+        std::vector<TextFragment> patch_fragments_;
+
+        std::vector<std::u16string> text_pieces_;
+        std::vector<long long> argument_references_;
+        std::vector<std::vector<BinaryData>> argument_strings_;
+
+        static BinaryData buffer_;
 
     private:
-        std::vector<std::u16string> MakePieces(const BinaryData &data, long long &offset);
+        void MakePieces(const BinaryData &data, long long &offset);
 
-        std::vector<long long> MakeArgumentReferences(const BinaryData &data, long long &offset);
+        void MakeArgumentReferences(const BinaryData &data, long long &offset);
 
-        std::vector<std::vector<BinaryData>> MakeArgumentStrings(const BinaryData &data, long long &offset);
+        void MakeArgumentStrings(const BinaryData &data, long long &offset);
 
-        std::unordered_map<long long, SubfileData> ParsePatchFragments(const SubfileData &data);
+        void ParsePatchFragments(const SubfileData &data);
 
-        BinaryData BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset);
+        BinaryData BuildPieces(const BinaryData &data, const TextFragment &new_data, long long &offset);
 
-        BinaryData BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data, long long &offset);
+        BinaryData BuildArgumentReferences(const BinaryData &data, const TextFragment &new_data, long long &offset);
 
-        BinaryData BuildArgumentStrings(const BinaryData &data, const SubfileData &new_data, long long &offset);
+        BinaryData BuildArgumentStrings(const BinaryData &data, const TextFragment &new_data, long long &offset);
 
         BinaryData GetPieceData(const BinaryData &data, long long &offset) const;
 

src/BinaryData.cpp

@@ -3,7 +3,7 @@
 //
 
 #include "BinaryData.h"
-#include "zlib.h"
+#include "ZLib/zlib.h"
 #include "EasyLogging++/easylogging++.h"
 
 #include <algorithm>
@@ -145,8 +145,9 @@ namespace LOTRO_DAT {
 
     bool BinaryData::WriteToFile(const char *filename) const {
         FILE *f;
-        f = fopen64(filename, "wb");
-        if (f == nullptr) {
+        //f = fopen64(filename, "wb");
+		f = fopen(filename, "wb");
+		if (f == nullptr) {
             LOG(ERROR) << "File " << std::string(filename) << " doesn't exist or is unreachable.. Cannot write data";
             return false;
         }

src/DatFile.cpp

@@ -1,7 +1,7 @@
 //
 // Created by Иван_Архипов on 31.10.2017.
 //
-
+#define NOMINMAX
 #include "DatFile.h"
 #include "BinaryData.h"
 
@@ -10,7 +10,7 @@
 #include "SubfileData.h"
 
 #include <EasyLogging++/easylogging++.h>
-#include <unistd.h>
+//#include <unistd.h>
 #include <algorithm>
 #include <iterator>
 #include <locale>
@@ -1122,28 +1122,29 @@ namespace LOTRO_DAT {
     }
 
     bool DatFile::CheckIfUpdatedByGame() {
-        LOG(INFO) << "Checking if DatFile was updated by LotRO";
-        if (!pending_patch_.empty())
-            return true;
+//        LOG(INFO) << "Checking if DatFile was updated by LotRO";
+//        if (!pending_patch_.empty())
+//            return true;
 
-        bool updated = false;
+//        bool updated = false;
 
-        for (const auto& i : dictionary_) {
-            long long file_id = i.first;
-            std::shared_ptr<SubFile> subfile = i.second;
-            if (patch_dict_.count(file_id) == 0)
-                continue;
-
-            if (*subfile != *patch_dict_[file_id] && *subfile != *orig_dict_[file_id]) {
-                orig_dict_.clear();
-                patch_dict_.clear();
-                LOG(INFO) << "DAT FILE WAS UPDATED!!!! CLEARING PATCH DATA";
-                pending_patch_.insert(file_id);
-                WriteData(BinaryData::FromNumber<4>(0), 4, 300);
-                return true;
-            }
-        }
-        return updated;
+//        for (const auto& i : dictionary_) {
+//            long long file_id = i.first;
+//            std::shared_ptr<SubFile> subfile = i.second;
+//            if (patch_dict_.count(file_id) == 0)
+//                continue;
+//
+//            if (*subfile != *patch_dict_[file_id] && *subfile != *orig_dict_[file_id]) {
+//                //orig_dict_.clear();
+//                //patch_dict_.clear();
+//                LOG(INFO) << "DAT FILE WAS UPDATED!!!! CLEARING PATCH DATA";
+//                pending_patch_.insert(file_id);
+//                WriteData(BinaryData::FromNumber<4>(0), 4, 300);
+//                return true;
+//            }
+//        }
+//        return updated;
+        return false;
     }
 
     bool DatFile::CheckIfNotPatched() {

src/Examples/patcher_example.cpp

@@ -15,7 +15,7 @@ using namespace LOTRO_DAT;
 using namespace std;
 
 int main() {
-    std::cout << "Gi1dor's LotRO .dat patcher ver. 5.2.0" << std::endl;
+    std::cout << "Gi1dor's LotRO .dat patcher ver. 6.0.0" << std::endl;
     freopen("patcher_errors.log", "w", stderr);
 
     setbuf(stdout, nullptr);

src/Subfiles/TextSubFile.cpp

@@ -47,7 +47,10 @@ std::string argumentsFromUtf16(const std::u16string &args) {
     return res;
 }
 
+
 namespace LOTRO_DAT {
+    BinaryData TextSubFile::buffer_ = BinaryData(10 * 1024 * 1024);
+
     TextSubFile::TextSubFile(DatFile &dat, long long dictionary_offset, long long unknown1,
                              long long file_id, long long file_offset, long long file_size,
                              long long timestamp, long long version, long long block_size, long long unknown2)
@@ -65,6 +68,11 @@ namespace LOTRO_DAT {
 
     SubfileData TextSubFile::PrepareForExport(const BinaryData &file_data) {
         if (file_data.Empty()) {
+            patch_fragments_.clear();
+            text_pieces_.clear();
+            argument_references_.clear();
+            argument_strings_.clear();
+
             return SubfileData();
         }
 
@@ -82,20 +90,20 @@ namespace LOTRO_DAT {
             long long fragment_id = file_data.ToNumber<8>(offset);
             offset += 8;
 
-            std::vector<std::u16string> text_pieces = MakePieces(file_data, offset);
-            std::vector<long long> arg_references = MakeArgumentReferences(file_data, offset);
-            std::vector<std::vector<BinaryData>> arg_strings = MakeArgumentStrings(file_data, offset);
+            MakePieces(file_data, offset);
+            MakeArgumentReferences(file_data, offset);
+            MakeArgumentStrings(file_data, offset);
 
             std::u16string text = u"[";
-            for (size_t j = 0; j + 1 < text_pieces.size(); j++)
-                text += text_pieces[j] + u"<--DO_NOT_TOUCH!-->";
-            text += text_pieces[text_pieces.size() - 1] + u"]";
+            for (size_t j = 0; j + 1 < text_pieces_.size(); j++)
+                text += text_pieces_[j] + u"<--DO_NOT_TOUCH!-->";
+            text += text_pieces_[text_pieces_.size() - 1] + u"]";
 
             std::u16string arguments;
-            for (size_t j = 0; j + 1 < arg_references.size(); j++)
-                arguments += to_utf16(arg_references[j]) + u"-";
-            if (!arg_references.empty())
-                arguments += to_utf16(arg_references[arg_references.size() - 1]);
+            for (size_t j = 0; j + 1 < argument_references_.size(); j++)
+                arguments += to_utf16(argument_references_[j]) + u"-";
+            if (!argument_references_.empty())
+                arguments += to_utf16(argument_references_[argument_references_.size() - 1]);
 
             if (result.text_data.length() > 0)
                 result.text_data += u"|||";
@@ -106,17 +114,29 @@ namespace LOTRO_DAT {
         }
         result.options["fid"] = file_id();
         result.options["ext"] = Extension();
+
+        patch_fragments_.clear();
+        text_pieces_.clear();
+        argument_references_.clear();
+        argument_strings_.clear();
         return result;
     }
 
     BinaryData TextSubFile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {
         LOG(DEBUG) << "Preparing text file " << file_id() << " for import.";
-        std::unordered_map<long long, SubfileData> patch_fragments = ParsePatchFragments(data);
+        ParsePatchFragments(data);
 
-        BinaryData new_data;
 
-        if (file_size() <= 10 + 8) // File is empty, nothing to do;
+        if (file_size() <= 10 + 8) {// File is empty, nothing to do;
+            patch_fragments_.clear();
+            text_pieces_.clear();
+            argument_references_.clear();
+            argument_strings_.clear();
+
             return old_data;
+        }
+
+        BinaryData new_data;
 
         long long offset = 9 + 8; // first 8 bytes - file_info. After them:
         // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
@@ -136,7 +156,12 @@ namespace LOTRO_DAT {
 
             new_data = new_data + old_data.CutData(offset - 8, offset);
 
-            if (patch_fragments.count(fragment_id) == 0) {
+            TextFragment id_comp;
+            id_comp.fragment_id = fragment_id;
+
+            auto fragment_iterator = std::lower_bound(patch_fragments_.begin(), patch_fragments_.end(), id_comp);
+
+            if (fragment_iterator == patch_fragments_.end()) {
                 // Retrieving old pieces
                 new_data = new_data + GetPieceData(old_data, offset);
                 // Retrieving old references
@@ -145,68 +170,74 @@ namespace LOTRO_DAT {
                 new_data = new_data + GetArgumentStringsData(old_data, offset);
             } else {
                 // Making and adding new pieces
-                new_data = new_data + BuildPieces(old_data, patch_fragments[fragment_id], offset);
+                new_data = new_data + BuildPieces(old_data, *fragment_iterator, offset);
                 // Making and adding new references
-                new_data = new_data + BuildArgumentReferences(old_data, patch_fragments[fragment_id], offset);
+                new_data = new_data + BuildArgumentReferences(old_data, *fragment_iterator, offset);
                 // Making and adding new strings
-                new_data = new_data + BuildArgumentStrings(old_data, patch_fragments[fragment_id], offset);
+                new_data = new_data + BuildArgumentStrings(old_data, *fragment_iterator, offset);
             }
         }
         new_data = new_data + old_data.CutData(offset); // Adding elapsed file data
+
+        patch_fragments_.clear();
+        text_pieces_.clear();
+        argument_references_.clear();
+        argument_strings_.clear();
+
         return new_data;
     }
 
-    std::unordered_map<long long, SubfileData> TextSubFile::ParsePatchFragments(const SubfileData &data) {
+    void TextSubFile::ParsePatchFragments(const SubfileData &data) {
         LOG(DEBUG) << "Started parsing patch fragments";
-        std::unordered_map<long long, SubfileData> res;
-        std::u16string text = data.text_data;
 
         size_t pointer = 0;
-        while (pointer < text.length()) {
+        while (pointer < data.text_data.length()) {
             // Parsing fragment_id
-            size_t pointer1 = text.find(u":::", pointer);
+            size_t pointer1 = data.text_data.find(u":::", pointer);
             if (pointer1 == std::u16string::npos) {
                 LOG(ERROR) << "Unable to parse fragment id! Cannot find '...' divider. File_id = " << file_id_;
-                return res;
+                return;
             }
-            long long fragment_id = from_utf16(text.substr(pointer, pointer1 - pointer));
+            long long fragment_id = from_utf16(data.text_data.substr(pointer, pointer1 - pointer));
             pointer = pointer1 + 3;
-            res[fragment_id] = SubfileData();
-            res[fragment_id].options["gid"] = fragment_id;
+
+            TextFragment fragment;
+            fragment.fragment_id = fragment_id;
+
 
             // Parsing arguments
-            pointer1 = text.find(u":::", pointer);
+            pointer1 = data.text_data.find(u":::", pointer);
             if (pointer1 == std::u16string::npos) {
                 LOG(ERROR) << "Unable to parse arguments! Cannot find '...' divider. File_id = " << file_id_;
-                return res;
+                return;
             }
-            std::u16string arguments = text.substr(pointer, pointer1 - pointer);
+            std::u16string arguments = data.text_data.substr(pointer, pointer1 - pointer);
             pointer = pointer1 + 3;
             if (arguments.length() > 0) {
-                res[fragment_id].options["args"] = argumentsFromUtf16(arguments);
+                fragment.args = argumentsFromUtf16(arguments);
             }
 
             // Parsing text
-            pointer1 = text.find(u"|||", pointer);
+            pointer1 = data.text_data.find(u"|||", pointer);
             if (pointer1 == std::u16string::npos)
-                pointer1 = text.length();
-            std::u16string text_data = text.substr(pointer, pointer1 - pointer);
+                pointer1 = data.text_data.length();
+            fragment.text = data.text_data.substr(pointer, pointer1 - pointer);
             pointer = pointer1  + 3;
-            res[fragment_id].text_data = text_data;
+
+            patch_fragments_.push_back(fragment);
         }
+        std::sort(patch_fragments_.begin(), patch_fragments_.end());
         LOG(DEBUG) << "Finished parsing text patch fragments";
-        return res;
     }
 
     // Make pieces/arguments/argument strings functions
 
-    std::vector<std::u16string> TextSubFile::MakePieces(const BinaryData &data, long long &offset) {
+    void TextSubFile::MakePieces(const BinaryData &data, long long &offset) {
         LOG(DEBUG) << "Started making pieces";
         long long num_pieces = data.ToNumber<4>(offset);
         offset += 4;
 
-        std::vector<std::u16string> text_pieces;
-
+        text_pieces_.resize(num_pieces);
         for (long long j = 0; j < num_pieces; j++) {
             long long piece_size = data.ToNumber<1>(offset);
             if ((piece_size & 128) != 0) {
@@ -224,39 +255,37 @@ namespace LOTRO_DAT {
                 piece += c;
             }
 
-            text_pieces.push_back(piece);
+            text_pieces_[j] = piece;
             offset += piece_size * 2;
         }
         LOG(DEBUG) << "Finished making pieces";
-        return text_pieces;
     }
 
-    std::vector<long long> TextSubFile::MakeArgumentReferences(const BinaryData &data, long long &offset) {
+    void TextSubFile::MakeArgumentReferences(const BinaryData &data, long long &offset) {
         LOG(DEBUG) << "Started making argument references";
-        std::vector<long long> arg_references;
 
         long long num_references = data.ToNumber<4>(offset);
         offset += 4;
 
+        argument_references_.resize(num_references);
         for (long long j = 0; j < num_references; j++) {
-            arg_references.emplace_back(data.ToNumber<4>(offset));
+            argument_references_[j] = data.ToNumber<4>(offset);
             offset += 4;
         }
         LOG(DEBUG) << "Finished making argument references";
-        return arg_references;
     }
 
-    std::vector<std::vector<BinaryData>> TextSubFile::MakeArgumentStrings(const BinaryData &data, long long &offset) {
+    void TextSubFile::MakeArgumentStrings(const BinaryData &data, long long &offset) {
         LOG(DEBUG) << "Started making argument strings";
-        std::vector<std::vector<BinaryData> > arg_strings;
         long long num_arg_strings = data.ToNumber<1>(offset);
         offset += 1;
 
+        argument_strings_.resize(num_arg_strings);
         for (long long j = 0; j < num_arg_strings; j++) {
             long long num_args = data.ToNumber<4>(offset);
             offset += 4;
 
-            arg_strings.emplace_back();
+            argument_strings_[j].resize(num_args);
             for (long long k = 0; k < num_args; k++) {
                 long long string_size = data.ToNumber<1>(offset);
                 if ((string_size & 0x80) != 0) {
@@ -265,24 +294,27 @@ namespace LOTRO_DAT {
                 }
                 offset += 1;
 
-                arg_strings[unsigned(j)].emplace_back(data.CutData(offset, offset + string_size * 2));
+                argument_strings_[j][k] = data.CutData(offset, offset + string_size * 2);
                 offset += string_size * 2;
             }
         }
         LOG(DEBUG) << "Finished making argument strings";
-        return arg_strings;
     }
 
     // Build pieces/arguments/argument strings functions from fragment SubfileData
 
-    BinaryData TextSubFile::BuildPieces(const BinaryData &data, const SubfileData &new_data, long long &offset) {
+    BinaryData TextSubFile::BuildPieces(const BinaryData &data, const TextFragment &new_data, long long &offset) {
         LOG(DEBUG) << "Started building pieces";
         // Moving &offset pointer in &data
         GetPieceData(data, offset);
 
+        std::u16string file_data = u" fid:" + to_utf16(file_id_)
+                                   + u" gid:" + to_utf16(new_data.fragment_id);
+
         // Deleting '[' and ']' brackets
-        std::u16string text_data = new_data.text_data.substr(1, new_data.text_data.size() - 2);
-        std::vector<std::u16string> pieces;
+        std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2) + file_data;
+
+        text_pieces_.clear();
 
         const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
         size_t prev = 0;
@@ -290,67 +322,69 @@ namespace LOTRO_DAT {
 
         while (next != std::string::npos) {
             std::u16string piece = text_data.substr(prev, next - prev);
-            pieces.emplace_back(piece);
+            text_pieces_.push_back(piece);
             prev = next + DNT.length();
             next = text_data.find(DNT, prev);
         }
 
-        pieces.emplace_back(text_data.substr(prev));
+        text_pieces_.push_back(text_data.substr(prev));
 
         // Building BinaryData from pieces
-        BinaryData result;
-        BinaryData temp_data = BinaryData::FromNumber<4>(pieces.size());
-        result = result + temp_data;
+        unsigned buffer_offset = 0;
 
-        for (auto piece : pieces) {
+        buffer_.Append(BinaryData::FromNumber<4>(text_pieces_.size()), buffer_offset);
+        buffer_offset += 4;
+
+        for (const std::u16string &piece : text_pieces_) {
             long long piece_size = piece.length();
             if (piece_size < 128) {
-                temp_data = BinaryData::FromNumber<1>(piece_size);
+                buffer_.Append(BinaryData::FromNumber<1>(piece_size), buffer_offset);
+                buffer_offset += 1;
             } else {
-                temp_data = BinaryData::FromNumberRAW<2>((piece_size | 32768));
+                buffer_.Append(BinaryData::FromNumberRAW<2>((piece_size | 32768)), buffer_offset);
+                buffer_offset += 2;
             }
-            result = result + temp_data;
 
             for (long long j = 0; j < piece_size; j++) {
-                temp_data = BinaryData::FromNumber<2>(short(piece[j]));
-                result = result + temp_data;
+                buffer_.Append(BinaryData::FromNumber<2>(short(piece[j])), buffer_offset);
+                buffer_offset += 2;
             }
         }
         LOG(DEBUG) << "Pieces built successfully";
-        return result;
+        return buffer_.CutData(0, buffer_offset);
     }
 
-    BinaryData TextSubFile::BuildArgumentReferences(const BinaryData &data, const SubfileData &new_data,
+    BinaryData TextSubFile::BuildArgumentReferences(const BinaryData &data, const TextFragment &new_data,
                                                     long long &offset) {
         LOG(DEBUG) << "Started building argument refs";
         // Moving &offset pointer in &data
         GetArgumentReferenceData(data, offset);
 
         // If there are no args - making 4 null-bytes and return;
-        if (!new_data.options["args"]) {
+        if (new_data.args.empty()) {
             BinaryData result = BinaryData::FromNumber<4>(0);
             return result;
         }
 
         // Parsing arguments from list in options["args"]
-        std::string args_list = new_data.options["args"].as<std::string>();
-        std::vector<long long> arguments;
+        std::string args_list = new_data.args;
+        argument_references_.clear();
 
         size_t prev = 0;
         size_t next = args_list.find('-', prev);
         while (next != std::string::npos) {
             std::string argument = args_list.substr(prev, next - prev);
-            arguments.push_back(std::stoll(argument));
+            argument_references_.push_back(std::stoll(argument));
             prev = next + 1;
             next = args_list.find('-', prev);
         }
         std::string argument = args_list.substr(prev);
-        arguments.push_back(std::stoll(argument));
+        argument_references_.push_back(std::stoll(argument));
 
         BinaryData result;
-        BinaryData temp_data = BinaryData::FromNumber<4>(arguments.size());
+        BinaryData temp_data = BinaryData::FromNumber<4>(argument_references_.size());
         result = result + temp_data;
-        for (auto arg_reference : arguments) {
+        for (const long long &arg_reference : argument_references_) {
             temp_data = BinaryData::FromNumber<4>(arg_reference);
             result = result + temp_data;
         }
@@ -358,7 +392,7 @@ namespace LOTRO_DAT {
         return result;
     }
 
-    BinaryData TextSubFile::BuildArgumentStrings(const BinaryData &data, const SubfileData &, long long &offset) {
+    BinaryData TextSubFile::BuildArgumentStrings(const BinaryData &data, const TextFragment &, long long &offset) {
         LOG(DEBUG) << "Started building argument strings";
         LOG(DEBUG) << "Built arg strings successfully";
         return GetArgumentStringsData(data, offset);