LotroDat/src/Subfiles/TextSubFile.cpp

//
// Created by Иван_Архипов on 24.11.2017.
//

#include "Subfiles/TextSubFile.h"

#include "BinaryData.h"
#include "DatFile.h"
#include "SubfileData.h"
#include "EasyLogging++/easylogging++.h"

#include <codecvt>
#include <locale>

std::u16string to_utf16(long long x) {
    std::u16string res;
    while (x > 0) {
        res += char16_t(u'0' + x % 10);
        x /= 10ll;
    }
    std::reverse(res.begin(), res.end());
    return res;
}

long long from_utf16(const std::u16string &num) {
    long long res = 0;
    for (auto c : num) {
        res = res * 10ll + (c - u'0');
    }
    return res;
}

std::string argumentsFromUtf16(const std::u16string &args) {
    std::string res;

    size_t pointer = 0;
    while (pointer < args.length()) {
        size_t pointer1 = args.find(u'-', pointer);
        if (pointer1 == std::u16string::npos)
            pointer1 = args.length();
        if (!res.empty())
            res += "-";
        res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
        pointer = pointer1 + 1;
    }

    return res;
}


namespace LOTRO_DAT {
    BinaryData TextSubFile::buffer_ = BinaryData(10 * 1024 * 1024);

    TextSubFile::TextSubFile(DatFile &dat, long long dictionary_offset, long long unknown1,
                             long long file_id, long long file_offset, long long file_size,
                             long long timestamp, long long version, long long block_size, long long unknown2)
            : SubFile(dat, dictionary_offset, unknown1, file_id, file_offset, file_size,
                      timestamp, version, block_size, unknown2) {
    }

    FILE_TYPE TextSubFile::FileType() const {
        return TEXT;
    }

    std::string TextSubFile::Extension() const {
        return std::string(".txt");
    }

    SubfileData TextSubFile::PrepareForExport(const BinaryData &file_data) {
        if (file_data.Empty()) {
            patch_fragments_.clear();
            text_pieces_.clear();
            argument_references_.clear();
            argument_strings_.clear();

            return SubfileData();
        }

        SubfileData result;
        long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

        long long text_fragment_num = file_data.ToNumber<1>(offset);
        if ((text_fragment_num & 0x80) != 0) {
            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | file_data.ToNumber<1>(offset + 1));
            offset += 1;
        }
        offset += 1;

        for (long long i = 0; i < text_fragment_num; i++) {
            long long fragment_id = file_data.ToNumber<8>(offset);
            offset += 8;

            MakePieces(file_data, offset);
            MakeArgumentReferences(file_data, offset);
            MakeArgumentStrings(file_data, offset);

            std::u16string text = u"[";
            for (size_t j = 0; j + 1 < text_pieces_.size(); j++)
                text += text_pieces_[j] + u"<--DO_NOT_TOUCH!-->";
            text += text_pieces_[text_pieces_.size() - 1] + u"]";

            std::u16string arguments;
            for (size_t j = 0; j + 1 < argument_references_.size(); j++)
                arguments += to_utf16(argument_references_[j]) + u"-";
            if (!argument_references_.empty())
                arguments += to_utf16(argument_references_[argument_references_.size() - 1]);

            if (result.text_data.length() > 0)
                result.text_data += u"|||";

            result.text_data += to_utf16(fragment_id) + u":::";
            result.text_data += arguments + u":::";
            result.text_data += text;
        }
        result.options["fid"] = file_id();
        result.options["ext"] = Extension();

        patch_fragments_.clear();
        text_pieces_.clear();
        argument_references_.clear();
        argument_strings_.clear();
        return result;
    }

    BinaryData TextSubFile::MakeForImport(const BinaryData &old_data, const SubfileData &data) {
        LOG(DEBUG) << "Preparing text file " << file_id() << " for import.";
        ParsePatchFragments(data);


        if (file_size() <= 10 + 8) {// File is empty, nothing to do;
            patch_fragments_.clear();
            text_pieces_.clear();
            argument_references_.clear();
            argument_strings_.clear();

            return old_data;
        }

        BinaryData new_data;

        long long offset = 9 + 8; // first 8 bytes - file_info. After them:
        // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

        long long text_fragment_num = old_data.ToNumber<1>(offset);
        if ((text_fragment_num & 0x80) != 0) {
            text_fragment_num = (((text_fragment_num ^ 0x80) << 8) | old_data.ToNumber<1>(offset + 1));
            offset += 1;
        }
        offset += 1;

        new_data = new_data + old_data.CutData(0, offset); // Adding file info

        for (long long i = 0; i < text_fragment_num; i++) {
            long long fragment_id = old_data.ToNumber<8>(offset);
            offset += 8;

            new_data = new_data + old_data.CutData(offset - 8, offset);

            TextFragment id_comp;
            id_comp.fragment_id = fragment_id;

            auto fragment_iterator = std::lower_bound(patch_fragments_.begin(), patch_fragments_.end(), id_comp);

            if (fragment_iterator == patch_fragments_.end()) {
                // Retrieving old pieces
                new_data = new_data + GetPieceData(old_data, offset);
                // Retrieving old references
                new_data = new_data + GetArgumentReferenceData(old_data, offset);
                // Retrieving old ref_strings
                new_data = new_data + GetArgumentStringsData(old_data, offset);
            } else {
                // Making and adding new pieces
                new_data = new_data + BuildPieces(old_data, *fragment_iterator, offset);
                // Making and adding new references
                new_data = new_data + BuildArgumentReferences(old_data, *fragment_iterator, offset);
                // Making and adding new strings
                new_data = new_data + BuildArgumentStrings(old_data, *fragment_iterator, offset);
            }
        }
        new_data = new_data + old_data.CutData(offset); // Adding elapsed file data

        patch_fragments_.clear();
        text_pieces_.clear();
        argument_references_.clear();
        argument_strings_.clear();

        return new_data;
    }

    void TextSubFile::ParsePatchFragments(const SubfileData &data) {
        LOG(DEBUG) << "Started parsing patch fragments";

        size_t pointer = 0;
        while (pointer < data.text_data.length()) {
            // Parsing fragment_id
            size_t pointer1 = data.text_data.find(u":::", pointer);
            if (pointer1 == std::u16string::npos) {
                LOG(ERROR) << "Unable to parse fragment id! Cannot find '...' divider. File_id = " << file_id_;
                return;
            }
            long long fragment_id = from_utf16(data.text_data.substr(pointer, pointer1 - pointer));
            pointer = pointer1 + 3;

            TextFragment fragment;
            fragment.fragment_id = fragment_id;


            // Parsing arguments
            pointer1 = data.text_data.find(u":::", pointer);
            if (pointer1 == std::u16string::npos) {
                LOG(ERROR) << "Unable to parse arguments! Cannot find '...' divider. File_id = " << file_id_;
                return;
            }
            std::u16string arguments = data.text_data.substr(pointer, pointer1 - pointer);
            pointer = pointer1 + 3;
            if (arguments.length() > 0) {
                fragment.args = argumentsFromUtf16(arguments);
            }

            // Parsing text
            pointer1 = data.text_data.find(u"|||", pointer);
            if (pointer1 == std::u16string::npos)
                pointer1 = data.text_data.length();
            fragment.text = data.text_data.substr(pointer, pointer1 - pointer);
            pointer = pointer1  + 3;

            patch_fragments_.push_back(fragment);
        }
        std::sort(patch_fragments_.begin(), patch_fragments_.end());
        LOG(DEBUG) << "Finished parsing text patch fragments";
    }

    // Make pieces/arguments/argument strings functions

    void TextSubFile::MakePieces(const BinaryData &data, long long &offset) {
        LOG(DEBUG) << "Started making pieces";
        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        text_pieces_.resize(num_pieces);
        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;

            BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
            std::u16string piece;

            for (long long k = 0; k < piece_size; k++) {
                char16_t c = char16_t(
                        ((short(piece_data[2 * unsigned(k) + 1])) << 8) | (short(piece_data[2 * unsigned(k)])));
                piece += c;
            }

            text_pieces_[j] = piece;
            offset += piece_size * 2;
        }
        LOG(DEBUG) << "Finished making pieces";
    }

    void TextSubFile::MakeArgumentReferences(const BinaryData &data, long long &offset) {
        LOG(DEBUG) << "Started making argument references";

        long long num_references = data.ToNumber<4>(offset);
        offset += 4;

        argument_references_.resize(num_references);
        for (long long j = 0; j < num_references; j++) {
            argument_references_[j] = data.ToNumber<4>(offset);
            offset += 4;
        }
        LOG(DEBUG) << "Finished making argument references";
    }

    void TextSubFile::MakeArgumentStrings(const BinaryData &data, long long &offset) {
        LOG(DEBUG) << "Started making argument strings";
        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        argument_strings_.resize(num_arg_strings);
        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            argument_strings_[j].resize(num_args);
            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;

                argument_strings_[j][k] = data.CutData(offset, offset + string_size * 2);
                offset += string_size * 2;
            }
        }
        LOG(DEBUG) << "Finished making argument strings";
    }

    // Build pieces/arguments/argument strings functions from fragment SubfileData

    BinaryData TextSubFile::BuildPieces(const BinaryData &data, const TextFragment &new_data, long long &offset) {
        LOG(DEBUG) << "Started building pieces";
        // Moving &offset pointer in &data
        GetPieceData(data, offset);

        std::u16string file_data = u" fid:" + to_utf16(file_id_)
                                   + u" gid:" + to_utf16(new_data.fragment_id);

        // Deleting '[' and ']' brackets
        std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2) + file_data;

        text_pieces_.clear();

        const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
        size_t prev = 0;
        size_t next = text_data.find(DNT, prev);

        while (next != std::string::npos) {
            std::u16string piece = text_data.substr(prev, next - prev);
            text_pieces_.push_back(piece);
            prev = next + DNT.length();
            next = text_data.find(DNT, prev);
        }

        text_pieces_.push_back(text_data.substr(prev));

        // Building BinaryData from pieces
        unsigned buffer_offset = 0;

        buffer_.Append(BinaryData::FromNumber<4>(text_pieces_.size()), buffer_offset);
        buffer_offset += 4;

        for (const std::u16string &piece : text_pieces_) {
            long long piece_size = piece.length();
            if (piece_size < 128) {
                buffer_.Append(BinaryData::FromNumber<1>(piece_size), buffer_offset);
                buffer_offset += 1;
            } else {
                buffer_.Append(BinaryData::FromNumberRAW<2>((piece_size | 32768)), buffer_offset);
                buffer_offset += 2;
            }

            for (long long j = 0; j < piece_size; j++) {
                buffer_.Append(BinaryData::FromNumber<2>(short(piece[j])), buffer_offset);
                buffer_offset += 2;
            }
        }
        LOG(DEBUG) << "Pieces built successfully";
        return buffer_.CutData(0, buffer_offset);
    }

    BinaryData TextSubFile::BuildArgumentReferences(const BinaryData &data, const TextFragment &new_data,
                                                    long long &offset) {
        LOG(DEBUG) << "Started building argument refs";
        // Moving &offset pointer in &data
        GetArgumentReferenceData(data, offset);

        // If there are no args - making 4 null-bytes and return;
        if (new_data.args.empty()) {
            BinaryData result = BinaryData::FromNumber<4>(0);
            return result;
        }

        // Parsing arguments from list in options["args"]
        std::string args_list = new_data.args;
        argument_references_.clear();

        size_t prev = 0;
        size_t next = args_list.find('-', prev);
        while (next != std::string::npos) {
            std::string argument = args_list.substr(prev, next - prev);
            argument_references_.push_back(std::stoll(argument));
            prev = next + 1;
            next = args_list.find('-', prev);
        }
        std::string argument = args_list.substr(prev);
        argument_references_.push_back(std::stoll(argument));

        BinaryData result;
        BinaryData temp_data = BinaryData::FromNumber<4>(argument_references_.size());
        result = result + temp_data;
        for (const long long &arg_reference : argument_references_) {
            temp_data = BinaryData::FromNumber<4>(arg_reference);
            result = result + temp_data;
        }
        LOG(DEBUG) << "Argument refs built successfully";
        return result;
    }

    BinaryData TextSubFile::BuildArgumentStrings(const BinaryData &data, const TextFragment &, long long &offset) {
        LOG(DEBUG) << "Started building argument strings";
        LOG(DEBUG) << "Built arg strings successfully";
        return GetArgumentStringsData(data, offset);
    }

    // Get BinaryData contents of pieces/arguments/argument strings

    BinaryData TextSubFile::GetPieceData(const BinaryData &data, long long &offset) const {
        LOG(DEBUG) << "Started getting piece data";
        long long old_offset = offset;

        long long num_pieces = data.ToNumber<4>(offset);
        offset += 4;

        for (long long j = 0; j < num_pieces; j++) {
            long long piece_size = data.ToNumber<1>(offset);
            if ((piece_size & 128) != 0) {
                piece_size = (((piece_size ^ 128) << 8) | data.ToNumber<1>(offset + 1));
                offset += 1;
            }
            offset += 1;
            offset += piece_size * 2;
        }
        LOG(DEBUG) << "Got piece data";
        return data.CutData(old_offset, offset);
    }

    BinaryData TextSubFile::GetArgumentReferenceData(const BinaryData &data, long long &offset) const {
        LOG(DEBUG) << "Started getting arg refs data";
        long long old_offset = offset;
        long long num_references = data.ToNumber<4>(offset);
        offset += 4;
        offset += 4 * num_references;
        LOG(DEBUG) << "Finished getting arg refs data";
        return data.CutData(old_offset, offset);
    }

    BinaryData TextSubFile::GetArgumentStringsData(const BinaryData &data, long long &offset) const {
        LOG(DEBUG) << "Started getting arg strings data";
        long long old_offset = offset;

        long long num_arg_strings = data.ToNumber<1>(offset);
        offset += 1;

        for (long long j = 0; j < num_arg_strings; j++) {
            long long num_args = data.ToNumber<4>(offset);
            offset += 4;

            for (long long k = 0; k < num_args; k++) {
                long long string_size = data.ToNumber<1>(offset);
                if ((string_size & 0x80) != 0) {
                    string_size = (((string_size ^ 0x80) << 8) | data.ToNumber<1>(offset + 1));
                    offset += 1;
                }
                offset += 1;
                offset += string_size * 2;
            }
        }
        LOG(DEBUG) << "Finished getting arg strings data";
        return data.CutData(old_offset, offset);
    }

};