LotroDatAlternate/src/subfiles/textsubfile.cpp

#include "subfiles/textsubfile.h"

#include "EasyLogging++/easylogging++.h"

#include <codecvt>

std::u16string to_utf16(long long x) {
    std::u16string res;
    while (x > 0) {
        res += char16_t(u'0' + x % 10);
        x /= 10ll;
    }
    std::reverse(res.begin(), res.end());
    return res;
}

long long from_utf16(const std::u16string &num) {
    long long res = 0;
    for (auto c : num) {
        res = res * 10ll + (c - u'0');
    }
    return res;
}

std::string argumentsFromUtf16(const std::u16string &args) {
    std::string res;

    size_t pointer = 0;
    while (pointer < args.length()) {
        size_t pointer1 = args.find(u'-', pointer);
        if (pointer1 == std::u16string::npos)
            pointer1 = args.length();
        if (!res.empty())
            res += "-";
        res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
        pointer = pointer1 + 1;
    }

    return res;
}

namespace LOTRO_DAT {
namespace Subfiles {

SubfileData Subfile<TEXT>::BuildForExport(const BinaryData &file_data) {
    SubfileData result;
    long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown

    auto text_fragment_num = unsigned(file_data.ToNumber<1>(offset));
    if ((text_fragment_num & 0x80u) != 0) {
        text_fragment_num = (((text_fragment_num ^ 0x80u) << 8u) | unsigned(file_data.ToNumber<1>(offset + 1)));
        offset += 1;
    }
    offset += 1;

    for (long long i = 0; i < text_fragment_num; i++) {
        long long fragment_id = file_data.ToNumber<8>(offset);
        offset += 8;

        // Making pieces
        auto pieces = MakePieces(file_data, offset);
        std::u16string text = u"[";
        for (size_t j = 0; j + 1 < pieces.size(); j++)
            text += pieces[j] + u"<--DO_NOT_TOUCH!-->";
        text += pieces[pieces.size() - 1] + u"]";

        // Making argument references
        auto arg_refs = MakeArgumentReferences(file_data, offset);
        std::u16string arguments;
        for (size_t j = 0; j + 1 < arg_refs.size(); j++)
            arguments += to_utf16(arg_refs[j]) + u"-";
        if (!arg_refs.empty())
            arguments += to_utf16(arg_refs[arg_refs.size() - 1]);

        // Through argument strings are not used, we need to call this function to correctly move offset
        MakeArgumentStrings(file_data, offset);

        if (result.text_data.length() > 0)
            result.text_data += u"|||";

        result.text_data += to_utf16(fragment_id) + u":::";
        result.text_data += arguments + u":::";
        result.text_data += text;
    }

    result.options["ext"] = ".txt";
    return result;
}

BinaryData Subfile<TEXT>::BuildForImport(const BinaryData &old_data, const SubfileData &data) {
    BinaryData new_file_data;
    long long offset = 9; // first 8 bytes - file_info. After them:
                              // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
    

    auto text_fragment_num = unsigned(old_data.ToNumber<1>(offset));
    if ((text_fragment_num & 0x80u) != 0) {
        text_fragment_num = (((text_fragment_num ^ 0x80u) << 8u) | unsigned(old_data.ToNumber<1>(offset + 1)));
        offset += 1;
    }
    offset += 1;

    new_file_data = new_file_data + old_data.CutData(0, offset); // Adding file info

    auto patch_fragments = ParsePatchFragments(data);

    for (long long i = 0; i < text_fragment_num; i++) {
        long long fragment_id = old_data.ToNumber<8>(offset);
        offset += 8;

        new_file_data = new_file_data + BinaryData::FromNumber<8>(fragment_id);

        TextFragment id_comp;
        id_comp.fragment_id = fragment_id;

        auto fragment_iterator = std::lower_bound(patch_fragments.begin(), patch_fragments.end(), id_comp);

        if (fragment_iterator == patch_fragments.end() || fragment_iterator->fragment_id != id_comp.fragment_id) {
            // Retrieving old pieces
            new_file_data = new_file_data + GetPieceData(old_data, offset);
            // Retrieving old references
            new_file_data = new_file_data + GetArgumentReferenceData(old_data, offset);
            // Retrieving old ref_strings
            new_file_data = new_file_data + GetArgumentStringsData(old_data, offset);
        } else {
            // Making and adding new pieces
            new_file_data = new_file_data + BuildPieces(old_data, *fragment_iterator, offset);
            // Making and adding new references
            new_file_data = new_file_data + BuildArgumentReferences(old_data, *fragment_iterator, offset);
            // Making and adding new strings
            new_file_data = new_file_data + BuildArgumentStrings(old_data, *fragment_iterator, offset);
        }
    }

    new_file_data = new_file_data + old_data.CutData(offset); // Adding elapsed file data

    return new_file_data;
}

std::vector<TextFragment> Subfile<TEXT>::ParsePatchFragments(const SubfileData &data) {
    std::vector<TextFragment> result;

    size_t pointer = 0;
    while (pointer < data.text_data.length()) {
        // Parsing fragment_id
        size_t pointer1 = data.text_data.find(u":::", pointer);
        long long fragment_id = from_utf16(data.text_data.substr(pointer, pointer1 - pointer));
        pointer = pointer1 + 3;

        TextFragment fragment;
        fragment.fragment_id = fragment_id;


        // Parsing arguments
        pointer1 = data.text_data.find(u":::", pointer);
        std::u16string arguments = data.text_data.substr(pointer, pointer1 - pointer);
        pointer = pointer1 + 3;
        if (arguments.length() > 0) {
            fragment.args = argumentsFromUtf16(arguments);
        }

        // Parsing text
        pointer1 = data.text_data.find(u"|||", pointer);
        if (pointer1 == std::u16string::npos)
            pointer1 = data.text_data.length();
        fragment.text = data.text_data.substr(pointer, pointer1 - pointer);
        pointer = pointer1 + 3;

        result.push_back(fragment);
    }
    std::sort(result.begin(), result.end());
    return result;
}

std::vector<std::u16string> Subfile<TEXT>::MakePieces(const BinaryData &data, long long &offset) {
    std::vector<std::u16string> result;
    
    auto num_pieces = unsigned(data.ToNumber<4>(offset));
    offset += 4;

    result.resize(num_pieces);
    for (long long j = 0; j < num_pieces; j++) {
        auto piece_size = unsigned(data.ToNumber<1>(offset));
        if ((piece_size & 128u) != 0) {
            piece_size = (((piece_size ^ 128u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
            offset += 1;
        }
        offset += 1;

        BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
        std::u16string piece;

        for (unsigned k = 0; k < piece_size; k++) {
            auto c = char16_t((unsigned(piece_data[2u * k + 1u])) << 8u); // First byte
            c |= (short(piece_data[2u * k]));                    // Second byte
            piece += c;
        }

        result[j] = piece;
        offset += piece_size * 2;
    }
    return result;
}

std::vector<long long> Subfile<TEXT>::MakeArgumentReferences(const BinaryData &data, long long &offset) {
    std::vector<long long> result;

    auto num_references = unsigned(data.ToNumber<4>(offset));
    offset += 4;

    result.resize(num_references);
    for (long long j = 0; j < num_references; j++) {
        result[j] = data.ToNumber<4>(offset);
        offset += 4;
    }
    return result;
}

std::vector<std::vector<BinaryData>> Subfile<TEXT>::MakeArgumentStrings(const BinaryData &data, long long &offset) {
    std::vector<std::vector<BinaryData>> result;

    auto num_arg_strings = unsigned(data.ToNumber<1>(offset));
    offset += 1;

    result.resize(num_arg_strings);
    for (long long j = 0; j < num_arg_strings; j++) {
        auto num_args = unsigned(data.ToNumber<4>(offset));
        offset += 4;

        result[j].resize(num_args);
        for (long long k = 0; k < num_args; k++) {
            auto string_size = unsigned(data.ToNumber<1>(offset));
            if ((string_size & 0x80u) != 0) {
                string_size = (((string_size ^ 0x80u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
                offset += 1;
            }
            offset += 1;

            result[j][k] = data.CutData(offset, offset + string_size * 2);
            offset += string_size * 2;
        }
    }

    return result;
}

BinaryData Subfile<TEXT>::BuildPieces(const BinaryData& data, const TextFragment& new_data, long long& offset) {
    // Moving &offset pointer in &data
    GetPieceData(data, offset);

    // Deleting '[' and ']' brackets
    std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2);

    std::vector<std::u16string> text_pieces;

    const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
    size_t prev = 0;
    size_t next = text_data.find(DNT, prev);

    while (next != std::string::npos) {
        std::u16string piece = (next - prev == 0) ? u"" : text_data.substr(prev, next - prev);
        text_pieces.push_back(piece);
        prev = next + DNT.length();
        next = text_data.find(DNT, prev);
    }

    text_pieces.push_back(text_data.substr(prev));

    // Building BinaryData from pieces
    BinaryData result;
    result = result + BinaryData::FromNumber<4>(text_pieces.size());

    for (const std::u16string &piece : text_pieces) {
        unsigned piece_size = piece.length();
        if (piece_size < 128) {
            result = result + BinaryData::FromNumber<1>(piece_size);
        } else {
            result = result + BinaryData::FromNumberRAW<2>((piece_size | 32768u)); 
        }

        for (long long j = 0; j < piece_size; j++) {
            result = result + BinaryData::FromNumber<2>(short(piece[j]));
        }
    }

    return result;
}

BinaryData Subfile<TEXT>::BuildArgumentReferences(const BinaryData& data, const TextFragment& new_data,
                                                  long long &offset) {
    // Moving &offset pointer in &data
    GetArgumentReferenceData(data, offset);

    // If there are no args - making 4 null-bytes and return;
    if (new_data.args.empty()) {
        BinaryData result = BinaryData::FromNumber<4>(0);
        return result;
    }

    // Parsing arguments from list in options["args"]
    std::string args_list = new_data.args;
    std::vector<long long> argument_references;

    size_t prev = 0;
    size_t next = args_list.find('-', prev);
    while (next != std::string::npos) {
        std::string argument = args_list.substr(prev, next - prev);
        argument_references.push_back(std::stoll(argument));
        prev = next + 1;
        next = args_list.find('-', prev);
    }
    std::string argument = args_list.substr(prev);
    argument_references.push_back(std::stoll(argument));

    BinaryData result;
    result = result + BinaryData::FromNumber<4>(argument_references.size());
    for (const long long &arg_reference : argument_references) {
        result = result + BinaryData::FromNumber<4>(arg_reference);
    }

    return result;
}

BinaryData Subfile<TEXT>::BuildArgumentStrings(const BinaryData& data, const TextFragment&, long long& offset) {
    // TODO: IMPLEMENT (never user)
    GetArgumentStringsData(data, offset);
    return BinaryData::FromNumber<1>(0);
}

// Get BinaryData contents of pieces/arguments/argument strings

BinaryData Subfile<TEXT>::GetPieceData(const BinaryData& data, long long& offset) {
    long long old_offset = offset;

    long long num_pieces = data.ToNumber<4>(offset);
    offset += 4;

    for (long long j = 0; j < num_pieces; j++) {
        auto piece_size = unsigned(data.ToNumber<1>(offset));
        if ((piece_size & 128u) != 0) {
            piece_size = (((piece_size ^ 128u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
            offset += 1;
        }
        offset += 1;
        offset += piece_size * 2;
    }
    return data.CutData(old_offset, offset);
}

BinaryData Subfile<TEXT>::GetArgumentReferenceData(const BinaryData& data, long long& offset) {
    long long old_offset = offset;
    long long num_references = data.ToNumber<4>(offset);
    offset += 4;
    offset += 4 * num_references;
    return data.CutData(old_offset, offset);
}

BinaryData Subfile<TEXT>::GetArgumentStringsData(const BinaryData& data, long long& offset) {
    long long old_offset = offset;

    long long num_arg_strings = data.ToNumber<1>(offset);
    offset += 1;

    for (long long j = 0; j < num_arg_strings; j++) {
        long long num_args = data.ToNumber<4>(offset);
        offset += 4;

        for (long long k = 0; k < num_args; k++) {
            auto string_size = unsigned(data.ToNumber<1>(offset));
            if ((string_size & 0x80u) != 0) {
                string_size = (((string_size ^ 0x80u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
                offset += 1;
            }
            offset += 1;
            offset += string_size * 2;
        }
    }
    return data.CutData(old_offset, offset);
}

}; // namespace Subfiles
}; // namespace LOTRO_DAT