textsubfile.cpp 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383
  1. #include "subfiles/textsubfile.h"
  2. #include "EasyLogging++/easylogging++.h"
  3. #include <codecvt>
  4. #include <locale>
  5. std::u16string to_utf16(long long x) {
  6. std::u16string res;
  7. while (x > 0) {
  8. res += char16_t(u'0' + x % 10);
  9. x /= 10ll;
  10. }
  11. std::reverse(res.begin(), res.end());
  12. return res;
  13. }
  14. long long from_utf16(const std::u16string &num) {
  15. long long res = 0;
  16. for (auto c : num) {
  17. res = res * 10ll + (c - u'0');
  18. }
  19. return res;
  20. }
  21. std::string argumentsFromUtf16(const std::u16string &args) {
  22. std::string res;
  23. size_t pointer = 0;
  24. while (pointer < args.length()) {
  25. size_t pointer1 = args.find(u'-', pointer);
  26. if (pointer1 == std::u16string::npos)
  27. pointer1 = args.length();
  28. if (!res.empty())
  29. res += "-";
  30. res += std::to_string(from_utf16(args.substr(pointer, pointer1 - pointer)));
  31. pointer = pointer1 + 1;
  32. }
  33. return res;
  34. }
  35. namespace LOTRO_DAT {
  36. namespace Subfiles {
  37. SubfileData Subfile<TEXT>::BuildForExport(const BinaryData &file_data) {
  38. SubfileData result;
  39. long long offset = 9; // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
  40. auto text_fragment_num = unsigned(file_data.ToNumber<1>(offset));
  41. if ((text_fragment_num & 0x80u) != 0) {
  42. text_fragment_num = (((text_fragment_num ^ 0x80u) << 8u) | unsigned(file_data.ToNumber<1>(offset + 1)));
  43. offset += 1;
  44. }
  45. offset += 1;
  46. for (long long i = 0; i < text_fragment_num; i++) {
  47. long long fragment_id = file_data.ToNumber<8>(offset);
  48. offset += 8;
  49. // Making pieces
  50. auto pieces = MakePieces(file_data, offset);
  51. std::u16string text = u"[";
  52. for (size_t j = 0; j + 1 < pieces.size(); j++)
  53. text += pieces[j] + u"<--DO_NOT_TOUCH!-->";
  54. text += pieces[pieces.size() - 1] + u"]";
  55. // Making argument references
  56. auto arg_refs = MakeArgumentReferences(file_data, offset);
  57. std::u16string arguments;
  58. for (size_t j = 0; j + 1 < arg_refs.size(); j++)
  59. arguments += to_utf16(arg_refs[j]) + u"-";
  60. if (!arg_refs.empty())
  61. arguments += to_utf16(arg_refs[arg_refs.size() - 1]);
  62. // Through argument strings are not used, we need to call this function to correctly move offset
  63. MakeArgumentStrings(file_data, offset);
  64. if (result.text_data.length() > 0)
  65. result.text_data += u"|||";
  66. result.text_data += to_utf16(fragment_id) + u":::";
  67. result.text_data += arguments + u":::";
  68. result.text_data += text;
  69. }
  70. result.options["ext"] = ".txt";
  71. return result;
  72. }
  73. BinaryData Subfile<TEXT>::BuildForImport(const BinaryData &old_data, const SubfileData &data) {
  74. BinaryData new_file_data;
  75. long long offset = 9; // first 8 bytes - file_info. After them:
  76. // first 4 bytes - file_id, then 4 bytes - unknown, then 1 byte - unknown
  77. auto text_fragment_num = unsigned(old_data.ToNumber<1>(offset));
  78. if ((text_fragment_num & 0x80u) != 0) {
  79. text_fragment_num = (((text_fragment_num ^ 0x80u) << 8u) | unsigned(old_data.ToNumber<1>(offset + 1)));
  80. offset += 1;
  81. }
  82. offset += 1;
  83. new_file_data = new_file_data + old_data.CutData(0, offset); // Adding file info
  84. auto patch_fragments = ParsePatchFragments(data);
  85. for (long long i = 0; i < text_fragment_num; i++) {
  86. long long fragment_id = old_data.ToNumber<8>(offset);
  87. offset += 8;
  88. new_file_data = new_file_data + BinaryData::FromNumber<8>(fragment_id);
  89. TextFragment id_comp;
  90. id_comp.fragment_id = fragment_id;
  91. auto fragment_iterator = std::lower_bound(patch_fragments.begin(), patch_fragments.end(), id_comp);
  92. if (fragment_iterator == patch_fragments.end() || fragment_iterator->fragment_id != id_comp.fragment_id) {
  93. // Retrieving old pieces
  94. new_file_data = new_file_data + GetPieceData(old_data, offset);
  95. // Retrieving old references
  96. new_file_data = new_file_data + GetArgumentReferenceData(old_data, offset);
  97. // Retrieving old ref_strings
  98. new_file_data = new_file_data + GetArgumentStringsData(old_data, offset);
  99. } else {
  100. // Making and adding new pieces
  101. new_file_data = new_file_data + BuildPieces(old_data, *fragment_iterator, offset);
  102. // Making and adding new references
  103. new_file_data = new_file_data + BuildArgumentReferences(old_data, *fragment_iterator, offset);
  104. // Making and adding new strings
  105. new_file_data = new_file_data + BuildArgumentStrings(old_data, *fragment_iterator, offset);
  106. }
  107. }
  108. new_file_data = new_file_data + old_data.CutData(offset); // Adding elapsed file data
  109. return new_file_data;
  110. }
  111. std::vector<TextFragment> Subfile<TEXT>::ParsePatchFragments(const SubfileData &data) {
  112. std::vector<TextFragment> result;
  113. size_t pointer = 0;
  114. while (pointer < data.text_data.length()) {
  115. // Parsing fragment_id
  116. size_t pointer1 = data.text_data.find(u":::", pointer);
  117. long long fragment_id = from_utf16(data.text_data.substr(pointer, pointer1 - pointer));
  118. pointer = pointer1 + 3;
  119. TextFragment fragment;
  120. fragment.fragment_id = fragment_id;
  121. // Parsing arguments
  122. pointer1 = data.text_data.find(u":::", pointer);
  123. std::u16string arguments = data.text_data.substr(pointer, pointer1 - pointer);
  124. pointer = pointer1 + 3;
  125. if (arguments.length() > 0) {
  126. fragment.args = argumentsFromUtf16(arguments);
  127. }
  128. // Parsing text
  129. pointer1 = data.text_data.find(u"|||", pointer);
  130. if (pointer1 == std::u16string::npos)
  131. pointer1 = data.text_data.length();
  132. fragment.text = data.text_data.substr(pointer, pointer1 - pointer);
  133. pointer = pointer1 + 3;
  134. result.push_back(fragment);
  135. }
  136. std::sort(result.begin(), result.end());
  137. return result;
  138. }
  139. std::vector<std::u16string> Subfile<TEXT>::MakePieces(const BinaryData &data, long long &offset) {
  140. std::vector<std::u16string> result;
  141. auto num_pieces = unsigned(data.ToNumber<4>(offset));
  142. offset += 4;
  143. result.resize(num_pieces);
  144. for (long long j = 0; j < num_pieces; j++) {
  145. auto piece_size = unsigned(data.ToNumber<1>(offset));
  146. if ((piece_size & 128u) != 0) {
  147. piece_size = (((piece_size ^ 128u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
  148. offset += 1;
  149. }
  150. offset += 1;
  151. BinaryData piece_data = data.CutData(offset, offset + piece_size * 2);
  152. std::u16string piece;
  153. for (unsigned k = 0; k < piece_size; k++) {
  154. auto c = char16_t((unsigned(piece_data[2u * k + 1u])) << 8u); // First byte
  155. c |= (short(piece_data[2u * k])); // Second byte
  156. piece += c;
  157. }
  158. result[j] = piece;
  159. offset += piece_size * 2;
  160. }
  161. return result;
  162. }
  163. std::vector<long long> Subfile<TEXT>::MakeArgumentReferences(const BinaryData &data, long long &offset) {
  164. std::vector<long long> result;
  165. auto num_references = unsigned(data.ToNumber<4>(offset));
  166. offset += 4;
  167. result.resize(num_references);
  168. for (long long j = 0; j < num_references; j++) {
  169. result[j] = data.ToNumber<4>(offset);
  170. offset += 4;
  171. }
  172. return result;
  173. }
  174. std::vector<std::vector<BinaryData>> Subfile<TEXT>::MakeArgumentStrings(const BinaryData &data, long long &offset) {
  175. std::vector<std::vector<BinaryData>> result;
  176. auto num_arg_strings = unsigned(data.ToNumber<1>(offset));
  177. offset += 1;
  178. result.resize(num_arg_strings);
  179. for (long long j = 0; j < num_arg_strings; j++) {
  180. auto num_args = unsigned(data.ToNumber<4>(offset));
  181. offset += 4;
  182. result[j].resize(num_args);
  183. for (long long k = 0; k < num_args; k++) {
  184. auto string_size = unsigned(data.ToNumber<1>(offset));
  185. if ((string_size & 0x80u) != 0) {
  186. string_size = (((string_size ^ 0x80u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
  187. offset += 1;
  188. }
  189. offset += 1;
  190. result[j][k] = data.CutData(offset, offset + string_size * 2);
  191. offset += string_size * 2;
  192. }
  193. }
  194. return result;
  195. }
  196. BinaryData Subfile<TEXT>::BuildPieces(const BinaryData& data, const TextFragment& new_data, long long& offset) {
  197. // Moving &offset pointer in &data
  198. GetPieceData(data, offset);
  199. // Deleting '[' and ']' brackets
  200. std::u16string text_data = new_data.text.substr(1, new_data.text.size() - 2);
  201. std::vector<std::u16string> text_pieces;
  202. const std::u16string DNT = u"<--DO_NOT_TOUCH!-->";
  203. size_t prev = 0;
  204. size_t next = text_data.find(DNT, prev);
  205. while (next != std::string::npos) {
  206. std::u16string piece = (next - prev == 0) ? u" " : text_data.substr(prev, next - prev);
  207. text_pieces.push_back(piece);
  208. prev = next + DNT.length();
  209. next = text_data.find(DNT, prev);
  210. }
  211. text_pieces.push_back(text_data.substr(prev));
  212. // Building BinaryData from pieces
  213. BinaryData result;
  214. result = result + BinaryData::FromNumber<4>(text_pieces.size());
  215. for (const std::u16string &piece : text_pieces) {
  216. unsigned piece_size = piece.length();
  217. if (piece_size < 128) {
  218. result = result + BinaryData::FromNumber<1>(piece_size);
  219. } else {
  220. result = result + BinaryData::FromNumberRAW<2>((piece_size | 32768u));
  221. }
  222. for (long long j = 0; j < piece_size; j++) {
  223. result = result + BinaryData::FromNumber<2>(short(piece[j]));
  224. }
  225. }
  226. return result;
  227. }
  228. BinaryData Subfile<TEXT>::BuildArgumentReferences(const BinaryData& data, const TextFragment& new_data,
  229. long long &offset) {
  230. // Moving &offset pointer in &data
  231. GetArgumentReferenceData(data, offset);
  232. // If there are no args - making 4 null-bytes and return;
  233. if (new_data.args.empty()) {
  234. BinaryData result = BinaryData::FromNumber<4>(0);
  235. return result;
  236. }
  237. // Parsing arguments from list in options["args"]
  238. std::string args_list = new_data.args;
  239. std::vector<long long> argument_references;
  240. size_t prev = 0;
  241. size_t next = args_list.find('-', prev);
  242. while (next != std::string::npos) {
  243. std::string argument = args_list.substr(prev, next - prev);
  244. argument_references.push_back(std::stoll(argument));
  245. prev = next + 1;
  246. next = args_list.find('-', prev);
  247. }
  248. std::string argument = args_list.substr(prev);
  249. argument_references.push_back(std::stoll(argument));
  250. BinaryData result;
  251. result = result + BinaryData::FromNumber<4>(argument_references.size());
  252. for (const long long &arg_reference : argument_references) {
  253. result = result + BinaryData::FromNumber<4>(arg_reference);
  254. }
  255. return result;
  256. }
  257. BinaryData Subfile<TEXT>::BuildArgumentStrings(const BinaryData& data, const TextFragment&, long long& offset) {
  258. // TODO: IMPLEMENT (never user)
  259. GetArgumentStringsData(data, offset);
  260. return BinaryData::FromNumber<1>(0);
  261. }
  262. // Get BinaryData contents of pieces/arguments/argument strings
  263. BinaryData Subfile<TEXT>::GetPieceData(const BinaryData& data, long long& offset) {
  264. long long old_offset = offset;
  265. long long num_pieces = data.ToNumber<4>(offset);
  266. offset += 4;
  267. for (long long j = 0; j < num_pieces; j++) {
  268. auto piece_size = unsigned(data.ToNumber<1>(offset));
  269. if ((piece_size & 128u) != 0) {
  270. piece_size = (((piece_size ^ 128u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
  271. offset += 1;
  272. }
  273. offset += 1;
  274. offset += piece_size * 2;
  275. }
  276. return data.CutData(old_offset, offset);
  277. }
  278. BinaryData Subfile<TEXT>::GetArgumentReferenceData(const BinaryData& data, long long& offset) {
  279. long long old_offset = offset;
  280. long long num_references = data.ToNumber<4>(offset);
  281. offset += 4;
  282. offset += 4 * num_references;
  283. return data.CutData(old_offset, offset);
  284. }
  285. BinaryData Subfile<TEXT>::GetArgumentStringsData(const BinaryData& data, long long& offset) {
  286. long long old_offset = offset;
  287. long long num_arg_strings = data.ToNumber<1>(offset);
  288. offset += 1;
  289. for (long long j = 0; j < num_arg_strings; j++) {
  290. long long num_args = data.ToNumber<4>(offset);
  291. offset += 4;
  292. for (long long k = 0; k < num_args; k++) {
  293. auto string_size = unsigned(data.ToNumber<1>(offset));
  294. if ((string_size & 0x80u) != 0) {
  295. string_size = (((string_size ^ 0x80u) << 8u) | unsigned(data.ToNumber<1>(offset + 1)));
  296. offset += 1;
  297. }
  298. offset += 1;
  299. offset += string_size * 2;
  300. }
  301. }
  302. return data.CutData(old_offset, offset);
  303. }
  304. }; // namespace Subfiles
  305. }; // namespace LOTRO_DAT