From 1bd90756d1adb62b94ba41d50570c34241b9e16f Mon Sep 17 00:00:00 2001 From: Mr_Goldberg Date: Mon, 6 Dec 2021 16:52:15 -0500 Subject: [PATCH] Add script to generate achievements and stats configs from UserGameStatsSchema. --- .../achievements_gen.py | 86 +++ .../vdf/__init__.py | 521 ++++++++++++++++++ .../stats_schema_achievement_gen/vdf/vdict.py | 221 ++++++++ 3 files changed, 828 insertions(+) create mode 100644 scripts/stats_schema_achievement_gen/achievements_gen.py create mode 100644 scripts/stats_schema_achievement_gen/vdf/__init__.py create mode 100644 scripts/stats_schema_achievement_gen/vdf/vdict.py diff --git a/scripts/stats_schema_achievement_gen/achievements_gen.py b/scripts/stats_schema_achievement_gen/achievements_gen.py new file mode 100644 index 0000000..b1bf91f --- /dev/null +++ b/scripts/stats_schema_achievement_gen/achievements_gen.py @@ -0,0 +1,86 @@ +import vdf +import sys +import os +import json + + +if len(sys.argv) < 2: + print("format: {} UserGameStatsSchema_480.bin".format(sys.argv[0])) + exit(0) + + +with open(sys.argv[1], 'rb') as f: + schema = vdf.binary_loads(f.read()) + +language = 'english' + +STAT_TYPE_INT = '1' +STAT_TYPE_FLOAT = '2' +STAT_TYPE_AVGRATE = '3' +STAT_TYPE_BITS = '4' + +achievements_out = [] +stats_out = [] + +for appid in schema: + sch = schema[appid] + stat_info = sch['stats'] + for s in stat_info: + stat = stat_info[s] + if stat['type'] == STAT_TYPE_BITS: + achs = stat['bits'] + for ach_num in achs: + out = {} + ach = achs[ach_num] + out["hidden"] = '0' + for x in ach['display']: + value = ach['display'][x] + if x == 'name': + x = 'displayName' + if x == 'desc': + x = 'description' + if x == 'Hidden': + x = 'hidden' + if type(value) is dict: + if language in value: + value = value[language] + else: + value = '' + out[x] = value + out['name'] = ach['name'] + achievements_out += [out] + else: + out = {} + out['default'] = 0 + out['name'] = stat['name'] + if stat['type'] == STAT_TYPE_INT: + out['type'] = 'int' + elif stat['type'] == STAT_TYPE_FLOAT: + out['type'] = 'float' + elif stat['type'] == STAT_TYPE_AVGRATE: + out['type'] = 'avgrate' + if 'Default' in stat: + out['default'] = stat['Default'] + + stats_out += [out] + # print(stat_info[s]) + + + +output_ach = json.dumps(achievements_out, indent=4) +output_stats = "" +for s in stats_out: + output_stats += "{}={}={}\n".format(s['name'], s['type'], s['default']) + +# print(output_ach) +# print(output_stats) + +config_directory = os.path.join(sys.argv[1] + "_output", "steam_settings") +if not os.path.exists(config_directory): + os.makedirs(config_directory) + +with open(os.path.join(config_directory, "achievements.json"), 'w') as f: + f.write(output_ach) + +with open(os.path.join(config_directory, "stats.txt"), 'w') as f: + f.write(output_stats) diff --git a/scripts/stats_schema_achievement_gen/vdf/__init__.py b/scripts/stats_schema_achievement_gen/vdf/__init__.py new file mode 100644 index 0000000..6b47213 --- /dev/null +++ b/scripts/stats_schema_achievement_gen/vdf/__init__.py @@ -0,0 +1,521 @@ +""" +Module for deserializing/serializing to and from VDF +""" +__version__ = "3.4" +__author__ = "Rossen Georgiev" + +import re +import sys +import struct +from binascii import crc32 +from io import BytesIO +from io import StringIO as unicodeIO + +try: + from collections.abc import Mapping +except: + from collections import Mapping + +from vdf.vdict import VDFDict + +# Py2 & Py3 compatibility +if sys.version_info[0] >= 3: + string_type = str + int_type = int + BOMS = '\ufffe\ufeff' + + def strip_bom(line): + return line.lstrip(BOMS) +else: + from StringIO import StringIO as strIO + string_type = basestring + int_type = long + BOMS = '\xef\xbb\xbf\xff\xfe\xfe\xff' + BOMS_UNICODE = '\\ufffe\\ufeff'.decode('unicode-escape') + + def strip_bom(line): + return line.lstrip(BOMS if isinstance(line, str) else BOMS_UNICODE) + +# string escaping +_unescape_char_map = { + r"\n": "\n", + r"\t": "\t", + r"\v": "\v", + r"\b": "\b", + r"\r": "\r", + r"\f": "\f", + r"\a": "\a", + r"\\": "\\", + r"\?": "?", + r"\"": "\"", + r"\'": "\'", +} +_escape_char_map = {v: k for k, v in _unescape_char_map.items()} + +def _re_escape_match(m): + return _escape_char_map[m.group()] + +def _re_unescape_match(m): + return _unescape_char_map[m.group()] + +def _escape(text): + return re.sub(r"[\n\t\v\b\r\f\a\\\?\"']", _re_escape_match, text) + +def _unescape(text): + return re.sub(r"(\\n|\\t|\\v|\\b|\\r|\\f|\\a|\\\\|\\\?|\\\"|\\')", _re_unescape_match, text) + +# parsing and dumping for KV1 +def parse(fp, mapper=dict, merge_duplicate_keys=True, escaped=True): + """ + Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a VDF) + to a Python object. + + ``mapper`` specifies the Python object used after deserializetion. ``dict` is + used by default. Alternatively, ``collections.OrderedDict`` can be used if you + wish to preserve key order. Or any object that acts like a ``dict``. + + ``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the + same key into one instead of overwriting. You can se this to ``False`` if you are + using ``VDFDict`` and need to preserve the duplicates. + """ + if not issubclass(mapper, Mapping): + raise TypeError("Expected mapper to be subclass of dict, got %s" % type(mapper)) + if not hasattr(fp, 'readline'): + raise TypeError("Expected fp to be a file-like object supporting line iteration") + + stack = [mapper()] + expect_bracket = False + + re_keyvalue = re.compile(r'^("(?P(?:\\.|[^\\"])*)"|(?P#?[a-z0-9\-\_\\\?$%<>]+))' + r'([ \t]*(' + r'"(?P(?:\\.|[^\\"])*)(?P")?' + r'|(?P(?:(? ])+)' + r'|(?P{[ \t]*)(?P})?' + r'))?', + flags=re.I) + + for lineno, line in enumerate(fp, 1): + if lineno == 1: + line = strip_bom(line) + + line = line.lstrip() + + # skip empty and comment lines + if line == "" or line[0] == '/': + continue + + # one level deeper + if line[0] == "{": + expect_bracket = False + continue + + if expect_bracket: + raise SyntaxError("vdf.parse: expected openning bracket", + (getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 1, line)) + + # one level back + if line[0] == "}": + if len(stack) > 1: + stack.pop() + continue + + raise SyntaxError("vdf.parse: one too many closing parenthasis", + (getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line)) + + # parse keyvalue pairs + while True: + match = re_keyvalue.match(line) + + if not match: + try: + line += next(fp) + continue + except StopIteration: + raise SyntaxError("vdf.parse: unexpected EOF (open key quote?)", + (getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line)) + + key = match.group('key') if match.group('qkey') is None else match.group('qkey') + val = match.group('qval') + if val is None: + val = match.group('val') + if val is not None: + val = val.rstrip() + if val == "": + val = None + + if escaped: + key = _unescape(key) + + # we have a key with value in parenthesis, so we make a new dict obj (level deeper) + if val is None: + if merge_duplicate_keys and key in stack[-1]: + _m = stack[-1][key] + # we've descended a level deeper, if value is str, we have to overwrite it to mapper + if not isinstance(_m, mapper): + _m = stack[-1][key] = mapper() + else: + _m = mapper() + stack[-1][key] = _m + + if match.group('eblock') is None: + # only expect a bracket if it's not already closed or on the same line + stack.append(_m) + if match.group('sblock') is None: + expect_bracket = True + + # we've matched a simple keyvalue pair, map it to the last dict obj in the stack + else: + # if the value is line consume one more line and try to match again, + # until we get the KeyValue pair + if match.group('vq_end') is None and match.group('qval') is not None: + try: + line += next(fp) + continue + except StopIteration: + raise SyntaxError("vdf.parse: unexpected EOF (open quote for value?)", + (getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line)) + + stack[-1][key] = _unescape(val) if escaped else val + + # exit the loop + break + + if len(stack) != 1: + raise SyntaxError("vdf.parse: unclosed parenthasis or quotes (EOF)", + (getattr(fp, 'name', '<%s>' % fp.__class__.__name__), lineno, 0, line)) + + return stack.pop() + + +def loads(s, **kwargs): + """ + Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + """ + if not isinstance(s, string_type): + raise TypeError("Expected s to be a str, got %s" % type(s)) + + try: + fp = unicodeIO(s) + except TypeError: + fp = strIO(s) + + return parse(fp, **kwargs) + + +def load(fp, **kwargs): + """ + Deserialize ``fp`` (a ``.readline()``-supporting file-like object containing + a JSON document) to a Python object. + """ + return parse(fp, **kwargs) + + +def dumps(obj, pretty=False, escaped=True): + """ + Serialize ``obj`` to a VDF formatted ``str``. + """ + if not isinstance(obj, Mapping): + raise TypeError("Expected data to be an instance of``dict``") + if not isinstance(pretty, bool): + raise TypeError("Expected pretty to be of type bool") + if not isinstance(escaped, bool): + raise TypeError("Expected escaped to be of type bool") + + return ''.join(_dump_gen(obj, pretty, escaped)) + + +def dump(obj, fp, pretty=False, escaped=True): + """ + Serialize ``obj`` as a VDF formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + """ + if not isinstance(obj, Mapping): + raise TypeError("Expected data to be an instance of``dict``") + if not hasattr(fp, 'write'): + raise TypeError("Expected fp to have write() method") + if not isinstance(pretty, bool): + raise TypeError("Expected pretty to be of type bool") + if not isinstance(escaped, bool): + raise TypeError("Expected escaped to be of type bool") + + for chunk in _dump_gen(obj, pretty, escaped): + fp.write(chunk) + + +def _dump_gen(data, pretty=False, escaped=True, level=0): + indent = "\t" + line_indent = "" + + if pretty: + line_indent = indent * level + + for key, value in data.items(): + if escaped and isinstance(key, string_type): + key = _escape(key) + + if isinstance(value, Mapping): + yield '%s"%s"\n%s{\n' % (line_indent, key, line_indent) + for chunk in _dump_gen(value, pretty, escaped, level+1): + yield chunk + yield "%s}\n" % line_indent + else: + if escaped and isinstance(value, string_type): + value = _escape(value) + + yield '%s"%s" "%s"\n' % (line_indent, key, value) + + +# binary VDF +class BASE_INT(int_type): + def __repr__(self): + return "%s(%d)" % (self.__class__.__name__, self) + +class UINT_64(BASE_INT): + pass + +class INT_64(BASE_INT): + pass + +class POINTER(BASE_INT): + pass + +class COLOR(BASE_INT): + pass + +BIN_NONE = b'\x00' +BIN_STRING = b'\x01' +BIN_INT32 = b'\x02' +BIN_FLOAT32 = b'\x03' +BIN_POINTER = b'\x04' +BIN_WIDESTRING = b'\x05' +BIN_COLOR = b'\x06' +BIN_UINT64 = b'\x07' +BIN_END = b'\x08' +BIN_INT64 = b'\x0A' +BIN_END_ALT = b'\x0B' + +def binary_loads(b, mapper=dict, merge_duplicate_keys=True, alt_format=False, raise_on_remaining=True): + """ + Deserialize ``b`` (``bytes`` containing a VDF in "binary form") + to a Python object. + + ``mapper`` specifies the Python object used after deserializetion. ``dict` is + used by default. Alternatively, ``collections.OrderedDict`` can be used if you + wish to preserve key order. Or any object that acts like a ``dict``. + + ``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the + same key into one instead of overwriting. You can se this to ``False`` if you are + using ``VDFDict`` and need to preserve the duplicates. + """ + if not isinstance(b, bytes): + raise TypeError("Expected s to be bytes, got %s" % type(b)) + + return binary_load(BytesIO(b), mapper, merge_duplicate_keys, alt_format, raise_on_remaining) + +def binary_load(fp, mapper=dict, merge_duplicate_keys=True, alt_format=False, raise_on_remaining=False): + """ + Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + binary VDF) to a Python object. + + ``mapper`` specifies the Python object used after deserializetion. ``dict` is + used by default. Alternatively, ``collections.OrderedDict`` can be used if you + wish to preserve key order. Or any object that acts like a ``dict``. + + ``merge_duplicate_keys`` when ``True`` will merge multiple KeyValue lists with the + same key into one instead of overwriting. You can se this to ``False`` if you are + using ``VDFDict`` and need to preserve the duplicates. + """ + if not hasattr(fp, 'read') or not hasattr(fp, 'tell') or not hasattr(fp, 'seek'): + raise TypeError("Expected fp to be a file-like object with tell()/seek() and read() returning bytes") + if not issubclass(mapper, Mapping): + raise TypeError("Expected mapper to be subclass of dict, got %s" % type(mapper)) + + # helpers + int32 = struct.Struct(' 1: + stack.pop() + continue + break + + key = read_string(fp) + + if t == BIN_NONE: + if merge_duplicate_keys and key in stack[-1]: + _m = stack[-1][key] + else: + _m = mapper() + stack[-1][key] = _m + stack.append(_m) + elif t == BIN_STRING: + stack[-1][key] = read_string(fp) + elif t == BIN_WIDESTRING: + stack[-1][key] = read_string(fp, wide=True) + elif t in (BIN_INT32, BIN_POINTER, BIN_COLOR): + val = int32.unpack(fp.read(int32.size))[0] + + if t == BIN_POINTER: + val = POINTER(val) + elif t == BIN_COLOR: + val = COLOR(val) + + stack[-1][key] = val + elif t == BIN_UINT64: + stack[-1][key] = UINT_64(uint64.unpack(fp.read(int64.size))[0]) + elif t == BIN_INT64: + stack[-1][key] = INT_64(int64.unpack(fp.read(int64.size))[0]) + elif t == BIN_FLOAT32: + stack[-1][key] = float32.unpack(fp.read(float32.size))[0] + else: + raise SyntaxError("Unknown data type at offset %d: %s" % (fp.tell() - 1, repr(t))) + + if len(stack) != 1: + raise SyntaxError("Reached EOF, but Binary VDF is incomplete") + if raise_on_remaining and fp.read(1) != b'': + fp.seek(-1, 1) + raise SyntaxError("Binary VDF ended at offset %d, but there is more data remaining" % (fp.tell() - 1)) + + return stack.pop() + +def binary_dumps(obj, alt_format=False): + """ + Serialize ``obj`` to a binary VDF formatted ``bytes``. + """ + buf = BytesIO() + binary_dump(obj, buf, alt_format) + return buf.getvalue() + +def binary_dump(obj, fp, alt_format=False): + """ + Serialize ``obj`` to a binary VDF formatted ``bytes`` and write it to ``fp`` filelike object + """ + if not isinstance(obj, Mapping): + raise TypeError("Expected obj to be type of Mapping") + if not hasattr(fp, 'write'): + raise TypeError("Expected fp to have write() method") + + for chunk in _binary_dump_gen(obj, alt_format=alt_format): + fp.write(chunk) + +def _binary_dump_gen(obj, level=0, alt_format=False): + if level == 0 and len(obj) == 0: + return + + int32 = struct.Struct('= 3: + _iter_values = 'values' + _range = range + _string_type = str + import collections.abc as _c + class _kView(_c.KeysView): + def __iter__(self): + return self._mapping.iterkeys() + class _vView(_c.ValuesView): + def __iter__(self): + return self._mapping.itervalues() + class _iView(_c.ItemsView): + def __iter__(self): + return self._mapping.iteritems() +else: + _iter_values = 'itervalues' + _range = xrange + _string_type = basestring + _kView = lambda x: list(x.iterkeys()) + _vView = lambda x: list(x.itervalues()) + _iView = lambda x: list(x.iteritems()) + + +class VDFDict(dict): + def __init__(self, data=None): + """ + This is a dictionary that supports duplicate keys and preserves insert order + + ``data`` can be a ``dict``, or a sequence of key-value tuples. (e.g. ``[('key', 'value'),..]``) + The only supported type for key is str. + + Get/set duplicates is done by tuples ``(index, key)``, where index is the duplicate index + for the specified key. (e.g. ``(0, 'key')``, ``(1, 'key')``...) + + When the ``key`` is ``str``, instead of tuple, set will create a duplicate and get will look up ``(0, key)`` + """ + self.__omap = [] + self.__kcount = Counter() + + if data is not None: + if not isinstance(data, (list, dict)): + raise ValueError("Expected data to be list of pairs or dict, got %s" % type(data)) + self.update(data) + + def __repr__(self): + out = "%s(" % self.__class__.__name__ + out += "%s)" % repr(list(self.iteritems())) + return out + + def __len__(self): + return len(self.__omap) + + def _verify_key_tuple(self, key): + if len(key) != 2: + raise ValueError("Expected key tuple length to be 2, got %d" % len(key)) + if not isinstance(key[0], int): + raise TypeError("Key index should be an int") + if not isinstance(key[1], _string_type): + raise TypeError("Key value should be a str") + + def _normalize_key(self, key): + if isinstance(key, _string_type): + key = (0, key) + elif isinstance(key, tuple): + self._verify_key_tuple(key) + else: + raise TypeError("Expected key to be a str or tuple, got %s" % type(key)) + return key + + def __setitem__(self, key, value): + if isinstance(key, _string_type): + key = (self.__kcount[key], key) + self.__omap.append(key) + elif isinstance(key, tuple): + self._verify_key_tuple(key) + if key not in self: + raise KeyError("%s doesn't exist" % repr(key)) + else: + raise TypeError("Expected either a str or tuple for key") + super(VDFDict, self).__setitem__(key, value) + self.__kcount[key[1]] += 1 + + def __getitem__(self, key): + return super(VDFDict, self).__getitem__(self._normalize_key(key)) + + def __delitem__(self, key): + key = self._normalize_key(key) + result = super(VDFDict, self).__delitem__(key) + + start_idx = self.__omap.index(key) + del self.__omap[start_idx] + + dup_idx, skey = key + self.__kcount[skey] -= 1 + tail_count = self.__kcount[skey] - dup_idx + + if tail_count > 0: + for idx in _range(start_idx, len(self.__omap)): + if self.__omap[idx][1] == skey: + oldkey = self.__omap[idx] + newkey = (dup_idx, skey) + super(VDFDict, self).__setitem__(newkey, self[oldkey]) + super(VDFDict, self).__delitem__(oldkey) + self.__omap[idx] = newkey + + dup_idx += 1 + tail_count -= 1 + if tail_count == 0: + break + + if self.__kcount[skey] == 0: + del self.__kcount[skey] + + return result + + def __iter__(self): + return iter(self.iterkeys()) + + def __contains__(self, key): + return super(VDFDict, self).__contains__(self._normalize_key(key)) + + def __eq__(self, other): + if isinstance(other, VDFDict): + return list(self.items()) == list(other.items()) + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def clear(self): + super(VDFDict, self).clear() + self.__kcount.clear() + self.__omap = list() + + def get(self, key, *args): + return super(VDFDict, self).get(self._normalize_key(key), *args) + + def setdefault(self, key, default=None): + if key not in self: + self.__setitem__(key, default) + return self.__getitem__(key) + + def pop(self, key): + key = self._normalize_key(key) + value = self.__getitem__(key) + self.__delitem__(key) + return value + + def popitem(self): + if not self.__omap: + raise KeyError("VDFDict is empty") + key = self.__omap[-1] + return key[1], self.pop(key) + + def update(self, data=None, **kwargs): + if isinstance(data, dict): + data = data.items() + elif not isinstance(data, list): + raise TypeError("Expected data to be a list or dict, got %s" % type(data)) + + for key, value in data: + self.__setitem__(key, value) + + def iterkeys(self): + return (key[1] for key in self.__omap) + + def keys(self): + return _kView(self) + + def itervalues(self): + return (self[key] for key in self.__omap) + + def values(self): + return _vView(self) + + def iteritems(self): + return ((key[1], self[key]) for key in self.__omap) + + def items(self): + return _iView(self) + + def get_all_for(self, key): + """ Returns all values of the given key """ + if not isinstance(key, _string_type): + raise TypeError("Key needs to be a string.") + return [self[(idx, key)] for idx in _range(self.__kcount[key])] + + def remove_all_for(self, key): + """ Removes all items with the given key """ + if not isinstance(key, _string_type): + raise TypeError("Key need to be a string.") + + for idx in _range(self.__kcount[key]): + super(VDFDict, self).__delitem__((idx, key)) + + self.__omap = list(filter(lambda x: x[1] != key, self.__omap)) + + del self.__kcount[key] + + def has_duplicates(self): + """ + Returns ``True`` if the dict contains keys with duplicates. + Recurses through any all keys with value that is ``VDFDict``. + """ + for n in getattr(self.__kcount, _iter_values)(): + if n != 1: + return True + + def dict_recurse(obj): + for v in getattr(obj, _iter_values)(): + if isinstance(v, VDFDict) and v.has_duplicates(): + return True + elif isinstance(v, dict): + return dict_recurse(v) + return False + + return dict_recurse(self)