WyOJ Datums 可用，添加流式输出

autoconf.py

@@ -0,0 +1,117 @@
+import re
+import sys
+import os
+
+# 输入、输出文件后缀。可更改
+insuf, outsuf = "in", "out"
+
+print ("problem.conf 自助配置")
+path = 'data'
+
+spjp, stdp, valp, subtaskp = False, False, False, False
+
+def continuep ():
+    pass
+
+for s in os.listdir ("."):
+    if s == 'problem.conf':
+        print ("警告：problem.conf 已存在，将会被覆盖")
+        continuep ()
+
+    if s == 'chk.cpp':
+        spjp = True
+    if s == 'std.cpp':
+        stdp = True
+    if s == 'val.cpp':
+        valp = True
+    if os.path.isdir (s) and re.match ("subtask[0-9]*", s):
+        subtaskp = True
+
+output = open ("problem.conf", "w")
+chker = None
+if spjp:
+    print ("识别到 chk.cpp，已启用 Special Judge")
+else:
+#    chker = input ("未识别到 chk.cpp，请输入自定义校验器类型：")
+    chker = 'wcmp'
+print ("use_builtin_judger on", file=output)
+if chker:
+    print ("use_builtin_checker %s" % chker, file=output)
+
+if subtaskp:
+    print ("识别到 subtaskX 目录，自动识别 Subtask 中")
+else:
+    print ("未识别到 subtaskX 目录，自动尝试普通配置（如果你认为 Subtask 存在，但文件名格式为 subXX_XX.in/out，尝试使用 subxx2datax.py 脚本）")
+
+
+tlim = int (sys.argv[1])
+mlim = int (sys.argv[2])
+print ("time_limit %d\nmemory_limit %d" % (tlim, mlim), file=output)
+patt = re.compile ("([A-Z]*[a-z]*[A-Z]*)([0-9]+).(%s|%s)" % (insuf, outsuf))
+
+def processdir (path):
+    tcases = 0
+    pre = None
+
+    for s in os.listdir (path):
+        if os.path.isdir (s):
+            continue
+        q = re.findall (patt, s)
+        if len (q) != 1:
+            continue
+        print ("识别到数据文件 %s" % s)
+        q = q[0]
+        if not pre:
+            pre = q[0]
+        elif pre != q[0]:
+            raise Exception ("数据文件前缀不相同！同时找到了 %s 与 %s，自动配置失败。" % (pre, q[0]))
+        if int (q[1]) > tcases:
+            tcases = int (q[1])
+    if tcases == 0 or not pre:
+        raise Exception ("无法自动识别数据文件")
+    return (tcases, pre)
+
+if not subtaskp:
+    tcases, pre = processdir (".")
+    print ("找到 %d 个测试点" % tcases)
+    print ("n_tests %d" % tcases, file=output)
+    print ("n_ex_tests 0\nn_sample_tests 0", file=output)
+    print ("input_pre %s\ninput_suf %s" % (pre, insuf), file=output)
+    print ("output_pre %s\noutput_suf %s" % (pre, outsuf), file=output)
+    print ("自动配置结束！")
+else:
+    tcases, subs = 0, 0
+    pre = None
+    subtptt = re.compile ("subtask([0-9]*)")
+    end = {  }
+
+    for s in os.listdir ("."):
+        if os.path.isdir (s):
+            q = re.findall (subtptt, s)
+            if len (q) != 1:
+                print ("忽略非 Subtask 目录 %s" % s)
+                continue
+            print ("找到 subtask 目录 %s" % s)
+            subs += 1
+            nc, spre = processdir (s)
+            if not pre:
+                pre = spre
+            elif pre != spre:
+                raise Exception ("Subtask 间文件前缀不同！同时找到了 %s 与 %s，自动配置失败。" % (pre, spre))
+            end[int (q[0])] = nc
+            if nc > tcases:
+                tcases = nc
+    print ("n_tests %d\nn_ex_tests 0\nn_sample_tests 0" % tcases, file=output)
+    print ("input_pre %s\ninput_suf %s" % (pre, insuf), file=output)
+    print ("output_pre %s\noutput_suf %s" % (pre, outsuf), file=output)
+    print ("找到 %d 个 Subtask，共 %d 个测试点" % (subs, tcases))
+
+    scs = 100.0 / subs
+    print ("正在按照每个 Subtask 等分配置（%.2f 分）" % scs)
+    print ("n_subtasks %d" % subs, file=output)
+    for i in range (1, subs + 1):
+        print ("subtask_end_%d %d" % (i, end[i]), file=output)
+        print ("subtask_score_%d %.2f" % (i, scs), file=output)
+
+output.close ()
+

daemon.py

@@ -0,0 +1,108 @@
+import os
+import traceback
+import threading
+import pymysql
+import time
+import subprocess
+
+GAP = 2
+
+def execute(s):
+    print('Running: ', s)
+    try:
+        subprocess.run(s, shell=True, check=True)
+    except subprocess.CalledProcessError as e:
+        raise Exception('Failed to execute the command')
+
+def watch_stream():
+    db = pymysql.connect(host='uoj-db', user='root', password='root', database='app_uoj233')
+    db.autocommit(True)
+    with open('did', 'r') as f:
+        did = int(f.read().strip())
+    print(f'Watching request id {did}')
+    last = ''
+    curs = db.cursor()
+    curs.execute('insert ignore into datum_streams (id) values (%s)', (did, ))
+
+    while True:
+        if not os.path.exists('stream'):
+            continue
+        with open('stream', 'r') as f:
+            d = f.read()
+            if last == d:
+                if open('flag').read() == 'done':
+                    print('Finished!')
+                    return
+                continue
+        curs = db.cursor()
+        curs.execute('update datum_streams set stream = %s where id = %s', (d, did))
+        last = d
+        time.sleep(GAP)
+
+def generate_data_zip(p, stat):
+    watcher = threading.Thread(target=watch_stream)
+    open('flag', 'w').write('no')
+    open('did', 'w').write('')
+    open('stream', 'w').write('')
+
+    did, pid, std, timelim, memlim = p
+    print(f'Received request for problem {pid}')
+    with open('stat.md', 'w') as f:
+        f.write(stat)
+    with open('std.cpp', 'w') as f:
+        f.write(std)
+    with open('did', 'w') as f:
+        print(did, file=f)
+    watcher.start()
+    execute(f'sudo -u datum python3 datum.py stat.md --time {timelim} --mem {memlim}')
+    open('flag', 'w').write('done')
+    watcher.join()
+    open('flag', 'w').write('no')
+    open('did', 'w').write('')
+    open('stream', 'w').write('')
+
+def move_to_upload(pid):
+    execute(f'rm -r /var/uoj_data/web/data/upload/{pid} || true')
+    execute(f'mkdir /var/uoj_data/web/data/upload/{pid} || true')
+    execute(f'mv data.zip /var/uoj_data/web/data/upload/{pid}/')
+    execute(f'cd /var/uoj_data/web/data/upload/{pid}/; unzip data.zip; rm data.zip')
+
+if __name__ == '__main__':
+    db = pymysql.connect(host='uoj-db', user='root', password='root', database='app_uoj233')
+    db.autocommit(True)
+
+    while True:
+        curs = db.cursor()
+        curs.execute('select id, pid, std, timelimit, memlimit from datum_requests where status = "Waiting" order by id asc limit 1')
+        res = curs.fetchall()
+        if not res:
+            time.sleep(GAP)
+            continue
+
+        curs = db.cursor()
+        curs.execute(f'update datum_requests set status = "Generating" where id = {res[0][0]}')
+        db.commit()
+
+        curs = db.cursor()
+        curs.execute(f'select statement_md from problems_contents where id = {res[0][1]}')
+
+        try:
+            generate_data_zip(res[0], curs.fetchall()[0][0])
+            move_to_upload(res[0][1])
+        except Exception as e:
+            print(f'Exception {e}')
+            traceback.print_exc()
+            curs = db.cursor()
+            curs.execute(f'update datum_requests set status = "Failed" where id = {res[0][0]}')
+            curs.execute(f'update datum_requests set finish_time = now() where id = {res[0][0]}')
+            db.commit()
+            print(f'Failed {res[0][0]}')
+        else:
+            curs = db.cursor()
+            generator = open('gen.py', 'r').read()
+            curs.execute(f'update datum_requests set status = "Done" where id = {res[0][0]}')
+            curs.execute('update datum_requests set generator = %s where id = %s', (generator, res[0][0]))
+            curs.execute(f'update datum_requests set finish_time = now() where id = {res[0][0]}')
+            db.commit()
+            print(f'Done {res[0][0]}')
+

datum.py

@@ -0,0 +1,156 @@
+import os
+import time
+import subprocess
+import sys
+import argparse
+import time
+from openai import OpenAI
+
+def fetch_generator(statement, model, maxtokens):
+    APIKEY = os.environ.get('DEEPSEEK_API_KEY')
+    if not APIKEY:
+        raise Exception('需要设置 DEEPSEEK_API_KEY 环境变量')
+
+    client = OpenAI(
+        api_key=APIKEY,
+        base_url='https://api.deepseek.com')
+
+    start_time = time.time()
+    msgs = [{ 'role': 'user', 'content': 
+             """
+请根据以下信息学竞赛题目生成数据生成器。只输出 gen.py 代码。
+要求：
+1. 解析题目中的输入格式、数据范围和子任务划分
+2. 创建 data/ 目录并生成所有 .in 文件
+3. 数据严格符合题目约束
+4. 使用合理的数据梯度，包含边界情况
+5. 数据文件名一定为 'data{编号}.in'，编号不含前导零
+6. 没有子任务的，数据文件直接放在 data/ 下；有子任务的，为每个子任务创建 subtask{编号}/ 目录，把数据文件移动到对应的子任务目录中
+7. 数据点在 10 到 20 个内
+8. 务必确保你给出的数据是正确的！
+输出只包含可以直接运行的 gen.py 的代码，无其他内容。
+题目描述：""" + statement}]
+    resp = client.chat.completions.create(
+            model=model,
+            messages=msgs,
+            stream=True,
+            temperature=0.0,
+            max_tokens=maxtokens)
+
+    response = ''
+    for c in resp:
+        d = c.choices[0].delta
+        if hasattr(d, 'reasoning_content') and d.reasoning_content is not None:
+            delta = d.reasoning_content
+        elif hasattr(d, 'content') and d.content is not None:
+            delta = d.content
+            if response == '':
+                delta = '\nThinking done\n' + delta
+            response += d.content
+        else:
+            continue
+        
+        with open('stream', 'a') as f:
+            f.write(delta)
+        if hasattr(c, 'usage') and c.usage:
+            tokens = c.usage
+
+    if response.startswith('```python'):
+        response = response[9:-3]
+    return (response, tokens, time.time() - start_time)
+
+def execute(s):
+    print('正在执行命令：', s)
+
+    try:
+        subprocess.run(['sh', '-c', s], check=True, timeout=20)
+    except subprocess.CalledProcessError as e:
+        raise Exception('命令执行错误')
+    except subprocess.TimeoutExpired:
+        raise Exception('命令执行超时（20s）')
+
+def generate_in():
+    # 其实应该用个沙盒比较安全，但是我懒得写了。可以最后扔到容器里头跑。
+    execute('python gen.py')
+
+def flatten_data():
+    execute('find -type f | xargs -I {} mv -n {} . || true')
+    execute('rmdir subtask* || true')
+
+def run_std():
+    if not os.path.exists('../std.cpp'):
+        raise Exception('未找到 std.cpp')
+    execute('g++ ../std.cpp -o std -Wall -Wextra -O2 -std=c++14')
+
+    for i in os.listdir():
+        if not i.endswith('.in'):
+            print(f'警告：未知文件 {i}')
+            continue
+        out = i[:-2] + 'ans'
+        execute(f'./std < {i} > {out}')
+    execute('rm std')
+
+def gen_problem_conf(time, mem):
+    execute(f'python ../autoconf.py {time} {mem}')
+
+def compress_data():
+    execute('zip ../data.zip -r .')
+
+def cleanup():
+    os.chdir('..')
+    execute('rm -r data/')
+
+def pretty_size(d):
+    if d < 1024:
+        return f'{d}'
+    if d < 1048576:
+        return f'{d/1024:.2f} KB'
+    return f'{d/1048576:.2f} MB'
+
+if __name__ == '__main__':
+    start_time = time.time()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('statement', help='题面文件')
+    parser.add_argument('--model', type=str, default='deepseek-v4-pro', help='使用的模型（默认为 deepseek-v4-pro，可以改为 deepseek-v4-flash）')
+    parser.add_argument('--maxtokens', type=int, default=20000, help='token 上限（默认为 20000）')
+    parser.add_argument('--time', type=int, default=1, help='时间限制（秒，默认为 1）')
+    parser.add_argument('--mem', type=int, default=256, help='空间限制（MB，默认为 256）')
+    parser.add_argument('--offline', help='不生成新的 gen.py', action='store_true')
+    args = parser.parse_args()
+
+    if os.path.exists('data'):
+        print('正在清空已有的 data 目录')
+        execute('rm -r data')
+    if os.path.exists('data.zip'):
+        print('删除已存在的 data.zip')
+        execute('rm data.zip')
+
+    tokens = 0
+
+    with open(args.statement) as f:
+        if not args.offline:
+            print(f'正在请求 {args.model}，token 上限为 {args.maxtokens} 个')
+            resp, tokens, duration = fetch_generator(f.read(), args.model, args.maxtokens)
+            with open('gen.py', 'w') as fw:
+                fw.write(resp)
+            print(f'--- tokens: {tokens}, time cost: {duration:.6f} seconds')
+        else:
+            if not os.path.exists('gen.py'):
+                raise Exception('离线模式下必须提供已有的 gen.py')
+            print('（离线模式）')
+
+        generate_in()
+        os.chdir('data')
+        gen_problem_conf(args.time, args.mem)
+
+        # run_std 需要一个扁平的目录，也就是所有文件放在 data/ 下，而 autoconf.py 不要求这么做
+        flatten_data()
+        run_std()
+        compress_data()
+        cleanup()
+        prettysize = pretty_size(os.path.getsize('data.zip'))
+        print(f'已生成 data.zip（{prettysize}），耗时 {(time.time() - start_time):.2f} 秒，消耗 {tokens} 个 token')
+
+
+