Repository navigation
Expand file tree
/
Copy pathinference.py
More file actions
122 lines (104 loc) · 4.07 KB
/
Copy pathinference.py
File metadata and controls
122 lines (104 loc) · 4.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import asyncio
import os
from loguru import logger
from sglang.utils import launch_server_cmd, terminate_process, wait_for_server
from tap import Tap
from tortoise import run_async
from tqdm.auto import tqdm
import confidence
from confidence.data import Data, Template
from confidence.dataset import DatasetName
from confidence.logger import Logger
from confidence.model import ChatResponse, Model, ModelName
from confidence.result import Result
from confidence.utils import last_git_hash, limit_concurrency
class Argument(Tap):
model: ModelName = ModelName.QWEN3_8B_THINK
model_name_or_path: str = "Qwen/Qwen3-8B"
dataset: DatasetName = DatasetName.TimeTabling
template: Template = "simple"
temperature: float = 0.2
max_completion_tokens: int = 20480
force_update: bool = False
concurrency: int = 100
turn: int = 0
async def request(
model: Model,
template: Template,
data: Data,
temperature: float,
max_completion_tokens: int = 20480,
) -> tuple[Data, Result[ChatResponse, str]]:
response_result = await confidence.request(
model, data, template, temperature=temperature, max_completion_tokens=max_completion_tokens
)
return data, response_result
async def main(args: Argument):
record_cls = args.dataset.record_cls
title = f"{args.dataset}--{args.template}--{args.model}--{args.temperature}--{args.turn}".replace("/", "_")
db_logger = Logger(db_name=title, table_name=title, record_cls=record_cls, force_update=args.force_update)
async with db_logger:
# ===== model =====
model = Model(args.model, args.model_name_or_path)
# ===== dataset =====
dataset_cls = args.dataset.dataset_cls
dataset = dataset_cls().load_resume_dataset(
await db_logger.already_processed_question_ids(),
force_restart=args.force_update,
)
if len(dataset) > 0:
logger.info(f"Example: {dataset[0]}")
# ===== task =====
tasks = [
request(
model=model,
template=args.template,
data=data,
temperature=args.temperature,
max_completion_tokens=args.max_completion_tokens,
)
for data in dataset
]
tasks = limit_concurrency(tasks, args.concurrency)
async for zip_response in tqdm(asyncio.as_completed(tasks), total=len(tasks), desc=title):
data: Data
response_result: Result[ChatResponse, str]
data, response_result = await zip_response
if response_result.is_err():
logger.error(response_result.err_value)
continue
thinking = response_result.ok_value.thinking
record = record_cls(
**data.model_dump(),
chat_history=response_result.ok_value.messages,
thinking_history=thinking if thinking is not None else [],
model=args.model.value,
dataset=args.dataset.value,
template=args.template,
temperature=args.temperature,
git_hash=last_git_hash(),
)
await db_logger.insert(record)
if __name__ == "__main__":
args = Argument().parse_args()
os.environ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] = "1"
server_process, port = launch_server_cmd(
(
"python3 -m sglang.launch_server "
"--tp 8 "
"--dp 1 "
f"--model-path {args.model_name_or_path} "
f"--served-model-name {args.model} "
"--reasoning-parser qwen3 "
"--context-length 131072 "
"""--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}} """
"--host 0.0.0.0 "
"--port 33333 "
"--log-level warning "
)
)
wait_for_server(f"http://localhost:{port}")
os.environ["BASE_URL"] = f"http://localhost:{port}/v1"
os.environ["API_KEY"] = "sglang"
run_async(main(args))
terminate_process(server_process)