Skip to content

Commit 03e0ebc

Browse files
authored
fix(eval): close raw tunnels at the layer that owns the sockets (#3017)
* 修复:仅通过 next_layer 契约关闭原始隧道 * 修复:保留 TCP 钩子直到层分类完成 * 测试:使用真实 mitmproxy 覆盖原始隧道 * 修复:由套接字所有层关闭原始隧道 * 修复:排除 HTTP 前缀后再关闭原始 TCP * 测试:缺少源镜像时让代理实测失败 * 修复:审计未保持 WebSocket 的 101 升级 * 让出口代理实测进入持续集成 * 补充出口代理实测许可头 * 修复出口代理碎片化 TLS 与关闭验证 * 修复 TLS 分片握手测试死锁 * 修复出口代理的碎片化 TLS 分类
1 parent 3f5c4bb commit 03e0ebc

7 files changed

Lines changed: 995 additions & 10 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,18 @@ jobs:
192192
fi
193193
node scripts/run-workspace-tests-parallel.mjs --concurrency=3 --workspaces="$WORKSPACES"
194194
195+
- name: Run live Eval egress proxy test
196+
if: contains(steps.plan.outputs.standard_workspaces, 'packages/eval')
197+
env:
198+
MAKA_EVAL_EGRESS_PROXY_TEST: '1'
199+
run: |
200+
docker pull python:3.12-slim
201+
docker build \
202+
--tag maka-eval-egress-proxy:12.2.3 \
203+
--file packages/eval/harbor/egress-proxy/Dockerfile \
204+
packages/eval/harbor
205+
npm --workspace @maka/eval run test:egress-proxy:live
206+
195207
- name: Run Runtime Host tests
196208
if: steps.plan.outputs.runtime_host == 'true'
197209
run: npm --workspace @maka/runtime-host run test:dist

‎packages/eval/harbor/egress-proxy/entrypoint.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,5 +57,6 @@ exec mitmdump \
5757
--listen-host 0.0.0.0 \
5858
--listen-port 8080 \
5959
--set block_global=false \
60+
--set rawtcp=false \
6061
--set confdir="$STATE_DIR" \
6162
--scripts /opt/maka-eval/egress_filter.py

‎packages/eval/harbor/egress_filter.py‎

Lines changed: 139 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -120,14 +120,52 @@ def public_trajectory_repository(host: str, path_query: str) -> bool:
120120

121121
try:
122122
from mitmproxy.proxy import commands as proxy_commands
123+
from mitmproxy.proxy.layer import Layer
124+
from mitmproxy.proxy.layers import ClientTLSLayer, ServerTLSLayer
125+
from mitmproxy.proxy.layers.tcp import TCPLayer
123126
except ImportError:
124127
proxy_commands = None
128+
Layer = object
129+
ClientTLSLayer = None
130+
ServerTLSLayer = None
131+
TCPLayer = None
132+
133+
try:
134+
from mitmproxy.net.tls import starts_like_tls_record
135+
except ImportError:
136+
def starts_like_tls_record(data: bytes) -> bool:
137+
return len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03
138+
139+
140+
def configure(updated: object) -> None:
141+
# HTTP 101 upgrades construct TCPLayer inside the HTTP layer without
142+
# another next_layer hook. rawtcp=false makes that path CloseConnection
143+
# itself; WebSocket upgrades stay on the websocket layer.
144+
try:
145+
from mitmproxy import ctx
146+
except ImportError:
147+
return
148+
if getattr(ctx.options, "rawtcp", False):
149+
ctx.options.rawtcp = False
125150

126151

127152
def request(flow: object) -> None:
128153
apply_http_policy(flow, flow.request.pretty_url)
129154

130155

156+
def response(flow: object) -> None:
157+
response = getattr(flow, "response", None)
158+
if getattr(response, "status_code", None) != 101:
159+
return
160+
# mitmproxy 12.2.3 sets flow.websocket before HttpResponseHook only when
161+
# the 101 is a real WebSocket upgrade (Upgrade + version 13 + option on).
162+
# A websocket Upgrade header alone still falls through to CloseConnection
163+
# under rawtcp=false and must be audited.
164+
if getattr(flow, "websocket", None) is not None:
165+
return
166+
record_raw_tunnel(flow)
167+
168+
131169
def http_connect(flow: object) -> None:
132170
try:
133171
raw_url = connect_target_url(flow)
@@ -137,6 +175,9 @@ def http_connect(flow: object) -> None:
137175

138176

139177
def tcp_start(flow: object) -> None:
178+
# Last resort if a TCPLayer is still admitted (tcp_hosts / ignore).
179+
# CONNECT raw is closed by next_layer → CloseRawLayer; HTTP 101 raw is
180+
# closed by rawtcp=false. Neither of those paths starts a TCPLayer.
140181
record_raw_tunnel(flow)
141182
kill_flow(flow)
142183

@@ -149,12 +190,31 @@ def tcp_message(flow: object) -> None:
149190

150191

151192
def next_layer(nextlayer: object) -> None:
193+
# Script addons run before the built-in classifier assigns layer. If we
194+
# set CloseRawLayer here, NextLayer leaves it in place. Waiting for
195+
# isinstance(..., TCPLayer) never fires on the production CONNECT path.
152196
current = getattr(nextlayer, "layer", None)
153-
if current is None or type(current).__name__ != "TCPLayer":
154-
return
155197
context = getattr(nextlayer, "context", None)
198+
if current is None:
199+
data_client = _next_layer_bytes(nextlayer, "data_client")
200+
if _is_fragmented_tls_record_prefix(data_client):
201+
if ClientTLSLayer is None or ServerTLSLayer is None:
202+
return
203+
server_tls = ServerTLSLayer(context)
204+
server_tls.child_layer = ClientTLSLayer(context)
205+
nextlayer.layer = server_tls
206+
return
207+
if not looks_like_raw_tcp(nextlayer):
208+
return
209+
record_raw_tunnel(context)
210+
nextlayer.layer = CloseRawLayer(context)
211+
return
212+
if TCPLayer is None or not isinstance(current, TCPLayer):
213+
return
156214
record_raw_tunnel(context)
157-
nextlayer.layer = CloseRawLayer(context)
215+
closer = CloseRawLayer(context)
216+
replace_layer(context, current, closer)
217+
nextlayer.layer = closer
158218

159219

160220
def apply_http_policy(flow: object, raw_url: str) -> None:
@@ -197,6 +257,58 @@ def connect_target_url(flow: object) -> str:
197257
return f"https://{host}:{port}/"
198258

199259

260+
def looks_like_raw_tcp(nextlayer: object) -> bool:
261+
"""Close only when the bytes cannot still become HTTP or TLS.
262+
263+
Script next_layer runs before mitmproxy 12.2.3's classifier. Copying its
264+
`probably_no_http` test here would treat `GET` / `GET ` as raw and assign
265+
CloseRawLayer while the request line is still arriving. With rawtcp=false
266+
the built-in path would have kept waiting for HttpLayer.
267+
"""
268+
data_client = _next_layer_bytes(nextlayer, "data_client")
269+
data_server = _next_layer_bytes(nextlayer, "data_server")
270+
if _could_start_tls_record(data_client):
271+
return False
272+
if not data_client and not data_server:
273+
return False
274+
if data_server or data_client.startswith(b"SSH"):
275+
return True
276+
return not _still_could_be_http(data_client)
277+
278+
279+
def _could_start_tls_record(data: bytes) -> bool:
280+
"""Keep a fragmented ClientHello undecided until its 3-byte prefix exists."""
281+
if starts_like_tls_record(data):
282+
return True
283+
return _is_fragmented_tls_record_prefix(data)
284+
285+
286+
def _is_fragmented_tls_record_prefix(data: bytes) -> bool:
287+
return 0 < len(data) < 3 and b"\x16\x03".startswith(data)
288+
289+
290+
def _still_could_be_http(data: bytes) -> bool:
291+
first_line, newline, _rest = data.partition(b"\n")
292+
line = first_line.rstrip(b"\r")
293+
method, space, _remainder = line.partition(b" ")
294+
if not method.isascii() or not method.isalpha():
295+
return False
296+
if newline and not space:
297+
return False
298+
return True
299+
300+
301+
def _next_layer_bytes(nextlayer: object, name: str) -> bytes:
302+
getter = getattr(nextlayer, name, None)
303+
if not callable(getter):
304+
return b""
305+
try:
306+
data = getter()
307+
except Exception:
308+
return b""
309+
return bytes(data) if isinstance(data, (bytes, bytearray)) else b""
310+
311+
200312
def tcp_peer(flow: object) -> tuple[str, str]:
201313
server = getattr(flow, "server_conn", None) or getattr(flow, "server", None)
202314
address = getattr(server, "address", None) if server is not None else None
@@ -224,9 +336,31 @@ def kill_flow(flow: object) -> None:
224336
pass
225337

226338

227-
class CloseRawLayer:
339+
def replace_layer(context: object, current: object, closer: object) -> None:
340+
layers = getattr(context, "layers", None)
341+
if not isinstance(layers, list):
342+
return
343+
try:
344+
index = layers.index(current)
345+
except ValueError:
346+
index = len(layers)
347+
if closer in layers:
348+
layers.remove(closer)
349+
if current in layers:
350+
layers.remove(current)
351+
layers.insert(min(index, len(layers)), closer)
352+
353+
354+
class CloseRawLayer(Layer):
228355
def __init__(self, context: object) -> None:
229-
self.context = context
356+
if Layer is object:
357+
self.context = context
358+
return
359+
if getattr(context, "layers", None) is None:
360+
context.layers = []
361+
if getattr(context, "options", None) is None:
362+
context.options = type("Options", (), {"proxy_debug": False})()
363+
super().__init__(context)
230364

231365
def handle_event(self, event: object):
232366
if proxy_commands is None:

0 commit comments

Comments
 (0)