使用mitmdump-map local功能修改雪球行情信息,报编码错误

环境:win10,mumu模拟器,python3.7,mitmdump 5.3.0,已配置好证书

问题描述:
1.执行
C:\Users\Cioffi>mitmdump -p 8999 -s F:\Users\Cioffi\PycharmProjects\selenium_hogwarts\mitm_proxy\maplocal_xueqiu.py,第一次刷新雪球APP数据可与json文件同步展示,当修改json文件中的信息,再次刷新APP的行情也,报错信息具体见log:
Traceback (most recent call last):
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\net\http\encoding.py", line 67, in decode
    decoded = custom_decode[encoding](encoded)
KeyError: 'UTF-8'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\net\http\encoding.py", line 69, in decode
    decoded = codecs.decode(encoded, encoding, errors)  # type: ignore
  File "f:\program files\python37\lib\encodings\utf_8.py", line 16, in decode
    return codecs.utf_8_decode(input, errors, True)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 190: invalid start byte

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "f:\program files\python37\lib\threading.py", line 917, in _bootstrap_inner
    self.run()
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http2.py", line 722, in run
    layer()
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http.py", line 203, in __call__
    if not self._process_flow(flow):
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http.py", line 374, in _process_flow
    get_response()
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http.py", line 359, in get_response
    self.send_request_headers(f.request)
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http2.py", line 392, in wrapper
    result = func(self, *args, **kwargs)
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http2.py", line 612, in send_request_headers
    raise e
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\proxy\protocol\http2.py", line 606, in send_request_headers
    end_stream=(False if request.content or request.trailers or request.stream else True),
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\net\http\message.py", line 134, in get_content
    content = encoding.decode(self.raw_content, ce)
  File "c:\users\cioffi\.local\pipx\venvs\mitmproxy\lib\site-packages\mitmproxy\net\http\encoding.py", line 80, in decode
    repr(e),
ValueError: UnicodeDecodeError when decoding b'--lu5TPf with 'UTF-8': UnicodeDecodeError('utf-8', b'--lu5TPfj9hPUKYa9VVEgPVzSVfen6C5\r\nContent-Disposition: form-data; name="file"; filename="1621442194331.tar.gz"\r\nContent-Type: application/octet-stream\r\nContent-Transfer-Encoding: binary\r\n\r\n\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x00-\x8e9\x0e\xc20\x10\x00{\x9e\xe1\x1aE\xdeup\x1c:\x8a\x94I\x03=Z\xec\x8d\x14\x91K\xe0\x14\x08S!\xfe\x92\x8fQ\xf0\x0b\x0cJ9\xa3)\x064B\x9a"\x98\\\xa1\x0e\x80R\x06\x90A\x86\xbb\x18[\xf2\xf5p\xe9\xc4VT\xec\x99\xae|,\xa7r\x12k\xd1S\xc7\xd1\xbe_\xf3\xe79G&\xeb\x9b\xa1\x8f\xc6\xb6\x8d=G\xe1o\xe3/\x00\xf1\x08P\x15\x87b\xb7/LMd\xc8\xd4\x8e\x98\t\x1dK\xe7R\xd2\xec\xc0\x12\xa3u\xa7\x042T\x98\x81T\x9b\x04\xfeO\x90g\xb9R\xb8\xd0r\xb8\xfa\x02\x85\xa7\xdf\x02\xb0\x00\x00\x00\r\n--lu5TPfj9hPUKYa9VVEgPVzSVfen6C5--\r\n', 190, 191, 'invalid start byte')


2.python代码如下:
from mitmproxy import http


# 报错:待debug,数据未同步更新

# request 方法名不能修改,当有请求来的时候,就会回调request方法
# flow:请求接口中的数据
def request(flow: http.HTTPFlow):
    # 发起请求,获取当前接口的url,判断是否等于指定的路径
    if "quote.json" in flow.request.pretty_url:
        # 打开本地的数据文件
        with open(r"G:\hogwarts_workspace\apitest\map_local\quote2.json","r",encoding="utf-8") as f:
            # 创造一个response
            flow.response = http.HTTPResponse.make(
                200,  # (optional) status code
                f.read(),
                {"Content-Type": "application/json"}  # (optional) headers
            )
3.json数据如下:
{
	"data": {
		"items": [{
			"market": {
				"status_id": 5,
				"region": "US",
				"status": "交易中",
				"time_zone": "America/New_York",
				"time_zone_desc": null,
				"delay_tag": 0
			},
			"quote": {
				"symbol": "PDD",
				"code": "PDD",
				"exchange": "NASDAQ",
				"name": "hogwarts-afeifei",
				"type": 0,
				"sub_type": "1536",
				"status": 1,
				"current": 1200.93,
				"currency": "USD",
				"percent": -1200.519,
				"chg": -3.125,
				"timestamp": 1620918354007,
				"time": 1620918354007,
				"lot_size": 1,
				"tick_size": 0.01,
				"open": 124.32,
				"last_close": 124.055,
				"high": 124.63,
				"low": 120.8,
				"avg_price": 122.49,
				"volume": 1831825,
				"amount": 2.24430788676072E8,
				"turnover_rate": 0.15,
				"amplitude": 3.09,
				"market_capital": 1.51560214705E11,
				"float_market_capital": null,
				"total_shares": 1253288801,
				"float_shares": null,
				"issue_date": 1532534400000,
				"lock_set": 1,
				"current_year_percent": -31.94,
				"high52w": 212.5965,
				"low52w": 57.02,
				"variable_tick_size": "0.0001 1 0.01",
				"volume_ratio": 1.07,
				"eps": -0.887307142024273,
				"pe_ttm": -136.289,
				"pe_lyr": -136.2888,
				"navps": 7.5244025215680805,
				"pb": 16.261,
				"dividend": null,
				"dividend_yield": null,
				"psr": 18.32,
				"short_ratio": null,
				"inst_hld": null,
				"beta": null,
				"timestamp_ext": 1620912601010,
				"current_ext": 124.22,
				"percent_ext": 0.133,
				"chg_ext": 0.165,
				"contract_size": 100,
				"pe_forecast": -136.289,
				"profit_forecast": -1.1120521041463377E9,
				"profit": -1.1120521041463377E9,
				"profit_four": -1.1120521041463377E9,
				"pledge_ratio": null,
				"goodwill_in_net_assets": null,
				"shareholder_funds": 9.320491303068321E9
			},
			"others": {
				"cyb_switch": true
			},
			"tags": []
		}, {
			"market": {
				"status_id": 7,
				"region": "CN",
				"status": "已收盘",
				"time_zone": "Asia/Shanghai",
				"time_zone_desc": null,
				"delay_tag": 0
			},
			"quote": {
				"symbol": "SH601318",
				"code": "601318",
				"exchange": "SH",
				"name": "中国平安-afei",
				"type": 11,
				"sub_type": "ASH",
				"status": 1,
				"current": 690.14,
				"currency": "CNY",
				"percent": -1001.13,
				"chg": -0.79,
				"timestamp": 1620889200000,
				"time": 1620889200000,
				"lot_size": 100,
				"tick_size": 0.01,
				"open": 69.49,
				"last_close": 69.93,
				"high": 69.83,
				"low": 68.88,
				"avg_price": 69.228,
				"volume": 50644837,
				"amount": 3.50604801E9,
				"turnover_rate": 0.47,
				"amplitude": 1.36,
				"market_capital": 1.263895891087E12,
				"float_market_capital": 7.48970423392E11,
				"total_shares": 18280241410,
				"float_shares": 10832664498,
				"issue_date": 1172678400000,
				"lock_set": null,
				"current_year_percent": -19.01,
				"high52w": 92.8649,
				"low52w": 67.043,
				"limit_up": 76.92,
				"limit_down": 62.94,
				"volume_ratio": 0.87,
				"eps": 7.89,
				"pe_ttm": 8.761,
				"pe_forecast": 11.607,
				"pe_lyr": 8.832,
				"navps": 42.24,
				"pb": 1.637,
				"dividend": 2.2,
				"dividend_yield": 3.182,
				"profit": 1.43099E11,
				"profit_four": 1.44259E11,
				"profit_forecast": 1.08892E11,
				"pledge_ratio": 3.18,
				"goodwill_in_net_assets": 3.0222267116821357,
				"timestamp_ext": null,
				"current_ext": null,
				"volume_ext": null,
				"traded_amount_ext": null,
				"no_profit": null,
				"no_profit_desc": null,
				"weighted_voting_rights": null,
				"weighted_voting_rights_desc": null,
				"is_registration": null,
				"is_registration_desc": null,
				"is_vie": null,
				"is_vie_desc": null,
				"security_status": null
			},
			"others": {
				"cyb_switch": true
			},
			"tags": []
		}, {
			"market": {
				"status_id": 5,
				"region": "US",
				"status": "交易中",
				"time_zone": "America/New_York",
				"time_zone_desc": null,
				"delay_tag": 0
			},
			"quote": {
				"symbol": "BABA",
				"code": "BABA",
				"exchange": "NYSE",
				"name": "阿里巴巴",
				"type": 0,
				"sub_type": "1536",
				"status": 1,
				"current": 209.23,
				"currency": "USD",
				"percent": -4.8522,
				"chg": -10.67,
				"timestamp": 1620918356221,
				"time": 1620918356221,
				"lot_size": 1,
				"tick_size": 0.01,
				"open": 213.15,
				"last_close": 219.9,
				"high": 215.5,
				"low": 208.96,
				"avg_price": 212.2865,
				"volume": 19776541,
				"amount": 4.2033524370845E9,
				"turnover_rate": 0.73,
				"amplitude": 2.97,
				"market_capital": 5.67247344678E11,
				"float_market_capital": null,
				"total_shares": 2711118600,
				"float_shares": null,
				"issue_date": 1411056000000,
				"lock_set": 1,
				"current_year_percent": -10.1,
				"high52w": 319.32,
				"low52w": 194.03,
				"variable_tick_size": "0.0001 1 0.01",
				"volume_ratio": 5.46,
				"eps": 8.602591600976101,
				"pe_ttm": 24.322,
				"pe_lyr": 24.3217,
				"navps": 53.53473351610056,
				"pb": 3.9066,
				"dividend": null,
				"dividend_yield": null,
				"psr": 6.01,
				"short_ratio": null,
				"inst_hld": null,
				"beta": null,
				"timestamp_ext": 1620912600942,
				"current_ext": 213.19,
				"percent_ext": -3.0514,
				"chg_ext": -6.71,
				"contract_size": 100,
				"pe_forecast": 24.322,
				"profit_forecast": 2.332264609761009E10,
				"profit": 2.332264609761009E10,
				"profit_four": 2.332264609761009E10,
				"pledge_ratio": null,
				"goodwill_in_net_assets": 31.229906023659424,
				"shareholder_funds": 1.4520236048510757E11
			},
			"others": {
				"cyb_switch": true
			},
			"tags": []
		}, {
			"market": {
				"status_id": 5,
				"region": "US",
				"status": "交易中",
				"time_zone": "America/New_York",
				"time_zone_desc": null,
				"delay_tag": 0
			},
			"quote": {
				"symbol": "JD",
				"code": "JD",
				"exchange": "NASDAQ",
				"name": "京东",
				"type": 0,
				"sub_type": "1536",
				"status": 1,
				"current": 70.0,
				"currency": "USD",
				"percent": -1.7544,
				"chg": -1.25,
				"timestamp": 1620918355175,
				"time": 1620918355175,
				"lot_size": 1,
				"tick_size": 0.01,
				"open": 71.07,
				"last_close": 71.25,
				"high": 71.23,
				"low": 70.0,
				"avg_price": 70.5428,
				"volume": 3648027,
				"amount": 2.57386013699498E8,
				"turnover_rate": 0.23,
				"amplitude": 1.73,
				"market_capital": 1.0912896813E11,
				"float_market_capital": null,
				"total_shares": 1558985259,
				"float_shares": null,
				"issue_date": 1400688000000,
				"lock_set": 1,
				"current_year_percent": -20.36,
				"high52w": 108.29,
				"low52w": 46.37,
				"variable_tick_size": "0.0001 1 0.01",
				"volume_ratio": 1.41,
				"eps": 4.908481437556622,
				"pe_ttm": 14.261,
				"pe_lyr": 14.261,
				"navps": 18.719591716617877,
				"pb": 3.7568,
				"dividend": null,
				"dividend_yield": null,
				"psr": 0.97,
				"short_ratio": null,
				"inst_hld": null,
				"beta": null,
				"timestamp_ext": 1620912597135,
				"current_ext": 71.19,
				"percent_ext": -0.0842,
				"chg_ext": -0.06,
				"contract_size": 100,
				"pe_forecast": 14.261,
				"profit_forecast": 7.6522502052259035E9,
				"profit": 7.6522502052259035E9,
				"profit_four": 7.6522502052259035E9,
				"pledge_ratio": null,
				"goodwill_in_net_assets": 5.8143422296169,
				"shareholder_funds": 2.9048107275064667E10
			},
			"others": {
				"cyb_switch": true
			},
			"tags": []
		}],
		"items_size": 4
	},
	"error_code": 0,
	"error_description": ""
}










同学,你这个报错信息好像没有传好,主要还是要看log是什么原因

已经重新粘贴了log的内容,麻烦帮忙看下,多谢啦

这个解码问题不确定是不是win系统的问题,看看这个帖子能不能帮到你:

我好像看到你哪里问题了,你的这行代码:

with open(r"G:\hogwarts_workspace\apitest\map_local\quote2.json","r",encoding="utf-8") as f:

把“with open(r"")”里面那个r去掉吧,看看,这里应该是直接贴你的json文件绝对路径就行

此处需要防止字符转义,路径中包含\a

你可以把你路径里那个 apitest换一个名字,就不用转义了,这是最快解决问题的办法。

如果你要坚持这个路径,那就还是要看看这句里有没有别的问题,总之报错就是出在这个处理上哈

已修改路径,报错依然存在

用来转义的r已经去掉了对吗

是的

image

把要判断的语句换成这个试试:

if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t=" in flow.request.pretty_url:
报错依然存在

没辙了,看不出来什么问题,那看看我之前发那个链接尝试别的解决方法哈

你的文件内容是gbk吧,然后你尝试用utf8解析?

我也是报这个错误,然后我单独执行with open那个命令不报错,执行windows命令就报这个错误

json的编码格式也是utf-8的

我之前也遇到类似的问题,实际是安装的mimtproxy版本为5.3.0导致的,你卸载重装mimtproxy为5.2.0版本试试

问题已解决,感谢啊。果然5.2版本用着明显比5.3顺畅,之前使用5.3版本,加载修改后的json或python文件时会明显卡顿一会儿。