From 05585803434c043c5e6d29996d8738727eff43a8 Mon Sep 17 00:00:00 2001 From: hailin Date: Sun, 29 Jun 2025 18:55:37 +0800 Subject: [PATCH] first commit @ sglang v0.4.5.post1 --- .clang-format-ignore | 1 + .devcontainer/Dockerfile | 35 + .devcontainer/devcontainer.json | 24 + .editorconfig | 25 + .gitignore | 229 ++ .gitmodules | 12 + .isort.cfg | 3 + .pre-commit-config.yaml | 48 + 3rdparty/amd/profiling/PROFILING.md | 425 +++ 3rdparty/amd/profiling/client.sh | 27 + 3rdparty/amd/profiling/install_rpd.sh | 10 + 3rdparty/amd/profiling/loadTracer.sh | 43 + 3rdparty/amd/profiling/rpd.patch | 12 + .../profiling/rpd_profile_server_enable.patch | 49 + ...rofile_server_enable_wCPU_activities.patch | 126 + 3rdparty/amd/profiling/server.sh | 20 + 3rdparty/amd/profiling/torch_profiler.patch | 25 + 3rdparty/amd/tuning/TUNING.md | 118 + 3rdparty/amd/tuning/benchmark_moe_rocm.py | 380 +++ LICENSE | 201 ++ Makefile | 43 + README.md | 75 + assets/logo.png | Bin 0 -> 402692 bytes .../bench_in_batch_prefix.py | 130 + benchmark/benchmark_vllm_060/README.md | 89 + benchmark/blog_v0_2/405b_sglang.sh | 24 + benchmark/blog_v0_2/405b_trt.sh | 17 + benchmark/blog_v0_2/405b_vllm.sh | 24 + benchmark/blog_v0_2/README.md | 164 ++ benchmark/blog_v0_2/config.md | 100 + benchmark/deepseek_v3/README.md | 285 ++ benchmark/dspy/README.md | 51 + benchmark/dspy/bench_dspy_intro.py | 192 ++ benchmark/generative_agents/README.md | 38 + .../generative_agents/agent_functions.py | 300 ++ benchmark/generative_agents/bench_other.py | 80 + benchmark/generative_agents/bench_sglang.py | 74 + benchmark/gsm8k/README.md | 47 + benchmark/gsm8k/bench_other.py | 151 + benchmark/gsm8k/bench_sglang.py | 141 + benchmark/hellaswag/README.md | 47 + benchmark/hellaswag/bench_other.py | 118 + benchmark/hellaswag/bench_sglang.py | 109 + benchmark/hicache/README.md | 91 + benchmark/hicache/bench_multiturn.py | 395 +++ benchmark/hicache/bench_serving.py | 1029 +++++++ benchmark/hicache/data_processing.py | 589 ++++ benchmark/hicache/download.sh | 66 + benchmark/hicache/nextqa.py | 159 + benchmark/json_decode_regex/README.md | 60 + benchmark/json_decode_regex/bench_other.py | 98 + benchmark/json_decode_regex/bench_sglang.py | 101 + benchmark/json_decode_regex/build_dataset.py | 58 + benchmark/json_jump_forward/README.md | 88 + benchmark/json_jump_forward/bench_other.py | 288 ++ benchmark/json_jump_forward/bench_sglang.py | 143 + benchmark/json_jump_forward/build_dataset.py | 58 + benchmark/json_jump_forward/dataset.txt | 50 + benchmark/json_schema/README.md | 15 + benchmark/json_schema/bench_sglang.py | 146 + .../triton_flashinfer_cudnn.py | 403 +++ benchmark/kernels/deepseek/README.md | 19 + .../deepseek/benchmark_deepgemm_fp8_gemm.py | 398 +++ .../benchmark_deepgemm_fp8_group_gemm.py | 486 ++++ benchmark/kernels/fused_moe_triton/README.md | 49 + .../benchmark_torch_compile_fused_moe.py | 294 ++ ...nchmark_vllm_vs_sglang_fused_moe_triton.py | 299 ++ .../tuning_fused_moe_triton.py | 564 ++++ .../benchmark_lightning_attention_decode.py | 576 ++++ .../benchmark_lightning_attention_prefill.py | 603 ++++ .../kernels/quantization/bench_int8_quant.py | 94 + .../quantization/tuning_block_wise_kernel.py | 474 +++ .../kernels/rmsnorm/benchmark_rmsnorm.py | 230 ++ ...enchmark_write_req_to_token_pool_triton.py | 342 +++ benchmark/line_retrieval/README.md | 37 + benchmark/line_retrieval/bench_sglang.py | 149 + benchmark/line_retrieval/gen_data.py | 139 + benchmark/llava_bench/README.md | 61 + benchmark/llava_bench/bench_hf_llava_bench.sh | 9 + benchmark/llava_bench/bench_hf_mme.sh | 9 + benchmark/llava_bench/bench_sglang.py | 96 + benchmark/llava_bench/bench_sglang_mme.sh | 2 + benchmark/llava_bench/download_images.py | 20 + benchmark/llm_judge/README.md | 33 + benchmark/llm_judge/bench_other.py | 151 + benchmark/llm_judge/bench_sglang.py | 97 + benchmark/long_json_decode/README.md | 33 + benchmark/long_json_decode/bench_other.py | 89 + benchmark/long_json_decode/bench_sglang.py | 81 + benchmark/long_json_decode/build_dataset.py | 27 + benchmark/lora/launch_server.py | 68 + benchmark/lora/lora_bench.py | 475 +++ benchmark/mmlu/README.md | 59 + benchmark/mmlu/bench_other.py | 173 ++ benchmark/mmlu/bench_sglang.py | 173 ++ benchmark/mmlu/download_data.sh | 2 + benchmark/mmmu/README.md | 30 + benchmark/mmmu/bench_hf.py | 105 + benchmark/mmmu/bench_sglang.py | 93 + benchmark/mmmu/data_utils.py | 221 ++ benchmark/mmmu/eval_utils.py | 561 ++++ benchmark/mmmu/prompt_format.yaml | 15 + benchmark/mtbench/README.md | 48 + benchmark/mtbench/bench_other.py | 111 + benchmark/mtbench/bench_sglang.py | 99 + benchmark/mtbench/bench_sglang_eagle.py | 137 + benchmark/multi_chain_reasoning/README.md | 49 + .../multi_chain_reasoning/bench_other.py | 186 ++ .../multi_chain_reasoning/bench_sglang.py | 140 + benchmark/multi_document_qa/README.md | 47 + benchmark/multi_document_qa/bench_other.py | 114 + benchmark/multi_document_qa/bench_sglang.py | 93 + benchmark/multi_document_qa/build_dataset.py | 70 + benchmark/multi_turn_chat/README.md | 66 + benchmark/multi_turn_chat/bench_other.py | 93 + benchmark/multi_turn_chat/bench_sglang.py | 79 + benchmark/multi_turn_chat/data_gen.py | 29 + .../multi_turn_chat/long_prompt_multi_turn.py | 129 + benchmark/react/README.md | 34 + benchmark/react/bench_other.py | 202 ++ benchmark/react/bench_sglang.py | 153 + benchmark/reasoning_benchmark/README.md | 77 + .../reasoning_benchmark/answer_extraction.py | 269 ++ benchmark/reasoning_benchmark/bench_sglang.py | 135 + benchmark/reasoning_benchmark/eval_utils.py | 206 ++ benchmark/tip_suggestion/.gitignore | 1 + benchmark/tip_suggestion/README.md | 33 + benchmark/tip_suggestion/bench_other.py | 133 + benchmark/tip_suggestion/bench_sglang.py | 100 + benchmark/tip_suggestion/lmql_funcs.py | 50 + benchmark/tip_suggestion/topic.jsonl | 50 + benchmark/tree_of_thought_deep/README.md | 51 + benchmark/tree_of_thought_deep/bench_other.py | 222 ++ .../tree_of_thought_deep/bench_sglang.py | 171 ++ benchmark/tree_of_thought_deep/lmql_funcs.py | 82 + benchmark/tree_of_thought_v0/README.md | 43 + benchmark/tree_of_thought_v0/bench_other.py | 179 ++ benchmark/tree_of_thought_v0/bench_sglang.py | 159 + docker/Dockerfile | 77 + docker/Dockerfile.bak | 48 + docker/Dockerfile.deepep | 77 + docker/Dockerfile.dev | 224 ++ docker/Dockerfile.ds-14b | 77 + docker/Dockerfile.ds-8b | 77 + docker/Dockerfile.llama3.1-8b | 77 + docker/Dockerfile.qwq-32b | 77 + docker/Dockerfile.rocm | 85 + docker/Dockerfile.sagemaker | 78 + docker/compose.yaml | 35 + docker/k8s-sglang-distributed-sts.yaml | 104 + docker/k8s-sglang-service.yaml | 81 + docker/serve | 31 + docker/start.sh | 6 + docs/Makefile | 57 + docs/README.md | 110 + docs/_static/css/custom_log.css | 29 + docs/_static/css/readthedocs.css | 9 + docs/_static/image/logo.ico | Bin 0 -> 38078 bytes docs/_static/image/logo.png | Bin 0 -> 402692 bytes docs/backend/__init__.py | 0 docs/backend/custom_chat_template.md | 42 + docs/backend/function_calling.ipynb | 544 ++++ docs/backend/hyperparameter_tuning.md | 63 + docs/backend/native_api.ipynb | 515 ++++ docs/backend/offline_engine_api.ipynb | 231 ++ docs/backend/openai_api_completions.ipynb | 534 ++++ docs/backend/openai_api_embeddings.ipynb | 191 ++ docs/backend/openai_api_vision.ipynb | 291 ++ docs/backend/patch.py | 53 + docs/backend/quantization.md | 152 + docs/backend/sampling_params.md | 282 ++ docs/backend/send_request.ipynb | 259 ++ docs/backend/separate_reasoning.ipynb | 427 +++ docs/backend/server_arguments.md | 195 ++ docs/backend/speculative_decoding.ipynb | 305 ++ docs/backend/structured_outputs.ipynb | 827 ++++++ docs/conf.py | 205 ++ docs/deploy.py | 22 + .../development_guide_using_docker.md | 52 + docs/developer/release_process.md | 18 + docs/developer/setup_github_runner.md | 49 + docs/frontend/__init__.py | 0 docs/frontend/choices_methods.md | 77 + docs/frontend/frontend.ipynb | 464 +++ docs/frontend/patch.py | 49 + docs/index.rst | 65 + docs/references/accuracy_evaluation.md | 60 + docs/references/advanced_deploy.rst | 7 + docs/references/amd.md | 146 + docs/references/benchmark_and_profiling.md | 142 + docs/references/contribution_guide.md | 46 + docs/references/deepseek.md | 166 ++ docs/references/deepseek.rst | 6 + docs/references/deploy_on_k8s.md | 345 +++ docs/references/faq.md | 15 + docs/references/general.rst | 14 + docs/references/hardware.rst | 7 + docs/references/learn_more.md | 3 + docs/references/modelscope.md | 28 + docs/references/multi_node.md | 92 + docs/references/nvidia_jetson.md | 76 + docs/references/performance_tuning.rst | 7 + docs/references/production_metrics.md | 173 ++ docs/references/supported_models.md | 145 + docs/references/troubleshooting.md | 14 + docs/requirements.txt | 20 + docs/router/router.md | 165 ++ docs/serve.sh | 3 + docs/start/install.md | 160 ++ .../quick_start/anthropic_example_chat.py | 73 + .../quick_start/anthropic_example_complete.py | 68 + .../quick_start/azure_openai_example_chat.py | 83 + .../quick_start/gemini_example_chat.py | 73 + .../quick_start/gemini_example_complete.py | 68 + .../gemini_example_multimodal_chat.py | 30 + .../quick_start/images/cat.jpeg | Bin 0 -> 345163 bytes .../quick_start/images/dog.jpeg | Bin 0 -> 416580 bytes .../quick_start/local_example_chat.py | 75 + .../quick_start/local_example_complete.py | 70 + .../quick_start/local_example_llava_next.py | 78 + .../quick_start/openai_example_chat.py | 74 + .../quick_start/openai_example_complete.py | 68 + .../quick_start/openai_example_n.py | 71 + .../quick_start/openai_example_o1.py | 57 + .../quick_start/openrouter_example_chat.py | 81 + .../quick_start/together_example_chat.py | 81 + .../quick_start/together_example_complete.py | 76 + .../frontend_language/usage/chinese_regex.py | 53 + .../usage/choices_logprob.py | 44 + .../frontend_language/usage/cot_decoding.py | 115 + .../frontend_language/usage/json_decode.py | 83 + .../frontend_language/usage/json_logprobs.py | 103 + .../usage/llava_video/srt_example_llava_v.py | 260 ++ .../usage/llava_video/srt_example_llava_v.sh | 131 + .../usage/openai_chat_speculative.py | 155 + .../usage/openai_speculative.py | 54 + .../usage/parallel_sample.py | 40 + .../trace_and_evaluate_rag_using_parea.ipynb | 408 +++ .../usage/readme_examples.py | 109 + .../usage/sgl_gen_min_tokens.py | 35 + examples/frontend_language/usage/streaming.py | 49 + .../frontend_language/usage/triton/Dockerfile | 10 + .../frontend_language/usage/triton/README.md | 35 + .../models/character_generation/config.pbtxt | 23 + examples/monitoring/docker-compose.yaml | 16 + examples/monitoring/grafana.json | 962 +++++++ examples/monitoring/prometheus.yaml | 10 + examples/runtime/README.md | 42 + examples/runtime/engine/custom_server.py | 53 + examples/runtime/engine/embedding.py | 27 + examples/runtime/engine/launch_engine.py | 17 + .../runtime/engine/offline_batch_inference.py | 43 + .../engine/offline_batch_inference_async.py | 65 + .../engine/offline_batch_inference_eagle.py | 38 + .../offline_batch_inference_torchrun.py | 81 + .../engine/offline_batch_inference_vlm.py | 52 + examples/runtime/engine/readme.md | 49 + examples/runtime/engine/save_remote_state.py | 51 + examples/runtime/engine/save_sharded_state.py | 74 + .../hidden_states/hidden_states_engine.py | 65 + .../hidden_states/hidden_states_server.py | 81 + .../llava_onevision/http_llama3_llava_test.py | 111 + .../http_llava_onevision_test.py | 264 ++ .../llava_onevision/http_qwen_llava_test.py | 111 + examples/runtime/lora.py | 37 + examples/runtime/multimodal_embedding.py | 21 + examples/runtime/openai_batch_chat.py | 94 + examples/runtime/openai_batch_complete.py | 93 + .../openai_chat_with_response_prefill.py | 33 + examples/runtime/reward_model.py | 32 + .../token_in_token_out_llm_engine.py | 43 + .../token_in_token_out_llm_server.py | 68 + .../token_in_token_out_vlm_engine.py | 75 + .../token_in_token_out_vlm_server.py | 78 + examples/runtime/vertex_predict.py | 66 + python/pyproject.toml | 134 + python/sglang/README.md | 16 + python/sglang/__init__.py | 78 + python/sglang/api.py | 279 ++ python/sglang/bench_offline_throughput.py | 424 +++ python/sglang/bench_one_batch.py | 531 ++++ python/sglang/bench_one_batch_server.py | 192 ++ python/sglang/bench_serving.py | 1579 ++++++++++ python/sglang/check_env.py | 305 ++ python/sglang/global_config.py | 49 + python/sglang/lang/__init__.py | 0 python/sglang/lang/backend/__init__.py | 0 python/sglang/lang/backend/anthropic.py | 77 + python/sglang/lang/backend/base_backend.py | 82 + python/sglang/lang/backend/litellm.py | 90 + python/sglang/lang/backend/openai.py | 471 +++ .../sglang/lang/backend/runtime_endpoint.py | 500 ++++ python/sglang/lang/backend/vertexai.py | 149 + python/sglang/lang/chat_template.py | 555 ++++ python/sglang/lang/choices.py | 164 ++ python/sglang/lang/compiler.py | 237 ++ python/sglang/lang/interpreter.py | 1021 +++++++ python/sglang/lang/ir.py | 608 ++++ python/sglang/lang/tracer.py | 283 ++ python/sglang/launch_server.py | 16 + python/sglang/llama3_eval.py | 316 ++ python/sglang/srt/_custom_ops.py | 150 + python/sglang/srt/aio_rwlock.py | 100 + python/sglang/srt/code_completion_parser.py | 174 ++ python/sglang/srt/configs/__init__.py | 13 + python/sglang/srt/configs/chatglm.py | 78 + python/sglang/srt/configs/dbrx.py | 279 ++ python/sglang/srt/configs/deepseekvl2.py | 676 +++++ python/sglang/srt/configs/device_config.py | 17 + python/sglang/srt/configs/exaone.py | 195 ++ python/sglang/srt/configs/janus_pro.py | 628 ++++ python/sglang/srt/configs/load_config.py | 89 + python/sglang/srt/configs/model_config.py | 512 ++++ python/sglang/srt/configs/utils.py | 25 + python/sglang/srt/connector/__init__.py | 51 + python/sglang/srt/connector/base_connector.py | 112 + python/sglang/srt/connector/redis.py | 85 + python/sglang/srt/connector/s3.py | 122 + python/sglang/srt/connector/serde/__init__.py | 31 + .../sglang/srt/connector/serde/safe_serde.py | 29 + python/sglang/srt/connector/serde/serde.py | 43 + python/sglang/srt/connector/utils.py | 35 + .../srt/constrained/base_grammar_backend.py | 194 ++ .../srt/constrained/llguidance_backend.py | 151 + .../srt/constrained/outlines_backend.py | 188 ++ .../srt/constrained/outlines_jump_forward.py | 187 ++ .../srt/constrained/xgrammar_backend.py | 183 ++ python/sglang/srt/conversation.py | 736 +++++ python/sglang/srt/custom_op.py | 101 + python/sglang/srt/disaggregation/conn.py | 81 + python/sglang/srt/disaggregation/decode.py | 495 ++++ python/sglang/srt/disaggregation/mini_lb.py | 285 ++ python/sglang/srt/disaggregation/prefill.py | 249 ++ python/sglang/srt/disaggregation/utils.py | 44 + python/sglang/srt/distributed/__init__.py | 3 + .../srt/distributed/communication_op.py | 35 + .../device_communicators/cuda_wrapper.py | 183 ++ .../device_communicators/custom_all_reduce.py | 568 ++++ .../custom_all_reduce_utils.py | 291 ++ .../device_communicators/hpu_communicator.py | 49 + .../device_communicators/pynccl.py | 283 ++ .../device_communicators/pynccl_wrapper.py | 472 +++ .../device_communicators/shm_broadcast.py | 501 ++++ .../device_communicators/xpu_communicator.py | 48 + .../sglang/srt/distributed/parallel_state.py | 1363 +++++++++ python/sglang/srt/distributed/utils.py | 224 ++ python/sglang/srt/entrypoints/engine.py | 577 ++++ python/sglang/srt/entrypoints/http_server.py | 840 ++++++ python/sglang/srt/entrypoints/verl_engine.py | 149 + python/sglang/srt/function_call_parser.py | 629 ++++ python/sglang/srt/hf_transformers_utils.py | 267 ++ python/sglang/srt/layers/activation.py | 174 ++ .../srt/layers/attention/base_attn_backend.py | 108 + .../attention/double_sparsity_backend.py | 257 ++ .../attention/flashattention_backend.py | 434 +++ .../layers/attention/flashinfer_backend.py | 1259 ++++++++ .../attention/flashinfer_mla_backend.py | 842 ++++++ .../srt/layers/attention/flashmla_backend.py | 284 ++ .../layers/attention/torch_native_backend.py | 262 ++ .../srt/layers/attention/triton_backend.py | 727 +++++ .../attention/triton_ops/decode_attention.py | 733 +++++ .../triton_ops/double_sparsity_attention.py | 1106 +++++++ .../attention/triton_ops/extend_attention.py | 456 +++ .../attention/triton_ops/prefill_attention.py | 213 ++ .../triton_ops/rocm_mla_decode_rope.py | 439 +++ python/sglang/srt/layers/attention/utils.py | 92 + python/sglang/srt/layers/attention/vision.py | 366 +++ python/sglang/srt/layers/dp_attention.py | 262 ++ python/sglang/srt/layers/elementwise.py | 411 +++ python/sglang/srt/layers/layernorm.py | 146 + python/sglang/srt/layers/linear.py | 1292 +++++++++ python/sglang/srt/layers/logits_processor.py | 586 ++++ .../sglang/srt/layers/moe/ep_moe/__init__.py | 0 .../sglang/srt/layers/moe/ep_moe/kernels.py | 567 ++++ python/sglang/srt/layers/moe/ep_moe/layer.py | 1046 +++++++ .../srt/layers/moe/ep_moe/token_dispatcher.py | 416 +++ .../sglang/srt/layers/moe/fused_moe_native.py | 132 + .../layers/moe/fused_moe_triton/__init__.py | 40 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 218 ++ ...072,device_name=NVIDIA_H100_80GB_HBM3.json | 218 ++ ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...344,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...344,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...344,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...688,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...688,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...192,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...evice_name=NVIDIA_L20,dtype=int8_w8a8.json | 146 + ...vice_name=NVIDIA_L40S,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...280,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...280,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...280,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=1280,device_name=NVIDIA_H200.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=2560,device_name=NVIDIA_H200.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...320,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=320,device_name=NVIDIA_H200.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...640,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...640,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...640,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=640,device_name=NVIDIA_H200.json | 146 + ...14336,device_name=AMD_Instinct_MI300X.json | 200 ++ ...14336,device_name=AMD_Instinct_MI325X.json | 200 ++ ...14336,device_name=AMD_Radeon_Graphics.json | 200 ++ ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 138 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=14336,device_name=NVIDIA_H200.json | 146 + ...=1792,device_name=AMD_Instinct_MI300X.json | 200 ++ ...=1792,device_name=AMD_Instinct_MI325X.json | 200 ++ ...=1792,device_name=AMD_Radeon_Graphics.json | 200 ++ ...792,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...792,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=1792,device_name=NVIDIA_H200.json | 146 + ...048,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=2048,device_name=NVIDIA_H200.json | 146 + ...=3584,device_name=AMD_Instinct_MI300X.json | 200 ++ ...=3584,device_name=AMD_Instinct_MI325X.json | 200 ++ ...=3584,device_name=AMD_Radeon_Graphics.json | 200 ++ ...584,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...584,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=3584,device_name=NVIDIA_H200.json | 146 + .../E=8,N=3584,device_name=NVIDIA_L40S.json | 173 ++ ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 178 ++ ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 178 ++ ...me=AMD_Radeon_Graphics,dtype=fp8_w8a8.json | 178 ++ ...096,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...096,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=4096,device_name=NVIDIA_H200.json | 146 + ...=7168,device_name=AMD_Instinct_MI300X.json | 200 ++ ...=7168,device_name=AMD_Instinct_MI325X.json | 200 ++ ...=7168,device_name=AMD_Radeon_Graphics.json | 200 ++ ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=7168,device_name=NVIDIA_H200.json | 146 + ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 175 ++ ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 175 ++ ...me=AMD_Radeon_Graphics,dtype=fp8_w8a8.json | 175 ++ ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../moe/fused_moe_triton/configs/README | 12 + .../layers/moe/fused_moe_triton/fused_moe.py | 1251 ++++++++ .../srt/layers/moe/fused_moe_triton/layer.py | 701 +++++ python/sglang/srt/layers/moe/router.py | 342 +++ python/sglang/srt/layers/moe/topk.py | 260 ++ python/sglang/srt/layers/parameter.py | 459 +++ python/sglang/srt/layers/pooler.py | 50 + .../srt/layers/quantization/__init__.py | 322 +++ python/sglang/srt/layers/quantization/awq.py | 200 ++ .../srt/layers/quantization/base_config.py | 140 + .../srt/layers/quantization/blockwise_int8.py | 408 +++ .../quantization/compressed_tensors/README.md | 6 + .../compressed_tensors/__init__.py | 0 .../compressed_tensors/compressed_tensors.py | 652 +++++ .../compressed_tensors_moe.py | 658 +++++ .../compressed_tensors/schemes/__init__.py | 9 + .../schemes/compressed_tensors_scheme.py | 56 + .../schemes/compressed_tensors_w8a8_fp8.py | 162 ++ .../quantization/compressed_tensors/utils.py | 218 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 ++ ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + python/sglang/srt/layers/quantization/fp8.py | 1002 +++++++ .../srt/layers/quantization/fp8_kernel.py | 817 ++++++ .../srt/layers/quantization/fp8_utils.py | 564 ++++ python/sglang/srt/layers/quantization/gptq.py | 433 +++ .../srt/layers/quantization/int8_kernel.py | 381 +++ .../srt/layers/quantization/int8_utils.py | 73 + .../srt/layers/quantization/kv_cache.py | 98 + .../srt/layers/quantization/modelopt_quant.py | 196 ++ .../sglang/srt/layers/quantization/utils.py | 153 + .../srt/layers/quantization/w8a8_fp8.py | 179 ++ .../srt/layers/quantization/w8a8_int8.py | 266 ++ python/sglang/srt/layers/radix_attention.py | 70 + python/sglang/srt/layers/rotary_embedding.py | 1271 ++++++++ python/sglang/srt/layers/sampler.py | 275 ++ python/sglang/srt/layers/torchao_utils.py | 124 + .../srt/layers/vocab_parallel_embedding.py | 566 ++++ python/sglang/srt/lora/backend/__init__.py | 25 + .../sglang/srt/lora/backend/base_backend.py | 117 + .../srt/lora/backend/flashinfer_backend.py | 131 + .../sglang/srt/lora/backend/triton_backend.py | 88 + python/sglang/srt/lora/layers.py | 347 +++ python/sglang/srt/lora/lora.py | 181 ++ python/sglang/srt/lora/lora_config.py | 47 + python/sglang/srt/lora/lora_manager.py | 228 ++ python/sglang/srt/lora/mem_pool.py | 239 ++ python/sglang/srt/lora/triton_ops/__init__.py | 11 + .../srt/lora/triton_ops/gate_up_lora_b.py | 176 ++ .../sglang/srt/lora/triton_ops/qkv_lora_b.py | 187 ++ .../srt/lora/triton_ops/sgemm_lora_a.py | 154 + .../srt/lora/triton_ops/sgemm_lora_b.py | 164 ++ python/sglang/srt/lora/utils.py | 158 + .../sglang/srt/managers/cache_controller.py | 501 ++++ .../sglang/srt/managers/configure_logging.py | 47 + .../srt/managers/data_parallel_controller.py | 280 ++ .../srt/managers/detokenizer_manager.py | 267 ++ .../srt/managers/expert_distribution.py | 81 + python/sglang/srt/managers/io_struct.py | 763 +++++ python/sglang/srt/managers/mm_utils.py | 373 +++ .../srt/managers/multimodal_processor.py | 68 + .../multimodal_processors/base_processor.py | 275 ++ .../managers/multimodal_processors/clip.py | 63 + .../multimodal_processors/deepseek_vl_v2.py | 119 + .../managers/multimodal_processors/gemma3.py | 83 + .../multimodal_processors/janus_pro.py | 84 + .../managers/multimodal_processors/llava.py | 147 + .../managers/multimodal_processors/minicpm.py | 167 ++ .../managers/multimodal_processors/mlama.py | 59 + .../managers/multimodal_processors/qwen_vl.py | 167 ++ python/sglang/srt/managers/schedule_batch.py | 1568 ++++++++++ python/sglang/srt/managers/schedule_policy.py | 510 ++++ python/sglang/srt/managers/scheduler.py | 2022 +++++++++++++ .../scheduler_output_processor_mixin.py | 614 ++++ .../sglang/srt/managers/session_controller.py | 149 + .../sglang/srt/managers/tokenizer_manager.py | 1211 ++++++++ python/sglang/srt/managers/tp_worker.py | 230 ++ .../srt/managers/tp_worker_overlap_thread.py | 240 ++ python/sglang/srt/managers/utils.py | 49 + .../sglang/srt/mem_cache/base_prefix_cache.py | 50 + python/sglang/srt/mem_cache/chunk_cache.py | 65 + python/sglang/srt/mem_cache/flush_cache.py | 33 + python/sglang/srt/mem_cache/hiradix_cache.py | 442 +++ python/sglang/srt/mem_cache/memory_pool.py | 853 ++++++ .../sglang/srt/mem_cache/paged_allocator.py | 283 ++ python/sglang/srt/mem_cache/radix_cache.py | 464 +++ python/sglang/srt/metrics/collector.py | 281 ++ python/sglang/srt/metrics/func_timer.py | 106 + python/sglang/srt/mm_utils.py | 339 +++ .../srt/model_executor/cuda_graph_runner.py | 573 ++++ .../srt/model_executor/forward_batch_info.py | 525 ++++ .../sglang/srt/model_executor/model_runner.py | 1100 +++++++ python/sglang/srt/model_loader/__init__.py | 34 + python/sglang/srt/model_loader/loader.py | 1396 +++++++++ python/sglang/srt/model_loader/utils.py | 41 + .../sglang/srt/model_loader/weight_utils.py | 849 ++++++ python/sglang/srt/model_parallel.py | 155 + python/sglang/srt/models/baichuan.py | 439 +++ python/sglang/srt/models/chatglm.py | 426 +++ python/sglang/srt/models/clip.py | 563 ++++ python/sglang/srt/models/commandr.py | 417 +++ python/sglang/srt/models/dbrx.py | 453 +++ python/sglang/srt/models/deepseek.py | 474 +++ .../sglang/srt/models/deepseek_janus_pro.py | 2070 +++++++++++++ python/sglang/srt/models/deepseek_nextn.py | 340 +++ python/sglang/srt/models/deepseek_v2.py | 1511 ++++++++++ python/sglang/srt/models/deepseek_vl2.py | 358 +++ python/sglang/srt/models/exaone.py | 371 +++ python/sglang/srt/models/gemma.py | 361 +++ python/sglang/srt/models/gemma2.py | 461 +++ python/sglang/srt/models/gemma2_reward.py | 70 + python/sglang/srt/models/gemma3_causal.py | 694 +++++ python/sglang/srt/models/gemma3_mm.py | 468 +++ python/sglang/srt/models/gpt2.py | 287 ++ python/sglang/srt/models/gpt_bigcode.py | 303 ++ python/sglang/srt/models/granite.py | 529 ++++ python/sglang/srt/models/grok.py | 798 +++++ python/sglang/srt/models/internlm2.py | 353 +++ python/sglang/srt/models/internlm2_reward.py | 64 + python/sglang/srt/models/llama.py | 641 +++++ .../sglang/srt/models/llama_classification.py | 81 + python/sglang/srt/models/llama_eagle.py | 146 + python/sglang/srt/models/llama_eagle3.py | 196 ++ python/sglang/srt/models/llama_embedding.py | 87 + python/sglang/srt/models/llama_reward.py | 126 + python/sglang/srt/models/llava.py | 574 ++++ python/sglang/srt/models/llavavid.py | 274 ++ python/sglang/srt/models/minicpm.py | 400 +++ python/sglang/srt/models/minicpm3.py | 688 +++++ python/sglang/srt/models/minicpmo.py | 1995 +++++++++++++ python/sglang/srt/models/minicpmv.py | 1186 ++++++++ python/sglang/srt/models/mistral.py | 23 + python/sglang/srt/models/mixtral.py | 407 +++ python/sglang/srt/models/mixtral_quant.py | 425 +++ python/sglang/srt/models/mllama.py | 1006 +++++++ python/sglang/srt/models/olmo.py | 376 +++ python/sglang/srt/models/olmo2.py | 412 +++ python/sglang/srt/models/olmoe.py | 431 +++ python/sglang/srt/models/phi3_small.py | 461 +++ python/sglang/srt/models/qwen.py | 318 ++ python/sglang/srt/models/qwen2.py | 441 +++ python/sglang/srt/models/qwen2_5_vl.py | 657 +++++ .../sglang/srt/models/qwen2_classification.py | 75 + python/sglang/srt/models/qwen2_eagle.py | 143 + python/sglang/srt/models/qwen2_moe.py | 492 ++++ python/sglang/srt/models/qwen2_rm.py | 78 + python/sglang/srt/models/qwen2_vl.py | 616 ++++ python/sglang/srt/models/registry.py | 99 + python/sglang/srt/models/stablelm.py | 330 +++ .../sglang/srt/models/torch_native_llama.py | 524 ++++ python/sglang/srt/models/xverse.py | 381 +++ python/sglang/srt/models/xverse_moe.py | 471 +++ python/sglang/srt/models/yivl.py | 115 + python/sglang/srt/openai_api/adapter.py | 1773 ++++++++++++ python/sglang/srt/openai_api/protocol.py | 459 +++ python/sglang/srt/patch_torch.py | 71 + python/sglang/srt/reasoning_parser.py | 154 + .../srt/sampling/custom_logit_processor.py | 38 + .../srt/sampling/penaltylib/__init__.py | 11 + .../sampling/penaltylib/frequency_penalty.py | 65 + .../srt/sampling/penaltylib/min_new_tokens.py | 94 + .../srt/sampling/penaltylib/orchestrator.py | 197 ++ .../sampling/penaltylib/presence_penalty.py | 65 + .../srt/sampling/sampling_batch_info.py | 312 ++ python/sglang/srt/sampling/sampling_params.py | 153 + python/sglang/srt/server.py | 18 + python/sglang/srt/server_args.py | 1255 ++++++++ .../srt/speculative/build_eagle_tree.py | 388 +++ .../eagle_draft_cuda_graph_runner.py | 239 ++ python/sglang/srt/speculative/eagle_utils.py | 701 +++++ python/sglang/srt/speculative/eagle_worker.py | 590 ++++ python/sglang/srt/speculative/spec_info.py | 27 + .../sglang/srt/torch_memory_saver_adapter.py | 81 + python/sglang/srt/utils.py | 1776 ++++++++++++ python/sglang/srt/warmup.py | 47 + python/sglang/test/__init__.py | 0 python/sglang/test/attention/__init__.py | 0 .../test/attention/test_flashattn_backend.py | 312 ++ python/sglang/test/few_shot_gsm8k.py | 148 + python/sglang/test/few_shot_gsm8k_engine.py | 144 + python/sglang/test/long_prompt.txt | 1 + python/sglang/test/run_eval.py | 126 + python/sglang/test/runners.py | 791 +++++ python/sglang/test/send_one.py | 88 + python/sglang/test/simple_eval_common.py | 466 +++ python/sglang/test/simple_eval_gpqa.py | 93 + python/sglang/test/simple_eval_humaneval.py | 133 + python/sglang/test/simple_eval_math.py | 73 + python/sglang/test/simple_eval_mgsm.py | 203 ++ python/sglang/test/simple_eval_mmlu.py | 121 + python/sglang/test/test_activation.py | 56 + python/sglang/test/test_block_fp8.py | 432 +++ python/sglang/test/test_block_fp8_ep.py | 362 +++ python/sglang/test/test_custom_ops.py | 88 + python/sglang/test/test_dynamic_grad_mode.py | 58 + python/sglang/test/test_layernorm.py | 113 + python/sglang/test/test_programs.py | 575 ++++ python/sglang/test/test_utils.py | 1035 +++++++ python/sglang/utils.py | 514 ++++ python/sglang/version.py | 1 + scripts/ci_install_dependency.sh | 26 + scripts/ci_install_rust.sh | 23 + scripts/deprecated/convert_yi_vl.py | 38 + scripts/deprecated/convert_yi_vl.sh | 13 + scripts/deprecated/test_curl.sh | 9 + scripts/deprecated/test_flashinfer.py | 217 ++ .../deprecated/test_httpserver_concurrent.py | 56 + scripts/deprecated/test_httpserver_decode.py | 55 + .../test_httpserver_decode_stream.py | 68 + scripts/deprecated/test_httpserver_llava.py | 88 + scripts/deprecated/test_httpserver_reuse.py | 42 + scripts/deprecated/test_jump_forward.py | 138 + scripts/deprecated/test_robust.py | 132 + scripts/export_deepseek_nextn.py | 115 + scripts/fix_corrupted_json.py | 40 + scripts/killall_sglang.sh | 28 + scripts/playground/bench_speculative.py | 280 ++ scripts/playground/launch_tgi.sh | 7 + scripts/playground/load_tokenizer.py | 14 + scripts/playground/lora/analyzer.py | 77 + scripts/playground/lora/lora_hf_play.py | 62 + scripts/playground/lora/lora_vllm_play.py | 30 + scripts/playground/reference_hf.py | 197 ++ scripts/playground/router/test_tree.py | 207 ++ scripts/playground/router/tree.py | 292 ++ scripts/update_kernel_whl_index.py | 31 + scripts/version_branch_to_tag.sh | 30 + sgl-kernel/.clang-format | 15 + sgl-kernel/CMakeLists.txt | 187 ++ sgl-kernel/LICENSE | 201 ++ sgl-kernel/Makefile | 69 + sgl-kernel/README.md | 73 + sgl-kernel/THIRDPARTYNOTICES.txt | 455 +++ sgl-kernel/benchmark/bench_awq_dequant.py | 118 + .../benchmark/bench_cublas_grouped_gemm.py | 262 ++ .../benchmark/bench_fp8_blockwise_gemm.py | 148 + sgl-kernel/benchmark/bench_fp8_gemm.py | 164 ++ sgl-kernel/benchmark/bench_int8_gemm.py | 146 + .../bench_lightning_attention_decode.py | 299 ++ .../benchmark/bench_moe_align_block_size.py | 385 +++ .../benchmark/bench_moe_topk_softmax.py | 120 + .../benchmark/bench_per_tensor_quant_fp8.py | 98 + .../bench_per_token_group_quant_8bit.py | 218 ++ .../benchmark/bench_per_token_quant_fp8.py | 90 + sgl-kernel/build.sh | 30 + .../csrc/allreduce/custom_all_reduce.hip | 180 ++ .../csrc/allreduce/custom_all_reduce_hip.cuh | 582 ++++ .../csrc/allreduce/trt_reduce_internal.cu | 532 ++++ .../csrc/allreduce/trt_reduce_kernel.cu | 226 ++ .../lightning_attention_decode_kernel.cu | 154 + .../epilogue/epilogue_per_row_per_col_scale.h | 309 ++ .../gemm/collective/collective_builder.hpp | 125 + ..._warpspecialized_fp8_blockwise_scaling.hpp | 733 +++++ .../gemm/dispatch_policy.hpp | 37 + .../gemm/gemm_universal_base_compat.h | 356 +++ .../gemm/gemm_with_epilogue_visitor.h | 492 ++++ sgl-kernel/csrc/elementwise/activation.cu | 82 + .../elementwise/fused_add_rms_norm_kernel.cu | 55 + sgl-kernel/csrc/elementwise/rope.cu | 89 + sgl-kernel/csrc/gemm/awq_kernel.cu | 219 ++ sgl-kernel/csrc/gemm/bmm_fp8.cu | 76 + sgl-kernel/csrc/gemm/cublas_grouped_gemm.cu | 172 ++ .../csrc/gemm/fp8_blockwise_gemm_kernel.cu | 406 +++ sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu | 1146 ++++++++ sgl-kernel/csrc/gemm/int8_gemm_kernel.cu | 747 +++++ sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu | 29 + sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu | 399 +++ sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu | 39 + .../csrc/gemm/nvfp4_scaled_mm_kernels.cu | 365 +++ sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu | 123 + .../csrc/gemm/per_token_group_quant_8bit.cu | 167 ++ sgl-kernel/csrc/gemm/per_token_quant_fp8.cu | 106 + sgl-kernel/csrc/moe/moe_align_kernel.cu | 160 ++ .../csrc/moe/moe_topk_softmax_kernels.cu | 505 ++++ sgl-kernel/csrc/speculative/eagle_utils.cu | 286 ++ sgl-kernel/csrc/speculative/packbit.cu | 47 + .../pytorch_extension_utils_rocm.h | 20 + .../csrc/speculative/speculative_sampling.cu | 145 + .../csrc/speculative/speculative_sampling.cuh | 223 ++ sgl-kernel/csrc/torch_extension.cc | 202 ++ sgl-kernel/csrc/torch_extension_rocm.cc | 82 + sgl-kernel/include/sgl_kernel_ops.h | 290 ++ sgl-kernel/include/trt_reduce_internal.cuh | 109 + sgl-kernel/include/utils.h | 348 +++ sgl-kernel/pyproject.toml | 39 + sgl-kernel/pyproject_rocm.toml | 32 + sgl-kernel/python/sgl_kernel/__init__.py | 57 + sgl-kernel/python/sgl_kernel/allreduce.py | 77 + sgl-kernel/python/sgl_kernel/attention.py | 7 + sgl-kernel/python/sgl_kernel/elementwise.py | 153 + sgl-kernel/python/sgl_kernel/gemm.py | 219 ++ sgl-kernel/python/sgl_kernel/moe.py | 34 + sgl-kernel/python/sgl_kernel/sampling.py | 210 ++ sgl-kernel/python/sgl_kernel/speculative.py | 101 + sgl-kernel/python/sgl_kernel/utils.py | 41 + sgl-kernel/python/sgl_kernel/version.py | 1 + sgl-kernel/rename_wheels.sh | 17 + sgl-kernel/setup.py | 249 ++ sgl-kernel/setup_rocm.py | 86 + .../tests/speculative/test_eagle_utils.py | 98 + .../speculative/test_speculative_sampling.py | 122 + sgl-kernel/tests/test_activation.py | 39 + sgl-kernel/tests/test_awq_dequant.py | 132 + sgl-kernel/tests/test_bmm_fp8.py | 43 + sgl-kernel/tests/test_cublas_grouped_gemm.py | 49 + sgl-kernel/tests/test_deep_gemm.py | 263 ++ sgl-kernel/tests/test_fp4_gemm.py | 151 + sgl-kernel/tests/test_fp4_quantize.py | 164 ++ sgl-kernel/tests/test_fp8_blockwise_gemm.py | 112 + sgl-kernel/tests/test_fp8_gemm.py | 67 + sgl-kernel/tests/test_int8_gemm.py | 56 + .../tests/test_lightning_attention_decode.py | 88 + sgl-kernel/tests/test_moe_align.py | 226 ++ sgl-kernel/tests/test_moe_topk_softmax.py | 53 + sgl-kernel/tests/test_norm.py | 133 + sgl-kernel/tests/test_per_tensor_quant_fp8.py | 67 + .../tests/test_per_token_group_quant_8bit.py | 178 ++ sgl-kernel/tests/test_per_token_quant_fp8.py | 55 + sgl-kernel/tests/test_rotary_embedding.py | 200 ++ sgl-kernel/tests/test_sampling.py | 141 + sgl-kernel/tests/test_trt_allreduce.py | 244 ++ sgl-router/Cargo.lock | 2557 +++++++++++++++++ sgl-router/Cargo.toml | 31 + sgl-router/MANIFEST.in | 3 + sgl-router/README.md | 97 + sgl-router/py_src/sglang_router/__init__.py | 7 + .../py_src/sglang_router/launch_router.py | 268 ++ .../py_src/sglang_router/launch_server.py | 202 ++ sgl-router/py_src/sglang_router/router.py | 73 + sgl-router/py_src/sglang_router/version.py | 1 + sgl-router/py_test/run_suite.py | 20 + sgl-router/py_test/test_launch_router.py | 83 + sgl-router/py_test/test_launch_server.py | 394 +++ sgl-router/pyproject.toml | 30 + sgl-router/src/lib.rs | 123 + sgl-router/src/router.rs | 828 ++++++ sgl-router/src/server.rs | 226 ++ sgl-router/src/tree.rs | 1483 ++++++++++ sgl-router/v0.1.0.md | 63 + test/README.md | 47 + test/lang/run_suite.py | 38 + test/lang/test_anthropic_backend.py | 25 + test/lang/test_bind_cache.py | 51 + test/lang/test_choices.py | 91 + test/lang/test_litellm_backend.py | 25 + test/lang/test_openai_backend.py | 92 + test/lang/test_srt_backend.py | 86 + test/lang/test_tracing.py | 129 + test/lang/test_vertexai_backend.py | 53 + test/srt/configs/random_config.yaml | 25 + .../random_flashinfer_vs_triton_config.yaml | 25 + test/srt/configs/sharegpt_config.yaml | 7 + ...sparsity-config-Llama-3.1-8B-Instruct.json | 1 + test/srt/experiment_runner.py | 359 +++ test/srt/kv_cache_scales_llama3_1_8b.json | 42 + test/srt/kv_cache_scales_llama3_8b.json | 42 + test/srt/kv_cache_scales_qwen2_1_5b.json | 38 + test/srt/models/compare.py | 52 + test/srt/models/lora/test_lora.py | 304 ++ test/srt/models/lora/test_lora_backend.py | 255 ++ test/srt/models/lora/test_lora_tp.py | 257 ++ .../models/lora/test_multi_lora_backend.py | 238 ++ test/srt/models/lora/utils.py | 49 + test/srt/models/test_clip_models.py | 80 + test/srt/models/test_embedding_models.py | 109 + test/srt/models/test_generation_models.py | 156 + test/srt/models/test_gme_qwen_models.py | 85 + test/srt/models/test_grok_models.py | 53 + test/srt/models/test_qwen_models.py | 77 + test/srt/models/test_reward_models.py | 92 + test/srt/run_suite.py | 188 ++ test/srt/test_abort.py | 54 + test/srt/test_awq.py | 45 + test/srt/test_bench_one_batch.py | 58 + test/srt/test_bench_serving.py | 204 ++ test/srt/test_block_int8.py | 222 ++ test/srt/test_cache_report.py | 211 ++ test/srt/test_chunked_prefill.py | 31 + test/srt/test_create_kvindices.py | 75 + test/srt/test_custom_allreduce.py | 166 ++ test/srt/test_data_parallelism.py | 78 + test/srt/test_double_sparsity.py | 65 + test/srt/test_dp_attention.py | 62 + test/srt/test_eagle_infer.py | 539 ++++ test/srt/test_ebnf_constrained.py | 242 ++ test/srt/test_embedding_openai_server.py | 87 + test/srt/test_eval_accuracy_large.py | 92 + test/srt/test_eval_fp8_accuracy.py | 159 + test/srt/test_expert_distribution.py | 124 + test/srt/test_fim_completion.py | 72 + test/srt/test_fp8_kernel.py | 128 + test/srt/test_fp8_kvcache.py | 114 + test/srt/test_function_calling.py | 299 ++ test/srt/test_fused_moe.py | 178 ++ test/srt/test_get_weights_by_name.py | 183 ++ test/srt/test_gguf.py | 27 + test/srt/test_gptqmodel_dynamic.py | 214 ++ test/srt/test_health_check.py | 28 + test/srt/test_hicache.py | 45 + test/srt/test_hicache_mla.py | 57 + test/srt/test_hidden_states.py | 137 + test/srt/test_input_embeddings.py | 151 + test/srt/test_int8_kernel.py | 164 ++ test/srt/test_json_constrained.py | 147 + test/srt/test_large_max_new_tokens.py | 103 + test/srt/test_matched_stop.py | 140 + test/srt/test_metrics.py | 88 + test/srt/test_mla.py | 62 + test/srt/test_mla_deepseek_v3.py | 111 + test/srt/test_mla_flashinfer.py | 166 ++ test/srt/test_mla_fp8.py | 49 + test/srt/test_mla_int8_deepseek_v3.py | 210 ++ test/srt/test_mla_tp.py | 66 + test/srt/test_modelopt_fp8kvcache.py | 30 + test/srt/test_models_from_modelscope.py | 40 + test/srt/test_moe_deepep.py | 89 + test/srt/test_moe_ep.py | 114 + test/srt/test_moe_eval_accuracy_large.py | 94 + test/srt/test_nightly_gsm8k_eval.py | 188 ++ test/srt/test_nightly_human_eval.py | 105 + test/srt/test_nightly_math_eval.py | 47 + test/srt/test_no_chunked_prefill.py | 30 + test/srt/test_no_overlap_scheduler.py | 35 + test/srt/test_openai_server.py | 680 +++++ test/srt/test_page_size.py | 47 + test/srt/test_patch_torch.py | 133 + test/srt/test_penalty.py | 95 + test/srt/test_pytorch_sampling_backend.py | 90 + test/srt/test_radix_attention.py | 134 + test/srt/test_reasoning_content.py | 343 +++ test/srt/test_regex_constrained.py | 187 ++ test/srt/test_release_memory_occupation.py | 98 + test/srt/test_request_length_validation.py | 88 + test/srt/test_retract_decode.py | 60 + test/srt/test_sagemaker_server.py | 179 ++ test/srt/test_schedule_policy.py | 65 + test/srt/test_server_args.py | 262 ++ test/srt/test_session_control.py | 778 +++++ test/srt/test_skip_tokenizer_init.py | 227 ++ test/srt/test_srt_endpoint.py | 518 ++++ test/srt/test_srt_engine.py | 190 ++ test/srt/test_srt_engine_with_quant_args.py | 60 + test/srt/test_torch_compile.py | 75 + test/srt/test_torch_compile_moe.py | 74 + .../test_torch_native_attention_backend.py | 59 + test/srt/test_torch_tp.py | 24 + test/srt/test_torchao.py | 74 + test/srt/test_triton_attention_backend.py | 64 + test/srt/test_triton_attention_kernels.py | 390 +++ test/srt/test_triton_attention_rocm_mla.py | 259 ++ test/srt/test_update_weights_from_disk.py | 293 ++ .../test_update_weights_from_distributed.py | 605 ++++ test/srt/test_update_weights_from_tensor.py | 90 + test/srt/test_verl_engine.py | 298 ++ test/srt/test_vertex_endpoint.py | 54 + test/srt/test_vision_chunked_prefill.py | 179 ++ test/srt/test_vision_openai_server.py | 713 +++++ test/srt/test_vllm_dependency.py | 168 ++ test/srt/test_vlm_accuracy.py | 215 ++ test/srt/test_w8a8_quantization.py | 75 + 1106 files changed, 224894 insertions(+) create mode 100644 .clang-format-ignore create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .editorconfig create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 3rdparty/amd/profiling/PROFILING.md create mode 100644 3rdparty/amd/profiling/client.sh create mode 100644 3rdparty/amd/profiling/install_rpd.sh create mode 100644 3rdparty/amd/profiling/loadTracer.sh create mode 100644 3rdparty/amd/profiling/rpd.patch create mode 100644 3rdparty/amd/profiling/rpd_profile_server_enable.patch create mode 100644 3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch create mode 100644 3rdparty/amd/profiling/server.sh create mode 100644 3rdparty/amd/profiling/torch_profiler.patch create mode 100644 3rdparty/amd/tuning/TUNING.md create mode 100644 3rdparty/amd/tuning/benchmark_moe_rocm.py create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 assets/logo.png create mode 100644 benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py create mode 100644 benchmark/benchmark_vllm_060/README.md create mode 100644 benchmark/blog_v0_2/405b_sglang.sh create mode 100644 benchmark/blog_v0_2/405b_trt.sh create mode 100644 benchmark/blog_v0_2/405b_vllm.sh create mode 100644 benchmark/blog_v0_2/README.md create mode 100644 benchmark/blog_v0_2/config.md create mode 100644 benchmark/deepseek_v3/README.md create mode 100644 benchmark/dspy/README.md create mode 100644 benchmark/dspy/bench_dspy_intro.py create mode 100644 benchmark/generative_agents/README.md create mode 100644 benchmark/generative_agents/agent_functions.py create mode 100644 benchmark/generative_agents/bench_other.py create mode 100644 benchmark/generative_agents/bench_sglang.py create mode 100644 benchmark/gsm8k/README.md create mode 100644 benchmark/gsm8k/bench_other.py create mode 100644 benchmark/gsm8k/bench_sglang.py create mode 100644 benchmark/hellaswag/README.md create mode 100644 benchmark/hellaswag/bench_other.py create mode 100644 benchmark/hellaswag/bench_sglang.py create mode 100644 benchmark/hicache/README.md create mode 100644 benchmark/hicache/bench_multiturn.py create mode 100644 benchmark/hicache/bench_serving.py create mode 100644 benchmark/hicache/data_processing.py create mode 100644 benchmark/hicache/download.sh create mode 100644 benchmark/hicache/nextqa.py create mode 100644 benchmark/json_decode_regex/README.md create mode 100644 benchmark/json_decode_regex/bench_other.py create mode 100644 benchmark/json_decode_regex/bench_sglang.py create mode 100644 benchmark/json_decode_regex/build_dataset.py create mode 100644 benchmark/json_jump_forward/README.md create mode 100644 benchmark/json_jump_forward/bench_other.py create mode 100644 benchmark/json_jump_forward/bench_sglang.py create mode 100644 benchmark/json_jump_forward/build_dataset.py create mode 100644 benchmark/json_jump_forward/dataset.txt create mode 100644 benchmark/json_schema/README.md create mode 100644 benchmark/json_schema/bench_sglang.py create mode 100644 benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py create mode 100644 benchmark/kernels/deepseek/README.md create mode 100644 benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py create mode 100644 benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py create mode 100644 benchmark/kernels/fused_moe_triton/README.md create mode 100644 benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py create mode 100644 benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py create mode 100644 benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py create mode 100644 benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py create mode 100644 benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py create mode 100644 benchmark/kernels/quantization/bench_int8_quant.py create mode 100644 benchmark/kernels/quantization/tuning_block_wise_kernel.py create mode 100644 benchmark/kernels/rmsnorm/benchmark_rmsnorm.py create mode 100644 benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py create mode 100644 benchmark/line_retrieval/README.md create mode 100644 benchmark/line_retrieval/bench_sglang.py create mode 100644 benchmark/line_retrieval/gen_data.py create mode 100644 benchmark/llava_bench/README.md create mode 100644 benchmark/llava_bench/bench_hf_llava_bench.sh create mode 100644 benchmark/llava_bench/bench_hf_mme.sh create mode 100644 benchmark/llava_bench/bench_sglang.py create mode 100644 benchmark/llava_bench/bench_sglang_mme.sh create mode 100644 benchmark/llava_bench/download_images.py create mode 100644 benchmark/llm_judge/README.md create mode 100644 benchmark/llm_judge/bench_other.py create mode 100644 benchmark/llm_judge/bench_sglang.py create mode 100644 benchmark/long_json_decode/README.md create mode 100644 benchmark/long_json_decode/bench_other.py create mode 100644 benchmark/long_json_decode/bench_sglang.py create mode 100644 benchmark/long_json_decode/build_dataset.py create mode 100644 benchmark/lora/launch_server.py create mode 100644 benchmark/lora/lora_bench.py create mode 100644 benchmark/mmlu/README.md create mode 100644 benchmark/mmlu/bench_other.py create mode 100644 benchmark/mmlu/bench_sglang.py create mode 100644 benchmark/mmlu/download_data.sh create mode 100644 benchmark/mmmu/README.md create mode 100644 benchmark/mmmu/bench_hf.py create mode 100644 benchmark/mmmu/bench_sglang.py create mode 100644 benchmark/mmmu/data_utils.py create mode 100644 benchmark/mmmu/eval_utils.py create mode 100644 benchmark/mmmu/prompt_format.yaml create mode 100644 benchmark/mtbench/README.md create mode 100644 benchmark/mtbench/bench_other.py create mode 100644 benchmark/mtbench/bench_sglang.py create mode 100644 benchmark/mtbench/bench_sglang_eagle.py create mode 100644 benchmark/multi_chain_reasoning/README.md create mode 100644 benchmark/multi_chain_reasoning/bench_other.py create mode 100644 benchmark/multi_chain_reasoning/bench_sglang.py create mode 100644 benchmark/multi_document_qa/README.md create mode 100644 benchmark/multi_document_qa/bench_other.py create mode 100644 benchmark/multi_document_qa/bench_sglang.py create mode 100644 benchmark/multi_document_qa/build_dataset.py create mode 100644 benchmark/multi_turn_chat/README.md create mode 100644 benchmark/multi_turn_chat/bench_other.py create mode 100644 benchmark/multi_turn_chat/bench_sglang.py create mode 100644 benchmark/multi_turn_chat/data_gen.py create mode 100644 benchmark/multi_turn_chat/long_prompt_multi_turn.py create mode 100644 benchmark/react/README.md create mode 100644 benchmark/react/bench_other.py create mode 100644 benchmark/react/bench_sglang.py create mode 100644 benchmark/reasoning_benchmark/README.md create mode 100644 benchmark/reasoning_benchmark/answer_extraction.py create mode 100644 benchmark/reasoning_benchmark/bench_sglang.py create mode 100644 benchmark/reasoning_benchmark/eval_utils.py create mode 100644 benchmark/tip_suggestion/.gitignore create mode 100644 benchmark/tip_suggestion/README.md create mode 100644 benchmark/tip_suggestion/bench_other.py create mode 100644 benchmark/tip_suggestion/bench_sglang.py create mode 100644 benchmark/tip_suggestion/lmql_funcs.py create mode 100644 benchmark/tip_suggestion/topic.jsonl create mode 100644 benchmark/tree_of_thought_deep/README.md create mode 100644 benchmark/tree_of_thought_deep/bench_other.py create mode 100644 benchmark/tree_of_thought_deep/bench_sglang.py create mode 100644 benchmark/tree_of_thought_deep/lmql_funcs.py create mode 100644 benchmark/tree_of_thought_v0/README.md create mode 100644 benchmark/tree_of_thought_v0/bench_other.py create mode 100644 benchmark/tree_of_thought_v0/bench_sglang.py create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.bak create mode 100644 docker/Dockerfile.deepep create mode 100644 docker/Dockerfile.dev create mode 100644 docker/Dockerfile.ds-14b create mode 100644 docker/Dockerfile.ds-8b create mode 100644 docker/Dockerfile.llama3.1-8b create mode 100644 docker/Dockerfile.qwq-32b create mode 100644 docker/Dockerfile.rocm create mode 100644 docker/Dockerfile.sagemaker create mode 100644 docker/compose.yaml create mode 100644 docker/k8s-sglang-distributed-sts.yaml create mode 100644 docker/k8s-sglang-service.yaml create mode 100644 docker/serve create mode 100755 docker/start.sh create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/_static/css/custom_log.css create mode 100644 docs/_static/css/readthedocs.css create mode 100644 docs/_static/image/logo.ico create mode 100644 docs/_static/image/logo.png create mode 100644 docs/backend/__init__.py create mode 100644 docs/backend/custom_chat_template.md create mode 100644 docs/backend/function_calling.ipynb create mode 100644 docs/backend/hyperparameter_tuning.md create mode 100644 docs/backend/native_api.ipynb create mode 100644 docs/backend/offline_engine_api.ipynb create mode 100644 docs/backend/openai_api_completions.ipynb create mode 100644 docs/backend/openai_api_embeddings.ipynb create mode 100644 docs/backend/openai_api_vision.ipynb create mode 100644 docs/backend/patch.py create mode 100644 docs/backend/quantization.md create mode 100644 docs/backend/sampling_params.md create mode 100644 docs/backend/send_request.ipynb create mode 100644 docs/backend/separate_reasoning.ipynb create mode 100644 docs/backend/server_arguments.md create mode 100644 docs/backend/speculative_decoding.ipynb create mode 100644 docs/backend/structured_outputs.ipynb create mode 100644 docs/conf.py create mode 100644 docs/deploy.py create mode 100644 docs/developer/development_guide_using_docker.md create mode 100644 docs/developer/release_process.md create mode 100644 docs/developer/setup_github_runner.md create mode 100644 docs/frontend/__init__.py create mode 100644 docs/frontend/choices_methods.md create mode 100644 docs/frontend/frontend.ipynb create mode 100644 docs/frontend/patch.py create mode 100644 docs/index.rst create mode 100644 docs/references/accuracy_evaluation.md create mode 100644 docs/references/advanced_deploy.rst create mode 100644 docs/references/amd.md create mode 100644 docs/references/benchmark_and_profiling.md create mode 100644 docs/references/contribution_guide.md create mode 100644 docs/references/deepseek.md create mode 100644 docs/references/deepseek.rst create mode 100644 docs/references/deploy_on_k8s.md create mode 100644 docs/references/faq.md create mode 100644 docs/references/general.rst create mode 100644 docs/references/hardware.rst create mode 100644 docs/references/learn_more.md create mode 100644 docs/references/modelscope.md create mode 100644 docs/references/multi_node.md create mode 100644 docs/references/nvidia_jetson.md create mode 100644 docs/references/performance_tuning.rst create mode 100644 docs/references/production_metrics.md create mode 100644 docs/references/supported_models.md create mode 100644 docs/references/troubleshooting.md create mode 100644 docs/requirements.txt create mode 100644 docs/router/router.md create mode 100644 docs/serve.sh create mode 100644 docs/start/install.md create mode 100644 examples/frontend_language/quick_start/anthropic_example_chat.py create mode 100644 examples/frontend_language/quick_start/anthropic_example_complete.py create mode 100644 examples/frontend_language/quick_start/azure_openai_example_chat.py create mode 100644 examples/frontend_language/quick_start/gemini_example_chat.py create mode 100644 examples/frontend_language/quick_start/gemini_example_complete.py create mode 100644 examples/frontend_language/quick_start/gemini_example_multimodal_chat.py create mode 100644 examples/frontend_language/quick_start/images/cat.jpeg create mode 100644 examples/frontend_language/quick_start/images/dog.jpeg create mode 100644 examples/frontend_language/quick_start/local_example_chat.py create mode 100644 examples/frontend_language/quick_start/local_example_complete.py create mode 100644 examples/frontend_language/quick_start/local_example_llava_next.py create mode 100644 examples/frontend_language/quick_start/openai_example_chat.py create mode 100644 examples/frontend_language/quick_start/openai_example_complete.py create mode 100644 examples/frontend_language/quick_start/openai_example_n.py create mode 100644 examples/frontend_language/quick_start/openai_example_o1.py create mode 100644 examples/frontend_language/quick_start/openrouter_example_chat.py create mode 100644 examples/frontend_language/quick_start/together_example_chat.py create mode 100644 examples/frontend_language/quick_start/together_example_complete.py create mode 100644 examples/frontend_language/usage/chinese_regex.py create mode 100644 examples/frontend_language/usage/choices_logprob.py create mode 100644 examples/frontend_language/usage/cot_decoding.py create mode 100644 examples/frontend_language/usage/json_decode.py create mode 100644 examples/frontend_language/usage/json_logprobs.py create mode 100644 examples/frontend_language/usage/llava_video/srt_example_llava_v.py create mode 100644 examples/frontend_language/usage/llava_video/srt_example_llava_v.sh create mode 100644 examples/frontend_language/usage/openai_chat_speculative.py create mode 100644 examples/frontend_language/usage/openai_speculative.py create mode 100644 examples/frontend_language/usage/parallel_sample.py create mode 100644 examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb create mode 100644 examples/frontend_language/usage/readme_examples.py create mode 100644 examples/frontend_language/usage/sgl_gen_min_tokens.py create mode 100644 examples/frontend_language/usage/streaming.py create mode 100644 examples/frontend_language/usage/triton/Dockerfile create mode 100644 examples/frontend_language/usage/triton/README.md create mode 100644 examples/frontend_language/usage/triton/models/character_generation/config.pbtxt create mode 100644 examples/monitoring/docker-compose.yaml create mode 100644 examples/monitoring/grafana.json create mode 100644 examples/monitoring/prometheus.yaml create mode 100644 examples/runtime/README.md create mode 100644 examples/runtime/engine/custom_server.py create mode 100644 examples/runtime/engine/embedding.py create mode 100644 examples/runtime/engine/launch_engine.py create mode 100644 examples/runtime/engine/offline_batch_inference.py create mode 100644 examples/runtime/engine/offline_batch_inference_async.py create mode 100644 examples/runtime/engine/offline_batch_inference_eagle.py create mode 100644 examples/runtime/engine/offline_batch_inference_torchrun.py create mode 100644 examples/runtime/engine/offline_batch_inference_vlm.py create mode 100644 examples/runtime/engine/readme.md create mode 100644 examples/runtime/engine/save_remote_state.py create mode 100644 examples/runtime/engine/save_sharded_state.py create mode 100644 examples/runtime/hidden_states/hidden_states_engine.py create mode 100644 examples/runtime/hidden_states/hidden_states_server.py create mode 100644 examples/runtime/llava_onevision/http_llama3_llava_test.py create mode 100644 examples/runtime/llava_onevision/http_llava_onevision_test.py create mode 100644 examples/runtime/llava_onevision/http_qwen_llava_test.py create mode 100644 examples/runtime/lora.py create mode 100644 examples/runtime/multimodal_embedding.py create mode 100644 examples/runtime/openai_batch_chat.py create mode 100644 examples/runtime/openai_batch_complete.py create mode 100644 examples/runtime/openai_chat_with_response_prefill.py create mode 100644 examples/runtime/reward_model.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_llm_server.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py create mode 100644 examples/runtime/vertex_predict.py create mode 100644 python/pyproject.toml create mode 100644 python/sglang/README.md create mode 100644 python/sglang/__init__.py create mode 100644 python/sglang/api.py create mode 100644 python/sglang/bench_offline_throughput.py create mode 100644 python/sglang/bench_one_batch.py create mode 100644 python/sglang/bench_one_batch_server.py create mode 100644 python/sglang/bench_serving.py create mode 100644 python/sglang/check_env.py create mode 100644 python/sglang/global_config.py create mode 100644 python/sglang/lang/__init__.py create mode 100644 python/sglang/lang/backend/__init__.py create mode 100644 python/sglang/lang/backend/anthropic.py create mode 100644 python/sglang/lang/backend/base_backend.py create mode 100644 python/sglang/lang/backend/litellm.py create mode 100644 python/sglang/lang/backend/openai.py create mode 100644 python/sglang/lang/backend/runtime_endpoint.py create mode 100644 python/sglang/lang/backend/vertexai.py create mode 100644 python/sglang/lang/chat_template.py create mode 100644 python/sglang/lang/choices.py create mode 100644 python/sglang/lang/compiler.py create mode 100644 python/sglang/lang/interpreter.py create mode 100644 python/sglang/lang/ir.py create mode 100644 python/sglang/lang/tracer.py create mode 100644 python/sglang/launch_server.py create mode 100644 python/sglang/llama3_eval.py create mode 100644 python/sglang/srt/_custom_ops.py create mode 100644 python/sglang/srt/aio_rwlock.py create mode 100644 python/sglang/srt/code_completion_parser.py create mode 100644 python/sglang/srt/configs/__init__.py create mode 100644 python/sglang/srt/configs/chatglm.py create mode 100644 python/sglang/srt/configs/dbrx.py create mode 100644 python/sglang/srt/configs/deepseekvl2.py create mode 100644 python/sglang/srt/configs/device_config.py create mode 100644 python/sglang/srt/configs/exaone.py create mode 100644 python/sglang/srt/configs/janus_pro.py create mode 100644 python/sglang/srt/configs/load_config.py create mode 100644 python/sglang/srt/configs/model_config.py create mode 100644 python/sglang/srt/configs/utils.py create mode 100644 python/sglang/srt/connector/__init__.py create mode 100644 python/sglang/srt/connector/base_connector.py create mode 100644 python/sglang/srt/connector/redis.py create mode 100644 python/sglang/srt/connector/s3.py create mode 100644 python/sglang/srt/connector/serde/__init__.py create mode 100644 python/sglang/srt/connector/serde/safe_serde.py create mode 100644 python/sglang/srt/connector/serde/serde.py create mode 100644 python/sglang/srt/connector/utils.py create mode 100644 python/sglang/srt/constrained/base_grammar_backend.py create mode 100644 python/sglang/srt/constrained/llguidance_backend.py create mode 100644 python/sglang/srt/constrained/outlines_backend.py create mode 100644 python/sglang/srt/constrained/outlines_jump_forward.py create mode 100644 python/sglang/srt/constrained/xgrammar_backend.py create mode 100644 python/sglang/srt/conversation.py create mode 100644 python/sglang/srt/custom_op.py create mode 100644 python/sglang/srt/disaggregation/conn.py create mode 100644 python/sglang/srt/disaggregation/decode.py create mode 100644 python/sglang/srt/disaggregation/mini_lb.py create mode 100644 python/sglang/srt/disaggregation/prefill.py create mode 100644 python/sglang/srt/disaggregation/utils.py create mode 100644 python/sglang/srt/distributed/__init__.py create mode 100644 python/sglang/srt/distributed/communication_op.py create mode 100644 python/sglang/srt/distributed/device_communicators/cuda_wrapper.py create mode 100644 python/sglang/srt/distributed/device_communicators/custom_all_reduce.py create mode 100644 python/sglang/srt/distributed/device_communicators/custom_all_reduce_utils.py create mode 100644 python/sglang/srt/distributed/device_communicators/hpu_communicator.py create mode 100644 python/sglang/srt/distributed/device_communicators/pynccl.py create mode 100644 python/sglang/srt/distributed/device_communicators/pynccl_wrapper.py create mode 100644 python/sglang/srt/distributed/device_communicators/shm_broadcast.py create mode 100644 python/sglang/srt/distributed/device_communicators/xpu_communicator.py create mode 100644 python/sglang/srt/distributed/parallel_state.py create mode 100644 python/sglang/srt/distributed/utils.py create mode 100644 python/sglang/srt/entrypoints/engine.py create mode 100644 python/sglang/srt/entrypoints/http_server.py create mode 100644 python/sglang/srt/entrypoints/verl_engine.py create mode 100644 python/sglang/srt/function_call_parser.py create mode 100644 python/sglang/srt/hf_transformers_utils.py create mode 100644 python/sglang/srt/layers/activation.py create mode 100644 python/sglang/srt/layers/attention/base_attn_backend.py create mode 100644 python/sglang/srt/layers/attention/double_sparsity_backend.py create mode 100644 python/sglang/srt/layers/attention/flashattention_backend.py create mode 100644 python/sglang/srt/layers/attention/flashinfer_backend.py create mode 100644 python/sglang/srt/layers/attention/flashinfer_mla_backend.py create mode 100644 python/sglang/srt/layers/attention/flashmla_backend.py create mode 100644 python/sglang/srt/layers/attention/torch_native_backend.py create mode 100644 python/sglang/srt/layers/attention/triton_backend.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/decode_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/extend_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/prefill_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py create mode 100644 python/sglang/srt/layers/attention/utils.py create mode 100644 python/sglang/srt/layers/attention/vision.py create mode 100644 python/sglang/srt/layers/dp_attention.py create mode 100644 python/sglang/srt/layers/elementwise.py create mode 100644 python/sglang/srt/layers/layernorm.py create mode 100644 python/sglang/srt/layers/linear.py create mode 100644 python/sglang/srt/layers/logits_processor.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/__init__.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/kernels.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/layer.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/token_dispatcher.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_native.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/__init__.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=64,device_name=NVIDIA_L20,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=256,N=64,device_name=NVIDIA_L40S,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=1280,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=2560,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=320,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=64,N=640,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=14336,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=1792,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=2048,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=3584,device_name=NVIDIA_L40S.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=4096,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=7168,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/README create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/layer.py create mode 100644 python/sglang/srt/layers/moe/router.py create mode 100644 python/sglang/srt/layers/moe/topk.py create mode 100644 python/sglang/srt/layers/parameter.py create mode 100644 python/sglang/srt/layers/pooler.py create mode 100644 python/sglang/srt/layers/quantization/__init__.py create mode 100644 python/sglang/srt/layers/quantization/awq.py create mode 100644 python/sglang/srt/layers/quantization/base_config.py create mode 100644 python/sglang/srt/layers/quantization/blockwise_int8.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/README.md create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/__init__.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/utils.py create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/fp8.py create mode 100644 python/sglang/srt/layers/quantization/fp8_kernel.py create mode 100644 python/sglang/srt/layers/quantization/fp8_utils.py create mode 100644 python/sglang/srt/layers/quantization/gptq.py create mode 100644 python/sglang/srt/layers/quantization/int8_kernel.py create mode 100644 python/sglang/srt/layers/quantization/int8_utils.py create mode 100644 python/sglang/srt/layers/quantization/kv_cache.py create mode 100644 python/sglang/srt/layers/quantization/modelopt_quant.py create mode 100644 python/sglang/srt/layers/quantization/utils.py create mode 100644 python/sglang/srt/layers/quantization/w8a8_fp8.py create mode 100644 python/sglang/srt/layers/quantization/w8a8_int8.py create mode 100644 python/sglang/srt/layers/radix_attention.py create mode 100644 python/sglang/srt/layers/rotary_embedding.py create mode 100644 python/sglang/srt/layers/sampler.py create mode 100644 python/sglang/srt/layers/torchao_utils.py create mode 100644 python/sglang/srt/layers/vocab_parallel_embedding.py create mode 100644 python/sglang/srt/lora/backend/__init__.py create mode 100644 python/sglang/srt/lora/backend/base_backend.py create mode 100644 python/sglang/srt/lora/backend/flashinfer_backend.py create mode 100644 python/sglang/srt/lora/backend/triton_backend.py create mode 100644 python/sglang/srt/lora/layers.py create mode 100644 python/sglang/srt/lora/lora.py create mode 100644 python/sglang/srt/lora/lora_config.py create mode 100644 python/sglang/srt/lora/lora_manager.py create mode 100644 python/sglang/srt/lora/mem_pool.py create mode 100644 python/sglang/srt/lora/triton_ops/__init__.py create mode 100644 python/sglang/srt/lora/triton_ops/gate_up_lora_b.py create mode 100644 python/sglang/srt/lora/triton_ops/qkv_lora_b.py create mode 100644 python/sglang/srt/lora/triton_ops/sgemm_lora_a.py create mode 100644 python/sglang/srt/lora/triton_ops/sgemm_lora_b.py create mode 100644 python/sglang/srt/lora/utils.py create mode 100644 python/sglang/srt/managers/cache_controller.py create mode 100644 python/sglang/srt/managers/configure_logging.py create mode 100644 python/sglang/srt/managers/data_parallel_controller.py create mode 100644 python/sglang/srt/managers/detokenizer_manager.py create mode 100644 python/sglang/srt/managers/expert_distribution.py create mode 100644 python/sglang/srt/managers/io_struct.py create mode 100644 python/sglang/srt/managers/mm_utils.py create mode 100644 python/sglang/srt/managers/multimodal_processor.py create mode 100644 python/sglang/srt/managers/multimodal_processors/base_processor.py create mode 100644 python/sglang/srt/managers/multimodal_processors/clip.py create mode 100644 python/sglang/srt/managers/multimodal_processors/deepseek_vl_v2.py create mode 100644 python/sglang/srt/managers/multimodal_processors/gemma3.py create mode 100644 python/sglang/srt/managers/multimodal_processors/janus_pro.py create mode 100644 python/sglang/srt/managers/multimodal_processors/llava.py create mode 100644 python/sglang/srt/managers/multimodal_processors/minicpm.py create mode 100644 python/sglang/srt/managers/multimodal_processors/mlama.py create mode 100644 python/sglang/srt/managers/multimodal_processors/qwen_vl.py create mode 100644 python/sglang/srt/managers/schedule_batch.py create mode 100644 python/sglang/srt/managers/schedule_policy.py create mode 100644 python/sglang/srt/managers/scheduler.py create mode 100644 python/sglang/srt/managers/scheduler_output_processor_mixin.py create mode 100644 python/sglang/srt/managers/session_controller.py create mode 100644 python/sglang/srt/managers/tokenizer_manager.py create mode 100644 python/sglang/srt/managers/tp_worker.py create mode 100644 python/sglang/srt/managers/tp_worker_overlap_thread.py create mode 100644 python/sglang/srt/managers/utils.py create mode 100644 python/sglang/srt/mem_cache/base_prefix_cache.py create mode 100644 python/sglang/srt/mem_cache/chunk_cache.py create mode 100644 python/sglang/srt/mem_cache/flush_cache.py create mode 100644 python/sglang/srt/mem_cache/hiradix_cache.py create mode 100644 python/sglang/srt/mem_cache/memory_pool.py create mode 100644 python/sglang/srt/mem_cache/paged_allocator.py create mode 100644 python/sglang/srt/mem_cache/radix_cache.py create mode 100644 python/sglang/srt/metrics/collector.py create mode 100644 python/sglang/srt/metrics/func_timer.py create mode 100644 python/sglang/srt/mm_utils.py create mode 100644 python/sglang/srt/model_executor/cuda_graph_runner.py create mode 100644 python/sglang/srt/model_executor/forward_batch_info.py create mode 100644 python/sglang/srt/model_executor/model_runner.py create mode 100644 python/sglang/srt/model_loader/__init__.py create mode 100644 python/sglang/srt/model_loader/loader.py create mode 100644 python/sglang/srt/model_loader/utils.py create mode 100644 python/sglang/srt/model_loader/weight_utils.py create mode 100644 python/sglang/srt/model_parallel.py create mode 100644 python/sglang/srt/models/baichuan.py create mode 100644 python/sglang/srt/models/chatglm.py create mode 100644 python/sglang/srt/models/clip.py create mode 100644 python/sglang/srt/models/commandr.py create mode 100644 python/sglang/srt/models/dbrx.py create mode 100644 python/sglang/srt/models/deepseek.py create mode 100644 python/sglang/srt/models/deepseek_janus_pro.py create mode 100644 python/sglang/srt/models/deepseek_nextn.py create mode 100644 python/sglang/srt/models/deepseek_v2.py create mode 100644 python/sglang/srt/models/deepseek_vl2.py create mode 100644 python/sglang/srt/models/exaone.py create mode 100644 python/sglang/srt/models/gemma.py create mode 100644 python/sglang/srt/models/gemma2.py create mode 100644 python/sglang/srt/models/gemma2_reward.py create mode 100644 python/sglang/srt/models/gemma3_causal.py create mode 100644 python/sglang/srt/models/gemma3_mm.py create mode 100644 python/sglang/srt/models/gpt2.py create mode 100644 python/sglang/srt/models/gpt_bigcode.py create mode 100644 python/sglang/srt/models/granite.py create mode 100644 python/sglang/srt/models/grok.py create mode 100644 python/sglang/srt/models/internlm2.py create mode 100644 python/sglang/srt/models/internlm2_reward.py create mode 100644 python/sglang/srt/models/llama.py create mode 100644 python/sglang/srt/models/llama_classification.py create mode 100644 python/sglang/srt/models/llama_eagle.py create mode 100644 python/sglang/srt/models/llama_eagle3.py create mode 100644 python/sglang/srt/models/llama_embedding.py create mode 100644 python/sglang/srt/models/llama_reward.py create mode 100644 python/sglang/srt/models/llava.py create mode 100644 python/sglang/srt/models/llavavid.py create mode 100644 python/sglang/srt/models/minicpm.py create mode 100644 python/sglang/srt/models/minicpm3.py create mode 100644 python/sglang/srt/models/minicpmo.py create mode 100644 python/sglang/srt/models/minicpmv.py create mode 100644 python/sglang/srt/models/mistral.py create mode 100644 python/sglang/srt/models/mixtral.py create mode 100644 python/sglang/srt/models/mixtral_quant.py create mode 100644 python/sglang/srt/models/mllama.py create mode 100644 python/sglang/srt/models/olmo.py create mode 100644 python/sglang/srt/models/olmo2.py create mode 100644 python/sglang/srt/models/olmoe.py create mode 100644 python/sglang/srt/models/phi3_small.py create mode 100644 python/sglang/srt/models/qwen.py create mode 100644 python/sglang/srt/models/qwen2.py create mode 100644 python/sglang/srt/models/qwen2_5_vl.py create mode 100644 python/sglang/srt/models/qwen2_classification.py create mode 100644 python/sglang/srt/models/qwen2_eagle.py create mode 100644 python/sglang/srt/models/qwen2_moe.py create mode 100644 python/sglang/srt/models/qwen2_rm.py create mode 100644 python/sglang/srt/models/qwen2_vl.py create mode 100644 python/sglang/srt/models/registry.py create mode 100644 python/sglang/srt/models/stablelm.py create mode 100644 python/sglang/srt/models/torch_native_llama.py create mode 100644 python/sglang/srt/models/xverse.py create mode 100644 python/sglang/srt/models/xverse_moe.py create mode 100644 python/sglang/srt/models/yivl.py create mode 100644 python/sglang/srt/openai_api/adapter.py create mode 100644 python/sglang/srt/openai_api/protocol.py create mode 100644 python/sglang/srt/patch_torch.py create mode 100644 python/sglang/srt/reasoning_parser.py create mode 100644 python/sglang/srt/sampling/custom_logit_processor.py create mode 100644 python/sglang/srt/sampling/penaltylib/__init__.py create mode 100644 python/sglang/srt/sampling/penaltylib/frequency_penalty.py create mode 100644 python/sglang/srt/sampling/penaltylib/min_new_tokens.py create mode 100644 python/sglang/srt/sampling/penaltylib/orchestrator.py create mode 100644 python/sglang/srt/sampling/penaltylib/presence_penalty.py create mode 100644 python/sglang/srt/sampling/sampling_batch_info.py create mode 100644 python/sglang/srt/sampling/sampling_params.py create mode 100644 python/sglang/srt/server.py create mode 100644 python/sglang/srt/server_args.py create mode 100644 python/sglang/srt/speculative/build_eagle_tree.py create mode 100644 python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py create mode 100644 python/sglang/srt/speculative/eagle_utils.py create mode 100644 python/sglang/srt/speculative/eagle_worker.py create mode 100644 python/sglang/srt/speculative/spec_info.py create mode 100644 python/sglang/srt/torch_memory_saver_adapter.py create mode 100644 python/sglang/srt/utils.py create mode 100644 python/sglang/srt/warmup.py create mode 100644 python/sglang/test/__init__.py create mode 100644 python/sglang/test/attention/__init__.py create mode 100644 python/sglang/test/attention/test_flashattn_backend.py create mode 100644 python/sglang/test/few_shot_gsm8k.py create mode 100644 python/sglang/test/few_shot_gsm8k_engine.py create mode 100644 python/sglang/test/long_prompt.txt create mode 100644 python/sglang/test/run_eval.py create mode 100644 python/sglang/test/runners.py create mode 100644 python/sglang/test/send_one.py create mode 100644 python/sglang/test/simple_eval_common.py create mode 100644 python/sglang/test/simple_eval_gpqa.py create mode 100644 python/sglang/test/simple_eval_humaneval.py create mode 100644 python/sglang/test/simple_eval_math.py create mode 100644 python/sglang/test/simple_eval_mgsm.py create mode 100644 python/sglang/test/simple_eval_mmlu.py create mode 100644 python/sglang/test/test_activation.py create mode 100644 python/sglang/test/test_block_fp8.py create mode 100644 python/sglang/test/test_block_fp8_ep.py create mode 100644 python/sglang/test/test_custom_ops.py create mode 100644 python/sglang/test/test_dynamic_grad_mode.py create mode 100644 python/sglang/test/test_layernorm.py create mode 100644 python/sglang/test/test_programs.py create mode 100644 python/sglang/test/test_utils.py create mode 100644 python/sglang/utils.py create mode 100644 python/sglang/version.py create mode 100644 scripts/ci_install_dependency.sh create mode 100644 scripts/ci_install_rust.sh create mode 100644 scripts/deprecated/convert_yi_vl.py create mode 100644 scripts/deprecated/convert_yi_vl.sh create mode 100644 scripts/deprecated/test_curl.sh create mode 100644 scripts/deprecated/test_flashinfer.py create mode 100644 scripts/deprecated/test_httpserver_concurrent.py create mode 100644 scripts/deprecated/test_httpserver_decode.py create mode 100644 scripts/deprecated/test_httpserver_decode_stream.py create mode 100644 scripts/deprecated/test_httpserver_llava.py create mode 100644 scripts/deprecated/test_httpserver_reuse.py create mode 100644 scripts/deprecated/test_jump_forward.py create mode 100644 scripts/deprecated/test_robust.py create mode 100644 scripts/export_deepseek_nextn.py create mode 100644 scripts/fix_corrupted_json.py create mode 100644 scripts/killall_sglang.sh create mode 100644 scripts/playground/bench_speculative.py create mode 100644 scripts/playground/launch_tgi.sh create mode 100644 scripts/playground/load_tokenizer.py create mode 100644 scripts/playground/lora/analyzer.py create mode 100644 scripts/playground/lora/lora_hf_play.py create mode 100644 scripts/playground/lora/lora_vllm_play.py create mode 100644 scripts/playground/reference_hf.py create mode 100644 scripts/playground/router/test_tree.py create mode 100644 scripts/playground/router/tree.py create mode 100644 scripts/update_kernel_whl_index.py create mode 100644 scripts/version_branch_to_tag.sh create mode 100644 sgl-kernel/.clang-format create mode 100644 sgl-kernel/CMakeLists.txt create mode 100644 sgl-kernel/LICENSE create mode 100644 sgl-kernel/Makefile create mode 100644 sgl-kernel/README.md create mode 100644 sgl-kernel/THIRDPARTYNOTICES.txt create mode 100644 sgl-kernel/benchmark/bench_awq_dequant.py create mode 100644 sgl-kernel/benchmark/bench_cublas_grouped_gemm.py create mode 100644 sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py create mode 100644 sgl-kernel/benchmark/bench_fp8_gemm.py create mode 100644 sgl-kernel/benchmark/bench_int8_gemm.py create mode 100644 sgl-kernel/benchmark/bench_lightning_attention_decode.py create mode 100644 sgl-kernel/benchmark/bench_moe_align_block_size.py create mode 100644 sgl-kernel/benchmark/bench_moe_topk_softmax.py create mode 100644 sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py create mode 100644 sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py create mode 100644 sgl-kernel/benchmark/bench_per_token_quant_fp8.py create mode 100644 sgl-kernel/build.sh create mode 100644 sgl-kernel/csrc/allreduce/custom_all_reduce.hip create mode 100644 sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh create mode 100644 sgl-kernel/csrc/allreduce/trt_reduce_internal.cu create mode 100644 sgl-kernel/csrc/allreduce/trt_reduce_kernel.cu create mode 100644 sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu create mode 100644 sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h create mode 100644 sgl-kernel/csrc/elementwise/activation.cu create mode 100644 sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu create mode 100644 sgl-kernel/csrc/elementwise/rope.cu create mode 100644 sgl-kernel/csrc/gemm/awq_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/bmm_fp8.cu create mode 100644 sgl-kernel/csrc/gemm/cublas_grouped_gemm.cu create mode 100644 sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/int8_gemm_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu create mode 100644 sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu create mode 100644 sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu create mode 100644 sgl-kernel/csrc/gemm/per_token_quant_fp8.cu create mode 100644 sgl-kernel/csrc/moe/moe_align_kernel.cu create mode 100644 sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu create mode 100644 sgl-kernel/csrc/speculative/eagle_utils.cu create mode 100644 sgl-kernel/csrc/speculative/packbit.cu create mode 100644 sgl-kernel/csrc/speculative/pytorch_extension_utils_rocm.h create mode 100644 sgl-kernel/csrc/speculative/speculative_sampling.cu create mode 100644 sgl-kernel/csrc/speculative/speculative_sampling.cuh create mode 100644 sgl-kernel/csrc/torch_extension.cc create mode 100644 sgl-kernel/csrc/torch_extension_rocm.cc create mode 100644 sgl-kernel/include/sgl_kernel_ops.h create mode 100644 sgl-kernel/include/trt_reduce_internal.cuh create mode 100644 sgl-kernel/include/utils.h create mode 100644 sgl-kernel/pyproject.toml create mode 100644 sgl-kernel/pyproject_rocm.toml create mode 100644 sgl-kernel/python/sgl_kernel/__init__.py create mode 100644 sgl-kernel/python/sgl_kernel/allreduce.py create mode 100644 sgl-kernel/python/sgl_kernel/attention.py create mode 100644 sgl-kernel/python/sgl_kernel/elementwise.py create mode 100644 sgl-kernel/python/sgl_kernel/gemm.py create mode 100644 sgl-kernel/python/sgl_kernel/moe.py create mode 100644 sgl-kernel/python/sgl_kernel/sampling.py create mode 100644 sgl-kernel/python/sgl_kernel/speculative.py create mode 100644 sgl-kernel/python/sgl_kernel/utils.py create mode 100644 sgl-kernel/python/sgl_kernel/version.py create mode 100644 sgl-kernel/rename_wheels.sh create mode 100644 sgl-kernel/setup.py create mode 100644 sgl-kernel/setup_rocm.py create mode 100644 sgl-kernel/tests/speculative/test_eagle_utils.py create mode 100644 sgl-kernel/tests/speculative/test_speculative_sampling.py create mode 100644 sgl-kernel/tests/test_activation.py create mode 100644 sgl-kernel/tests/test_awq_dequant.py create mode 100644 sgl-kernel/tests/test_bmm_fp8.py create mode 100644 sgl-kernel/tests/test_cublas_grouped_gemm.py create mode 100644 sgl-kernel/tests/test_deep_gemm.py create mode 100644 sgl-kernel/tests/test_fp4_gemm.py create mode 100644 sgl-kernel/tests/test_fp4_quantize.py create mode 100644 sgl-kernel/tests/test_fp8_blockwise_gemm.py create mode 100644 sgl-kernel/tests/test_fp8_gemm.py create mode 100644 sgl-kernel/tests/test_int8_gemm.py create mode 100644 sgl-kernel/tests/test_lightning_attention_decode.py create mode 100644 sgl-kernel/tests/test_moe_align.py create mode 100644 sgl-kernel/tests/test_moe_topk_softmax.py create mode 100644 sgl-kernel/tests/test_norm.py create mode 100644 sgl-kernel/tests/test_per_tensor_quant_fp8.py create mode 100644 sgl-kernel/tests/test_per_token_group_quant_8bit.py create mode 100644 sgl-kernel/tests/test_per_token_quant_fp8.py create mode 100644 sgl-kernel/tests/test_rotary_embedding.py create mode 100644 sgl-kernel/tests/test_sampling.py create mode 100644 sgl-kernel/tests/test_trt_allreduce.py create mode 100644 sgl-router/Cargo.lock create mode 100644 sgl-router/Cargo.toml create mode 100644 sgl-router/MANIFEST.in create mode 100644 sgl-router/README.md create mode 100644 sgl-router/py_src/sglang_router/__init__.py create mode 100644 sgl-router/py_src/sglang_router/launch_router.py create mode 100644 sgl-router/py_src/sglang_router/launch_server.py create mode 100644 sgl-router/py_src/sglang_router/router.py create mode 100644 sgl-router/py_src/sglang_router/version.py create mode 100644 sgl-router/py_test/run_suite.py create mode 100644 sgl-router/py_test/test_launch_router.py create mode 100644 sgl-router/py_test/test_launch_server.py create mode 100644 sgl-router/pyproject.toml create mode 100644 sgl-router/src/lib.rs create mode 100644 sgl-router/src/router.rs create mode 100644 sgl-router/src/server.rs create mode 100644 sgl-router/src/tree.rs create mode 100644 sgl-router/v0.1.0.md create mode 100644 test/README.md create mode 100644 test/lang/run_suite.py create mode 100644 test/lang/test_anthropic_backend.py create mode 100644 test/lang/test_bind_cache.py create mode 100644 test/lang/test_choices.py create mode 100644 test/lang/test_litellm_backend.py create mode 100644 test/lang/test_openai_backend.py create mode 100644 test/lang/test_srt_backend.py create mode 100644 test/lang/test_tracing.py create mode 100644 test/lang/test_vertexai_backend.py create mode 100644 test/srt/configs/random_config.yaml create mode 100644 test/srt/configs/random_flashinfer_vs_triton_config.yaml create mode 100644 test/srt/configs/sharegpt_config.yaml create mode 100644 test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json create mode 100644 test/srt/experiment_runner.py create mode 100644 test/srt/kv_cache_scales_llama3_1_8b.json create mode 100644 test/srt/kv_cache_scales_llama3_8b.json create mode 100644 test/srt/kv_cache_scales_qwen2_1_5b.json create mode 100644 test/srt/models/compare.py create mode 100644 test/srt/models/lora/test_lora.py create mode 100644 test/srt/models/lora/test_lora_backend.py create mode 100644 test/srt/models/lora/test_lora_tp.py create mode 100644 test/srt/models/lora/test_multi_lora_backend.py create mode 100644 test/srt/models/lora/utils.py create mode 100644 test/srt/models/test_clip_models.py create mode 100644 test/srt/models/test_embedding_models.py create mode 100644 test/srt/models/test_generation_models.py create mode 100644 test/srt/models/test_gme_qwen_models.py create mode 100644 test/srt/models/test_grok_models.py create mode 100644 test/srt/models/test_qwen_models.py create mode 100644 test/srt/models/test_reward_models.py create mode 100644 test/srt/run_suite.py create mode 100644 test/srt/test_abort.py create mode 100644 test/srt/test_awq.py create mode 100644 test/srt/test_bench_one_batch.py create mode 100644 test/srt/test_bench_serving.py create mode 100644 test/srt/test_block_int8.py create mode 100644 test/srt/test_cache_report.py create mode 100644 test/srt/test_chunked_prefill.py create mode 100644 test/srt/test_create_kvindices.py create mode 100644 test/srt/test_custom_allreduce.py create mode 100644 test/srt/test_data_parallelism.py create mode 100644 test/srt/test_double_sparsity.py create mode 100644 test/srt/test_dp_attention.py create mode 100644 test/srt/test_eagle_infer.py create mode 100644 test/srt/test_ebnf_constrained.py create mode 100644 test/srt/test_embedding_openai_server.py create mode 100644 test/srt/test_eval_accuracy_large.py create mode 100644 test/srt/test_eval_fp8_accuracy.py create mode 100644 test/srt/test_expert_distribution.py create mode 100644 test/srt/test_fim_completion.py create mode 100644 test/srt/test_fp8_kernel.py create mode 100644 test/srt/test_fp8_kvcache.py create mode 100644 test/srt/test_function_calling.py create mode 100644 test/srt/test_fused_moe.py create mode 100644 test/srt/test_get_weights_by_name.py create mode 100644 test/srt/test_gguf.py create mode 100644 test/srt/test_gptqmodel_dynamic.py create mode 100644 test/srt/test_health_check.py create mode 100644 test/srt/test_hicache.py create mode 100644 test/srt/test_hicache_mla.py create mode 100644 test/srt/test_hidden_states.py create mode 100644 test/srt/test_input_embeddings.py create mode 100644 test/srt/test_int8_kernel.py create mode 100644 test/srt/test_json_constrained.py create mode 100644 test/srt/test_large_max_new_tokens.py create mode 100644 test/srt/test_matched_stop.py create mode 100644 test/srt/test_metrics.py create mode 100644 test/srt/test_mla.py create mode 100644 test/srt/test_mla_deepseek_v3.py create mode 100644 test/srt/test_mla_flashinfer.py create mode 100644 test/srt/test_mla_fp8.py create mode 100644 test/srt/test_mla_int8_deepseek_v3.py create mode 100644 test/srt/test_mla_tp.py create mode 100644 test/srt/test_modelopt_fp8kvcache.py create mode 100644 test/srt/test_models_from_modelscope.py create mode 100644 test/srt/test_moe_deepep.py create mode 100644 test/srt/test_moe_ep.py create mode 100644 test/srt/test_moe_eval_accuracy_large.py create mode 100644 test/srt/test_nightly_gsm8k_eval.py create mode 100644 test/srt/test_nightly_human_eval.py create mode 100644 test/srt/test_nightly_math_eval.py create mode 100644 test/srt/test_no_chunked_prefill.py create mode 100644 test/srt/test_no_overlap_scheduler.py create mode 100644 test/srt/test_openai_server.py create mode 100644 test/srt/test_page_size.py create mode 100644 test/srt/test_patch_torch.py create mode 100644 test/srt/test_penalty.py create mode 100644 test/srt/test_pytorch_sampling_backend.py create mode 100644 test/srt/test_radix_attention.py create mode 100644 test/srt/test_reasoning_content.py create mode 100644 test/srt/test_regex_constrained.py create mode 100644 test/srt/test_release_memory_occupation.py create mode 100644 test/srt/test_request_length_validation.py create mode 100644 test/srt/test_retract_decode.py create mode 100644 test/srt/test_sagemaker_server.py create mode 100644 test/srt/test_schedule_policy.py create mode 100644 test/srt/test_server_args.py create mode 100644 test/srt/test_session_control.py create mode 100644 test/srt/test_skip_tokenizer_init.py create mode 100644 test/srt/test_srt_endpoint.py create mode 100644 test/srt/test_srt_engine.py create mode 100644 test/srt/test_srt_engine_with_quant_args.py create mode 100644 test/srt/test_torch_compile.py create mode 100644 test/srt/test_torch_compile_moe.py create mode 100644 test/srt/test_torch_native_attention_backend.py create mode 100644 test/srt/test_torch_tp.py create mode 100644 test/srt/test_torchao.py create mode 100644 test/srt/test_triton_attention_backend.py create mode 100644 test/srt/test_triton_attention_kernels.py create mode 100644 test/srt/test_triton_attention_rocm_mla.py create mode 100644 test/srt/test_update_weights_from_disk.py create mode 100644 test/srt/test_update_weights_from_distributed.py create mode 100644 test/srt/test_update_weights_from_tensor.py create mode 100644 test/srt/test_verl_engine.py create mode 100644 test/srt/test_vertex_endpoint.py create mode 100644 test/srt/test_vision_chunked_prefill.py create mode 100644 test/srt/test_vision_openai_server.py create mode 100644 test/srt/test_vllm_dependency.py create mode 100644 test/srt/test_vlm_accuracy.py create mode 100644 test/srt/test_w8a8_quantization.py diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 0000000..15c76cc --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1 @@ +sgl-kernel/3rdparty/tensorrt_llm/* diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..0c061cd --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,35 @@ +From lmsysorg/sglang:dev + +# Create non-root user with specified UID and GID +# NOTE: Replace with your own UID and GID. This is a workaround from https://github.com/microsoft/vscode-remote-release/issues/49#issuecomment-489060908. +ARG HOST_UID=1003 +ARG HOST_GID=1003 +RUN groupadd -g $HOST_GID devuser && \ + useradd -m -u $HOST_UID -g $HOST_GID -s /bin/zsh devuser + +# Give devuser sudo access +RUN apt-get update && apt-get install -y sudo && \ + echo "devuser ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/devuser && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean + +# Set up oh-my-zsh for devuser +RUN cp -r /root/.oh-my-zsh /home/devuser/.oh-my-zsh && \ + cp /root/.zshrc /home/devuser/.zshrc && \ + cp /root/.vimrc /home/devuser/.vimrc && \ + cp /root/.tmux.conf /home/devuser/.tmux.conf && \ + sed -i 's|/root/.oh-my-zsh|/home/devuser/.oh-my-zsh|g' /home/devuser/.zshrc && \ + chown -R devuser:devuser /home/devuser/ + +# Set workspace directory and ownership +WORKDIR /sgl-workspace/sglang +RUN chown -R devuser:devuser /sgl-workspace + +# Switch to devuser +USER devuser + +# Install uv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..5767aa2 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,24 @@ +{ + "name": "sglang", + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "devuser", + "customizations": { + "vscode": { + "extensions": [ + // Python development + "ms-python.python", + "charliermarsh.ruff", + // Rust development + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml" + ] + } + }, + "forwardPorts": [], + "runArgs": [ + "--gpus", + "all" + ] +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..030a729 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,25 @@ +# https://editorconfig.org/ + +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{json,yaml,yml}] +indent_size = 2 + +[*.md] +indent_size = 2 +x-soft-wrap-text = true + +[*.rst] +indent_size = 4 +x-soft-wrap-text = true + +[Makefile] +indent_style = tab diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75e29fa --- /dev/null +++ b/.gitignore @@ -0,0 +1,229 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# MacOS +.DS_Store + +# Vim +*.swp + +# Documentation +docs/_build + +# SGL +benchmark/mmlu/data +benchmark/mmlu/data.tar +benchmark/llava_bench/images +benchmark/llava_bench/mme_pack +*.jsonl +tmp*.txt + +# Plots +*.png +*.pdf + +# personnal +work_dirs/ +*.csv + +!logo.png + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +compile_commands.json + +*.iml + +# VSCode +.vscode + +1 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..265d8d9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,12 @@ +[submodule "sgl-kernel/3rdparty/cutlass"] + path = sgl-kernel/3rdparty/cutlass + url = https://github.com/NVIDIA/cutlass.git +[submodule "sgl-kernel/3rdparty/cccl"] + path = sgl-kernel/3rdparty/cccl + url = https://github.com/NVIDIA/cccl.git +[submodule "sgl-kernel/3rdparty/flashinfer"] + path = sgl-kernel/3rdparty/flashinfer + url = https://github.com/flashinfer-ai/flashinfer.git +[submodule "sgl-kernel/3rdparty/deepgemm"] + path = sgl-kernel/3rdparty/deepgemm + url = https://github.com/deepseek-ai/DeepGEMM diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..8350951 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile=black +known_first_party=sglang diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..eb30735 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +default_stages: [pre-commit, pre-push, manual] + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-symlinks + - id: destroyed-symlinks + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-toml + - id: check-ast + - id: check-added-large-files + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: detect-private-key + - id: debug-statements + - id: no-commit-to-branch + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.2 + hooks: + - id: ruff + args: [--select=F401, --fixable=F401] + files: ^(benchmark/|docs/|examples/) + exclude: \.ipynb$ + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black-jupyter + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.8 + hooks: + - id: clang-format + types_or: [c++, cuda] + args: [--style=file, --verbose] + - repo: https://github.com/kynan/nbstripout + rev: 0.8.1 + hooks: + - id: nbstripout + args: + - '--keep-output' + - '--extra-keys=metadata.kernelspec metadata.language_info.version' diff --git a/3rdparty/amd/profiling/PROFILING.md b/3rdparty/amd/profiling/PROFILING.md new file mode 100644 index 0000000..7e15ec8 --- /dev/null +++ b/3rdparty/amd/profiling/PROFILING.md @@ -0,0 +1,425 @@ +## Profiling SGLang Infer System with AMD GPUs +This AppNote describes the SGLang profiling technical, code augment and running steps for systems with AMD Instinct GPUs, nevertheless the same procedure may work with Nvidia GPUs too. +Examples and steps are provided in detail, to facilitate easy reproduce and use to localize performance problem towards optimizations. +Two primary methods are covered: +- [RPD](https://github.com/ROCm/rocmProfileData.git) +- [PyTorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) + +### Profiling SGLang Infer System with RPD Profiler +RPD profiler is a low-overhead cross-platform profiler. Therefore, the same RPD code augment not only works for profiling on ROCm/AMD GPUs, but also works for profiling on CUDA/Nvidia GPUs as well. To do RPD profiling on SGLang repository, please use scripts and patch files included in this directory and follow the steps below: +1. Install RPD with rpd.patch applied during installation using install_rpd.sh, both files are in this directory. + +install_rpd.sh + +```bash +# download and install RPD +apt update && apt install -y sqlite3 libsqlite3-dev libfmt-dev + +# install rpd module +git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData +cd rocmProfileData +git checkout 976899e9c6dbc6dd2bccf770818e4e44125590ac +git apply rpd.patch +make && make install +cd rocpd_python && python setup.py install && cd .. +cd rpd_tracer && make clean;make install && python setup.py install && cd .. +``` + +rpd.patch + +```bash +diff --git a/rpd_tracer/Makefile b/rpd_tracer/Makefile +index e9d9feb..b2e9e1a 100644 +--- a/rpd_tracer/Makefile ++++ b/rpd_tracer/Makefile +@@ -16,7 +16,7 @@ ifneq (,$(HIP_PATH)) + $(info Building with roctracer) + RPD_LIBS += -L/opt/rocm/lib -lroctracer64 -lroctx64 -lamdhip64 -lrocm_smi64 + RPD_INCLUDES += -I/opt/rocm/include -I/opt/rocm/include/roctracer -I/opt/rocm/include/hsa +- RPD_SRCS += RoctracerDataSource.cpp RocmSmiDataSource.cpp ++ RPD_SRCS += RoctracerDataSource.cpp + RPD_INCLUDES += -D__HIP_PLATFORM_AMD__ + endif +``` +2. Add loadTracer.sh file included in this directory to /sglang/python/sglang. + +loadTracer.sh + +```bash +#!/bin/bash +################################################################################ +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ +OUTPUT_FILE="trace.rpd" + +if [ "$1" = "-o" ] ; then + OUTPUT_FILE=$2 + shift + shift +fi + +if [ -e ${OUTPUT_FILE} ] ; then + rm ${OUTPUT_FILE} +fi + +python3 -m rocpd.schema --create ${OUTPUT_FILE} +if [ $? != 0 ] ; then + echo "Error: Could not create rpd file. Please run 'python setup.py install' from the rocpd_python dir" + exit +fi + +export RPDT_FILENAME=${OUTPUT_FILE} +export RPDT_AUTOSTART=0 +LD_PRELOAD=librocm-smi_64:librpd_tracer.so "$@" +``` +3. Apply patch (provided in this directory) with "git apply rpd_profile_server_enable.patch" if the main profiling purpose is to get info on gpu kernels as well as limited cpu activity info. + +#### Common Notes 1 +Please note that although we are doing TP=8 in the example, we purposely only log RPD profiling on 2 ranks in the patch file (i.e.tp_rank=0/1) for profiling/visualization convenience, as even Perfetto streaming mode can only load maximal 8GB json file for visualization. With 2 ranks logged in RPD profiling, we could still check whether there are issues among ranks (e.g. load imbalance issue, nccl issue), and at the same time, we could log relatively longer time duration before the json file generated from RPD file hits 8GB size. + +rpd_profile_server_enable.patch + +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..9021c01 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,24 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() #block pytorch profiler for rpd profiler enabling ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.start() ++ self.rpd.rangePush("", "rpd profile range", "") ++ logger.info("rpd is enabled") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done") + logger.info("Profiler is done") +``` + +#### Advanced Debugging with RPD Profiler +Sometimes, we want to use rpd profiler to capture more CPU and python activities in order to debug some challenging issues (e.g. root cause of load imbalance across gpu processes, root cause of bubbles, etc). Only in such cases, we need to apply patch "git apply rpd_profile_server_enable_wCPU_activities.patch", where 3 files are modified. + +rpd_profile_server_enable_wCPU_activities.patch + +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..2edb427 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,26 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() ++ logger.info("torch profiler is disabled") ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.setPythonTrace(True) ++ self.rpd.start() ++ self.rpd.rangePush("", "scheduler", "") ++ logger.info("rpd is enabled inside scheduler profiling") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done inside scheduler") + logger.info("Profiler is done") + + +diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py +index 2621ccd..181df85 100644 +--- a/python/sglang/srt/managers/tokenizer_manager.py ++++ b/python/sglang/srt/managers/tokenizer_manager.py +@@ -58,6 +58,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams + from sglang.srt.server_args import PortArgs, ServerArgs + from sglang.srt.utils import is_generation_model, is_multimodal_model + ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() ++ ++ + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + logger = logging.getLogger(__name__) +@@ -514,10 +518,20 @@ class TokenizerManager: + self.send_to_scheduler.send_pyobj(req) + + def start_profile(self): ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "tokenizer_manager", "") ++ logger.info("tokenizer_manager rpd profiling started!") + req = ProfileReq.START_PROFILE + self.send_to_scheduler.send_pyobj(req) + + def stop_profile(self): ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done inside tokenizer_manager!") + req = ProfileReq.STOP_PROFILE + self.send_to_scheduler.send_pyobj(req) + +diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py +index 7111c93..2bd722c 100644 +--- a/python/sglang/srt/server.py ++++ b/python/sglang/srt/server.py +@@ -30,6 +30,8 @@ import threading + import time + from http import HTTPStatus + from typing import Dict, List, Optional, Union ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + # Fix a bug of Python threading + setattr(threading, "_register_atexit", lambda *args, **kwargs: None) +@@ -152,6 +154,11 @@ async def flush_cache(): + @app.post("/start_profile") + async def start_profile(): + """Start profiling.""" ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "server rpd profile range", "") ++ logger.info("rpd profiling started in server.py!") + tokenizer_manager.start_profile() + return Response( + content="Start profiling.\n", +@@ -164,6 +171,11 @@ async def start_profile(): + async def stop_profile(): + """Stop profiling.""" + tokenizer_manager.stop_profile() ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done in server.py!") + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, +``` + +4. As an example for grok1 profiling, we create a dummy_grok1 directory with config.json (see content below) inside this directory and copy this directory to the right path for "--model-path" if you want to use the example server.sh file provided. +```bash +cat ../dummy_grok1/config.json +{ + "architectures": [ + "Grok1ModelForCausalLM" + ], + "embedding_multiplier_scale": 78.38367176906169, + "output_multiplier_scale": 0.5773502691896257, + "vocab_size": 131072, + "hidden_size": 6144, + "intermediate_size": 32768, + "max_position_embeddings": 8192, + "num_experts_per_tok": 2, + "num_local_experts": 8, + "num_attention_heads": 48, + "num_hidden_layers": 64, + "num_key_value_heads": 8, + "head_dim": 128, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "model_type": "mixtral", + "torch_dtype": "bfloat16" +} +``` +5. Launch server with rpd enabled script ./server.sh in one terminal inside the docker container. + +#### Common Notes 2 +- Remember to change model-path to the correct path +- loadTracer.sh is needed to conduct profiling +- SGLANG_TORCH_PROFILER_DIR is used for default torch profiler +- Do not use loadTracer.sh if you are using the torch profiler, simply use python3 -m sglang.launch_server. + + +server.sh + +```bash +#!/bin/bash + +# export SGLANG_TORCH_PROFILER_DIR=/data/sglang/ +export SGLANG_TORCH_PROFILER_DIR=/sgl-workspace/sglang/profile/ + +# Get the current timestamp +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +# Define the log file with a timestamp +LOGFILE="sglang_server_log_$TIMESTAMP.json" + +# Run the Python command and save the output to the log file +loadTracer.sh python3 -m sglang.launch_server \ + --model-path /sgl-workspace/sglang/dummy_grok1 \ + --tokenizer-path Xenova/grok-1-tokenizer \ + --load-format dummy \ + --quantization fp8 \ + --tp 8 \ + --port 30000 \ + --disable-radix-cache 2>&1 | tee "$LOGFILE" +``` +6. Open another terminal for the same docker container, and run the rpd enabled ./client.sh after you see "The server is fired up and is ready to roll!" message from server side terminal. + +#### Common Notes 3 +- Use curl http://localhost:30000/start_profile & curl http://localhost:30000/stop_profile to control the start and end of profiling. Check sglang/python/sglang/srt/managers/scheduler.py for more details. +- Please don't use RPD profiler together with PyTorch profiler to avoid interference. +- The rocmProfileData/tools/rpd2tracing.py file is used to generate json file from RPD file. + +client.sh + +```bash +#!/bin/bash + +# Start profiling via API +curl http://localhost:30000/start_profile -H "Content-Type: application/json" + +# Benchmark serving using sglang with random dataset and tokenizer +# Define the log file with a timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOGFILE="sglang_client_log_$TIMESTAMP.json" + +# Run the benchmark with specified parameters and save logs +python3 -m sglang.bench_serving \ + --backend sglang \ + --tokenizer Xenova/grok-1-tokenizer \ + --dataset-name random \ + --random-input 1024\ + --random-output 1024 \ + --num-prompts 120 \ + --request-rate 8 \ + --output-file online.jsonl 2>&1 | tee "$LOGFILE" + +# Stop profiling via API +curl http://localhost:30000/stop_profile -H "Content-Type: application/json" + +# Convert tracing file to csv & json +sqlite3 trace.rpd ".mode csv" ".header on" ".output trace.csv" "select * from top;" ".output stdout" +python3 ./rocmProfileData/tools/rpd2tracing.py trace.rpd trace.json +``` +7. Follow [Perfetto docs](https://perfetto.dev/docs/visualization/large-traces) to visualize large json files. Try to adjust parameters so that the trace.json file size is less than 9GB. + +### Profiling SGLang Infer System with PyTorch Profiler + +Please use the steps as follows: + +1. Apply the patch torch_profiler.patch. Note that you can modify "if self.tp_rank == 0" in the patch to allow more ranks be recorded in profiling. + +torch_profiler.patch +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..6ecd78c 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -240,7 +240,6 @@ class Scheduler: + ) + self.profiler = torch.profiler.profile( + activities=[ +- torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, +@@ -1033,9 +1032,11 @@ class Scheduler: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ if self.tp_rank == 0: ++ with open(f"stats_repro_{int(time.time())}.txt", "w") as f: ++ print(self.profiler.key_averages(group_by_input_shape=True).table(sort_by="cuda_time_total", row_limit=-1), file=f) ++ print("Profiling stats done.") ++ + logger.info("Profiler is done") +``` + +2. Create the model path directory and copy it to the right path for "--model-path" if you want to use the server.sh file provided. + +3. Modify the included server.sh by removing "loadTracer.sh" before python command and launch script ./server.sh in one terminal inside the docker container. + +4. Similar to step 6 in RPD profiling section, but remove the last 2 lines in client.sh, which converted rpd file into csv and json files. Run modified client.sh for PyTorch profiling. +------- +- [Torch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) diff --git a/3rdparty/amd/profiling/client.sh b/3rdparty/amd/profiling/client.sh new file mode 100644 index 0000000..150ea9f --- /dev/null +++ b/3rdparty/amd/profiling/client.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Start profiling via API +curl http://localhost:30000/start_profile -H "Content-Type: application/json" + +# Benchmark serving using sglang with random dataset and tokenizer +# Define the log file with a timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOGFILE="sglang_client_log_$TIMESTAMP.json" + +# Run the benchmark with specified parameters and save logs +python3 -m sglang.bench_serving \ + --backend sglang \ + --tokenizer Xenova/grok-1-tokenizer \ + --dataset-name random \ + --random-input 1024\ + --random-output 1024 \ + --num-prompts 240 \ + --request-rate 8 \ + --output-file online.jsonl 2>&1 | tee "$LOGFILE" + +# Stop profiling via API +curl http://localhost:30000/stop_profile -H "Content-Type: application/json" + +# Convert tracing file to csv & json +sqlite3 trace.rpd ".mode csv" ".header on" ".output trace.csv" "select * from top;" ".output stdout" +python3 /sgl-workspace/rocmProfileData/tools/rpd2tracing.py trace.rpd trace.json diff --git a/3rdparty/amd/profiling/install_rpd.sh b/3rdparty/amd/profiling/install_rpd.sh new file mode 100644 index 0000000..d1b04b9 --- /dev/null +++ b/3rdparty/amd/profiling/install_rpd.sh @@ -0,0 +1,10 @@ +# download and install RPD +apt update && apt install -y sqlite3 libsqlite3-dev libfmt-dev + +# install rpd module +git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData +cd rocmProfileData +git apply rpd.patch +make && make install +cd rocpd_python && python setup.py install && cd .. +cd rpd_tracer && make clean;make install && python setup.py install && cd .. diff --git a/3rdparty/amd/profiling/loadTracer.sh b/3rdparty/amd/profiling/loadTracer.sh new file mode 100644 index 0000000..8a95a33 --- /dev/null +++ b/3rdparty/amd/profiling/loadTracer.sh @@ -0,0 +1,43 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ +OUTPUT_FILE="trace.rpd" + +if [ "$1" = "-o" ] ; then + OUTPUT_FILE=$2 + shift + shift +fi + +if [ -e ${OUTPUT_FILE} ] ; then + rm ${OUTPUT_FILE} +fi + +python3 -m rocpd.schema --create ${OUTPUT_FILE} +if [ $? != 0 ] ; then + echo "Error: Could not create rpd file. Please run 'python setup.py install' from the rocpd_python dir" + exit +fi + +export RPDT_FILENAME=${OUTPUT_FILE} +export RPDT_AUTOSTART=0 +LD_PRELOAD=librocm-smi_64:librpd_tracer.so "$@" diff --git a/3rdparty/amd/profiling/rpd.patch b/3rdparty/amd/profiling/rpd.patch new file mode 100644 index 0000000..8791765 --- /dev/null +++ b/3rdparty/amd/profiling/rpd.patch @@ -0,0 +1,12 @@ +diff --git a/rpd_tracer/Makefile b/rpd_tracer/Makefile +index e9d9feb..b2e9e1a 100644 +--- a/rpd_tracer/Makefile ++++ b/rpd_tracer/Makefile +@@ -16,7 +16,7 @@ ifneq (,$(HIP_PATH)) + $(info Building with roctracer) + RPD_LIBS += -L/opt/rocm/lib -lroctracer64 -lroctx64 -lamdhip64 -lrocm_smi64 + RPD_INCLUDES += -I/opt/rocm/include -I/opt/rocm/include/roctracer -I/opt/rocm/include/hsa +- RPD_SRCS += RoctracerDataSource.cpp RocmSmiDataSource.cpp ++ RPD_SRCS += RoctracerDataSource.cpp + RPD_INCLUDES += -D__HIP_PLATFORM_AMD__ + endif diff --git a/3rdparty/amd/profiling/rpd_profile_server_enable.patch b/3rdparty/amd/profiling/rpd_profile_server_enable.patch new file mode 100644 index 0000000..3cd3915 --- /dev/null +++ b/3rdparty/amd/profiling/rpd_profile_server_enable.patch @@ -0,0 +1,49 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..9021c01 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,24 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() #block pytorch profiler for rpd profiler enabling ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.start() ++ self.rpd.rangePush("", "rpd profile range", "") ++ logger.info("rpd is enabled") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done") + logger.info("Profiler is done") diff --git a/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch b/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch new file mode 100644 index 0000000..5416f4d --- /dev/null +++ b/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch @@ -0,0 +1,126 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..2edb427 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,26 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() ++ logger.info("torch profiler is disabled") ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.setPythonTrace(True) ++ self.rpd.start() ++ self.rpd.rangePush("", "scheduler", "") ++ logger.info("rpd is enabled inside scheduler profiling") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done inside scheduler") + logger.info("Profiler is done") + + +diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py +index 2621ccd..181df85 100644 +--- a/python/sglang/srt/managers/tokenizer_manager.py ++++ b/python/sglang/srt/managers/tokenizer_manager.py +@@ -58,6 +58,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams + from sglang.srt.server_args import PortArgs, ServerArgs + from sglang.srt.utils import is_generation_model, is_multimodal_model + ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() ++ ++ + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + logger = logging.getLogger(__name__) +@@ -514,10 +518,20 @@ class TokenizerManager: + self.send_to_scheduler.send_pyobj(req) + + def start_profile(self): ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "tokenizer_manager", "") ++ logger.info("tokenizer_manager rpd profiling started!") + req = ProfileReq.START_PROFILE + self.send_to_scheduler.send_pyobj(req) + + def stop_profile(self): ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done inside tokenizer_manager!") + req = ProfileReq.STOP_PROFILE + self.send_to_scheduler.send_pyobj(req) + +diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py +index 7111c93..2bd722c 100644 +--- a/python/sglang/srt/server.py ++++ b/python/sglang/srt/server.py +@@ -30,6 +30,8 @@ import threading + import time + from http import HTTPStatus + from typing import Dict, List, Optional, Union ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + # Fix a bug of Python threading + setattr(threading, "_register_atexit", lambda *args, **kwargs: None) +@@ -152,6 +154,11 @@ async def flush_cache(): + @app.post("/start_profile") + async def start_profile(): + """Start profiling.""" ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "server rpd profile range", "") ++ logger.info("rpd profiling started in server.py!") + tokenizer_manager.start_profile() + return Response( + content="Start profiling.\n", +@@ -164,6 +171,11 @@ async def start_profile(): + async def stop_profile(): + """Stop profiling.""" + tokenizer_manager.stop_profile() ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done in server.py!") + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, diff --git a/3rdparty/amd/profiling/server.sh b/3rdparty/amd/profiling/server.sh new file mode 100644 index 0000000..f877e6c --- /dev/null +++ b/3rdparty/amd/profiling/server.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# export SGLANG_TORCH_PROFILER_DIR=/data/sglang/ +export SGLANG_TORCH_PROFILER_DIR=/sgl-workspace/sglang/profile/ + +# Get the current timestamp +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +# Define the log file with a timestamp +LOGFILE="sglang_server_log_$TIMESTAMP.json" + +# Run the Python command and save the output to the log file +loadTracer.sh python3 -m sglang.launch_server \ + --model-path /sgl-workspace/sglang/dummy_grok1 \ + --tokenizer-path Xenova/grok-1-tokenizer \ + --load-format dummy \ + --quantization fp8 \ + --tp 8 \ + --port 30000 \ + --disable-radix-cache 2>&1 | tee "$LOGFILE" diff --git a/3rdparty/amd/profiling/torch_profiler.patch b/3rdparty/amd/profiling/torch_profiler.patch new file mode 100644 index 0000000..40f5574 --- /dev/null +++ b/3rdparty/amd/profiling/torch_profiler.patch @@ -0,0 +1,25 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..6ecd78c 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -240,7 +240,6 @@ class Scheduler: + ) + self.profiler = torch.profiler.profile( + activities=[ +- torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, +@@ -1033,9 +1032,11 @@ class Scheduler: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ if self.tp_rank == 0: ++ with open(f"stats_repro_{int(time.time())}.txt", "w") as f: ++ print(self.profiler.key_averages(group_by_input_shape=True).table(sort_by="cuda_time_total", row_limit=-1), file=f) ++ print("Profiling stats done.") ++ + logger.info("Profiler is done") diff --git a/3rdparty/amd/tuning/TUNING.md b/3rdparty/amd/tuning/TUNING.md new file mode 100644 index 0000000..0638041 --- /dev/null +++ b/3rdparty/amd/tuning/TUNING.md @@ -0,0 +1,118 @@ +## Tuning SGLang Infer System with AMD GPUs +This AppNote describes the SGLang performance tuning technical, code harness and running steps for systems with AMD Instinct GPUs. +Harness code, examples and steps are provided in detail, to facilitate easy reproduce & use to tune performance towards workloads. +Three primary runtime areas are covered: + +## 1. Triton Kernels +To maximize Triton kernel efficiency, several strategies can be employed: + +### Key Environment Variables: +- **num_stages**: Adjusts the number of pipeline stages to optimize kernel efficiency based on the specific type of operations (e.g., General Matrix Multiplication - GEMM). +- **waves_per_eu**: Controls the usage of Vector General Purpose Registers (VGPR) to enhance occupancy, thereby improving latency or throughput. +- **BLOCK_M, BLOCK_N, BLOCK_K**: Tunable tile sizes that assist in balancing memory transfer and computational efficiency. +- **matrix_instr_nonkdim**: Optimizes the usage of Matrix-Fused Multiply-Add (MFMA) instructions for specific kernel types, such as Flash Attention. +- **OPTIMIZE_EPILOGUE**: An environment variable that can be set to `1` to enhance performance by eliminating the `convert_layout` operation in the kernel's epilogue. +```python +@triton.autotune(configs=[ + triton.Config({'waves_per_eu': 1}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 1}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 1}, num_warps=16, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=16, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=16, num_stages=1), + ], key=['BLOCK_N', 'NUM_TOKEN_BLKS'], use_cuda_graph=True) +@triton.jit +def _triton_kernel_funtion(): + ... +``` +## 2. Torch Tunable Operations +**TunableOp** is a feature in PyTorch that allows for the definition and optimization of custom kernels with tunable parameters. This feature is particularly useful for enhancing the performance of kernels by experimenting with different configurations. + +### Key Environment Variables: +1. **PYTORCH_TUNABLEOP_ENABLED**: + - Default: `0` + - Set to `1` to enable TunableOp. + +2. **PYTORCH_TUNABLEOP_TUNING**: + - Default: `1` + - Set to `0` to disable tuning. If a tuned entry is not found, it will run the tuning step and record the entry when PYTORCH_TUNABLEOP_ENABLED is enabled. + +3. **PYTORCH_TUNABLEOP_VERBOSE**: + - Default: `0` + - Set to `1` to enable verbose output for TunableOp. + +### Usage Example: +To enable TunableOp and tuning, and optionally enable verbose mode, you can run the following command in your terminal: + +```bash +#Tuning +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=1 your_script.sh + +#Inference with tuning op +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 your_script.sh + +#Print out the log +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 PYTORCH_TUNABLEOP_VERBOSE=1 your_script.sh + +``` +## 3. Torch Compilation + + +The following are suggestions for optimizing matrix multiplication (GEMM) and convolution (conv) operations in PyTorch using Inductor, a part of the PyTorch compilation framework. The goal is to leverage Triton to achieve better performance. + +To tune Triton kernels with GEMM and convolution ops (conv), use the `torch.compile` function with the max-autotune mode. This benchmarks a predefined list of Triton configurations and selects the fastest one for each shape. + +### Key Configurations: +1. **Max Autotune**: + - Set `torch._inductor.config.max_autotune = True` or `TORCHINDUCTOR_MAX_AUTOTUNE=1`. + +2. **Fine-Grained Control**: + - Enable GEMM tuning: `torch._inductor.config.max_autotune_gemm = True`. + - Enable tuning for pointwise/reduction ops: `torch._inductor.config.max_autotune.pointwise = True`. + +3. **Backend Selection**: + - Use `torch._inductor.max_autotune_gemm_backends` to limit backends to TRITON for better performance. + +4. **Freezing for Inference**: + - Use `torch._inductor.config.freezing=True` to enable constant folding optimizations. + +5. **Debugging**: + - Set `TORCH_COMPILE_DEBUG=1` to extract Triton kernels generated by Inductor. + +### Example Code Block: +```bash +#Gemm Tuning +TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 your_script.sh + +#Specify your backend to TRITON for Gemm Tuning +TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_BACKENDS=TRITON your_script.sh + +#Inference with large improvement on AMD GPU +TORCHINDUCTOR_FREEZING=1 your_script.sh +``` +## 4. Fused MOE kernel +To maximize moe kernel efficiency, need to use below scripts to find out the best launch configuration + +### Key parameters: +- **--model**: what moe model type to do tuning, it will automatically decide the size of d_model, model_intermediate_size, num_layers +- **--tp-size**: simulate the whole model run configuration to set the dimension size using tp correctly +- **--batch**: M dimension size of moe kernel, for prefill moe kernel the value is batch*input_len, for decode moe kernel the value is batch +- **--dtype**: computation type + +```bash +#Tuning +#for example, we have one case like this "python3 -m sglang.bench_latency --model dummy_grok1/ --load-format dummy --tokenizer-path Xenova/grok-1-tokenizer --tp 8 --batch-size 32 --input 1024 --output 8 --attention-backend triton --sampling-backend pytorch --quantization fp8" to run, it defined batch-size 32 input lenth 1024 and output length 8, from "--batch" in moe view point, the prefill batch is 32*1024 = 32768, the decode batch is 32*1(only one output token generated in each run). +#so we can tune decode moe use below command +python benchmark_moe_rocm.py --model grok1 --tp-size 8 --dtype float8 --batch "32" +# and use this command to tune prefill moe +python benchmark_moe_rocm.py --model grok1 --tp-size 8 --dtype float8 --batch "32768" +``` + +## Reference + +For more detailed information on tuning SGLang performance with AMD GPUs, please refer to the following link: + +[ROCm Documentation: Triton Kernel Performance Optimization](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html#triton-kernel-performance-optimization) diff --git a/3rdparty/amd/tuning/benchmark_moe_rocm.py b/3rdparty/amd/tuning/benchmark_moe_rocm.py new file mode 100644 index 0000000..5aff8c0 --- /dev/null +++ b/3rdparty/amd/tuning/benchmark_moe_rocm.py @@ -0,0 +1,380 @@ +import argparse +import json +import os +import sys + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from tqdm import tqdm +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe, + get_config_file_name, +) + +padding_size = 128 if bool(int(os.getenv("MOE_PADDING", "0"))) else 0 + + +def main(model, tp_size, dtype: str, batches): + method = fused_moe + + for bs in batches: + run_grid(int(bs), model=model, method=method, tp_size=tp_size, dtype=dtype) + + +def prune_configs(M, N, K, configs): + pruned_configs = [] + elemBytes_a = 1 # [DV Note] Hard-coded for float16 (2 bytes) + elemBytes_b = 1 # [DV Note] Hard-coded for float16 (2 bytes) + + mfma = 16 if M < 32 or N < 32 else 32 + + # TODO (zhanglx): figure out the boundary between large and small gemms + large_gemm = False + if M >= 2048 and N >= 2048: + large_gemm = True + + for config in configs: + BLOCK_SIZE_M = config.get("BLOCK_SIZE_M") + BLOCK_SIZE_N = config.get("BLOCK_SIZE_N") + BLOCK_SIZE_K = config.get("BLOCK_SIZE_K") + num_warps = config.get("num_warps") + matrix_instr_nonkdim = config.get("matrix_instr_nonkdim") + # kpack = config.get("kpack") + if matrix_instr_nonkdim > mfma: + continue + if mfma == 4 and BLOCK_SIZE_K < 64: + continue + # some layouts could not work properly in case + # number elements per thread is less 1 + if BLOCK_SIZE_M * BLOCK_SIZE_N < 64: + continue + SPLIT_K = 1 # config.get("SPLIT_K") + GROUP_M = config.get("GROUP_SIZE_M") + if matrix_instr_nonkdim > BLOCK_SIZE_M or matrix_instr_nonkdim > BLOCK_SIZE_N: + continue + if matrix_instr_nonkdim >= M and matrix_instr_nonkdim != BLOCK_SIZE_M: + continue + if matrix_instr_nonkdim >= N and matrix_instr_nonkdim != BLOCK_SIZE_N: + continue + # Skip BLOCK_SIZE that is too large compare to M/N + # unless BLOCK_SIZE is already small enough + if M * 2 < BLOCK_SIZE_M and BLOCK_SIZE_M != 16: + continue + if N * 2 < BLOCK_SIZE_N and BLOCK_SIZE_N != 16: + continue + # skip large split_k when not necessary + if SPLIT_K != 1 and not need_split_k(M, N, K): + continue + # skip split_k that leads to EVEN_K = false + leap = SPLIT_K * BLOCK_SIZE_K + modv = K % leap + if modv != 0: + continue + # skip large GROUP_M + if GROUP_M * BLOCK_SIZE_M > M and GROUP_M != 1: + continue + # out of shared memory resource + # TODO (zhanglx): This does not consider the LDS usage in the epilogue + LDS = ( + BLOCK_SIZE_K * BLOCK_SIZE_M * elemBytes_a + + BLOCK_SIZE_K * BLOCK_SIZE_N * elemBytes_b + ) + if LDS > 65536: + continue + # Skip small block sizes and num_warps for large gemm + # For fp16 and f8, we want to only use BLOCK_SIZE >= 64 + if large_gemm: + if BLOCK_SIZE_M < 64 or BLOCK_SIZE_N < 64: + continue + if BLOCK_SIZE_K < 64: + continue + if num_warps < 4: + continue + + pruned_configs.append(config) + + return pruned_configs + + +def union_of_list_of_dicts(l1, l2): + result = [] + temp_list = l1.copy() + temp_list.extend(l2) + for myDict in temp_list: + if myDict not in result: + result.append(myDict) + + return result + + +def run_grid(bs, model, method, tp_size, dtype: str): + + config = AutoConfig.from_pretrained(model) + + top_k = config.num_experts_per_tok + d_model = config.hidden_size + model_intermediate_size = config.intermediate_size + num_layers = config.num_hidden_layers + hidden_states_dtype = config.torch_dtype + + if config.num_experts_per_tok: + if config.architectures[0] == "Grok1ModelForCausalLM": + num_total_experts = config.num_experts + else: + num_total_experts = config.num_local_experts + else: + raise ValueError(f"Unsupported Mixtral model {model}") + + # tp_size = 2 + num_warmup_calls = 10 + num_calls = 30 + + num_warmup_trials = 1 + num_trials = 1 + + full_configs = [] + + block_m_range = [16, 32, 64, 128, 256] + block_n_range = [16, 32, 64, 128, 256] + block_k_range = [32, 64, 128, 256] # MUST >= 32 + num_warps_range = [1, 2, 4, 8] + group_m_range = [1, 4, 8, 16, 32] + # For now we see better perf with num_stages=0 for all gemm configs we care + # But keep this explicit so that we do not forget we may need to set it to + # other values in the future + num_stage_range = [2] + waves_per_eu_range = [0, 1, 2, 4, 8] + # Remove 32 because of triton compiling error + matrix_instr_nonkdim_range = [16] + kpack_range = [1, 2] + + for block_size_m in block_m_range: + for block_size_n in block_n_range: + for block_size_k in block_k_range: + for group_size_m in group_m_range: + for num_warps in num_warps_range: + for num_stages in num_stage_range: + for waves_per_eu in waves_per_eu_range: + for matrix_instr_nonkdim in matrix_instr_nonkdim_range: + for kpack in kpack_range: + full_configs.append( + { + "BLOCK_SIZE_M": block_size_m, + "BLOCK_SIZE_N": block_size_n, + "BLOCK_SIZE_K": block_size_k, + "GROUP_SIZE_M": group_size_m, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "kpack": kpack, + } + ) + + M1 = bs * 2 + N1 = model_intermediate_size * 2 // tp_size + K1 = d_model + prune_configs_1 = prune_configs(M1, N1, K1, full_configs) + + M2 = bs * 2 + N2 = d_model + K2 = model_intermediate_size // tp_size + prune_configs_2 = prune_configs(M2, N2, K2, full_configs) + + configs = union_of_list_of_dicts(prune_configs_1, prune_configs_2) + + print( + f"{bs=} || {len(full_configs)=} | {len(prune_configs_1)=} | \ + {len(prune_configs_2)=} | {len(configs)=}" + ) + + best_config = None + best_time_us = 1e20 + + print(f"{tp_size=} {bs=}") + + for config in tqdm(configs): + # warmup + try: + print(config) + for _ in range(num_warmup_trials): + run_timing( + num_calls=num_warmup_calls, + bs=bs, + d_model=d_model, + num_total_experts=num_total_experts, + top_k=top_k, + tp_size=tp_size, + model_intermediate_size=model_intermediate_size, + method=method, + config=config, + dtype=dtype, + hidden_states_dtype=hidden_states_dtype, + ) + except triton.runtime.autotuner.OutOfResources: + continue + + # trial + for _ in range(num_trials): + kernel_dur_ms = run_timing( + num_calls=num_calls, + bs=bs, + d_model=d_model, + num_total_experts=num_total_experts, + top_k=top_k, + tp_size=tp_size, + model_intermediate_size=model_intermediate_size, + method=method, + config=config, + dtype=dtype, + hidden_states_dtype=hidden_states_dtype, + ) + + kernel_dur_us = 1000 * kernel_dur_ms + model_dur_ms = kernel_dur_ms * num_layers + + if kernel_dur_us < best_time_us: + best_config = config + best_time_us = kernel_dur_us + + tqdm.write( + f"{kernel_dur_us=:.1f} {model_dur_ms=:.1f}" + f" {bs=} {tp_size=} {top_k=} {num_total_experts=} " + f"{d_model=} {model_intermediate_size=} {num_layers=}" + ) + + print("best_time_us", best_time_us) + print("best_config", best_config) + + # holds Dict[str, Dict[str, int]] + filename = get_config_file_name( + num_total_experts, + model_intermediate_size // tp_size, + "float8" if dtype == "float8" else None, + ) + print(f"writing config to file {filename}") + existing_content = {} + if os.path.exists(filename): + with open(filename, "r") as f: + existing_content = json.load(f) + existing_content[str(bs)] = best_config + with open(filename, "w") as f: + json.dump(existing_content, f, indent=4) + f.write("\n") + + +def run_timing( + num_calls: int, + bs: int, + d_model: int, + num_total_experts: int, + top_k: int, + tp_size: int, + model_intermediate_size: int, + method, + config, + dtype: str, + hidden_states_dtype, +) -> float: + shard_intermediate_size = model_intermediate_size // tp_size + + hidden_states = torch.rand( + (bs, d_model), + device="cuda:0", + dtype=hidden_states_dtype, + ) + + w1 = torch.rand( + (num_total_experts, 2 * shard_intermediate_size, d_model + padding_size), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + w2 = torch.rand( + (num_total_experts, d_model, shard_intermediate_size + padding_size), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + + if dtype == "float8": + w1 = w1.to(torch.float8_e4m3fnuz) + w2 = w2.to(torch.float8_e4m3fnuz) + w1_scale = torch.ones( + num_total_experts, device=hidden_states.device, dtype=torch.float32 + ) + w2_scale = torch.ones( + num_total_experts, device=hidden_states.device, dtype=torch.float32 + ) + a1_scale = torch.ones(1, device=hidden_states.device, dtype=torch.float32) + a2_scale = torch.ones(1, device=hidden_states.device, dtype=torch.float32) + + gating_output = F.softmax( + torch.rand( + (num_calls, bs, num_total_experts), + device=hidden_states.device, + dtype=torch.float32, + ), + dim=-1, + ) + + ################################## + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + start_event.record() + for i in range(num_calls): + hidden_states = method( + hidden_states=hidden_states, + w1=w1, + w2=w2, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + gating_output=gating_output[0], + topk=top_k, + renormalize=True, + inplace=True, + override_config=config, + use_fp8=dtype == "float8", + ) + + end_event.record() + end_event.synchronize() + + dur_ms = start_event.elapsed_time(end_event) / num_calls + return dur_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="benchmark_mixtral_moe", + description="Benchmark and tune the fused_moe kernel", + ) + parser.add_argument( + "--dtype", + type=str, + default="auto", + choices=["float8", "float16", "bfloat16"], + help="Data type used for fused_moe kernel computations", + ) + parser.add_argument("--model", type=str, default="hpcai-tech/grok-1") + + parser.add_argument("--tp-size", type=int, default=2, help="Tensor paralleli size") + parser.add_argument("-b", "--batches", type=str) + + args = parser.parse_args() + + batches = args.batches.split(",") + + sys.exit(main(args.model, args.tp_size, args.dtype, batches)) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9c42268 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2024 SGLang Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..23d1ddf --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +.PHONY: check-deps install-deps format update help + +# Show help for each target +help: + @echo "Available targets:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +check-deps: ## Check and install required Python formatting dependencies + @command -v isort >/dev/null 2>&1 || (echo "Installing isort..." && pip install isort) + @command -v black >/dev/null 2>&1 || (echo "Installing black..." && pip install black) + +install-deps: ## Install Python formatting tools (isort and black) + pip install isort black + +format: check-deps ## Format modified Python files using isort and black + @echo "Formatting modified Python files..." + git diff --name-only --diff-filter=M | grep '\.py$$' | xargs -I {} sh -c 'isort {} && black {}' + +FILES_TO_UPDATE = docker/Dockerfile.rocm \ + python/pyproject.toml \ + python/sglang/version.py \ + docs/developer/setup_github_runner.md \ + docs/start/install.md + +update: ## Update version numbers across project files. Usage: make update + @if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \ + echo "Version required. Usage: make update "; \ + exit 1; \ + fi + @OLD_VERSION=$$(grep "version" python/sglang/version.py | cut -d '"' -f2); \ + NEW_VERSION=$(filter-out $@,$(MAKECMDGOALS)); \ + echo "Updating version from $$OLD_VERSION to $$NEW_VERSION"; \ + for file in $(FILES_TO_UPDATE); do \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + else \ + sed -i -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + fi \ + done; \ + echo "Version update complete" + +%: + @: diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcb6771 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +
+logo + +[![PyPI](https://img.shields.io/pypi/v/sglang)](https://pypi.org/project/sglang) +![PyPI - Downloads](https://img.shields.io/pypi/dm/sglang) +[![license](https://img.shields.io/github/license/sgl-project/sglang.svg)](https://github.com/sgl-project/sglang/tree/main/LICENSE) +[![issue resolution](https://img.shields.io/github/issues-closed-raw/sgl-project/sglang)](https://github.com/sgl-project/sglang/issues) +[![open issues](https://img.shields.io/github/issues-raw/sgl-project/sglang)](https://github.com/sgl-project/sglang/issues) +[![](https://img.shields.io/badge/Gurubase-(experimental)-006BFF)](https://gurubase.io/g/sglang) + +
+ +-------------------------------------------------------------------------------- + +| [**Blog**](https://lmsys.org/blog/2024-07-25-sglang-llama3/) +| [**Documentation**](https://docs.sglang.ai/) +| [**Join Slack**](https://slack.sglang.ai/) +| [**Join Bi-Weekly Development Meeting**](https://meeting.sglang.ai/) +| [**Roadmap**](https://github.com/sgl-project/sglang/issues/4042) +| [**Slides**](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#slides) | + +## News +- [2025/03] Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html)) +- [2025/03] SGLang Joins PyTorch Ecosystem: Efficient LLM Serving Engine ([PyTorch blog](https://pytorch.org/blog/sglang-joins-pytorch/)) +- [2025/02] Unlock DeepSeek-R1 Inference Performance on AMD Instinct™ MI300X GPU ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1_Perf/README.html)) +- [2025/01] 🔥 SGLang provides day one support for DeepSeek V3/R1 models on NVIDIA and AMD GPUs with DeepSeek-specific optimizations. ([instructions](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3), [AMD blog](https://www.amd.com/en/developer/resources/technical-articles/amd-instinct-gpus-power-deepseek-v3-revolutionizing-ai-development-with-sglang.html), [10+ other companies](https://x.com/lmsysorg/status/1887262321636221412)) +- [2024/12] 🔥 v0.4 Release: Zero-Overhead Batch Scheduler, Cache-Aware Load Balancer, Faster Structured Outputs ([blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/)). +- [2024/09] v0.3 Release: 7x Faster DeepSeek MLA, 1.5x Faster torch.compile, Multi-Image/Video LLaVA-OneVision ([blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/)). +- [2024/07] v0.2 Release: Faster Llama3 Serving with SGLang Runtime (vs. TensorRT-LLM, vLLM) ([blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/)). + +
+More + +- [2024/10] The First SGLang Online Meetup ([slides](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#the-first-sglang-online-meetup)). +- [2024/02] SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)). +- [2024/01] SGLang provides up to **5x faster inference** with RadixAttention ([blog](https://lmsys.org/blog/2024-01-17-sglang/)). +- [2024/01] SGLang powers the serving of the official **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)). + +
+ +## About +SGLang is a fast serving framework for large language models and vision language models. +It makes your interaction with models faster and more controllable by co-designing the backend runtime and frontend language. +The core features include: + +- **Fast Backend Runtime**: Provides efficient serving with RadixAttention for prefix caching, zero-overhead CPU scheduler, continuous batching, token attention (paged attention), speculative decoding, tensor parallelism, chunked prefill, structured outputs, and quantization (FP8/INT4/AWQ/GPTQ). +- **Flexible Frontend Language**: Offers an intuitive interface for programming LLM applications, including chained generation calls, advanced prompting, control flow, multi-modal inputs, parallelism, and external interactions. +- **Extensive Model Support**: Supports a wide range of generative models (Llama, Gemma, Mistral, QWen, DeepSeek, LLaVA, etc.), embedding models (e5-mistral, gte, mcdse) and reward models (Skywork), with easy extensibility for integrating new models. +- **Active Community**: SGLang is open-source and backed by an active community with industry adoption. + +## Getting Started +- [Install SGLang](https://docs.sglang.ai/start/install.html) +- [Quick Start](https://docs.sglang.ai/backend/send_request.html) +- [Backend Tutorial](https://docs.sglang.ai/backend/openai_api_completions.html) +- [Frontend Tutorial](https://docs.sglang.ai/frontend/frontend.html) +- [Contribution Guide](https://docs.sglang.ai/references/contribution_guide.html) + +## Benchmark and Performance +Learn more in the release blogs: [v0.2 blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/), [v0.3 blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/), [v0.4 blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/) + +## Roadmap +[Development Roadmap (2025 H1)](https://github.com/sgl-project/sglang/issues/4042) + +## Adoption and Sponsorship +The project has been deployed to large-scale production, generating trillions of tokens every day. +It is supported by the following institutions: AMD, Atlas Cloud, Baseten, Cursor, DataCrunch, Etched, Hyperbolic, Iflytek, Jam & Tea Studios, LinkedIn, LMSYS, Meituan, Nebius, Novita AI, NVIDIA, RunPod, Stanford, UC Berkeley, UCLA, xAI, and 01.AI. + +logo + +## Contact Us + +For enterprises interested in adopting or deploying SGLang at scale, including technical consulting, sponsorship opportunities, or partnership inquiries, please contact us at contact@sglang.ai. + +## Acknowledgment and Citation +We learned the design and reused code from the following projects: [Guidance](https://github.com/guidance-ai/guidance), [vLLM](https://github.com/vllm-project/vllm), [LightLLM](https://github.com/ModelTC/lightllm), [FlashInfer](https://github.com/flashinfer-ai/flashinfer), [Outlines](https://github.com/outlines-dev/outlines), and [LMQL](https://github.com/eth-sri/lmql). Please cite the paper, [SGLang: Efficient Execution of Structured Language Model Programs](https://arxiv.org/abs/2312.07104), if you find the project useful. diff --git a/assets/logo.png b/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2a8bc258f6663112f02b40f04843789585d86d68 GIT binary patch literal 402692 zcmeFZcQ~AD7e2}uj7}It9a5B0qm2@b5JX9onbB*4h~7J+1S3I;lBkgoMkjjjqKoLn z=$#;XJF>I)_uJn&-}&?WcU+h2E#`f%=ULBs*1GR|Efe`jQ<;L4i4+eHk3v;NQ5z4B zL=O)SFa{#Jx)X9pCVzF%e4?jnrJ;exdvy-N1L8B`5ni3)U;WDBGymuOK0Y@d!C&_Q zS1+Lb`~3H({`$dxaCNx4`g6QNbf|!bCy%G92-o$(->7>QdvnzLmWN0ww%WJamyLyz zh;dHI0u;fhpvVZ)=Y(gBUM%Zr6%sz-3~rM1u-PA?@yLs(8yBGrKfiaO5#BNzyDEmK zL(LKStI&9oDU8mo^)>a5EuIKo@jXCuNXRUCR?X*{TwW3XzTR3yMM-I@@wBnAzra9T zVbY;hYFCbwdxLAg^$QF&H9$cg1j6Tl07HZR|BL^RY=BmULlWPT4Z8oo7}__w@#YJ% zPrd=8Ymff57;hpQ1Dr&hi@*BOyCME-nP{Ik2E^`@gvck{_+RUWYlIB!d!CtNUbqu- z_%c(`(h)B;4Tq8bz08T)!0K?q~?=JWB$=U>7j_9Xvy0`g#ab~YImTI~OP zzdVRMvax(-9-~MRiKE0i5;RQ5|9CD=Z$Jz9Gfby3>aqJYisbE=AUgudb%jSU?6zgx z{~oH46|mZAT=+0!>3`asg%T(gVEs`2Q)f*q8H0L zv2G*&{q%XyUj@ouyOcyEwfyX6kHYJIkr5;hkWzcpS`s8zhVb4p-aNTc64E`K^Uq^h zgy<2}&s4vij5jTPy+lhE`JPsXY-I`zzPq4}AJf@S{=L7@s`t|y?{Xb$yW9Ws{C_6$ zd#DFMbo5MD&9%%`nnc4Nv;49(cTB>uTpm7mrDbO1&Ldq@TjB!EBKFCQ*9_pu4(XZ9 zIZBTN(FSPDD5Zeb=l`toW1xX)rsXBzE@aA%&g5W&ab_^ZrMPa%JfHY>i`hm@FXIP< z&VP-77!M6-kikRcsnG%c>koe}gxG)^j&sFI2JLi->s+x+qD3A%YhGcdbWV|ND z?YuPxd6rw;+44A>?RS0G{(grLC&0J4KR&z?<}6}KG;4A0#mK?_JSxt)$eDC*Gh`%y zvi;yq`t+xf|FS|B1sV{ZTs}E;{y#TOXXt(D==X0y*+dk5?qoggv}zQb(FVISV}A3; zo*jZ%pVn>b&i&=^JUNwp<-P@r+^EfoAGW^M=J_bIB9y$-YXG`7vsu0`ntdIhKJm|V zi%_7k!Y{0WuU>ZcQ}RkM$0uwOGSt9uv_mhW+~>_hoQlY9`3t78t0$Ve*gi|>sPH(8 zC}$e}uQ1w(R&Jb+B)%|lBlsUSbgHDWId5l`ymm(+wt!C_bvt?O)_a4wQ!CRK*A;KJ z;U;oNqYRQ=tb4Ctm-VZ@5}nN&GO;Z2&I=Y-8M`+0Vdg zU-j}X89?9$Xw1jOc>5!NofT25^Opd(_`kzBnuFTL|L)7PMbVpb(h*a34c_CLm5knu zY2on7F-hFj1F6UNxRDwxH&Zb4qrUDNpqnhYPU&))i+R_QT^S*}B&5cq& z$2xz_G_0kZUx(k5^oniF=w3?H4@pNK;#vd|NljGQ9p@29Mjkvk=!rs}79gnhjO@o< zCwHT#P6kwA<5eH;&0Z_x0se8C^oT~qz0zBrWqPY^{Yz4z83k+5o%#N2Lmn2&Fjx2?b$wP9U%-wd=Nc&OA|dAfpZxGOjewJXJsM#7Va?=p^= z#y$xNvb)SUm)p*9Egq7G7}gAoe;SV@BS_+v0kS62SAWa;XMyhkH&9;(Od+t@=CFg& z0T=!5W7)KiFGJGG!gNBMMNHb$&+K#^#8qqUnAw@pQ7@ZY)-naMt`kpovX)?CR0U~} zazBQq~9k#7(i2 z>N{9TS1m6n;SN5+P-d75HhNEV5vUU&RB-R+BFV=tfa7Op@6qZ=tkcE`lF5&yJG#VP z+2dC`@-uAxafj;scqP(i=}7&v3fBFoW#ql_>{8^r^k*9fH(QbHZmAX(MnOgD*^6Xf zl3}@w4A6)#EL5&6FTCP+gu0tZgO4^9^0qkh z8~}L%$+^rk)s1Am-Hf5AIXy~s{(l+BDTw-Kdk^IRVqN=RcmMGkv-i}DQf3l)iP*H* zwd2#tIX+^$cP((`pn@nbEMxMXlFk!&m4iYK2*BgZpUREFugExB{pe#t75A_*sy{)3 zdDC*q>omQ>7FoA9DRy9GUpzvuIsLQdmv}UC(p_({8nvD1DeEa>GsZF1MG!izXwVx> zi&T5NNC+^dj@_)0HyedOpx)?42Y92yDoG6?V0;4>jJC7Zokf`AslwsGIh?;YQKAFz z`I5D|;(BsUe2v+B)HGLc+G6lQh8b3y)56hNBgv4L@`f!wkhSe0h_e5|sO|3}_Zi{j+icXdEY?^gnjvRnpvCOLXuY>o`wWS1;qQ zo6!#Bh}*uu-?wVzROoq^POIv>e;*ploxYnwVX%7SGON3gIb~^9a8H!5jP7FFDVmow zQvvkJ&zz3HF+ip-GWE>A!mCfggLdz@&d$m7!!IdhwylK3oG7eW>{J{1v1l7v-rHs* zIyNbr5cPS4OvBlAvRKRr!&UAA;Ft62^k>Ca*ifl;9dj@;-UJzFB`yY|r-Aw*5ygHS z@f3iP(y5nCRpxP$!bb42A82`TQ7@s`vh$#HcJhVd0DSn!xcQ@44L7qM*1ugxNDD+R zkS!H-*2D1dSW~Co2Fq=8NT{P_S={!~U~WIz?`2X_oO5{zS7uxHRMVXRaR7s*>w-~D zbKA+!q7N{=NX7u>!?l*00-T9|-!PAtAVW=pq>1~y{yA)<=nT70jX~`QBWV6lj%%#v z#5tGx*HO9){GDq4O_zqk8qKh;ThOTLY zr?UOHuy<$>IW71=-8Q@w!Bi>AMcs@{6SipE|>_ZPLW)8m?9B1`T2V6+RBA@nga_%?sBa z{O`?yCh*XFq=Xkw#$6dSP5a}*q^a7%r@_R6S4umMB*%}7cijyanT??&4hZV=6my!( zxihQ#&Zdh&olwNSNeA@dZH+tWr({?c;S8hGGvW!Wd@tK>2F2OXbeut1+~ZA#7N>oq zpv99UXQ)FY?OMR3!)0vuds>9IaDBc4b6GG96Rz)n+m3jwQZSP&F6!3~Y@$!Rw4Dd% zf>z)faT!igiR$ooMph4RedO-Sh-!J&yKw}u$P0kWi=vo7d|55m6@$w787m{qa7i2Z z+mp@eP@yq3ud+8mkkhKhgjyB1#n=Ju-%}TI27PH*@iCr=4aQOiJoiXzXy`R(YdT1` zI5b|gcD+vQa+Y@wCYD50uhM0D*rr_{;h#K}*V%nEax|6qBH@Yq1kBYpR=d5;E49DF zBPl9wS19%lSF9sRk(04QU1yV@@R^2P<*&iMxGxEK(A2L($PhQl*_yO~frochcEHp3 zE1dJSXNNt|rsr_n60az1!DXvtP=A4p77D65SV%^V=c(cn1QFO{xc*$hd z!Z>7Mc?-Wx{bFQhrE$L!LNAQ;YyEr3)3i66Urolh$|0$y`0M)u%k)@xyOHL;>lduV z#k0XaPw?-ul`hvj@Q_xy`;xvDkuq8KT-Dfp)Jp49ocM(BQY&eLRG5A}cQ+w`cKNN? z5(Kgn@hX0xktB+TBuYR}oO-M!tm&cLN{vvFF#~mNu=~tDHtnY^7e4%;+l9lBJjG<- zBBZ;Ib>YUXZ6&fl7Fh|FKP4+vBiv#OOWUc|XX=v$ZOU+~ItO|R=aETKu8p;5&#Z;kq1LG4B%e%tMB zc@bp9{Na0{dpTV3g7fA|&N5*;N|Z~jKQ+DDo|R@CO;L0+G;z7zXrttMZz}&_CFapg zWMOoH^`&U?Y23Eou$tGRYyO2yu94fY#+Yn&_pne7VO#Fe+iVsO;6-msM*^iZQT62x zhtuiDn3sjc90t|ruLQ}pd}Rdd1n=Kb z0GtW}Mjp`Xl{RzN9?e173c1Qgw*5z*Rvvg2wI|g6qR|xF+rt)IP;=|Xf6Lv&k z9({t3dE%nZdLk$N1D41pQ=GaYCsJ&%ZDlUSx^fQf&%QXIjNl(up4vEw0|YHiL!3l9 z+VleHWW`At_PiNS_mvD(`$$-+pqP@*FUh_+^JbdV+o5{0m%GNQDWGm=L4l~+T<&_8 zxrg-=G=CQPfUu=S8w%{$sKP){A!|wuI}X}CRq41!F9-_Oxmo7+^Ox)g&Ga# zPDZss>LP%Lcm2*}*be7Lr$uKbviaFWxY)dG7<{=M-=DJu;z}2ZJ)b&STAs(L z1zw~$ZfNfrCs8!xCQznF!!gy!fvYG82YJ=n1{XX?_$!6|-?Yv*q4I$YK=NS`vyP|( zW#Lo8jcO8Xvh*-_Q?(7$H7A&jEB!_st@h3YW47Tv`K^@nRjEIhsx7F^e%riU zs2gUVPoM5Mr<1K}G!U6F?Imzi(t0c1x2NZonybTa&)9$%d=ON!eepdYMqA8~ah6JOZ7Lohj*vmQE@d#rMMVOxWgNAGAKx`QU~&VJ7{ z{T?XdoXqEDDr~<64oaipN=x!z>zinBBpLl$Bhx3!D>2DnA`WEVtD@g=^2_C7aNP`v zI_BiZEV3?7G-GHxrBk28z!|q;iGr;Y!6#jjvr%VJoO$CmLs^jmv}Te&hBf>af!ooJ zk_AtU3uQhJIy6H&N{!w(6*!>0G4L5Jw?~5rIe5)HGGrH#*B6SYq zx7I26ldC3s<{b;kPLt(Y9(Vbo9O^E&L>%uPei!i_wH(p0%jh=sU2ReGoi03~1$>0y zcCxiJMC+`2@dt9m zsckQ}>Q1vzoyku#Uw*hCEN2bfeCtP^A-3MMe2-1u8D1u;8YnV36!4O^)ZX-T<)Q$= zIsY)$(OA9g+J)t_8>W)lQ6>J5T&LZ$N;bWU0|=!J0Q}n!0tXGpno-+ZeZi8Nu*K1Q zpx-a1BSEM@Q3wB)>vDOyy}!eU${;r0YFv`^Olp!emk@X7`Gu@+0RzYlE-!BQ%*w%l z99Z25IsAydVKbU;X}+FvxL$P#$6 zD-s~G?~=nyiuF0Rw_QtKFccx`?1;aJDrisEL4ou z%XQv)T1zf-W8UIIX*WZQ^QW+<;8d!!(Tk3shhIhFmhuVhuhN?c3xCD%+Paudk^%H_@%9s$7s*}DLZsNPP`w4yNku*5%7l4iWawg5 z%t>-L$#hf!YC05EKOZ-*uZfc-@8I>UUhB<&4lWf;>o4m25xG_P!_sypX-mvq5yV0c zObs|1o0&5=kzz#U<|exVZx&6Qd#V6(r#tlSzM#U7k$73CRd(}lQh}r995~UOj(Qb= zw7x#zJGjW=M+o90+B7@E`tH*7@AuYhovmoPm?WN?+kBmioA?}I+N$Ix26b|z0 z?$51C&JLIKyw(AGZ5W`l$PxGY|xL zC$uxzmH{GQDZQATD?$-&SV=89CZ(X{x#EW#sic6LUd{fOBHNyIBlaM)&M#(}*6>XtwTTZKz4W=z^) zxcjb~H>@en3yelc#)4$_e+u7iAl^+$>WPv)U$H!OF|W9L5_Wg{FmC&7JmIib$L(lA zZ8Wfw{T%y{Lx>c>&nesWRZ979X4A-_n10!~1Ppo|*t8zL0IX%G2L28;jrj8E+n$E+ zbcTzbJ~3+SlZ3c|uvk3m@6&<@ud1(cfA4oLjaMnC!? zG51~C3Ls2oxUBb{0Jk1JwOcda!72cKkUaG?m%AO?`VM;UAqRb!p;Id_rYM;}dGnef=S`9{!t>I$bXXh(aNd4UzTWS28wQJmhG%Dfd;zwTYWES9NG;9e=r zp^ERyjg)v*~WN**m6p*nbje8NSXy zx-2MYY4-7Bg+f@fl?ft*gV5;9UqU$S(^dH*%*2%RUL-ai7yQ%VApg6d87d?QlPF^5 z$h!3p*9jnaLsrxeb5rTWQF{CFAZ9>-Oc)~Ni1cCdG>XouSaVc9omOfQzkxJ3 zNMD|66;CZVNEailhq~QtD=7@&)>C4|1;K6j2FFv?b+#C)_>xF5=e0+ zk6w%_@Y%`Lp*E^pCn<>V?^gQKx@gO2f>_YC^QAUv^WmFAU8nR(F#@Lg936T48S+2e z0Y|I~orquv z!)T2ky*e%RxRbzlMTFLuS6vMH^ym=Y_;Sk;h}+E%Tz42wEwC*PX!&ZPF+8(3lw^}3 z=mRP4dIhK0W?s2XGr>`)9KAs06@eacX*mO{=cw@j-=#BY%hhSb)MFgG3G4kvQh(B(lR`*z!eo~Bf7~LA9*wSFj<1{aoN?T$>Gl5{ml?mKgkZ1r+fx$`cGJ2RNFz)I z%~n{Lk1`p$q7ya2c-phI>V3!%P`A6&LWK7*?PsVx!*e6Iws`@_N~nH-P!SlUh$onv zB>4m!5+q(pjMgm<5C(OM;|^X)^QCKC8C5o-={L4e$>`!;^Z^bFH|eqWPvL zEMEEw(B>-{BDC@G4Ixpv`OVmm6wUo--+Sh)-<(5OT61q~1D3l&QPg!a-o9-GL;7!7 za@`*;-0=P&Br55_~*IqUMgme~#Fh;k%c)dCgxIqc7dwmhsl@cOW!W*j@o@=m|<**R%fvK*}NcLjhA3uZw*@nNSz!&4n*cQL65@CFZFwR z-z98U-*s)+4M7VGbdY%pkSYnse+@Ix)VuNmn9Za@dh|{UlC(*HlaFhxB$}e0ES>@M z2@guatWQnd)R;jQLPkWNQiye`&X|v*04)`u$nhqg3DB5~f0xWl`^=QAM4Ai~An$x` z>BsfFBpresqYHVm>L``WXoYJndO!Hn1Pw1AH42>^PHHD-V&t2zXBg0|csnsyqp1HI zcKFMi*#xvdW6{-B8a$1eIUP=bq*FBCytv&Dmb*d9>l-GYd9C(NNCV!K&6dg$sgB%? zyJ2626U~S@9jSA7Qm3bUqJe;JZKqKzqP=ND(q9rhmFt%pA63Vl?lPB~v65nuOFz>U zrZ*)(Ya&^{pt!Z5&*O!B$;rQ)H??W2HwbP+DsgFHuG|6KGQPIr6Kg`O?pB3q$|VP) z+^VEVW>1L6OmA0m=C$ndrp{M_NBxDRkHKmZ(Q&N3Ca(2uFHPjs&UL;B0NaBbb*R{f zL?q~M4ufc5zriP<)jlS`y#LgHQ9AHo@$NPqtL*W&efBNohBckvA)&+hzP-ZoKRrS) zYS81sEhXPW%{JtI4MDO&ib5gEFCJBu7xYI8$m`j*7UG~|ERE|2lIB+5SJU(tM z!-~EgFYz;BJis+~=p@OPjpyCVB_d@E!Hqg1C6|nJ_DtY!aEH&~Bt*S(;UB&(NK11_ zHgUeu(ZLq33q;2*Ra`Y8xD(IA2N0CeL01(beb>@iC^-@5Qn)-sW?YXEsJPRnccpx= zT=(NZYB?=xPYD&RDC!XjklMY0%S|$bU`!yf06vEsBi6iOqOdlzbg;oFcsvL)nY&zoYzu-!4?_wpY~c)$JbgHQNEm$W^Yx8{i>wv5vPTp{F+IvzC^PD5 zCQQM767rL$j~FT~N+Yz^F6*j8AxHKF7J>!?hX)v+qlx z5WEkk&@zJ}g(h%);Lr_f(-cXBciT=XwO{yDWLGYI8i2Ue56j)knFfDKl5@4!JBPk@ zk>JM5=Z)W0^3s}?c{3i-tVuUXhneu#D_ty*6jnMMGgi?PZp%ckXTxZGJzk2(;l!P5 z5-fzivMOyJOEx^XtK|#WT9l4F5kTcuE+1<|6hPWDx`#OxkrEDgC2UU#m1hyH9s|Nk zBe(i3^kviP1jmubxs@nMfeGn=E9%N5mla;6M3U!mIi1jGdnk;gnpc7>i_Y8Y=Ktdp zi4b5l<>LdFEt^b*=9e;oavg!mw8N4$Kdxzkfx6rI5+OCaNtb3K#qRx6uQn^IX{gTY zx|^s|&$oQ_nbkixizR5^nw(61*={c}ba|x@@-OGk;Z%fww(bew3MYe90%pq{NB%C& zp(7O<1JpfFW4CT&=U9c>)YqjQzb1aZTw*k_KaTg0gCZ!y>BGVJM8MV7#?!PkD${y` zrp6Q?z9IY`5%Kj;AthK8WPoUI*Nh{jfV24ZCMgZ_NbgHyqg%+*Nw#uttBO-idTf|+>>srm5)o@)*$cym&$1(htKS8%$GhBM} z<$SGi7;?FS`0^+I#O6Hh-R!?`=>7ZY>v}*#IyvU+s1%qID9Ad$M>hm4Wy1Q745=An z{R+B6zbe{xtAgi?MX>{F%w~s&boOhY7kM+6KnmWE-CN0Onk{=#bGhPawpM;!k)=-2 zxzD^A%A=Zd@1(XJB$s9jI&gKY-LB@nlKmaT$hfrF9@c*ET|n_DB9V3o5}joiL_G2* z)7J43@6^3nCnddVlJPGk( z2iBL3TJTX766qYD|jr%k`cCo88hOcVBB3HuB_A1mD zcooROzVRQm`gC6zK1mc$;qh-YM-8A2kv~pQ4)CrEZOz;Lp41V~$+xp;)~4%}4O7g`aPR$2>~3958fol2#r}lLstLmU2QX+B2fCV?0`_wL~|k4!>JCcjD3#YBeUr;X_l2BS^X93z)tzb-h z_Q;yLxMY#uVKVI`g?tIk4Vc@=lD8qS&cIvT3wLQMb&w64L5sg}XNmjQ7E-_5gr8V{ z^4hPol_%af0R=78D7+@lwQ8Yw$Lza5Zq%7Yh`fxTr^@0#PXdXaYz3`BrG9YSe!Bst>1T5x$VTwdwJcakserD*f_ z^NBpq_YoT{&5AGE>}RT;^@_r}LcFUY7Xd^{Q{{%$?`%Bc`7PdYMAqRI)ox z75YydVsDbkQwR?Qz9Z1It1{m0wI7`*_#pOYC}T>%@<9EC6mVGU)TVZi%$^8v{bwZQ z_*TVHl>mHLp#Gli$pB}O5W@wbW2z>(^>#r(KOufqzoc*_B;ECQr_z^Y1>msDGj0;1 zJB=#B^aL1iL?PpVWn=VmiCo1dS>ei2V-O_FfX+ntXzYPlr~C(s3yOq`=ja9lj&llv z5I0M1DdAsaTr5X^qmlW31?!>6TxtWFVC@gX$9?i_a`gJBzp|Zo?1LV;A##Zdckjpa5JRnH+4A*1(3MZhV0(S_ z3Oj2sbI^4^!nxLSw>qZ#UP}02?}SCff*MX^!OSk|9GL-Gy;WV(>;W^B?0{k>pmFyV z*a$7GQgSP=DinTcY1QLZMrExV`x`7-RO%<70;2~_oQ0fLFPXI-Ec?xf%`BtxhI*%M zsf)q=CbBb0j!$b}ap!CR(Vy964=RpiO(s;CrBEQ)l@Gb7Z2Zu^l2BWa{M&2*Nz?H< z%I_urXeD>$(?th1q&dV1?ObiB+(>&Q&wwRFk#4DjxDpYrGEbH{Gq}2$M zDz$>p6Z%30X2hu=* z9?|lvo+-~pF8k(X1GeoI9wBD$Th#Xw%4V}KYyZXzbxP@JSL&Ei93enBMa0!o9wK=E z7rz2^;Yz@hX*$M%drkwS)yb6@{CjW9E3q{{G5AfdhGx7p$*+p|pJjwpByz74am;oV z@e8eFHgGh&Z1RuNB&RdFB{^zuF`ZerV7hzWAPQE7XyodEn{N zaxGj%a)VKa&;apU+tm)1oYdJ0PGUM@?^f1SDqWTyf3Xnk*BTX6IX;rMyq_mF=aMEJ%dsl-*{cPY=p)h;W%0uPu|axU|-yIgay-m4efSd0uD z0UXqI0|q)}l=SIT#bP*lNwc{NTsX02w`sM*G=LSB5ubOHe&l5~(NwUVS`wyY_gKWs z9SW6^TA#T3hpe+G!hc3it|DWlcEBNz=*)zN9{{YOT(uc{2tTrnT$x_X)dKe&399Db z>3I=oLaP!M(0C2>SBqk!7Y+A$L8&oLXOwE{?s?06Ob260q5Pw;doeMGcfSATY4eKd zS9O=HH*l^i4NcOl>>CcE(y?w+L0_rKQ>AHRqK!O%xR^N29G8V!K(ns#xlA4TBx>6zb%dRqiZtGE zb%^WilD+P8m3z!&KyHiHEHP#b-QIHiT5-VbdALbF2+an1oRLeNk=yFGwzX!eEfeOJ zbo-aKK!T*HWc^~)boW)Mq;q?$=wDNTR+0!y_q8NiNSF+(b!t}lbrgas z_*+Sk>7jC77Y1P4KJ?0F{!ojj8?OYJA74^gW?#Vz$eM8=Jk(2q@A$u^&N>c zb-@m;O=1sq%gR$juUdz8bU`%3QmwjL0$Fpiq&dkEyg^>Pp*FUCJ@5JXLTqoBG*c@- z3h%Uiekc(I4>!^aLl~Aj@tKy95hYiaL^d#mA@#;f*5O_L4kMT3(e^k9u{vfIX$R>_ zJ5D3cNXFTk%c@VwM>5|uhhXS&hKx67?L;Y~WxOnI`Lmrqr!iQR_z0)MW&P>M;s6j3 zZgFEt=iIKd*T!rY$)#NNs5Q#oxB@gPJP=Zova0}~{o!q7A$L$o2S$K=O!XCgzjFcV zNi&pa3loj9cLt|0Qde5aUppU=>@G*U%W=b0PM8mLa)2Z$jJC4}jb>#`x9gG_ZAHw! zYud%R8e#I6-s8R`KWcC7`_CCYpCQC8N^?i_ z@8)P8TMIvTVO-KV@C8UFSY z+o+D7NMbgo-Mkc(bwdaH(MBILQ$|>O`w2AR;!Jvmaqexg?thvP+P@_-(ExlS`N{Vk}O5rfQtk0(7*66SH+NW?u-c?@<-X$B0+&7f6TO^CuGpE>Knu?^f0kChX! zFCSd{ma=9ekUjCD^5rZIC_^D4C)i7T4ErwKs4lRG=35fhw4oq3f9=LTA&y?N&Irt1~G*KNg!s;=1IN>`uWe2>_Mbk}uz8Y`@M z{QEMB@cCi87wKRk6^Ese+xcd6Q~5T8Tf`qVdzGG6=wAvzh6U?L3-2~m*)?LM{{U)olI|8s7a>BtT91 zIg^zVET9G{vD%~F?r5E)r9eFT_=am)HPMIRW{bB%8scF{_Q|~4BfZ#|&aUKsp`xQX zLL;Qi2>L_jRqQmbx-Q4Fa+^0GEBvZD%L1fd%2)>cd{=d&=~)^~U%!|VwID%lqMzDg zxKoLIHxCX&$@cF~7aZNozokxWT9UzEt?NFEdY=51!C-M+9coj^wfxMj1*7dGu*+M% zzE<@m*t=`VwhUha%EI0_Yug^pkWs+(_>gt?Yaw^J#rq`-DYy&2!jrH~Mm5oG^E}$A zbxJ{Fkr?YOh?B0m$!@H%QQ!+}&6cZW_zJ(_oRGaQ|0X;DCqqr?i=8@xn$X5G{#&USC zNi$Y?;5b)>K*bzE*o-}Nslcx7z?@buN09Te8Q7TN7@`XCxmW>2QquTqN`w~h>or9! zcfNZdEO&;j=}Ff z@#MP|%qcIU=+nv&kEDqlzrYkV%FYF$1p=;|Wx%5roazgG8wctz2Rw3>ThvCwyXP`n z+W9?woDjl6grx(&v&iLn5S{hMs-{<23~%RdeH(tD0OkDfSG;2+102+R5Pct$@=(sl zFEW_-Ahfrm=NtPIqOk8y*Z)fuP5*=V@S1O<>ZYRZFk%9Ciy!%AEo_R7pb z;6b#!(|)d@gwFa{HB**hM@DG!!NOOAxxI1DgeQ=Gs=39`yDj^TvOLvJSJc2GqOn{U zG|JY&$&I}x=Ne(d1EdrH3}d|QvzY42tx_0S8vCh^)jUe|3Ro(e}ikSSm7V5Pi9JOz=FB{Pf%ox?s98Yf6*~zaF(woMiZk!U2$0 z%>OEAMn4%MnNUrtssRI0%5}QE^)+BFhkCMNunx&Lp0r?Smi@ui8RG53o*QAJ!Q<)x zON<3H?cr=B`bayEAlH0Adsn2vSa?faVyU@6Tgc0wL4?kTn*rkRmTaaLN;hNcV?xsnnRr0=|mY-XCUU`A5gw~W^B!yf4&79%GYxS zTV;u|zFKkUS~svmGvmdru*xrQ*rrlJma)eL3jfVA&`GE+akL5`d4&+IJLBp0x!P}G%4i`K2DB_P14 zhrOvXanE9~BRtz}G@}PTX17m($_X)gYmzUNk<&G2c2x@>)s{c^Sx;1vKP1%TOdsHV z-q__l;d4VB=8f~Z@!-dGjmNjp;51?|htm-?_o8*^?5gseL1|f*)_%$v*ALRNH$jh_ z#K@Y!j)(QWSn;}au$p;|u2&q{)YmkY4wnq*mZfz43T8`BV=GjFHkpeStMgrx1H8Vx zE*6n1RT;tvKevE#eW`9_!hH}vD97>AG);4{0w-LE&EPD{TPW@76Y zk2UDbaj9U8$5LP4UwC~=pG?XSv_S=6DM$S!c3cT&LOCEeO1oXIYa53N#aGIVvH+eA!mU{8k$-111!d_uAJ% zEjOLoUOpSbV4LasP(&dz9=BPRir0K0Hd)HaQAh+UGt!Z~mTE(O z56z@V*5G~v4B`2D-$28>Iqs7oBMd!;;Xxu_O~!UxSqi+9X1cZi5|S5a#~*n`KO5L5 zn_kh1C7SY!4t4p7ZpCiy{>THQ`#Op7Q;K+;B%pf=eLyivD`(~2E`Wn$uP*9kRk2M^N{7a7Uzv~Bu4r)Fi-Y1R$DX!s zy(P3vP2rx>R2ePRRTO%vobGzY(O}p;$R`()O?p4@C_?u)8Ntxli1u1D^&W}TwlSE* z1g3R|9j_ENYJNN2DuC2MLeaj zPNx^QXtJQVprMs08Tl;*^F3;cm$0Jb={w=R1ZTcvi#CM4iBcbM?yWP|_m4w}_FV3< zU>vDGva_!7CCVCDp)cL8elJCA0(4eEc&jg z>`n?K;LGh&SiNL|Nej659=RN<$)o!HA$}|EKi>u5(5NogzJr*$(SCiXi|>8#m@ONl zaZUWE&!b7JlVon?w$0;fb_TvfKnFjXx4qHACbb}E@bUHbfN9tbs;7Ccxc`XDfAN^p zmw3ccf1@&akUA4!l1Mz|SsX3WIJC%?rkU5|h(!l~DH<_K4Rd~qhkEAhg-s<|AHM14 zfJi8u^2tMUg!ShCYG^+qQbN!gL#lm>krbvlrPMl!vzfeGl@iZh!lBY*7Q~N*xaai4 zlzH-6U6Y(9LbwtaP1{TXzCSgx^qYKMyoCzK2a&?DJ>!@hbcjtF-Jd-lSCq4;hFxWr1A27ywgeZpdF;4C7+3$LA|d0))wE#h zAih4Oeqly0`rzDnG6DjZ2yz$cp*7b?1!p?414AyN@8&-RU30pkDb`!5z0b--gdz^c z9yUed!)2hAC4TYv_wC;FPmll?ia;O(lc;?g@fWn!dK|oZV&>`+&}zM5dZ_cqs`#Fk zuh?SnEp>*Y;$B+?O5bPxvgSVTm2vU%=r~tZZWq+qSZ7gD(8_PAhDWh6nuqT4!<&zh zX&SRN`G(g3mhz3H1mK%@@U-ykDm0^l;k9tJb@r2f`~m(1@>h2YuR$ zF;f_Ki-qGG%5#3rb>r@8^kC;rBgu;i!3{m zR*BEurI9MHz#G5peU^Y~H5BN0SE|e!Fl!+!AW>dV$rnxFTaD^L(!OT+EpXL7UEYUFyD6vi&6|9@uJK`oD-{!g)4;QEv^RJG0zxavjEMoySt*(5 z_%s47mGo@F(XwYxR*CO&5jp9mj*1A`5ct{)OqxmKiYtpmRo_P@$u#CMi&(U!bP)9Q2>x?}k?r2zK;(FW8Y&F3zfgMGLvqBcXNOoihh9h6n)R?UM>1@}b}5<=xtJw%Ds;G}XnMV$;=om%9&p?gjqhx;uFuc+nmBSc^<&wDeI%lKO6~1 zJtp4xw9hk(z^wpLSddH>t#1z2#%KBjc)>h7Wq0j6wU+*~o3qT0#HOt<#h()+y@7&Pj^o0Xe|o47+x!b-`* z{_T50zqk0+bBtgtJ3hB83@*wV9PpSOJCoI4^RXaKC^);r|D{>^h?pAzerPaA&XGzK zoiTbcVt#{57; zD{ZH}I69DMPjoY5cdky54W!owQ#{bO-v?S!%#`l~{77J^Q zEv<>e4AbYy00n);Y^f!K-aM?;P~ zW{M4Bm!u0-W*!9)1!Fa~`e>bsjC36>7fUg+N&yy2!M~bdEv9ZJwfjk2iC!sCse``N z80QAZkOIt0yq$|uhjbrC2IYsk;Vwg7i|DM|$#KB*urgu2-x)I)myy(zN)GuXz?d}T z&}-T(CuVoXNl=gIkYHeM`N77ut9aPEAT^dgvcS_2_F$MY!$rH^DxY~~t6w)C>Mz9w z9J7ZV&{}GvK6ZZzm}C2a1tzp_U%wN*KZ>gA`)-$RQ}XyB7i;^=I7a_8HPd?{<6nt6 zxUF42+dQ@^%|0=}ej2WfeKB}>>SFbsYRQ`!=U;+mMy;*0epu+7_C%<5!Y?E);0diH z$iuh$#BFU-X=CU`mZ*^ze=r6bmX-b=7&t@@%+-TN<`t5E_g#BFtdh4FQB4BN5}tP&IAT<@`dh-KT! zK3x3e0>Hq+FK?`@LHpzLmhCmiAAd_{4Q}s`=b!Hu@SxRu*k8{&R=ja-vgQyMO#}^T z57|SLuCO%l@kdTy_^qQzp+v@@P*na^*RYb`YHCvcyUQPq4pSE&KfY663C3M2r%U`o zyuwTl?Q1fQ|7SUjYb+il94ZtpFRo*1OeVPg9`8i~j-b|(e*)w+mXj%6JpBC(qL<)s z<}ncOx0z64OsSJ9=&xM1(8d^N$+mmM=RFF5c=q6Guc!FxsKRNp z!J|e}6g0)PahjrkAzpT|yF9&#D2ix~U?Y8{_0h7fKYGjK_k_sxQiLI@=eBATKCmR; z|1E-0M^E31yM6%ki0b?&P<ko6~FOCJg@vj|AtV9jjGMgY+cnQD`6haxN zVxU-vQ!N(``e*<)HGO4MiBmD{@a19zruN&x2nk~nPs4fDs4wKCz}ZVtw`x-pFcoml z(~L8#Wp1RD)I2p3tYsdhARF?aIIq}~_V5G^H%Q+!dL42bub6U+e-t6EZ#>RzxY)b5 z+clEBoTPcSXz`)V?}p*teN_A#?aj;kq|ts2qA3{lqh2cId7*1IRI6&c!I(a+9l zA~{4vxXnfQ-#=7d{x*)6fJHSyeeK&a-;Gc(>)$SA@;8>E1CD%x??$mMl^*CrgY2i{ zCU@F@qEdj(BhmWtsLokGdOzE$xI6n`8^Zje0YBy$k$8?-k)Z?~Kd6Y(Mu#>1g~jjt zHtl-BE(bw{Z)idr?XDo~s(WJVpPGc$H}b}qrjWo_Dv4&?qUe;Opgg9YkD5;$yK2Ck zux;p6kULs1tV&>Rgbuj@MwB?0hPqc*f%DqQu~elR8eV*f8>VaP!UhR3@GGyk|;F5FTRRZBVqv$)QDHe-@Y#{dToA17n3 z4^ukDPevJk-W+N2T(9zo>=l^cq~H;`8U^@QP#Ux`M4I}sgDw&lKf=_eQ(R|+*tRM- z<-hGh_wsv}f(m;0A^uQaP3_aDi+Rnq=&HAI3H6S~tiR8qt5$BE;ET@)277%zv(eYC z9BjMX%E$h-Kealxm)t2VbG}6h4^cAsGlS@c-ztpCYST$g;V4N_5!NQ>G0h?8aluVk z!TQS<@1k|6C817AEL~xc!O_yAx)D1@Ba;OBb0!dJ*X#6jy^0&|7aV=-TBO%oU!g8_ zf%Uk#CE`nna^SrgSG$4k%ui~gzm5xS)IB7#YYxz~UW+j#YkKiqMc?0V@92#}O#EDG zI!ZA`O%)mPZ7qdWM$?Pfk4_70Glj97VE!gOl=XV!kMGH9`J5aw`!W!_G5a@#^q-8b zbr^*IiG1yVrKQc~PtHBpj|aVf99(&~&YeijuU%~n^|FlA|EpPqph`rKA65UiZ_qnR z#vY6F+P$O1(uwg|Z!w(x{hcrAQs_!IcpHe~?G44-)YY`_!v>EicEB*EU&oX$wNT}Q zndisTSuOhaT(3ZPSt!RBpUvX!NXp_hQH)SCKHBw2q{+x~wJdjdiObN)->?-C>w9_a z5a|dcwJ66Y>uoiLIdc8>eXews%|B#d2Oo#Wx;Bib0}k;^UEu@0u(j2-5RXzT z-*WPN9EVK)9Qzz$>wMuA&PYK)7a&W|B5;Zk^tc@Y@J%N7H=(nWvFH{I5M~6mq++>< zl^c>{YjWV`q7lLzGM=LNyLx3HPR>$9E)}S@7JPyiZ!W4Y5?RzhuV7&7vO#EGYUqqr z$QQ6KTTnalf?GX%@gISY4GADLg*;GW9%~tY#R3f|q-mtb2n}i_9}#h{oBT9h_j3+) z9LFi`LrV2c;#5Xgc|z|?DCz)d084;u7P~(`B5tr2%Q7E(x*@wwuSmn-PVIy)gK2*tt zm2?TJ>B9x%)y4JawF6hZr}-M7Wd%Y!_F!uyOSH+9a)L!qo`5D)C`}d>=GAFely!o< z#zg9O3TMmYbrnUp^9UrpsHaXanB7MT@C`fWm^prl$4!UeDWBle>askU(!G4>)_NV$ z*6MX}G2ztw`)yh31a`rwV}|cLzYcX&Cv4GZV)Q*9yQ~qdAN>Sw`s}aryP^`Bccb6 zx3J@^6QV%)f$P}sJ8!#}A{AZrIK58s>;P@B(!BdnT2(2OQo5bQZF$u$WG$-RsaHOi z(n&{CUx+V@piss}Q3+1{s;o;O{bT65ff8R43Hxb}er1TBwFq(FLj6}(6xNx*%2Xe^ zo_DD7b%NDv5Clh^Frn&ds04}{K-<;H`qMIMiPx3yB}=_T&;n-6!V6zIzF6Q<@IZiE z+jr{>*|t47o*!TE&KhJ)ywmSGecerp?|(=rI5Hl^!2F$qiU22m~H{c-B97Ctm0u!V(yQ8LiAFkm=OFm$yP zpYPQ^#FPt55$2kaSfa+A92(7<9%IEHmraia-F<$-bo_IvNYdIFc`7?k$QfK&TF}8>UlwIz;W3yx9M`LqG9?7i85n-+ktWq^@8|jmNEUT%&Zzg6TDJ${m49kI z$`ND5yeIrh*pJ*6#seXI4w^!qNvt$Gq6JJFNTW^Cy8YoOzgp$+lir{|dLgcUS8Dvt zrL3{lIKwnXBta8ZYmj|?`$>Y8sxR@a*C4 z^{@$jM~TkL(hYkkZcomQ$8_3f-#&<9w5HeMe0#ss(r0TIhg6S&Wz(d#9#xEJ| zN-=3D0SgL@_QbXr)+-_#boF}69=C>OkH3$$OB_xj!&Mxa9IcksP#W64`ha__& z0@E_PFlR&(RxxgnrNV2;Y56#Lq=2_0b33OEoQ)50&!mHxQQF)El0_3Ef7~*4*OPpR zmlZo`??+KbC%4>!gAzCQP!hWwI7$^N&;K1{eNi;44#5tsswa2e**7>u`77j2yCT_Q%IbGL zm%4#;@l9K!et=ao^Bqupy`?uj^~$d_vT3cQeGFi}UHaIiFHBKgO}pR!J4ZkP+~| z&;$2q8o2UOG3$VJz2vx~(p;uOn5y1$J9eoTkVKX6Q51#2DT9iIV^zm2Y%#up<>0J1 zTUIq75jYD+wCO=f z%l55CHHHZ8)ai3orE8`!->)~Ri;5^se4jpv$))a|&+V4aMG#s$FV;oh6hdAuZQ2uJ z6xXMv`5W*MzH#V(dOD2Yl$Kl3l~QJZ1LMu#&tdG0r7&en!)ryoP53WBwt!6+slo3TP2G?OQi6~b@YA~U zyW`%0_EFVZtb|X^C88Qr*YStqXXr1(&z^KrUpx*zXUuIg7{l*1kY;bPibIk%HallW zeI$JdnsIUB`8^>l?!V;7^iNS+NMEVM$|(tdnGcUF273!TERfgSS$6b8_=#9&)soHJiBI{|lfZaFSD&LlQ$5}Dkh zwfZ@lpHqSAua8@U6jY}KgDacfcsZ)GX6vGjyjjsVfq)mKL8K8lcen&D+D5VErds^ z-8LLtBjEa_P%D9puq4<$nZSB$Tvl#(GO@1FXfVwACYN}K0q2dFTjlXv+(;+;W6JkSZ)u(MJ@27U^6!)RHc%Fgq^RzHst;$_=+Z8QMBnxf zWcf8WYwKpG=$XpBLGQ58=U+;I`yNYECL;xC^PfHf{4%d*VoVvOEj~<5%FN|%8#>u? z=$;C6@DJqjCR|iS{8*{|Lt6^wskVExGuY;Dy#Ep!@D2JkTuA1#{VyVgvlL(n_w%*b z*`5)+Y&R%@U%5F!CvIxMV|$DOJe7~aOA*^D=H&Y&_O^$vQNbvRV|9qG$6-1eWu*yo zS;RR~KhLz|1hUhiZ^A-i!Y%K#$)p$WK8s0x{U~k4Fz>W-wy?kg#8S$p*?h~)&Cx=# zLQOmAE3~72@-G7-Gy3iaDZN)_Q?8iaSZ#E4RmlY(JtNiTsHDe#VA@Qq z5c?@B;V~&21Q%(Q%D-X3o^9nSP>PAu!^UmJEj*_cCJv7q!7gb0L81p`sw_ctc5e%Q zF3altXu?MvVm`y{=3oi-%XOqGA6{v=!cf(2G(I>TYa)3MdQ2i_ru{}l9SZk zn^YfC>B-jmO`Q)Q2}6UInYsodFA*!MSttef_IWC$TF!G^c`(SVn@JM=QMuVyNPSpc z2#i2ff7m2E#7hr==p@b7gNtY}TpHw38)9{s^h{x>>EopiH&~2l|0Jl_8*X2qxu>bu z%O4ruj=Nq_8=}GwM7^qciLXiPOpW9?oT;KH5Gm}tsNy4HAfN8XgV>&K)CYMa5l$)uD^TL{Xgf9yF zJNc8ppgU1CSP(|Xa9Xi@d1E6nw$hQP5B0Tl2SeoT$9Uv66a`*zWcu=FhK*07AVdExS#Tu$PXTC>RWWur^`LkbA5=>f`8hV=M}cwBc``@Kz7+e15&tSjS&~oznI9$Xa;h9~@pOc6 zor`%d_|RCSzj(0@aB?SBuWMNhFtE*hGUtFpE&VIVRT=lP-3gyc`XaH6fduHDCiPI! zA6i+eIwd8NthWBx@c9sam20pv$mTm{yf}C_l?0w6KwV=C1lrU8^(YkuHb%fT+G_@Q zsDi%uGrb06N-pEywNg0W7$|76BHnK8t;Ng1{d^pEK80K!&ZAPdJ3xC0u#*;)UlRhO zB$H*uyW;GrVQjH;A)&#N zES3>wQx+)K)6xLd$%Sk6@SW}16*UIQF!cCHkC_y8DqXOi?aVyA6cNfSjur+atG91G z;Z@ExF`^+wsxtug#SSR&e`PRi_^b$d6xH{ z&n#IW_j4&j?#p;Bk&*I6v-)JI%p&7 zaSb4uF~UbWVUg3crEskfH)xPqsbG=jA3s_@vM8Y_FCt!0p;~t<1+C@-D2oRlUZ&ht zSOGkJ{T|ceuzF%`QFF|DDIKr&|)1tdfO{*WuTt}C3Tef<5cqsF(5w&8eBrk^l9?H zU{j7uRA=kUTnEVq^U(}vAd3=X7_(Kiu%Bpp{FdY1#j|gY!hNq#`oW9RGG}swT>)lb zuqCOoGh9%df?4(6;hE!1e0B>Jux6OG2q7FKG85&>n@7hB%kYnPw<0pw^mSCn0c4-P zc>R{UF`9{3_wt8lnB+IV(?u+>?)TH6!fO#hi)dc}oXGe1T)uX$RotS+Z?=ql2_75e z-khHjOrjf2@v5el2-U(`LzE4FsDK5xG0A2Z)<>X_nvodARBZfuFN5JN>?-qegQ?vz z*EvBBF5*XwZGG@5GsFN5OKL%m8*HbK89WR;Dnm;_C+Lxi_37S5q|OnhF->CiQ1TQj z4qG)R2CHZjSmpASP7LD)AREO<958oW={kkdKyUdiSH)jpWj5YA9B&V()xW=2wTpcU z)__?ub4$<=q1l;O%m6p<%678%Hm zwUcfReS>6cG}Fdgio1!1*LG#D;>tcNIY)M*cf*)UCW6q%g&o?^Bl)F3#gCdg82PG= z!f_XBDS6e}G_=HINn}NJm-#uDTs)prg1lv7#xDUbgnsRqnlQ|4G$Tu-QNt|Z*V>F~>G$h#RtA@PKy zPE0Sl;&E1}a*HpWPR!xGVia4gAz}m-<(hj41HuQwLFwY>hvC-Sf)8trQU&0n*4~}+ zhvo9Of1v$<_z^@W1wuO35b?95VP`WiY2|}6r%JqZciF|0{adz@UMCP@IpV$>udMqX z@W-U({*QNVYuhyji_3`Ww|q!51^_>q@e8CKiHpJQQW;?m?B*-P&(yJvQnjsWQ6wNo zAl{0=>$ULgtRn4rZiY?E^GLFi^}@c%ctENSXii3a1itTGH3XmiAk$m(s$k|03Lw65 zMfA3qxb%G{vb25(2Yq&&BNFi znF`|wwVST2G<-9Ib03=v|GFB154A1WBsG!zxOnr ze;FMDKUj~jQv^KyEQGwVLphD9W!^l!&Ys(T)CDuq+u)f1#hdd#6ENID=HaLf z6yW5o;%>7)<#w>{bT+i6$upT*#=X2|%4lJ-%W_el0lZ@c1ljPb0O!SC~UIVA=-OZlGY<(IDfR4@^ zTJJQ6!yM_Wgv`A+t$*K+u79Jy`Q>}mhlk$?J(oDENs?+7 zTu2dr0a7D;uuK^7&f)$CG1Tw?bf|4@zFS5c4(I*wvxLI$Y^3hU90k8gZ%WHnBJNFo zM_;sKAEeznuPJutBmjF>C{M!8f)RXofMw`pFp3%GVs$Rh@H?vw`S}z^$t0 z+pdt=5jTOSCgefP(WgVhb^>}~C~%@}%|BQ5`CY)tP7bM_>)AX>ukuKawWJb>p88>D z?h8@?Haufw|g&^B;|+!_)strqlt16 zUVDdW`(^X(G>IYqSTBntkud0eyYM&1IStNDC44X8D%Z}*{OV%h&S zTG8?*-sIw!#7n!)mx*Q>))n7)M0F5_j;NE@6Y2j1v3)%1*-WRQZ2!}m^!WMKQ+DzI z+uu6gI2Xf)4yQ9uQa4^JGuU_;@n#;s7D{-}Hs_qU3$B;YYf_(&!Yeq&-Gxsr<8I1O zIlJNOyC#wvKvUTHWIwcneDg#DsnQ&%mgRII?FFcNe=ihun#58a~X^4S7rvRe2rWDQR{ z((eBr98R9UFyDa^>`?=v3oeHulZ3PM(qmfu&PundD9=_APc@7?SCzxxQ($Q#+93&5 zCN#FdZw0c{ZA-&k8vL#JcdIUyB|^3`nn5u#&%&+2*JePT6p>!|j#bUL%)(bz-R7^X zN*ZQ<`xd_=5xklRPbnI~wQLD`UMb@++Z(z%_=Ed|2KX=pDq*|Hm_BF{Jkd`Z?5R2{ zIt)_t)+~S2ET=YfT*)*3@@#}W%=}|JL^K~cx=L^V+9o7@68!2nGdHcJ0r-L?JuE6< zO5c@^7QFWqnf;h{F5-{pbpDWT#Ddg&47eKubL7yT)(63oyy2QN{)d+C@;cDP(k|ak z`T98kPb}47#h`0C4J>*A`f>z6mmZ>2+IWvipHoQ?aJ^r0izb20rlW^ZW?>Kw&XK;; z(B-IDr$@+ZD`rl=ty4vNQpZ`#+*qE`5za^=WS-H;#ru?A8E;(8A;d9?<}JC^*EK3h z!k=72<%rnYd_MYUGf_QL1!7!;p+#Z=|3bSBivmn5*emTH5u1>v_3RxlOeCj1%Rq2r z`#G(TLlsWajjDK$+9{bV38Fd_YJqov7Sn)WX_RTB5i<3_{FKyHrO z;}94fGk2FNWxf0xyGG_Yz}>t^@Ptgp&kzR_;};i-A&ENMcmnG&1Mm0jAF`7@5G^@M zGURzPgmhMt_eRCkIZl9+*{AE;^M>hbhNtuC5$i_3l)qG&oBknbNP*Nz9CxvSD0e0$ zrXq5(Kgd1%e3h@^Fb!n+wpD^!NGADe=k9}FF0@o#l+3g$EJZ&^e}VA%G;Nx4L~wtd z?gBry)o|gbU96M%xpan|aS?3NxU}YO;W?V&a**O-C5j83_cR9y2GlXBubv!~0^vFp z>)IL+1J6GFvp3!6JYW98RSDv?A>!eqAzV4yVKBrue=n~+iZog!GxZr;+=0^S-PC_e*nlh80IciRw={B9u zz;IuylIHi}a9>Sl7p!X#<99zT@^ds@LgIP_<%eFZZ$tk>gC?Ax7~jogtMhY%%T`dY zSZJV%^{FTViUNO~jMG*E;mM8N+ClYNVPjkbR$M1moe252At^S>6k7@~X}@cS4~w_X z>?8dHBXa8}?ou>-?QTiYMSo*z8Wtann*-znWhAEYv6sYK@(Zxgv37M?OHxzov_;-3 z6K}CKif?J)w`}2N>Ds>d6@0$%2}#*z?oTMky%B|OpuiV13XuL;4zQHIA<3TX#VcZn zaX`f}Uzf(GWIE84jsgG%^mVWCm$xfCxvr31djz!`nJr*t@u}6`cf+Yxq>y>Ky`8U( zc&mcyzClxPn-jAOWj)mqsr2G-p|T@hKs)%b0piC^=g~eret8&{t@+jYH>mcNPy#Ac z<#E3w{4J=eL$qm{YAJm7mH(-Wz;4y%?_%J`3W(i;U?8Jgk=)m$O8&F9PYoAUh+*M2 zVLMq&4P95@oAAy=ve*2R)V$f_Ma(rO2}vZuHG&t@&q+d+g~9~`+Le@1eWGsCb83O! zo9Z71jhgG4UC=dPlj7jjtn~8++BN;DYcxDK#)lBC)R0*kZkwsMI^G1_F+~Dt0Xso?yUQ+2R%!)*H zmhg}7n&ccy;G!DJ+&BIwf+<>j{%BI69I%j3*f3cGfMqJ?$d7u4`C@gl8#h5Pi2aVS zbirW=OX>5Jx;b4V?_m>Zed&4agvl8pBKD-1)(klD+jZr}G3$1zK26)({Q2tkK%}`7 zzy1FI19a;+`2FS-``LfMn~l*lDIg&VZ7ckLlXwEgiQCHdXJXvaTB6l||19yGjd`5H zfQ>E8mQAX06qaZp?e6rKL67ym<#0Qx?44809q;aKiZa#3b&{WH;6v&7seiY~Mpe6B zn-m@;T|k_OlKE~Q&hyu1_@luUbRHGs$m~zWHs!=JjWVM6$u&jP!CKFf+q3(G6goGC zd{D|_j!JQyxalWMmB$%~N0nT8czFqs#s>laDj&;#oSEWI^KGyBTx2D1N(fITk>1j~ zpj{;sms;vKKOQeR`a97s5pLBgKOapcvTk5h^hbST2fu z8{&6cA6z~NtF3e$3*S6@izOM4{UCCnqybFwB2fMPi{ElXgWFmdSJ{!G{MUkbrR1L# zk~m?7CIe4eRtNGOdXB!%MPRgyEOSXv$!11aTHWtWFp{x^zorQl0OAiOJ6YKQY;MUvYsPKj3*lX4t3}IKokH zq{P38Ls4|-HwF8TOYM^VL_XK|&pl~=5k#hyaot(A_)UWNmAfrH^JCna1ppGeb)xi7 zmkzedJJhLeJ>OXDN&jATe9NaGh&LJ(ZDU#w3@Bx6|0VF=Yw@0skyQI9D@v*e4O%Vv zWD!uHhZC}S8upco0=6c)#r6urYO@UsomevA?Sv!PkvW)x+5%fJaog?#Pxyr55VpP#Liw1oo8#SzhLx@S1U_wWN5CqMJk=ANfd-(*}jOHnPRcKifrvzpKd|8%SL? zPnqb#g|2CZn94mux`0j}4OV{OA*6k!rs$fOXl^PQw`lSfy%J@f=Sr;rrIo*?p_E4O zTwTaSP9H4TYe_F_EW-YN0D&1-9!XA^H|$`y9-mfJe|wG+Qa!W;8S`E| z7#4c9mNPcjZ)W7m`}{CojPac={J+@4RWPH!@z@!*kaNvc!$1U_ohE#lT${k{T*K$L zf==08x98rU1m61GzHMvg!uF5n+O08409Ri8G6kHkloUAppulnz+ZR6NAIi+5(6Y)5 z1`^XEUWw|3KrBeQ#9%g10Z2GPSU6TZ_Vo@fGdzK#V&|%%>l31snY`A}rBaSSBD1sV zAE)jMyg~aRxP%P!m^|d&rVnoW!LkbQU*Es{hsQkAEcfEhF40@e=y2hOi=72`=8Q2r zh?f(+(#f>`?}T5w43q6+yQs(*MR{nwB^Oi^*AcM&$gRA|Z94FgcKUmWbOF93CLSF7 z;OU=kcn^?6>{_TbUA;BesQXTacNZ zu=7V(C@KTLBT7}lWJWy;Q_(;dls#G2pk8CLZG}Aq<|~*$4wLjhw!QgiG5hyzGPtd( z;$8B-&iTS~li%Jej?PiGd$s65nDg!GAO6!Br-H1u?Vzlik4?wPGl!fvp0}xi-qW|Y z+xXti55(O+5XJ+LVoRBeoC^7mbyQmyQ(M+ht*GmCl``IYRFl__`>l(*BbrNt{foTE z8ASu0@u}7JzfSAtpSG;piTFxJt^d$*W^28!s-xLR@PuV`#u$j|SvKgE(=UQ3d4O`F zI^*$ZKVa-pKku`?*Imc04^&||`&d{(BD;xRTc||SpYGNUIf*ES3u8W`+PcDCs>LiK zpya!9&FTH9`@ROxGw!y$>1AfCEG7PT{r0D@XEFA0;CCE58GXlLe6dQ#7Z|V7;~VFU z5Tip;O`A*MX^_OBD8#@(Ure7{eqX(-bJwjXasSot+YC|Z$sRm^S5Q0@?*%`j8QU;r z*+fkstjUR&mp)Z`x5nj1l`70DK2dgzTGF#u8Rq1UZQj;gQe7Y&v6v?uERzo|ipO}V z(WLq;H8l#g6$%ynr9^eZ3u_nC10j-bhm(49X(_mY6&1GggLqUV6;&`KtgTh^bnBX(iJ@OtX zIUaFc*n}gTwkOaNC+#|LHn`c*x+|0yh3$^O9dw~f8%jXFm&g`(m)6p5q@-L%DRc+kxduC;!h@ze+4{`=M2GCEHlyerg>M z5e8M4g+AEg%LKn85o6k$zdbLa@1x1MZTy>{q@nn~j#a}w7a55@)3Rcr9JK>oF#mAz zlq!Im0c#Ns2oHx2>U1;Y7Yq0Ia~@n6I`)GeRSk|l^=*!i^o$0-AdXqN>+pZgI4-@# zr*ORamKyfT3-xqXlmR88hs~3p>jJ(0J#AYxlO}w8fZ91%8~`OQe@XhxZAiivsN-bd zveeC}E~G`CFc~8`EKT;By*(A=m72?^EMKHU6^r0ednsHWe3QVzc=&A@KB{zvmQzeL zDuOj18MHlL9S=@^0W2{~x=h>L6PUN6C*DI5bz5O*_ZgyopKCZz3URk#=>)R}^&FSa zMs2ATY6EXKIxt^N(^sIY6b3>!7rr%qUw^45I3V2cCfu!35(E=%QqTAHSV|qK86czgH{nmNDoVP&35BMAXCC=%N=-$n=%HUl334)>axb@ol zzVRSmf~a-h!1HT_&W{`812UugANGSfXXlY4l%@MCH?1qn4Ew4^Y`_8{bYeuoL8gtu zztt;#&Uj$DT1LsGbEJN7PdWgdd|ujkROV}`T$<`pQ!=@9_)Rmg^{G_UYIQ7<&|ssG z2DO$??zeiQ@99+6-qY4+0#3Pj(#Hn7z1X#@BFbX}XGa$vMZ`JR#VEtP*x=+Hn8K?U zS1n#P+ckMmqJ{k}7+Xs#J?Pmi zwkxwdUi=r#9!X)6%?gh32*(R;rPq&1_D3_t*QLE*KWOL!RG7;fj{Yb#`WWg>;9vj1p$8@-|@DX2NE`HN4~J1 zR48Sx7zmr&iK+`s*%#Bm7>{6ux89XuJ1?X4M>rQNzPC8^pl&w#lt9>)i*M1Y(x4jn zs6!PeA6{e^BG8nG*w(BRA5w-@7wPz!ksXN?oT;_%Iz()bTF2>v%cBT8G{)49}jF%;TcS7IB~nPT(VGx^l@+ z6-f#|yrt(HMhckRCD8USK+mjw@rXkG!%_#jYx&sXN8PbqJ(-KW{EhP+wc(DIOw|-g z=HB%f2FKX-9(aoc^U>X&ay4>o$$E8zH_yI&+=v@{gL-jK!F{BqKBDbUK}Q@^*EQVZ zDJmyk9ndG0kWa6v`UeO&Rg7!jNv|2v?Fz;hKIMgun$~HmUY9OXzc4w_x{L_+Qp!~t zvL@L`!|sXY^TXsi|30xmt=~8ae{Knt@wafk;i0_LZk%0E`Vl~#t_p;`xh*86n*E)% zMC;uOrZi;mF5Ch~EgW!+mtBzoP!nMTp?ZBKMi2L&%qe&)7VcvsQ)no}_N#;|7@hVk z?$6zkh;44->q_SSZEYYkpn(epU!x>KHw)L9ExtN-pLzYp&h@vTqD^Zp$cQS zU5jY6t@%SJP(b{IDjw0&J3!cpx$kf2e){LdC$9@d6Jb|?3d|*{Ve>>|Q%5*!as2-0 z^KYuCBl{&%<_%rWQs#iI(dVYOE>Jk?XS7QG8ck0pVvG^@wNV}~r`d;|$Ct?<-hL9u zH&tQ-im!q##i3=m2ISB4kvWLpa==%#=&anqAcyDI!jj=UKCbMgj*F497M}8y-|)Q- zF>6ow7-uV5BMAQR)(R!PxE`onzzJwjY9obPT^2$d1=vX#e0=wv5ojnGR`xsK#Q{Ml z&W8!eH1gF1R7}{=SI#onr>AK7itp0_tCUx+&u0tB7|adN^~PGHj(Ib0e{%4oQ|-h; zx1ya#cIZd3L4)G6NAZtPttUmIHznVybVx z6ap~os-NzSul@^PuY+W+b|9Tczqm?tgk(!QUT*k_{Lw1C+Zg47V7cf@J{ynl4khAQ z$L>|>-B5-R5&pPRx-Y-;3pSK&SB>o6anLLcOyJ=&14c1dokKqTVb{ad<&&j@ zfYi#z%EQz{y=>?8eSQdc&^g%Fx4q}AHTF4qbrH5>%)0oV!>&#sk^6S%+Jc!7R5t(#Hvq*NK2@WE`V44iBX%-lzfT=^1okjMgU=ifw%P+SrD zd=x+CGf--L>uM=e5Rv^x@f8GLZAy0DzgH6hz!>DzXDh-RUwuWp@PRG|pi*i_eL@f_ zt1RNDSarH%wj5*idrY-94?60Ke$8 zJ{7t59XZfA_~TtUou>jb_FcY8y1$J?aTW@dmBceISJB}C+S7zAlMXfN{?nwSb%*6O zj^AO}RB&5!ypTX?nf?K0Lq>Almb$RBfe01yIw#7u544 z`vRLupWJ&|UnvowlQk#u^e*!(aYk#QuAT%&&CrE*m8;)kytIrdmwLEsCM;DE#RR#& z_`S5d;tH6-@AIx5<|aodQ`w+gS}Vm~z=%AZ3+mF3(pd4-&>y;=>ZylHZj`xR;1Ij#VOy9szYN z2W;&|<2WKuw|pl!l92LD>_j&(`giy#8J*=L8tUCz^WF0sN9Ju!wTe4t{=9C@im9y)IOHz_1YT!5o>^NkAkz& zIzSz-$$R;=u@}f$xtozQwd#)u=JE)Y=5jub(7axecFszrYqCi_^9Bt$83(Xl%tZe> z+YYrXl94KOUP9ORe6@!m)%&g|14}xC8wC_Dz{!dnnaa?0Z>gz-hmcNuaVF7xCjC>p zU_rJuCterR^fd`$qD!*x^%RdKF%(f0FB1}smInuUFn0pPf1}QRT)3hEkwr0undj)}yXs=cShld}#*%afo#P`q(8G#NA%1zByGAM9z} z3@9qogBY<=SM5*1>r_OB&v`)S=FaDmiZP}Lj|3mn*Ag4WP>TV zas(HigSPw?yx#Mwh~a^cy((IqiHMI=BJz05(~G-9#(XhI+NPgmD_?PFvKo2f!ivN) zn{BC5nx{M6qnBe>*x4KpP9=` z^i>EF57z9~B25hcPGoS4X=%*bdO43laW3{w1J9{p!IaDpxlepV2i2x^Hq*I__ zqWY9{-Ms0hS0<1UWm*ji_yN+a!EHGrDxYawxQ_b6mA}59Phsz5{zl~i5-|cp<49I<*1AyZLJtyHdx>1~+}L z?5Qm7c=*$!M(v*fH>d*is``)5RnPS334TA6)`frrxR*YAs=$nMPUUAlr`qLjJ?K6i zFPZd?_e2Jfe~t@N*9m*p>nM`eA$&Fny7VbVkN-Zg#4PjlJypW_IOBkgfuqI~0C(Bl zna6{;fKY$dGH0`0)Buk0<3XI*hB0=ZW9scDwnT`5~C|^KX#(R5r0;5-uqSHDG z4!P2JyXpCZhWi}RPvrrE+}dkT1adINJ~#UiN^u(7tI%7oJ+Ka5eVfjK0KE z*T6)|NBWqt4R$}(YeOCdaLF#RgD7;%IKGm9;C5iPVVn?ok|J$*&ph8lGQs`@F~dl( zZ)mj@KUzfz1FEQi)UW|wA;EtrwT>R;g=7Bq0e@vyXM8Ux)h>jZ6BUh`O^RISRYiDPW*#Iwld|WQ)E{ zCrV1{SP$Ag2?U0j5crJRp@|$90}L;(dd#;;g}cDFE0BcmssT#-%#`(5PU_)QP=k*z zjsCV%@XpS-EYn$h;rfL3Wp7AXtp_(0e^`)4;D_l`+0FFYmc(FY-3)TU4Czx)xQWndoifJ1+45>w=&@QCiz4ct6J!3v zPj~~YE&cW*dus}h<2kUS;iDyI^0X>2gZtBa(KF6?i<(QLV-cM!ynAuh{s?Y%FbV@k zloe}rqoEr+faSC$KY{Ns$+%ER_<5TTfj0M|jp=y4lX)74W&0)z z&6AqIZX``WLyIY_F3LRzaL`yBZ`|Frxv*&zAcZ=D25pm-Dy=GJhTb@4|wH^*-x~QpF=zeNNUa^cmIHcnmPcOCX&cd2WO>Ln<_K zr~Yde+nN%hZC)|N9;tagE+s=hEQ1r~clUC{u9yTl1yQi$5kr6;OOv;P0^(TN<_QY; zLcvAasQ4dqW7ubn=y8{mPEdGhmMGh|31Z4RYGM1e7w!0Sfm-=5f1kE`vquWwFE@d< z$gkz=nqSjVpPdxbVw8Oqpk5Y&F~lrwHE;h9o4}!nRuK7rrs!CMw=T$h!fR$Il6(m~ zyIZ|ChT`#HekazeQm=WNWXTz0A52zdEn1(p*|u}OPA(c@+N{cG^Up! zhyJEf^#OAed1M@VLXKGbdZ|<9@^Bk zAT1+@`hmgM1?^`cET5m7mzZH5LpuBYwY%3J*=Z5Bvm+6-IhN#4Yl2u`OYp4(=+;oq zQ+ze^e%~_gTC26^-(O6Zr}q&zK$T^6kKe~qs*S?b9mWpl^RIeaD5!zD_cjOu()-Vb zrt~)Dp0TW-k(It&sp_g+iF>95fJyxlkjH3@9j%u_54baoi5)ez{Jwk_P=R}y9>7o0 zG+xGGUhZ08HF;yPKG^9;v-S*?>~MkPu7S%lsLVJ$U#9b;EZH?7XSP|Rlf{Rg36>lnUB>oU>KOJiJLPr86^*~;7q zNS68i2sOR^!`5Nb7P^Dbd4GLf6*6 z5$U5VBfiCh-v_e8W;E79n~p=6ePJ5HKMp{KX-fD{pr>_w@*uXu_FEF(04T%uo|`a8 z9cTQVV<=qp?3rCxgWv?DD4LM(Ryesl`*VwmOQE%?|2Ay4D(DO3*vpHvQJOPda&3UP z} z2y)$&-bf+>>ib#W?bTcz_mm(97_NihXr1jy4?53`H&pxG>6-N$_XEe_o&rXI&ns>88h1g>Vw#^J{ zg{0+FTSeFWia2p-$2L5lz1FF?islSMNY+BW*5_oj`!Y7+mNu6LJpQqL>__!zSdq_E zt9~LTXvN?@FS4_gR$};v56|g`mGV`#Wt2*8N9HSN3brbrYAia+c-j_jhEjosKY;S+ z)z~g5_Ywc;K~$ragxa266g9C-`wvI!Qo=n*@<4C@{1)0KgEmDG{n}u(lEQ8F?y!zJ z3~ba>LmbS@8M|zwM+a>7SQ=E#B-x{L+lxjnmIq39K5y2NZym=q>#%4UXHJ-7DYXJHD20;WwY;gD})VXzvh+>W^xKWT)uMH@ab>_WmNXh&4wlF)@jNLD%}g+ z?aB&W#RW1>3aj41bQ1RjR}UB06DVyI2-PENJswmV1q8h^!vitxwOQ)PT1>GY!5e-7 zbUf-|+Y(v{jOobp1(rDk5DO4u-W`=Zi0= zyBQeuR?*)-Z&Jac}N* z4v)R*59CW^8#&2kw*xm=4spr~1ZAp{%U>U^wXA3WJu9BYlPvM;5a~|vJ40N{yGu)Z zO>N_;iWCD?xnfhk-Ebn7f_BuPomd<4yw`B&igBSc;f&!7tV zE}TL;swZ;jnxTUSH-H`6A;+PYnlRGl`0Qo57h!Y4NY$F0es-}%Wx@!5!Cei5REJ(i zs;_~9+s%c#y&G&C{u<4- zRkP&74um~$$fsRJD|Zv!$)Ayd@+DwriBhlzP`)-xyz3a4%zM^6P(ca1src_JYM2Yy z&(>6{gkdVrS|k8TW5~rz^>79T=m#d}H$QaZizfVkwul!XPUs@ap*P zn5>oDvx8o#PC0~D^|6LPFvW^iVW+Uxfm9p|5s3rj)WV%(wU%e7r#pKfGUPrRA4%Q? z{97zXj&Wtae37)0^^cq1aVRML&dsmn5(UHmBMK^0SJ3?x&11Hr%gxQC!gc|*xRe#x zuE@P?XZ6b+l1I_|hssigr2P2nEL8sn#Cu-Krh9sR&3RFhW zGG2wyeWfYy6za&%1F3}HK+UL(^hc7u|3%VCs4g5ql-MgH5D6F#!_c4h0>8q!shy22_8>CAWLTPCNTl+#4noN_T}B6>R~g7bmZlk<|+^F01Q;+lO!J2~j&_r-KrYe@b=dv|O{B4zwR+l8kMC zeg6mtY9IcB8pE~!J*c560iy}+9#3$2zZnpdV1crs&xFL-gMOb%1@RCe<%dMY(M7ExRAZ+@dhLO9r*yPHjN(A_ES?r}-kFe;w>DKAI6+7uJMd>6*ixgw)@@w9 z@Yjt3(25MiTSlD5v@JU84jitpMqH#P-!3^j}*^I_2;Qv?p z$%uE6l(jim^6qjlih6k1S$%(Tg6Yce?7jHnwJ%1R731;gwz1r%vK1bv=%H1fdygiz z60oEouyX8`Z$eJ#ioFr7axabP;X<)x$kg}x0&&0}MZ|$4BF;V)ZE_tj7ucp7=9Z_8 zM9+VpbyfX7SMp%8W;LwtXFVT2yroq&)ku+U11=VaPeX&V8SkZs@TR zCwT{}Q>KH{tWQJaw4yNhUwdOv-0g%+S~Pf@H$NsVdBF6wvb?CSd*h zk%>i7p%d+QGuY+WU1iY8vnmN##WdYq^k%mN4Knde2w(SX#ni9e*j>ozOVUcjDb1N?kX@(KCqwtNSNh4`^$(b z#@g3x)aUmgJMcdSKf|vfa2kdlBS_=s(dV5~?@xH%u>R z+?#FI-bV#YUCVn%DIx@^6+%Hs(j$poYy%^2HeT1Oin+sVx^2#GdY#?0{to^!DEY?1 z!)RH(YV^lB8vx*D&i!tkW6GJ{9r#jP3@Kk-_;8r5aw}&9$_L10ij<&*JY%2Xrf;hq zOp;%Ks{fbu-s-+2>Nq8{yXVdL(SIL7DBd;da$_ z3FB6`56&O2S~A=m+?HaUNPBL*n2~$U7Fwafpk<`*`pitQ!kJ1HBwJJ9ht#V(LRnK0; z7sj?Ij^*J{xKrL z5(^MXVj((>gb6<%dBq%H8}@5#s@<94v+rz*{YIn0cgOcsf|pNwtrQD4-#&64tS6qA zyzB6Ya^G)ejLxy8ioH5!IiuP5lJrAR&YX(L<@dbSnO!6`jZthPQ}QlSQD*WB>0n+~ zKjv6jrqS&1P)B_NCqDC@-xM*(Sjw!wKAenA?}nNS>1o?rX7gIR!5y}xSpgTx&ox$W z^Ge%9AhC}wHfIMa5d>Li9eeMUT7%*SuU08wJkG_)?ah<%qt~Hx^MOBo7h7r(O9ID7 z`QH|UT~GUeKN@4`ngaA=MSndl2!a~#qQ87SbP9XgKzDB>C9rR~S8f|NCCKVfB|Y7Za4{YZ7RBB+>g{+9}oyQRWae3?#VH7OFBt ztRr>9MBIdu)vYlUJ4_nUS-G7(w-u0|7g$-7eQ(ae!d;j6c2lc`eH-ryA zj*3Xdbu=zQ$CBf6f?pLlK~C?Ozp&HMGGgy>q-M8SC_`3fA6w2mi3m-2pG7gskKflH z8M@GT97b%Ecf^rnZrba3c$Mk1(0%!b^sTKLpX^9Nn{6F{k}h2dEFsw+xETx(JRWj@ z1VSSrp92wO@3z)oyO-0=|H``P_EtOUnr{F7y+IjFMv`6eItZ*<_3D>-As(zGntWug$CF5C?u`I=hg|7UI z6s{!|QQhY()7R*pAz^KX&D#G+#p@(^j&BpWb!EwjNCJn8xtjZ@s+nKceyb|IG7Y6Y zcmk{6wC);vGvFelk>;;mq!YQo|4}Pl7dR_ntA`;Rx7cde%n%Hs7oToEMyl63d-qeC z1lF6hqp{0$iJ1N^YQeOlnpr_-NeF%-CgzI4{2&+Zm|s-B3lIoA4k0lRi*aYuB<2Tj zDQ~bilrG6$Ey?23CdS?DZSUs?qvcP`q!u6IH^;#EeolR6ylbmWQC>w$t%B_@A9=4= zubtoUqJVxComM6b#h8)WP7R+5ndv;4E%K{6-FftLG5Zb8IT_qY;m*>0tI4oFPDW)1 zEtE;_zZcG8_l*$7D|Z8SBG6Kvy&H5bY>ez-k~3jeM#p9v&%Ns~(|4!pMffu7rAAjH z4NF{>GCh3WH``HRDEMRmb*i;YbQr^8fttnZ>IVEXqgX*PWSeh zV{c4oD^p_CBZ^*S=-}Rf;j?`G%ix-}2K%4e-WFTPB3*5kl(;KeUdHa9;q^JUc42R? zpx~POF>3y$ih6lJA2B^O0$AjlRRGr*@hV)SN%>9Wxxv#l2DHfz4@W9Cd)NAH4j57W zEBYpb3Y3qH+hgaai)w>RTy$JSm`d0x9GmJFYRQcsXI!2n(`mnNTnSV=cW6;1t$}=x zYU~)0d&7SScwW25M*wnw)lS%v>!Z;b{4zM18E+qPPc@IJetS$F-n+JqM?}l^rG6dA z`JS1UDKfo682CO2PuHV|&fa&Hfd942?C#TUK$h#^=g=c~=WKBpZCGvBC><=YDBNq5 zB{Ji~FcKr2J1_3qivTLzl!~7YTZL7NW-v3B*%W)|%+*<6c9XvWnN;%AbL5*!i$^=d zW9eSNDT;anNojvG-)M*Y`@(ko`v&)r0C(qE6^({DwrWiH9R(n;WRI6)X*R4@&Rn@Q zb8RD2X*JNC)iH3RO@;pBvKXYjv)~oaE;VxS{>!RZCr;6H6}?vU^>?3!XM?Eh`!d!cDJwY_ak_(y_iz`1Y(UK8 zHG1c^@+6z9f1&~(l`X;Q+7c_2vkDjz>_dlM@TA|Pks%90jgPIdKmlqDTMN`issh~e z9LGK3AY_@et6PXse~6JAT{Pr#%b!5r3#*n#weGF|ifoNWd{g`Q_GI=THUF2*w=`O^ zT-j0>82vm#@_?DSW<=paE#49sN z4Pu%KxJr>u+mlT$&t;3e$_l@W3onUs?CTCLL?@VYK z!7xEY0*Pi)=}}dBts|qit5uk9NVX=$U53`F55K<^P+;g*9X+dbl6m3za#kt&^V+w1 zI#9A=M>P07ezVN8-_L&`p00EHATF7m6a&bCH_57%>cxEVq3A9}g(gAxrZo=H5#qALhFt%Us&FA+^j9D zUAk~d)yAEJ!T@wC2speCR|YEt>XE%e-%wo|F7SdFNiK#wY=^x6AE1*6cW^wLD^$9K zPW9(uXAUvkJ$Y93zg|DSHTw=zU#1ua!_=f=e}9vVhCm^bj992(yEQWEwt!N^6%^hN zVofaZ6CqeLTv!8c#IfLC5Q8Lrl4@Ly-f8T5>@iF*ml1<(n zi|0184?qX3S)L%KI_?7+`7CN)x6jC2`ut7*TA5#hEW!wK2Z|%K39u9MTduv_uhAuH z;eT6Z2|>ed1qU}=C1BXuqK3=?M;nm-#zS5?=XU(J!CEneyky8ftG^Ho-jgKD4!JTHt`(OW>NIIgFVf-Y*~#?B0q54zd3I9O z3OGp9H&sy)jM4tAepeX>k{)od%+14;ax0_+-Ugy}x{pgE?3U_4tg$W>^K=_MSx%4- zUGNT%fj%=rRC_Vqi?Cz;CI&N;hGIsa_@=rcs)5L1Y@T&JQx170>n!_%?vsEL=wuEn z_-*jhdyKES1N^(d4|^!LLUiNkzG*@bY6n-jxBVuiqq|EdYf@2(^F5j2@?GoTFTPo zArfzQZmJ@cL0Go3_kyW=L4Hd|nnTDFEZ5AU8~kx3sgLZ1UpioUeo8F$2MVwVMPG^= zUB%`KbidbN6hM9>kh8sU<-ww98%G~Sm`*57Od*RcbVmw+!O^f@9`=a|Kz-e0<*o^L zCE~~#qPC}afHv-RytP-_o2`1T5s!tcN z2b;{ehtSc7W2n9d3nPBqX&W-P@6Uq+8GZP@*D_do7Ban{qA>luHkBH$K;}lhVStv} zN1njTY*Qmc+`{dr+`z}nTy}44nc9;`o2^Pqu~$$c_vi(`8X=MvfTgcp{>sdl48r70_XiXw48!7*QFrQNP9@}gU>0gV zZ8qMkd~lY(F6FXvDT@>Un#&31ic$<4u?1XibTNdXlVmhH_mliBLjAdft{oenq-**( zN-0ueS`x>qabUEx4M|a}0OGQ{IiWs04>=rb{LY_( z8k}DlZt@SLu=#FedVCTH!WK%c&yO0PG~Iz> zy6zt&_q=ugBUWj9)t{jhVF@=CFc}IjOQ|y(j;AAqb6(nYLkSXM?PZY2&htB{G_>NEfOPwA` zk8;(Pk?j7m{bwJMdVI8ra8U#rF&~p>#~G1~x6(hhvl;eWzdHd1Lr#_XjYjzPep^a0 zkC=VOM#;6Wtg{p53xrm)VrNo~{3hvxvM?e*r3b>f($?e0#*eHxTQ%nu&FpB?jfhs7)1X@0AGy0&xH#lL^--urbQuT{(EPweR6jC+R{f5zrXFNGt@7Wgd81C!QKHufykVZ7LD?|t^JOuGf&yi^V0D`C@sC# zNwP0Hw7p2lK@kKtBwex6cJ1v=WSirwWQtxcAt$fY%)ryuJRHeawthtE0SJgP)}0H$ zNy0^@3O5*ti%hnWQqoLK`iqX*xqjAVen2KX?IG;qCwyQLFv}6Ly+~d@F!z%yDxf5b z$Y@VrPW-Q9WlwkI74;J@zppw)I=7ku|ETYzmdbuHB+n35^g@FQDydNe{G>-@F4wk@ zzBAsRliE0)R+Ff+0lr`D*J8{HaZM@3kr0BXrGuJpdFj zBUkWb`~M9iaqHRtS^KW)B}RII`4?H>ys27syiUE&eqhLcw3BHbD7egISzJo6&wA}T zDSoNxyPob|;Z1@e_Lp#Vo=_pIuT>B5OE zm;yt@p+^jwz-5n+yp6^$fI>%)1B|BE>5fj|sP5mG7Z}hq!WoFjz-VS7<4xgbNC@>) zJrSZwGC%(ps7o={QYwQ_Pu64Ep0#AzelX~l;*HEZG2se4;bG)NQUa@R%6Xnc=2Bze z*j)#vmAJHM-AFnh^W#fuso9<$e^Ve34VC={)snRjtoZ5>Q6^HhAMug%c)TzBey7}USn(g@L^By;mD+8*O;3+Wf*?2uX=QaO%5gMh5v$r9SLw3 zv1WUWaR`w4al9#K*Y{VMm9ZlqvC5uV?Q*;w|5CL$^dg||Q@okhjVDu|Y!D=(VPOy> zQIBLX)k|!63dFk6``-0+`(Z)&_Nx95310IiTYNP9K)i*dlsKnrU{ImZ;CPD2SW=oS z_JXF}&#$7D=mNtq>hxkRRgE9X{GFxL(>r?DO)uuP`Sj8|7($lBRz)v2sj;@yq3YzN zVrflAQEDs2upohhIX}59k5TTsNOF;J|itsSh%UC_*apauLnEmH#Gb0 z&z_gNq;i41)cvp-QfZi_ggTfW?!iwqO`<~JrNylU5NlQJK1y<9>9zN7=~83;cl}YW z6cw}463#3$rJsBX(N(l|ixy-d=^eLx!+{8t@eBQQMZzeB^(!bX*`HQH)%UX>4`2P0 ziC-tKcKN`3IEC&m+sB@C;=vPEvy?VE`=m5jnB6=&5)r9zj=ImDPdr5U7kk7>PCZ0g ztElq)JbZ;~NO=w$rGG+oxCx_b>{I|UqMAEUUk#}Bi3kjFhkh#VH_BB7_j4Kx+*CWc z&Fp70Ax$mlKFsjuqBmirZa(uBl3Z;(9URQRpr4r1k3_&M6?-=%dXXbHk_1a)3LDu9 zV~_?gfi?jsnz?<)XJ<{g{GRbw!PsCpE{;3C*1z+G0)=baDO$Y!(f!!fe`EP$=(k}5 z%2yu$QvzoT(3X%#d)4=E-~>j%05hm$WSTUb(>I050_cCv>&yh(4V82@<`~fhdSt@= z=nteIUxXgK7INv)nSw{e0FivM0(JTZ3eGBx^o~V&RS&NSq7Z-l9XJt+m`g~@E7kw2 z7H?iBd3rH0f?AOm$j>FbZtxI6)EwwY$$Sfu%J3s0M98rB!AW>j94%OcbEtvzUxAo! z|2qa?$tRam00Fy8W3&9Hex-+qzrltDaBEVCQDGJZ{WTEGczNvN%1ycQxZ)4B^~y=I zlAmlHyY!7OOTH+jT<4fVR{U9yx~ZlD0Sz?+|5!O!8F-R1YJoSYTS+^`Kc-OP3P$0& z@oCk_@seg@if6IRC1j=i1U`MB;w4}eOaFfWmNE{oS}Yl{mpk_XSSEt9iS3cR58qX~ zG&L;mR|BNsQ^gFJY!uP<9fsTDT2E8cT|RXuqD|DjkW6T#UPFFRDwn1H5b^1a&a|E9 zo#2}I3gZFU#T=bK{SXv`nsw4DJTfn1ELyz_$1UKvmr#88S)ebzM(W;Hx`SK6I387& zkJ_?*K7`wvR;PGb1+2WY^jRT=yv9Ss#Sv(axhEX%b4J+BO2U&&`^^xN5_4roFbSi* zvl{UtSP2L ziRECd&ZotQ!d7(^J)u-w>qrm8_V52(zclV3s0mJwy&Q|JFE271B7bju>(P4bvR+=y zU&Dd~qHe^@aEh|eXC9lfp3~n;*&mE@AgZq@v^=!|S`fZWyLjE!>>w88pUHJhSNzP+ z#w}#Y+BxTc=!Y-9*w2ZBb7hdx@!0r#V6Wd`K-KgjCn$8H$fR}OcnNie6;N)ZVoTkY zQm%h`fj+@W_w-?qsyTO2PN0F zw(zVYF)ahWmqI-d!^K$wHwL6<#RS#H&r6-&Ugv!b7Z^Suij}$V)KkQ?EzUo!X7ZG) zdJ7g)_?&&<52gBiWLC$cXhp-wn8JrWY1?%nFW+iGd%FVL%}`Dch*j9Dz7Nl-SRh0C z0J39s0XT#)Obq#c6z{ga^RQhTbN+rEa0&Sp@NbG`9RHX2X)P{(THC0De;ItpWJV3& z9JK%8IUpTV0}BY_~^>X!R(D$ncGYTl~Q#z34eiqMAx1em)N z+zIv=iR1D+pWv~s6w|pDSH&^8s1WKosyK3&{Qx%O_3>6F?ym<=nK2_-Xe>o~5SUx7 zBHu866~6mV*NSxu5K@+6;G`T+I^^ZCneuBP3@D(DYesw^Ud6=XkOpX&0TX1qGPcG` zWE1h!&dX-T?&$<`=+S~m9aOxath-Y+zoeX66~qp;Mm`4qLly-=0CeoIrzNgj2&ZCo ziZqE|$Hf#DM|1M3CRLZ>vjLplM6lk z$CK06zo_!PO+q-zf}jE&qQcr(4n1?kZ{wgd4Db5dhC>JY5n4nq>q256NRXa8;cM9K zwN7x7{TjsB<6V&)e4BxFG#OpxnBF0wg+x>xs1gvEBctiRtP<|iqC}Xz452zv{>3V#oYh-Ef%rPn$3!CuN-7sc z8X@lZZ8=YE$#JeDs!3D=HLg@cXA+g{q_U&9Y=8gQGv^a3*&?#X=gTTw*W+c-*FgpC z%lI8sEjK2nFpi5{_C@2iSWKZl4XlSo%V|IALwhHTKAd?S?8!(`uQeh}A+IXEwJ?XndU z=yd#CXpKXKN zcdtG*Afkhi0|ddr8(s7;OhMm*&|8)C4#^Ou0-^!1nlD!m$5BT zlp+Hd<9V0)mXZ#QPPg9X6(eR90y1iyb>_}K4;}O7XZ2|-aYQ`;G|PHSNe|rgZ>doR zW>r1#PP7pB!RKlj#TaSH`>2MY87|AC6fPlNwebi7CWD)52D9D>*!5*}B`5E~BUs=9 zUs94>jxj9s;C=dSX|?gL!xwRPKk}`5%P@u-5#dpN97vDP_eeo){b+1HaIOwQnu^}w zG)|pzw0`n$&X?sHMM7)4S0jS@QbGEm#Lm!;6$~CHPgV%PG$Fv5&d{JGh0cX&B3K_W ziM$}fnpx>jSU>VJX9P(^5^@q9sV_=IPEMyxh0VX}0)f(Vsam&Pc8O#TxQ0@z3KX{b zPsjtwJnJ+HO}c{qQ~$RR!o3+j84cgRuaSq>zbZ9dg_xvaAF`%`1`QUS?vQs4lCk7G zT?MV@y-5K^tVBB_PdW-AJM(t;P41T&`4;5cU9Qm&c0l1ySz~r(e96g6bogi$rgz#l z2e?042O(>TXd}Rqp;HQ_!FbZD2A#;jw2&s~+ijd>Lo`|fDVp)TNz-wBNvz*)%9^x# zyD<0kOnw8U4$*)D){PvUI~ccnDP{sTb`tB?m>7YEgu^!>f3k0d+MoRmNQ%|G2~hzM z;m7Htv);PO6AV)WFv64*7`gzsw^}#}x^J#n>90Ex8+OR_Ik+gZQK6q0-~Y>}&G%*M zx!4UmZ_E*iPUc=0N^ecmsHYQbFh+b^l4u3;#AB_Y;hOo(o!AvdNhW4h2*Zr_ldsI0 zMuZXpq)E`Z_95~@e@WE}?ZnXyEXR_{WNh$fm~mgVq}${aZm>R5BiDJ^KH%*E@FmmC zEp{(cX3>QFszE>J421NQ-ZOhMVWrqZMD>o78AUpQ7jz2kc2Q?jXeKV88`=GL7Wyr) zTvRV>(K?)eM(XPleoc^HOx zVpI*MK&*3!^FK`EJ|V91*syXc#654+G9_eiB>Jt1F)?Zx=(oCKUF%dCU1eQrv6l9* zio`QP=uWPA1xFciMaz2v*97#e7L7qWS_cYrpo1lvuS1x=La}M#z{R)w;dHU5K!kDM z*j$o%EM}ku&0ozz3wfH-K5YRQ$;)q41qnIJ zDtJjA9Ju)HGO-)Vr(rQBA3iK>{H5l7Rha1yMaPk`-2yYH(WYVz$R-=c$Li>pS^Mo( z)lPosLvm2-3%}WV*WhZ_86e?k`A%JJJjZgO$|^C1BWLdyS$(AtsGzaBqWxSu@;kbF z5->gZfa$i4&sD%~^MBe>9^JHj`>An0EBf9^x$JGV2c7=~Bo?m)mse1OYM%nkt~TJK z*!{~7O+@+iyiVagv~=9gb%g*?U$Gq41B!1uby^tTg~BdPKf_3 z396Po%(tI>*y}>*qjs}1{iM$eTy4H&4@sfq^srdAKM*Ypzh;veZ=jG#s;!ml1+bDO z#g>?s&c3R_=;Sid%sZmj?+a%Ra(4E^fQJ8E4-t)@aAQUgtQNqJMFLEbuSl*98p#V= zx{`nZgi#y|FXB*qJtUsJqlL{E&@ddCs?KCWVl|8N^PzEBNzjq&W5mQpbt@JRI&QeS z^OEg|^dfiEAFvgayVo$Blbs#$FSbv-PMClC)dVsq__rhP=YQ_VTWQ8QdRBh|%sD{O zS9L!Q!Eo{C|Jx8&qGtWWJv2ovQ+j96=l3~hM^%`@9rh_~l^lLPbn$t?6J*PEQj(`D zevlj%C|0o{#;o3hato;x{4NpWN9*Fj@UaI%`SZc0{lRRW(;Q4UpbVUt1QeR7G~Kd# z(kI=|8-oD0VEy}&j0nTWk=YfgjNi*5(LlP62aP~Tz z;VM2MTX28X26vI>7SUm#+Bht^HecScT12B0N#Zb^=8f^G`d~ewR#CJ06!0RiDz_8! zm9HVS_QAGS)Kq>v` zonL6WqDF()uH@U4F8>uTd%hbcN)Wtnw>-u^a6eF&kb`ZaZNeSu4tuU=#{b|s?;ZbD z{1nMbvQxr_dfVi?Ioy;L3Y?nFk{2UU>Ht0O*yhxW5{-Mn4+sZ6k=pC-!Nsx~rTwWv zJ(;yjdb^!{P5Z(gZHe|DhFD65o50$H!0P(F*o5k=N8`+-&qBmJ$@r$9L^+oFkdDr) zv2=W}B?i37wacRqQ0$`aN&;L(&gRYBBRiiB9R{Oi89z42VIr4Ny=z25*q}z;ie^Wr z=N$tmREwpLm?K9i(;tkXh9~)zPy0els9V~%{N6*BGR#>#lbzkv|AJGzPeKq>_Wa-? zm}}4fFIRcdHu9dEjm%k0WanG>u(WF^ACX#>jKBk#$LqK>?z(#(H#=@!>BAJQA1N%w z=AkPWn#3o4TOI&AY{k?`Vn_5AL8d-_H&SyN+K1#wbE~RX!V>SlVve^|$P0o>YE>n zjRSCIA8wNI)cSv#@iqFPnZ&b$7ZdY~Q_-9xeu*$tmtG|jq0)1urUf{6fawZhiVUl8 zGH*u6Av2+?tYZ%c^>|d5`WhUbpicT7Fa(K=6RGoV8Ngig$owqA3j+ZzGXSV)5s=k2 z;AnOgaEv%*qvumI$j=w!2RUHxlFQOsUYgt#pxSuc`TCd~sFi^wRJ;Y&K4MBKx-;o6 z0?)AoT2Xlz*30QB&{R}tktsBo12hqGJZvG`okXT-456l1d`CPW^Dd8s8j=rH2CEOP zE%hl;oiAODApY!j^>Mv#!Qx5Dz0>Ul_7@?=lt8aqv3m}|Ws3~GEGU_nXN9NVLxiO$ zJZ=y#%bWC^)ED2xqwZ_EKbdp_I)GVCzyd_SB5e76)L!Gi_=i0#KfIo55#ICIkJF|dO|G{sFuKPST;?&pQexd1-qK{~E3H_pjRJL2_L_}&YMhWY> z;Yd!<|EK#SbPuv>nS3bjJ#@)Eg(M*}Q~J8~{GEfPJ!Qw}7acg@^320Kt*}T7DOF^Y za-~4evYSaC!Ecx>e~@na@+Eh^Q-PowO>l&6xpr)3a-j6(W-I_(CFMw3KyTtz56g2d zv=ILwsi|_QzGPCOL}FRxevdEriix@gK=9rUH1B_4q2BjY^Lr zL$*=%vYZF@zYwTySu@?gQrwmw2E@waf;HZBDggRznN=iI%KwcacM+%4^Y`!>{~5oN z*gSbc?_qwGc7Q8oevv#^{AJl9x_AHVyrMSjbunoz6=`YnBO zd!tU}g8#esE&#GV_g~VCuc(#V^8Vm zK>mqQwG9B8iF&Wy{B+gY<|+pbR(Z8cqMvC-($$0EMJNzYd;BEyVpB<18M%eG{R16K zkxq~E4QZO4ZNFkJnUCTrp!3GAl)ikKf3_B{u)}fK=-YDG!{Up7FF6t0mNcdR6NrO< zO%FdDicZcG6@1pTGdFK(2jtN_MCkGYJgzR=8&zHGdy*!*cBLQH;YTKaSA6)`0L*>2 zg0swW6`i~wkacEays zR*w-qm@hDrCWPP}9M4qY6N!se;t!0)J&bXD_t?=dJI9dbiPS6`@cv-=@czE|57A4( z&9SNFqzIP{oWf-TL%%a<6H;PXA0Lb{nC*o+*(Q?*e`HPtI?yh@J`iih1@pxjZ%p;a z;z)k%ywEk{OQVeuyxZQnuV0{xfB{*E5~bW&lapY=V;-BA*SosF{m3`zFpQt3LFzTL znbPp3r^YY~4pnPC70a~WO+G0n4!{BFq#CAk3(Hv%_t3?lf{|`~>yp5DL;w%Z>s;+) zg(`Vm5W~G;*)@9F1g5UO(M|P%!6^>i9*fs(dXi6p%)72}KIcR6`>zYI{PIhbkT+7C?viAv|3c*WGn%S9!E`5FWe*r zJPx5C94o$?B$T7kmy9-{O%jTK0WZ`+rP$;+P3YIdxdvsCe5}d#YPL|VLp_*0p_+je zL3LoGf5hopijc0F!E~a~NP7~6lmIA5|Ge($W(W?g0Kkt3JGn(BqkHtvtMeQJUf-yd za(U4l1c_bg!HPOlsD%e7VZYYc4R-XtFTSe`wG3c4uLrsyJHec)OL$1E8M}S`CGdak z_zV2nd+&Y~6D`bt4Z2kk)Tkl)iUZtTUb=toU7Z;s`Xl5)>1Fl92_yhTDZNtJ3?aPw zx^+*mN<8(;FSwTB`89YF8gHH)z9@vV2InKc(2&Q@_oR|UXUyVt_SN~@tgUtSox7Ge z*ewjL0d;FQuC7>^n^M|H)z>wEYVG_JG1OxLqt(+lvhs*DYTC?w2jZH)O!E$GOV9d7 z;W7(`NzizkMSWakw=_vmF>#W0w3tKys`bf*#JMm|=uf2ZEiRi!Sw&7Q-$ZAg3+3P! z72ke$WbgKdG#QrQ|(` z*z4)OSrKM-$LuJ`9b{<-Mw=QKM>Ft2D(3q=k{%C1#w0A`4fBo$^86_`zV>0c=qEVF zyJO_RWSTP_q8^NkDyhGnnRM5N8S~2Bz2$@Wd!hqYEe$!X^RfxV$mD?+nMl0e7df|2 z`e`SZ$W~c{kb%rL+}*^N+0d*}W>6t`;_CpYc;zbXXr&%z1UW6GPLwbRm^m;r5I-y# zcQjH{FWb}ok=kXw&mUA8VR6+-SL3F+pCWDDsUwDF9bwQ~&` z4#mLWZb0;=-H)f5a^so<3_AG z@ftGyXN~Xc5q)3~FbyRV!Yqw;_MpX0FQnZ~>b~^U(mBCDrh91SV;LB=O>M5TJzo4x zWD}NLad=>+UDqj$#3szLBkQ?bbef4HFom>W;fOv@AVi{;S?jZq{`}`2t_Hi(^EhUw zvLSBvy^VtJuYNlCO>nfnsrtKD(QUN?i_zZ0 z`cq-UlCY$5&UI7u1DZZ0i=)MQX1HivA-qvomA$jiBqCW$v1Ybk{5LYqQ7U5!<8;>S z>=3D_i#u$o^4d3C_49Cy*?OJwMprfyos9yyf@@=T#OdIYcY}FCM^t!u8s)ld12D1z zHs~0Ek#gfkRB{F?=%xc>isY3Ei%wx)$ICW1JI`l@=Z3RhvXvavzUatVY^gKrCVg}~ za4&TXJDceRj7SqOs)RrMdD$XOdZ)_DXija1z=qumNvVj&h~au9e;qq}U2fJ^IB7&j z2>X>}f>^KgrM0wAm=r=-FmfL z5SjKw*DS%V~jCFRB|)qLpUFVr}oKCbbMJrmCc|`u*fun{9R3mN{`OW3dd9*x#0*O2G|sw3*OCzSneXq^SVeKl8_zAowL0708rJQ^C03TIzM@y(=+&!;a81g0_BUk!G{iz#%~sL&=lY?IJUFX7}JP)+9{*6w20>o)r#v@$CG_bk*C z0ROoC#_Ytf_A_ejzCDy!p|`8t3#8}lUQ?D+!CAxX>*Ie#c8Q6x=PS%fOkxCGqmQ?2 z?eCey>@51tBGn-@T)=B0tN_%>mvC0gp(oxcH^yETlVm&`8t>z>T%pk(b03Nts|>|u%1y7@xplYWQgYsw|Id^h z>8$WUU7bf*M+Kk^DNuSm>OReEx&MzC27IJgYP{d2^PhkE6k6`?)>lT`{E!SOdGQ`D z)Ze%3rFUiUi}S~k9|t(z>f}n z_4S2{0?L1w!L?N5S!5u=6n-gV*lBj1O3f-u23(y1p;3PI8QP6($40c~8ro#}T&3wye`xW;H9_8_G2NHXhf@%#6<*d^B=nERg|tOc7`ySN3?)pd@7xC+cI`ljC4^AWSt?k zfd6kjf)aWRksYR7rxe+z+9H1iOqTrf5yBK8pb{TbXpqKsT2M$3*)hp38J?dOhEPivQgAW#Z|hTbs;hUOSN;FbaCd z2OTf13M*a%Uauf%J|b0)1Ag!2$OwoLn3}1u*OSfTc7#6JaJpp)8JzV_*B~#R`ko+s zs=aEs!OZMa)`G&kNQgeV+QS621Wf?R4oxH2Jz+6x5~fKsqsO#317{U#<)R}y+R%+e zfd77IU7FoPzwz)XOt+shP~Z9oz(% z$K&1uGJtdZM)J1d;PoV|wV5yt`?p+|)Tp@cFg~fdQd@_zgi+N{7V{eSg85OGuIc9w z=Z;PeC>vHYoZWalIB$=vSQPFogp+Ri&V;W!&brTgL{f)@L|Z|%RJ4N zXp_-VTEK7}8vYV%zhcs)Yd2vHLLdOktLl%J8f&{LyS?BYDmM*huRnU2# z29N93`wdj{ujSn2y#G2>)0h}AXW+iF;{ON~ICQxQVgLj;kR3(Q9hNzppkphhigs?A z@?Tb1Vh$lUF!qG-<*cyE)Kg7;_2t$nf@QA z3_Iq$IDR=Y_;L%WiO^Vr>o8%Y0>h(Gqo2{_#|l}pgc}xaM_+aIzMoEnFEM8N{ALE; z@+OMjNsRVh+$wq_XW)%c(Md5GF4DeF_RRh5?LRihJv{D8j&a0z@$HKz4_|_@Zd69U zn-+cHY~-N&R~hup@M}cC=&Y(TYh)y%AfGoXUZPOVsgk3x*42*dPZkFr11E(DM5^E_ zpI}`*ZD-G9th{Edv_Jifiqr~K1gXm(;i{Lv@tzvpwX*_%t-@*Pm6)*H64P(Rvt|vz z_iug-26r%Xjk(|Mte$u?%vz$;#&< zxKiNWr)Zg_(uwiTcm4%#hC4(kQ)Tu&_sk;ChgjT#1IQhAMV_R9PFmor9#Yx5pCXrO zWJ3KHqb!6jo$c7`cdo|oT}x`L{J_wbXK_qK$3EozR6rx{LxGVSRWsTmWa<~NraCth z%XH#CiDmblecc0wAF00CQZM}3#1{9#82V9gva^~RYteg)A|IG2e)xX~d#k88n{HdU zY1}PHa1HJR3r=vC#+~5q4vmvQkl+r%8+QvHAUJ^_!QCx1u7Br!zkT-DW9dnIFfw-B$)rAdPVfF!JBuHWUlf7ixVc0C6fy?x7AymDn=G9oW zp#W1NiQ4<1rSv`VdcSbX6+}`!_hNn|dvCd4heZKhnq^8&xaqI{8G##$E}fOpYrqE< zO5{H{o8}f>IFLWrCMT;%_8of0>D+DC`pu^C8oqtU@J|;NE#>|`9a%9KksWp&go$G( zP2%p-ZYPnMiDDUiY${L|Psf%>kD|9BYDb02+i(cOyh#&LpiB^04GazG*&Qq`R`U4Y z2aH2n8`gR%nhr)|r1x_Eqm?*s%s+Tkpq`rF(=FGtHrRa%*yZYyJwlxj8tT|cP^whvlt<3kK0s8rSGMbaey;n;?HLW zqRTw8=`~rAxOpFh83czW2bTDPg0*t%6FNo%JFgpJU^4zD8PQu;++FCkp-G=G=G!q& zFy;`nZH(|kxGy}cJP@`XO5{9KiCED_HhabizeBP2FrA~M9Xkv2_{Tl;eLCfxI}bXh z6$eRNK$+OXPL0fJ2}yR)f3;Fb|7$Cih5ncPk6Ege|7ZRTRtKbWAneyd(X>l5?hpme z71YK>#&JAO$!L3~z~^i=shA3$5b7bN-J-Ou5=XHN<6ES|5Mi7%SM;$P@nQ+;$=_>FxZBK0O#Gb=F2#9do%hdE{lzTNJ)98N-=LK#g*FIG z4>m~(e0I)gc@YRU;d9o#y}Q4A=;SzSx@pEf3J5Yj&NuccbGXNGxrTY^q8r3*0Brjc z-%HmMphDTQakf#QT9w?d&AQ+^sxxqRPl8NE%!0VGc6An#ID;RMZR{1p?=VtF&dv=? zn=X=mA^6k$Ro#hnqLt~B={*m&$8TGRQF)1(5pC&|Fyf#pwk@4<1jH3}|x4ATK$i}dmkLN4|E_#Uxe z$f4zS;XeZy>BNTAk86xR4pPu0Y+V;Ii$&oy>ypk9-O%Y!D;i)NYN4P;vw05wlgSw4 z7QqTSRNCsDhY1)y+x=lZ>;nZNJ=8j$-gPIY@TDWiMtoj@4J}cFS-}Fzwpf1=t$6JN zFA#;oq7Dqqf$3r;sGOm$6ZH==?t4&yU1*o*vq92}2*X8+td-w&=jjro%*__jQ8&5t zI|kb@n6kmGi20Srk{9Z*7Gu24kXgU0O#09!sg&53SXeEXb2tCJLcMUa4du1rhc+Z4 z7`tV(5X2(b_f1c>7zGNDu+{yW0N)-Lx2ybD-6f6?r~L*w7lIE9-oMC1D$ebEo`{~T zTzDc@TV?A4EqM`1mOjFWPY@xggo!j#`-hnb3fBGYRxRaZF$=)Yck=?Fr=%Z(J-W

U3Wp@(=H?VD41NZCQ>hjA>+p zfYslOvRi-l3bc9fsdetu!{5|#{wRaV+jyhsT4~p0z~an0KdI*<8>+_P=Oq46MUqIO zSKbO>CDkGp!wc-G66%hf5q7}~jp%=uG`A3tp5wM6+;IeaxnNj1?XH0ydyygBVnf;`$| z(~XqIIKaT~6TvT4*=QrM` zp9Nw=OoaS29{=V_Q3oE;{_)R~3K_lsZ!vkjiDX$^aYWeJr+ZKiPC7C|?jMec{-U?C z|F#%^Yqma=^VxReKX9+ZkUpZJ+Me#u^x3u`Sd)JkO$qN=z$H$jg*md-tu2mtr3-mg zx}RRUpKEztWG-mYOilhy_vW4G>khFD$b+GV?@_eX81qLYGP&nE7`ipaL5R_u9?nt;^LTI{`BS?C!|$l8dBQ6)`I=GoF!}=IEHx@S^l*0ZRnuQ+ z9V8OA|J$hBqUIx~*a(B1BPwhjDnFK}PH^;p7j}49|FvL*(EvZ0q-Ou)yAhGi4)er; z;a~?&cI^KrBLXgQ&gTqooTFj|cCx*`M^AJ}z0VA}@1^G0PjY$z>^WMdtjaI!84{qw z^i9N1OMWXBiCcelwb)^R97GKW36h`@4->*goJ@_^jueJUp?e)h)N*SrgW1T9bQGya zMI3!Xft zeVfy-&IzuyYDSLv6r-}IAE~L_{&|1GhZp6xTZBgIGt#83BfT^J9oHw1okZf3)!6E z?!m8mye%1M(}n3d+zpLC-A8I(6UW-EjXO`aj0(M^MWZ3{PTJ)C4C+^GVvkjd`hZw! z029s-;ARS7AA;m-v8ltS8r6w@HJ=Pn)sIrY!d0^2U>~}6vF=azY>Zgoc3$SZX0Nwj zNZy72+u3c2+NTyu^(=Wi)Kzwi6_QrzGo!<;KAb+}wZvFSJTjW|!fIDZ=mEAAEk}Qu+5iqRnk|_0*w2 zp*Pzbg8Dh#diI0w|5|FuVtt3TdtSHyn~xRauZn7Hp}oedzsd6OpE~b<=xP4m=+8J* z1!yniiiX6Sl9P3AtoP|9T*#}V);~%ll_bKmxN2i5EN=SniiW)t^oEdRbH3m(!iO9y zcWOL!)9KineT(hTIpOP^sao35IMK8Bf`0006d=kw^?J=k+fEI`CGk;l<&6&=%1D6z8va3H~aK@3>X58~Y?;kJg|1(}@YK z0~G;wIx#Fcrp*iqQ*mO|`|-!okuYAm@iPnM${>7LQrDAH|BvCu zQz#0#^gQ8T`>FsPtr{U}PL96=864jF(0}-VzdK}x@cjHj`V?OZ-Sdm;blm{f{n&bi zUpuLZNL6ZTA#hO^)bk#hGMYUwiOe&wKz-mf4sq0CsL7>hKd0vb8P$4@^FLTPOnz3g zxjsqUlqvOktv5nn05>!Rd+oSxQxuL=6x=Jlgz>T{P?}}7ke-jZXRt3htc$n#ngnN} z1?Ep+!z8!}){IkWm&Sojd6IR4pJ*HJUQy~)6GIE3m@&~jL(_}o)|+?Snf?j{f1PNA zxhZ4a^c5vh zH}2`z?w5U-eXlom4*n5_V5yGr4vWwD7ea`r2cIFz6|!u zJbjPVwq#d4!>AR`u(Coy_8(!%{r^pLFBRSMRWlOu_lW`+DTgL#r1chdARvq6vjP9q zj7-2sb^Vc15ngbyY4MN)F~Z3GpX4~i3hvu(FC<2nMjKXanulWF&~ME zUj>n-SQ&+T{++i#j3O#g z1RPEvul=C=w2`Ta0* z+|g=7kjKqq9{?T>9_L%pk77n4cIs-Iu+1hqjqG4klULLOn4SB2@g2CN_IHaU;QF(! z<15kQnXW1++)w@|x&!(b4ZJw>*G78FO-hol%rZ;lUeif`L_o#hTrm>6hKhiGYp%@9 zK*Nl}PIHy=OCj71^$-=1Rj$yU!CIjLge6ernjdc0Vo03%9obR&%pTsMTn)U^TiH&m z`{hkSvphj@ndlyH3KU8Zcp7xshJ9NwiTMsWkKX`qT~I%(iseG5!35Rtfg`wU4f0Uk zLgb%9;`j8hw&LyoP}NG*bGVY%Rl8PlZNxF!P~i`X>^!XNJS1xr9;U&r6)Kwwc*;lt zBkPXCd;o}c>DOTkjBs&|MhX8kHQJ}RZmgJJy>Qsh9{6^Tl>=7^w3x6Vo6qztZeYyX zRGo(*?sIiPI3W!BN#wzWl-r@q7fllswv0^Sw9U~@917V^O6!5Sj*9uu>Z;E3^Wo#+ zG{8zxi#P0JCgEv4;R>d;-GW1#oPgt#cO_q9pA-<@dw<3f8aqTXY}=Aooe{J#@Y5em zC3AHkR1vI#%eMP2p9{#uTMFs-UDrcJEuq3xhy5lv7dS zZBp>&`~rni2{xA1{YiN$7<{s;7-WfeKybK(Um@LX*r6GyfJas1o z950;Q*>`_^Z%DHG)rpJ_T(KIOsCB=*weQ}G^ck40)86O2GN29R}MD>GqM z!W~kT{9Q(^Qg3^w>z+N>MGvOcAz1f*_tmEyLwDe-p)7XcS28yyaR^S&yVrxpvpu+J z`8BwjoX)s&ZA%C^97q*dVi8FpPgK%T1_j@gzwMzbeio*7$X7(ez9necNaJA&(TLX< zgTz4BA%$NW#U>b0ebiJ#t6c&@K8C z9N=@xlP5y>C2KdP@`&~rnwI;TDHYo|(DuWf8M+RIzxliozw@^?``<%M(V;dLzn<6p z{qDw6A4x4#6rqly$~VKX{St?570U~m59)s}rD6oa-98#&(}Ab74Yp)1*>t7_+)E~M zlko!=#m7uxi3L50c+5tOM5|9K_Mza-tgUaUW=W8^HJ#}kOX z-!-!bycEPaj}T=@AgBZ(W%38IkJ0{K1?cSncKQFvu-CCaECEldSBdr<`w;vBoy5Ec z=KtWOW1{~0gWF8kpcCDbHGc%B`x(Z{-d$%kEjKcB{TctPj_K?)of+Vy-0wzt-C=6o+H2DN z*q~pda!r9Dc5O+nO<`aKC=aj0d)u}@Ql2m=YnJ-MEINdNhp>Vwevy-gHT+H9V=Z1DL!Q@@gm*RMkVr*Pqg zHVsGV6zz^uLpsL{fU4vjbr{!7UVXHf zb(P?}4w3TVcB)I)>ejE{1gZH&YjZw=h4J&(QXor!+C_x!jiqSH6R#OjlUcc#eD= zxCMPwjuUwe{(!@c&DX>HGoAm#Sg&ikx=hL@11+{pwpAheGbCiqiDk|F^-AAHEAcMs zs~w_EnIPu$QOfST!0iEAOit=o8P7FoJt6e#zbmj(+Wp?6hbOtp~?R1m}9f$u3cBS>ShDh zxv|&Cerk%y?QTcFUIV-z3il+Q7&bo|s`m0$wZN4w?d8~I@zJ&(=%y)P9Vq790gD+_ zJBT?Jilu~XP9}2YM0=LqN@&};B64UZAbA$*VruT zi%M395TLt9OV}#5{7Wt$t$A8rm1Z4M?OnT5+;{5vihuOLxj)X*i=Uv@yqW(}u(Mlk ze&!+JQgGG$LhD{%IuHFeLH!t1KN_*`9%B%W?tq7u>0Oj7+7c_;@=;W0Vcn1QLRRPx zc85)#P6dKb+?<^V*g3$Qo~nDgb;kG@B(!?8_Hx>aO?r%2>vvPi5&M*`mpeSq&)sW$ zweSNiiczWl2Ye+KHGlW$u>vfRtZr8CS)h35|5@oMsE>~vgD;!g>|?Qlo8~==X%}AX zd-bs*$>{dd3$N!mPt(h zYN$TfkT8X@d9i4OZ&K9>+7-BulJ@RwVEUqg;vWDI(FCvvxPWW+gJ}JGHnPK>nGA({ zq&Ov*58)X6lR=X_f|)0heD2D#S<-}2)e;A&r^~~2_MPjq+61q9coArYit9WD3<8>< zQsJ#T;6&?eaI<1$#z`m(Qkj;FWMvx=z5h7UVOoZek7uI@HIOuxGwvfJoX#I5dtR42HE=X&*h*Zzy=bza#-SfQX0ga=>FzP_*Nq?C&o9`d5u0HQi+^Vchk0YQ} z`yh+`jcRl`8gSHLa7Y?Pz-TkE8_nk%pdBZ6&n7}^Ap)p?td~6n)sTL@N)StdD!Qw0 z?hzL@L#{iyz^_^;{K?kHb73f?WEw-8`8wW})+L}vS4jH*N*fmo{WMVJ{#fNtSnj^^(89m zv5RxHMrs#M3-F0aUXQ|;@!nuKT;$~qEHPcp`$I}$V*5D~F4vz8X2%b>) zleI;l_GCdw<)`Of6<_p~%B7;SKZe%Rvo2GXX!QNiZb{#)I9Ids#BRxj9`~KsOv39r zULx;IIIHlemY?FAk!$a3B-bJMF|-ty_o}^=8|iZE0*5X`+jIA$!se)2z1l*TMS$u- z>PRb|R_k?$WT{j}(sda>lK&9X+)wtH(ZaDlghM0wCKwjvk7*)sQHOdaB@Zwl@ZAq8 zylg`VoS?6n6qF~6+PQW7uqO)ry+zep-Zgkr{ZyF>6$?JdNIVHTKUAb0#WrZYSitW% zZTH<8HyRaHk$Cl zL8Ea*&4u@|JldZ_>6cAzuigpMu;+U2BTa@!DE+?=?w--bY|I`zXpG^ zMR^fgyJZex>f(<5j!i`7L%beQ-5T?%LB+;#+U-X^5Ui!fh|J9BUE1O}K%s-RWGTox zGf}*_fX*MjA;iIj_ThaTa!plOa`p2{FlM9`xc;vo+3t zBS!40yC|G!OOf!$L=67J>nR0R5URT@MuI}65O3GG8i@PkyQ`#8NSRHH@IF3bI`e{; z4F@91*R~Ssq0kg~k^<45l5g@^MqaO?ArhT{(y%&ZSDO2grI6BgK&ievqA7&_9Ch9f zO1NIezNC&CeTdN0-dZLo&IrLzU1wUT_Na=?(rC*jS*(83&{8*b>bWH~-lyjW`z`C?r7d9z?huWC=hXtJ3O78bViUM>5XGY+% zbHVsUQ7IlRC;CMixn5aBJHPO|ni}2W(&u42QbSO^PPtfZ+LdCiGuP*Kp`JCiu{Sui zV}RJ}>>2N}c^rR`3?%@E5)DI@(g+70xRI@DHw;}voF)cD2k(D344Tg|Xfrg~vtz&E z_7;BV#Ovn%gO~99y}!*%i{Y>rBujs=6-b6%DRlo3KXFxmDdxTH^yTan(=+AK&5m9T z(U6>eK_Sq8*|zKT3O+ccp=r3;N|#wEp;~3+>uNz`LQsh}O{ds_)GK&x%zbzDx$!Sf zAE<%IS8Xjn_jVNntHVV389S~hmrw@1- z*LRx6U%TBMEQ{oC*}X;3ao*fgHhCUqqoi^9MsroA+=d|BV>jZQFBZrF2U{Wj^l$^C z#{SQfr?VSRuBCE7&u_vx!njZ3G%j)PG7PpG`_cCUKDr5IB`uL{b0(%!%f-^ULA)E0 z0W8_;qweZK&^H*A0u7T}+RMz{xnY^hq^>3Vf?3C!q;zydrBLgUbT_ivk&!WSwROeD(%AcBdv^f7 zWL@@bG#N7iE7t@BqR-F+g5;_SiD zxlfmaM+bZFR?eY|3iWv5DTvKR+J9n%=)}c7p8KF4(iO&2XaICX=;gso_n3O-He0#! zQRW@L0{p-%Tx#ltH|taCmDSgjN+n6CLB-#Bm^iX?Ev{7Sv_DG386NQi#WZXJ!&P(z zcC-fn(#pg8TmqL}X z$r4tB`s$}CXsnUeQKm(@dorbc&`f*kqSb`D&7SaPKWp<^0K-dV+{}!0M!&;^2eOsd zH+TLgk>6y=#bUrl@1*0KM&;HWMg4mZXIRu5*v8J+u@l$OW6^38Dt<4W~4@-I5(O zi9FBN#5akoEqKVSKWY~Kt~hM}q`nSmj~gDnl`(RRx8skM#I7cxr!w(k{o6kdT{J($ zT*C?h0hNE-M;!eP=K05E*LJ7iswC==yqAHzc`urXL}XctIr7&o+tdAXVU|2Rv=EH4 zjM(CcC*%?wjIvhjy8KK{4wRM)RbCQ3QU+s%-x+s|!Q}9h<zd3v zgMcC;0mVuXPz!hW?0(ZcU?rrSPKn4$=?LT8%?h>5%%U?Djb#@M;=Z)6bbfV<&GWeA z6q-a#%rdskqDfK92)_se6cRw07RUT*Ab7PDK{&A8M|tE*)Z3EplGh)H6)->o*xQn| z*PzvquusDS;@`rvBFNRD=t|l^VN^0nBe-tfHLVaWU@YTt25}k2`+>(KRN{eU+R|)z z)nQcXsb4gf-VIfziRWOT0;*CIN@2XXI@G)hC`f-kEZoBuQie8#PV^HCs)CyB#_?zl zSo9#Zt56ycb+Y#_zh5#SBN6&>*lggv&eqa7#&zn|d?b1={s^?Zmp?Yc{ylJAQRCe` zuOXrtdD_$@1>WGPNANJ)JaFeq zckTo0(#Reucnz`7ZgoU8!gY0{;9dNL7uw*@iI4WZ#Ye~8hkCNePD5JGtLQEuBdnY9 zF5KtsAsRUB%0nUxSXtkSZ8vuM%TvLie;R7puL32ig6eJAY>Y@ zsoKpEU5!2=su}Z#0T7}^SXd|@pEU7CWHe-lExch-YHeoB=8kObjnpB-m}1^Hj$8jG z$DyC7=&#^6tY&b}?q>_y!KRQF%I?&;&}t}x-M ziY5zdfjU1GN`?}l;&n-t+$ zcD38Z|BMpMedOdHCvNe4H?c?@T2CJ<;@$uFiW<1Kb4!#;a+fpSlkFMIWKHfHml;rI zeFxY(uAtEQ+k|cT?F|L4)O7xSTiVmQBm%k49A6jOvI+7kzMHS8)ok0cJDzKY*RE4W zisPkE$=IRT5{`V3FinX#ObVe43mOq6wgY92hZcO2X^e-fkfaiZV8hT8tz>*=W$`$Q zGGd35%jKwW z7s`YxTV6F|BQH}$D**72^Yz2c6!zGQw6sG@7QMd$Trh@c;yF6W6BJC@OIR$f)%PW| z!wl}F^nnQ)m5wigCU@CGUK0s#=K<1g7}}bXF13c^D{ap~L^^aSQJ|%5Bu1wK7XNMG z^$L8%Tf2`XWYI_Wq&XZPa?a6jom;7U_uABv`WAh8^BS@cPQ_+!WwXsLOE9{5uv7R< zYZvid2JmU*3MgKQ#IhHeVuPidbGKJr4?H?QbjGt?Q3tsxg`J~~;aaO$ozj&`YHy6C z@=RoJ3$(h{!zF)+4?P%Hfy*xdMB{=bL}rAALMq$QVF)-dIXaVYS%iF>G~C13B9PND zZ`<_aT5bU9uhMsz#vSPu!mV~{%t8KScmk(A%tK1dI0WLUIvF|_50f%A#A;zbjol-v z8d!TYr$f?j-%c*5y35p6xAj*ok@fLGW2vC8;Aq*GEOMp7;02u9N!+7H-p&(i&hpVR z4YBD~rTZWIWcqusg4LBLkX1C!FA&qaLdJV`>2?L`KD}w2eZ?6rG(FJHXa&@Gov+Z0 z>_G+BD0$J5U7A}Q#a_jg5dcG78yXrOTpa@-2V!PGE}no41f57l+UBHgbO1_oa${$XjT4 z&XbMob0e1in%&We;q%DMmzZ$XnE=?JKv9^@tR;%CR$q5Zx)wqM}H@FE8` zN62i9Dn*cWcWD~Ckrc1(Ph`)t7(*Mv^WVFjZG2C<8|oA875Fkaucg(LIA z%q_puL4#}zvj-$`EiCaS)}B|+dxN|K#T?vX@<3oKuR@R3Ef!kn zRh<4-t>cIFyy7Szx``Sb_lG#=zq|l2(vzvE=uT@rG#t-{ga9$A)3QJf%o2h)9-R+B z?go#FKC4-(?E(!J0fObzP@|bY$3?9a;Q<;E`85TXdI!yN{_b_AN4;(@pF2<|bEX=<7|nEgO&&gFccp)b`m!*7SmB+*w^{ zo`x<3)BSnw!&ifmuG*4r2JVJi6a53L6OR5oWYxq!WKIU|Hf%XfxA0q`?n^xhP2VXN zN+JC1=sn5!bbXIM8ZE7S`?FhZNm&Y>JmZr&dFd=on{-IwOI5$Z(r@Nndmfj#`E%8Y zU}Xe+;3}AC$9TN@H~Ed#BV00*rJ4bUf^vY9l_D?J7S8kh0;G6lEW+!3hPM#94}27_ z9eqc})nW=XWvD7^b}RcKVTEKv0x~}ZxcO--skS#hM=`a_v@;m1) z3#%zUpVZ-N%@k8Ipx63`QSrR}QF~>YjBG<@c=!*{D2uXBnP3P)8SpsK!|XiR@*96xT* zA8xyND5FF=ZqRr4b-~XVrVa!>`iN(1|KGfYop)E_pkLOZ8&5eI$VSR$FW+hCT~c<5 z4;(TT&Fetu#L}W0)}O1iET_pW(ZT&BIcS6LcT!B5^^3_Rfo@%y&KPnLNa^`4sASWh zgnUV#=**4wI95s`uuANczS5d@?sln}GBw3WHT0)M7btX%3m(>UYOBGWHdX&Zhc|M# z@E*Nb&K9a^_L6_OPCQ49yYACojGC-ptm(uJ5S8bKHz@e|EXs4X=WRP^9iPxcwV(jH zF&C;l<@tI5Pr#Jy=ZSp9kT!_gtkL3ea!6hkAjVSvi|1g!Il-y4Yglzp@@xylFs{kC zbop3u#^mr$glp2**|uO(+Q{_s68Y*xBJgx@=^Lp};KavOj!uNQ2!vOd#c_@Q@QWsp zty)@`^Iq4J728n2Ch6KT@$C8T+_Ih|LL9`cte)d#J5@;u#&!6hYSFEW@DP~Yg8s!o zUZ?eds?XY7jSI2y1ypSBo3Us@fNBv-*oyHOg2(oe6pkY{_KlC+=k;tQU&Lk9F)GkD z!kJJWu&%gztkBJZ-0-)su{v7H0?@2;CM+CN0y&WG2pn&>>0MC%Rt^T%oB_=0cxQT;$y_2uTH3t`Zn5;?yMU%7N+j2 zo!WP_01#@f`Am;y6(rSQ;PHBy8XUC9H8SYzYERGZcQmT4Dy zXYC2+G9vyZ44zEO#XHYL2y%LYF5&Vb{%$egj9p>_DXxR+)ob|z@d@a|i8frT+uj)# zDg3gVuOCdw_HLObfFl)QtbNpRCQ@9F(zTGHjpNp}_R9;gur_uqfp=QJ&MLqs1!)q{ zeow=H8MC;zPz3Br4*%Zx{=qDUflmg?6)mXtUI=@W?LtqM-S6uIadmQ3f5u9gd>vRD zLJO}Ezw*h`R-d`h*N$~uVNn^ld*Ep)%J7a`E`l>XNw;MCy*uBky!Fd41mR}jX|$;BwFi5a&}~0DV~WN zXhUqVw}cN>l8?3KRapmWs&m;Th3&J?4xIg{Te#Nr58x^we%Xy2%mjWp<7A6g*9q){ zszT(X=B?bE$-7t~JD6Gn*>H$VV5X2VHK1+9um-ri5#x;2CF;OipZg~IqZTx7q5K1B z!bO=7D)jS?=1`B;7+b+E`MBABfRTr2&k7_@mh+6xQ8^vFnvtY`O!>982d9m88TxOx zpUY|UB~<*v{J714xn~-i+()4uC;!6uhxIHr3bNyvPIK=wXrOJ3)X>Ju>wul;+mFFB zVMy&LZuj%$;n50*$I=E-a+Z}~L7K=p#6+^gS#Ui$^%TCevq!t5#j6%iMCr6s-d|zO zjZKRZ*Y>wC_Vuxz$kh+{5Keci#9h0NX^iOKvh3m)j*CE zR9+@dk6{cm{)$aub+c?!4EuI3s1RX$G_DGUY#uJG`UP+Ni@t9qg01UF;>oP*QiS^8 zVgESZZ5!Clz!57nm`h)4l*X-yd|`>aFMiph!DTS zsXmHpqUJMIy{rAjyuphQ?N)H5i$0+#JIWr{ty&wPNSfkd_VK*M((dR3(w8m@F_*98 zaQR3x^1i+!m)BPH{k^ouv+_D2J00{3BbF=JE{eloeOAN?2(W>y2WF0o=t|41v%3L*hZz ziCTi5YXjD9?T-Xh;}r3hoFs%+_D`9Sf1G*S%#xr-$ea){oKrfV+FEf%Ww5E8us|mz zyEpvGEZgHPytb_&mA>^GD;#*?%zi_W_+Tw1)1)L#tOSZQh${nP71%h^v=N&YRE+KrfwQ`z+P$=E5r zzz&8v5KRnrPX(!X&$M5*=RUsyM@$iZ5FCUs*V=J3GkZu_k$Om#;O=6e!!x?6aVZs8 z{LRzm#u8B$Z_G$&h4P!zo6_-ly=_Ktak@2uknd|Y`78j?XKk(;8wmGPJ+NdtXi#n- zn}>L}ZQS!?2!!bHHDHzozTw`2dPVwlTUai;H=Tx6dz`%6Sw$9*Ym)NaKt zV=-B!)Vt-xvnS?K`ouyg_ntj4<GE0%kuRL7k|$`%2^(n+=%?+%#C zaoTh7&%NOl;n=0n$5xrVeo+-yQbI#ab(!YAoVFN z--O~nYvDgDXyf}un!?mmt>Er{ND>J9i`A*2TkbDwvBe74sl2r0NnzZZdfD$f`jy>p z-Zih(`OJo1O~Xo}Y- zDOvP2fVw!gMnRLz>o3vExA}<4T4-NdvqMTM;AOd79_>&Bf@n zNf_05WeW4n?bDAEZxPnluk0MOb9rt@1!^4k@>wvhc1ak?+>=MuI7-QW2@RCRP17W# z!0|3gFH zE0rpahQZG!t4>RecL%1V3p-dk#ws|%`rFIxO_+Ut>8!{AcX2l4Y1lD`d5-zYgpXU&au+FldmEu%gK>iMbn{bT%ez5=)Qo$wCv_h}nC7e4wg1Mgd^;W0Xp%a|)m{np!f=P^F1l+*l~ zCE}%+n=>>{<1G<)Fp_s?SWF-UFo)(qtsGY#em5w|JA=1GV~h74)r#>XGRTq&)2^t; zyXmCw#dbIs2~rQXm|p*tdfDl=N|o&%Ovv>*UAl^I z(=y;A#mZb$Re0fo5h+s*2F!_FI&1paNzn=omC^G2KLv+_nh$EtkQo}ZVuKW;cZSaF z9|d#?aXRI)l=_{!g7#u^oNlFeiuA6<+rKZpyxYG<4~-KJS%GPSvL7xi=L{ai?ex7VS%Ziq6BRN2WS@0^f+{2&OekIx(Z$Y>yegHVq4B*OuTOsaoKeHS#_W<0){X4NUx++^!8-QHP z9JQKqPHMMiu4e;c@z~q^y7!B;81lW^^n5H95WII-N9DwBBt};7acE^D5zj}AyclhR z8qz-X1B6iN1Fa-?2ju)u_XQRu%>55nA>{(2Y-XVIl8+x1))ORfKm_6-MAaOi0R2=l z8MXo{Rk;cxooDqAETOFVPj7WmDUe+*%ri`k%Br&+Wm6_xDTK7Umudy>VL9Lh&@%RJ z;JR)7>+Ec5HrwnkU|ExXXd{`fLJuqQh+p~$NIVtZqOJTJ^7nSNaGqBz+63XAe+t%_ zl?_2Qed`YAr9#)E^V8NzD%Y%2RA8N%VK)Lk7s{>X-C=yWPpbgVksc8J`p9}1xLTnH z6u+TBjp$ocEQlHr9rc^IYQH8`ho)SzuP^*kJ=+6ueZ+}s>0%M4W_8ZJ=UKY64O7;m zWY_1tX1|t(&3cwTTC7ya+N4{xl`!EOX&D z+1Q)$P{?;u^8+zY7aY2U>%?!(Tk8NDfR%2nAZs{KRcb!-%G(25{ZrIRb1+e?&e?iN zH`glZl%#l>IkTxPm4PFAxr7m$Bc(FM81BB?ue8-U@f!uD8~lf|z83-|kYAb(@$Ync z8_aQh-!BZWeP|Bc2jyoGv(fXVcD&eaT(wIqK7MXQB`vplUoGEYPMK~8SMol@mtbTv z!k5sUMtnWL|FKYzQRc-CNCn3LE5%?$Qca}4mnl<-ECLE)t!0d_;sKp!(<}FKMRQJq zDZ2vjc&h_^?j;{qt(%pcQ3S}T=~F27yUZ^!2py%8uk`Qt3%P?4&!5SkP2u(GVo^F(@?L3{ z_f?>O6Hukp44xihJdic(3MKTZ|Fw6C8|`2E{}J|`K;P=b^wskBHqLk!(8fOLo;EhSxwl!72bgd2wExYyO0}*6+L4eSg-yaL+vV z#m3LTYquXe${XP{>30Bww=0(Dj-oH8A;VB}Eh4{1) zKhjv%UB6Z99k_>^YH2H6#T_E^ya6U&)xbmN-hd&C;YLD;)k*55 zJrQk#5QsaKlo8l|(>$-gMm4C)OEn|FB~U7W)J`P;>!kpmF6dp?Vwx7`*9dc8vPGsB zaLknyHI%8$MKgNRO>nzI(wP#!UhqQM#73j)*tm3(O$*C92nRs1rFXez`^wDwJM>5ijsZFlio^@}eHXS47 z+WYt@Jr=1NK9IEZ_A4c?YrnewG7YdQ=KauzDctvoZ0Ig#o(U)AC2bRszr^Fl8-J2N za`sU`PcEG%{B1<3B!Hks*|HACOl3rI#SkuC+8y;?Vn9xOuyJ=zEOvADRxbLAB_4B} z(;nAg(cN*=;kCI^FZtouNFnUyPv4t+_S{~XbH%A@Q}v97L87uacRUwe$ zxan!UN~5WOs1IV{x?H4wyd)W)YDg;8E+wBP8SQ!DQ&`DSx$BVHT|1>BV;@hGRL1d| zLo_?!sWHmW3wYBq$LT-FJuy`e^eqzDe*Q=yKSO=bS%F2t5&*Zdap(`*?N!_vqt{}@ z(qEYm4C2gx7ro&W4JXBq^Ja>6X?s>MbMns`$m*$%$Ls==;9*nCsx6!MimuKx!`?=V@Vuo)n3w z^jL9&Jgp)>F#P;K)}QV4g=d(rok?G=xi37KQt{i5m6Pm9A02^I{eN#g_}|hDFzWZy z|B>QMGmtB&qpF4hv8aN6SBAp8;WoZFLO%j_VFzakFoh<(Ns`&!GDx*jX|jNdm#tYb zgPX~IqcVosnc~+PpXOI8b+^EE8P3^trX30TkUa_^uIB1#2u||YsW>n#<&9mS z@|O_D>2k!Kdz_~Yc-1t%Gj~h-0BFCz5aZMV7y12=4VegWd6G4U-oL3fRt6LUp3Iv` zyx*ndKVhX@?itFjLQSRgQY~ zED**4D~dFgTlTZ>{j|C4k<(~oJ1}fRn1IrHvS2QUPxoD;2&b9#au=mV-q%&)_a-us z&-T|s3ILTi@0w#DmWkHJM}6>ESE{Hpl;6%5NI;R?_4yP9ESJGP1-@MrX6BkVyp!QO zXI5I*g;yBcAlN?H`0Y?cywEqFr+kSVRjrl3^(jfV`h14oF3WE8UDevo4?1_^gsr?( zd?GZFsq=(}Dx3nz%ODhYmkvA|b6B&9g?;*KB_cQJx?aXVzJdrr`DAK482rK#kfKZcCnc?-Y$7wbcxcnh|PWv%Zv$;uC zzw$Hjm%{VHRFs`I2V21%9W&4;8dmlPHC)}=#`EYqz0#kg-(`qB66>cAhlCs2$jbti zWW2)AoTyvkp%&fUHzL*j!nN(i^`@McTtoa@@}eShx63tn+Tl1g=Uhea0cG`-+|=nG zJxPrARo?d^HAPaBQvW^IzBB1e`TX%O;tpaw8LHgVa|BV4UHFAE#hBXuo0MP4mv32; zgEOx1RSN!l4(k(QBfNUj@vXH+5mDc2yaEAoty!L(I8U|tdNBgZjyE) z!tEq>^uBr_PIyBxi6f+GysX-m*mg`LjEQ#*@r_%(B4)%s*%s6IzQb}E)qj*LMLv4~ zpg+xz+P_a5CejuGCQ!Od;OpAV)taq<|}LeaMVSf)kfWpseEwp zybqVJyL_2Gu<`w=oL80~f1E|cH|2SiPXlF))B%?&@3sj<|0+p`C+w_I&Y!uE(NL9E zTxrBcysf)4EAU`U>Zzxl)?;*ZRfrAV2Lqt_c`s4cpYI`#F+bc7&Mz!{ll5@+l5&@irHP%|pVZi0sv%4X8qr+pHND^w z3{Ri1l`kpfliw`Kb&doBpHsnBHcn#4wr-iU1tEsBZ~JOj66zJ1w6kS54!=Y5_$g&i z+J!GZz7l@CTs^U-ATri0Z%1={2$2j>f7Y3&1;*VN)r`FPpx+eXm6nuMYk3|oe-QFK zIAtk$?&2lycbSwgKC{FC-bVy_ve(Z}CdA98i{j|)&h%oK-m#wGk5R$3?bKJ_0i^$-;4fgO_DLEv?e@lhbo1|Ha(b#Zs_MJj@reBN>sVizu`pWiBid;usk zQ?8nG!-#Z!6#hG!7M^g<$5^(Vlt2WPL8(||>YI@~e2fwCkGo=-V50|r*Cvj`vjOKG zPlCoC%zOdseI8=LZwEec>x%ZSZS(XGH`}i3?iiTn9vl(*@I#@xXAz?8+ox*E+E82O z^VcK_-lq!h1HgT_+Qech>&@`3hh6T+Z=bv_MVacOxXaqjXx-mZAt3RPcf)7muh&-Xe{r9`T=@s zMj~&`cf7jSpXIkWe_nDe`&yoPS&Gb>fOx?s<1JKnj=@mmkg}s@M{uBeb||p;CD8B37AiDR^xX+jw)D>sqKx)>%SXV zxV{pLd5%Nf?OAE0RzbhmY306dwAK65oPo0;aLTPDRf6SB`bw-OJ*a7FY03bT8B*x) zC(=aHWK@M%_i_;faD)Tq9h{)c-qc>qa+TK8w`eWrPxt=&M-zz%g@scoGR$Ij4w z79t%XACLk{6m|OoznU;-t`DW`1;U1H@eB&uSDhN~Kd9_4yH(_&SrfO&eWmj=+gp|R z`fBTEJki&rPVI8|;{+^D(QEd+K0!_`v@2ZHp+je)|94wC@6$V+PG6*Udj2v6&;HNJ zzje%izqKGORgwiVh)MHCgrM0Vz9)C96NO6&Y z<1`nIA9bl|fYR}*7058!+0jv71_n{n6Y_^7PV5-y%x9IXpw*@xjL(PR;xY+1c}OKB z+K^cST+X)TWi%Z2a+woBFGQHLWDG<*>)xZXt3-j&uixiV^U1{ZaOKvoMadn0gwi+F zGa#1e%5!tPSxz;6cZ*Ckkr~Xqp)tXqrk}?p*SpK^*P1N%eBRD;@p&(=nptbysgt83 zG9hmK!qkaMm$!^P>LF@a)#R{6bDIn4N+oxmxt|I?fDtssyPH*z)5rE{KN}Ks8W)6J z1b#fH=Bizu@Q^it>O!cMl;D{GvQUcNXT;pmZzs>#PjdH<<;+82zPQJeR-L_}FMNhf! zGQv>wF096}Ls2s#{o??rWOFx0`rAc#%1nRI*b_+xSj8) z@{jjMtW}Tc4d@jt-%m~;ODZev2TbwA3V}oTuOP939+FOgx1LQlcW0ze3RdqeJi-57 zAz`dF*A8Z}$7k24&9>uxI(Zr^;uBJ6zvZV=rFfhH)`OU8L#n?_6DN(UB0_mdZ!Ic8 zLO>yl=V}4IZw0U9boRux8K9lRJ);GX(jy)pZ(wt2K(K|%yq$;-ofiHl>GD&|0hR8$q4asHuhj`?}tT>KVc zN%c3XA*uK73uRvO!fXAWc0raOU;Qm6Qznh8|4uj2HhAgtUDP{ZRQ}n-?&#y4q=Mo2 z9_ue~Ua!@Klj(1@P5Z3v*)#@2Z$k6)B=u5TVhTu`0%AVnqG~r9eCJ&{WI{UtCM?~T z!wzLX&3WqAm|4DP_;Kyzx&1Zk4FW*H6&g==#aMsFZ9Rcx$%uaryhqo?LqbC;yv>9T z!3ub@VI3#&CO^lOqT)`;pAe?D5v|dkf{IcuFX3DiU=R3jm9B=B*c&dOciohT+UTn9Gs-)g-DK^ZhSptmOt*9e?}*#Gbc3ru5(eEDoX$|UwdUAPF~Kfm+!iWuuD0O z_*r`-7E2M6XP{g~>#N`A1`47j}G<<2`Z!(2CQ%pEPZ`qBsDMqR(Vv$d& zsMlxYCp(tmgdluU9)%wC4l$ShRHI*pmntE(R#NDuS4MSh`m_?SOq^{^%+f=a{b_Vy z2#Z2)E&0W~D^tFfZELNeHFpnQjl)t-NzUl|&*Eih{F66}G+9#V%q3FkVD*sNR1wD_ zv#825uqtnWfOblc0KuqW5#8}GA)s@=RH?xy1{MRXP%T897so&`p-h@d6SRfilb^Lc zt!1lY_ak92ib5H|QFz~m00Dl0oeKbjUS>)_GSlv^I}B?wpow3%D=ac8p5E2drfM>E zlks{9P^vffhTTmhM zc5^XC$Nv#@=aZjHTsbP;{%ys@9IZbKC1&F!9-`R(rx5)6Va3yMH<3dWZ~kweU&2R8 z_WS6C5OgDJ*RQes80pVz>X%_Zo(Z;&!aE*%Q%uPJ?jEiE_OKCIgVHXFPX^BbN-e`5 zF3BJ2nXx=@+@5TA6RxHr>?u>@PC^+p-jz2kF7%=AAAp0v>3Z<@#s)#Gg^oHP zpS2g<>sF;U6D@stNn}4n$fMChkyIA_zf%M!RH0!fZU8}GIny_CB2sz^gg`AF5wa(H z$>&2)kex2XZ9{j|41kXPLS=4)r=bQ)Ls9$nEHbQ-eM{{zp?6>)pfYeT_v;$(b1b4l zl9D5vkTO#*a~_Lb^CM-UYx-13o~Zscd*+MM7#+2N$U!4|x%fqZ`V2*Y0M;ZZ@2cc4 z1gjbj;EeU_Cr&GkdL(fADWtTrIi$5&rd@Cd`ApR5?*ox^vpy{R^|BRLBcy%5tTJF@bb8@FNl|75~KA}LPl_Q zIwC;Ox1RJMA(vbhLBuEE_U(+vAo)Q*`BghKK^u-?n}i8V(K+2kzDGhyZnT`r+e^A@ zW9@K9dBn`t4H$_ew%m-aO0UYUHo;;8{&ac-&=%G3@$)L}?$6wV_hGBEzwaLB^TTFa z9_$x1^4BQNhk8 zNg}X`@FFeyOpO&tS*dW>w#`*b8zuF?;xWFsS zCq^^7bB5qlRdNELy!CV&#JFgxN?ywvfZ$F6n#2K6i$aVQ=(I>dhar{n)%$dBz$RLQ z=8hbO6{dr_5QozJy->ghx+XH|>Or+FRg;**_a?QQ$aCBE1>q|BqF;m#5@ewu>U_Elc1_h}!LKBWT`=dJ}K?t#jX=DvUfIl-t8(@z z;2t3BG6aZ_Kc0DGiQMI?{Xu-{myg zBiaTbSX~*!YUfq)ocas>DW7(z0M@lkT=h)|Y+4Yd{XSk?Vd!m{547j$RO0><^Y;!x zPo)dsZ`256cGH)O`W_2m`GAimz_BJO5Y7j~q1nA20v%<_X3SZ$qSHDy{luFOx#NcO+t23Y5iqS@#=5W> zfvd>4V1;uE>!Jb*kYw1miuT^0v&~!85&wXW0-6fRuPOgVfNNvOx>Y^T*Y3q@{dk*7 z6p5+k1!~Wc%7VAOW8fL{p0wtMh+v*SOS40h)BNo3DtZ%IFkFLdoTU$F#)K5G6(Tgi zPa}DptI9+oOyC6_gU;UB2m|#+^#TLDtK|`p3HT}jT$&K0Z3FlL_?~w9)Q!Mt+rx>$ zM>kon5Mbi}EYKhm$<_x)JqMz4RGRv(Gm$SJ4rXsNWTPfxM#5n@p)l5y)J@~tm%lMtoQj{DSyCKn8tacII@&m>(~H7Lb+9uGNI_fB1+cp9cfIA4 z9+3u5!)8QzqG8?Z!ei(5z%Rv)$mzmz^>R5?um3I`C!Jn7+?W(FkWS+tbxO?wXV6*Hh ze~Lar37mC^-DTk0)yH!)4vE*&uG%8 zcyp@bffZvBT$6OCpIMTYRSgFwHUK?emh`ViA_UmqO>nh@!D+(>yP&%dP_UWsuMw=3 zm3y_QSWD;|yFm6YaVFbLs)Wd00SPXJPq_sfKX)}}_x)}R%QwP_@)TXuy&xSzKjUu+ z@yxxg>0*2zGd+BvuIBiN>R~=_3clIQy<%@Dr2}~y{dooW8%Z23 zHd7M}NN>W^<>i9T6P;*a)7V)68k7eX)IP?^SQ2y>SWN#xp(6~$I~j4vxT}trYWd#y z^V(&3K`Z}`{67(o_~(A7A#AB+vA<~Te@VN}|42I(kQLxxr$1-}f)9|)4gRiKsfVXn zmrYOF=C7f)N}ZRGZUY?2vP?l|eP+pzIbu?U)ZK=OMK5SMycXU~F-Z`_rT3zpxZiCS zz*>;Br{jnAJQJ(&`FH1qlFK@ek=f&wQslP3cH|6cD&k-DNQ&uxdHQ#r-&eE;(w9R) z;rnoYgA%WLYB$+QFB$J@%CxUGWzM$RX3X&`#gvE_>43ZYFDx!;<9Mq<{A(;g!ptp( zwKjx=zwBqENNUQBW@?K7MgRbxA+phEd76+DxAZ2@6Xf!BU-MF8MV0wif5P-~Vha7B z{1U1N^_a6GtKC2}Jhf5ctM#WBxE{p6l+G>lmFo>vZ`NZ6uHhc>r!Je96s=u0b4?|S zLz?l8MM`Gt9EV+kybu6`f1m@F#ZOP+@g~>741s%P6|ketFqYMCLNksg71+ix%zHupvtUdy(QRj#Ih8!MK-nIbP@5;UkPzy~T z1o!vG(cz8L`2)<0A88o8q{9l@)kL@8n)fO(+Ycv>NvYcKp1XQ|A8XCE4rtc2N3~64 zF8332qP13fm>D;agAmF;7C95s<;GEHxC}C!*Hn>o8}OKBP3+z6#N%ek^99L$H0PsB zR-K2(R5ouCF6@WdX8+Y~7v4VJJ6O$13>r0cd=RR&X0`7Sns441Mt53qCNaZ~UkRcr zhB=`5kMS0Bwl%lkvsX3(~jLBI_n&lS{&qC!5LpI~sI#Xv;}jKm@SqyH+kFfKsE z^hv)FZw20@;@M#>?RtYth~5Uw2JjM^(&wr<@mjF9*FISEpag zm?lh|%8#r$98);S>%?xg=T8=W-`GE|zFgD;wC%ecn@?}RfQTc4N9C@^t)%4OcZ4qq zR{uu+y|~$#r0e=vZ$^4ro+|rd@3X#47mtFh8;m;90$44ez`fl0Rr%h0qCvzJ%jc-7 zv(0CR1~IlDZ4Iwy5>u!hLuuH$Ia4_{3mnkw+7qjMqqWy?j?<3d6Tbf7y|yBEiAnLKPfp6PZA}9R6xQ zMPcw@fqia`}5O@A%k%=wV#i%>~6nHm@Ppx>2wz0uL8&Igw8t=k)2sZB>W zF1_w-^;8Vx*%8B{S5Bk)e$Iy85XOxc4+fOdJn+3eB95c#s2Jb5U2lxMu{-C@$WbvG zh?u4=d{Hyte(hDI7=<+9WPbLLqrjzr#KZI9Z&O@M4;c*Q#Eq{gcepFYey8oxa75o1lu%A(WuYRZT9QfUlg>bFaUr590Xkc=bFpcEmZQ_w7U7aU(CD}J>R^Kbrv zFGi2;D2(J(H`wX8-r*SnTo7uQ#NIwqfIfAJ5{|ZuBxuNQGg<}=(&E5NKiqL#bZjSm z61_HS8;5idB`iOUnTK0{BW&`}>;`{jbeh2p(vjYdnMg{$1=&gg7gj*;A1Z46R@nxV z9$7@=Xn9ConnIQUaNBZ7h48HkzIrMXDg=o_rN#>dh`oMY-s+GNmusC->d)xYBHN5E zdR=WYyX;DDzdGtGnBE{HU5Kid3<_cC$R{HhMJ>f}RVoLQe~xeXvGHt4)z4jrpN8r| z(2nhK`%8;?u4n3GMFOmZOqNHwAs~kWsrT$rvX9Hvx~E2|5Q0FqxCd7)2*InSoT|Ho zDU=|~R)6865~h>M)%VNHLy2GSC~a-(?f!()vnk0{Ip10a6r!+mQZXEc8wM)HLzodE*7eIz zlO|jcO(*KN&p-c+`PIL?&!LpF?DhX5kTCkhyZsAR?+?gm@AaD&<^D6~!V9u$WX5S) z7!M&nf7WY1ZM}a+6{4zhf}woqcxOWx%K2?^4KpYs__&~}n!aS%lI#If;HJGkqGlg2 zmhf(sjc$L9%Q{f-SMP|xQpAkT+7R!W`d7Brp=a`l^u&Cg|>Y`%o@xexiRR9aAe zC&YgRVw99*nQl{RN&Y-gjke z(TROkFI$np>ks&gfLSO(hc0kHWqSxuR@})9dTl4W;NC#f6#q0Z_<$?iG(7WhJ}A^X zG9KFP6WsoE|8PGnjI#Wl&cuS;Pyhtq@@8TudI!QuxY3NO)J&D8@L#|tS6T-7gl|Q_ zoRN!26-rMLr*$lBH^9( zI6}G>lrIWV6T;-RS+gR7P4jsT%lG!VuqqB8UMbhgR(&8;7y~=z;DbtHQGSOUom_6%0|fKEfCoHHkB`KtQY7?Jc$DINa`6=RL{Yy-L-LnVZ0VWl2aJaT?fR_3I(LwT`-a+$dcyy0W4;-~dHSLv2F!-vVE<tmKjc}p2$2P+crUj&X$euY*=`eR`Ul$?|lzcNi*AQQnNK#wY_kN3vO1J%;Oey)az5 zJ@4Z}{`Vsx57`qBj-BDYlxvRzvj|c?JWdYGM0*h1B1d=e6Qk30pwrI6E6E=f(=BSC z{SHF*U(7QhUm@&)PW_VozdG`Y>x1c58_CE!t^k%tw}k42~aT-Wm8>oXITErY|^;ptwdm}{h$|XM6f>ZWT2?P zuc3SsQFP5aLH0U7Kzq--8$b;zRr6OKIoNXf=Ic|ixp(r;?cNXAUpNWH+s~Zj9O>z| z)9F9et}YyDl%K4(BP+jmQ{f%8yUcqd1%sY=d;=>=V!e2=5IRC{EL+TnN{%Gb23W0~9^2KC zY%J26pAQh4?OiYEk(*lp(dn$eJ+j4{2AH88bm=0A4e|8S-1YQ&v z!VI@F<{qrx;`FH{;WeN6acGBvD5LIJpT?JRBT$0X$)(em8I4xM%`>Ah%!EtI`GIksO(A^f7MBPf_Ghg$mTA~_Bx*tOil zS)L4j&P^w?5E90#$3iO127tm)_ZeXjpA-JB99=Qyg#mPfOOVc~LYtW2*cw(KRVcxLmTyT8N zww_%D8Uw4!R{P$3r2BUB=2G`H-aZ-S?i(h)Jw4997T-09-g?L3Ra!Ml&b8F!HXh0! z+lS1_jZ>bsvjyVwL@#_7ToaJrz#5A|5A61&bbA}XEs4(d#8@G$i4SW62O$-N9}kfU zClVK@Njxf|ZJKUJqM$6^EImF{G#ApKmRfph3WH)CBl0_f6ftw33GD$I!xHcG0^Xih z^+I{Qsw`Knicq6HGSMeKkUL8UnervhzU@ObPhaAN{Hpd*K|6UabZXDMjyewq)@ToU z_|J8!!hYqqT`EoVul2Ly&ujmQMqJwoe@-EbZD7HE`+2Vz?!jx*uAUHGMU4~;vrSw% zv%xSI3`j_+WXJn)+O;IUJwg9mN?ImyvU*~v6mVeU=!kWPx3H4XpYl`{EtjsD-+wUFJsY{| zpm=4CJij;uflL3W0kBDJ2jsVfg>wJq^Z$x0AO$6fQttKTjB3VIB{wG4So&puwC7#R z+~o2n2eU9=4`l}v|MR0yUPk?|3zdbPiy%pw`7t|ayQPjlanT3iZ_$3; z*UQ7%soNK417A8B6C_N3WmofhW4SA#|M(f_Wo`Z`51Za%4|UjlW=9u3}ZTeA(^p3}@{<0*yCbbXYf&s@1o`E8!cH~cX0r}62n zOWtEi^*^dHr9ApsVPL4Cg%}U6JGPHA9>=jON!g@Dh-!fz2unFoV<9NTQ^YU47-?R znFoPyj)k|)R}V?~=LbDzVwbhw6QnMa_zUA1G?scr_k=O|M>Y9UfGi)%*bc`PiOYc5 zJS+ziq?61^XQRnbkfb&=dR)1Rwa1a zfUA&{cud)Y7L*6Ee&C<<2{YcN&s(LxXMCN$9hBS^w$$AMHMMWfKWbbi43T}1teZ78 z9HcMeCj~5Tpa_8$5qI)EyYI-tlC8#nE}|~T^Mz68p@Mc%Mvi7CV?VxmgBq|b_J)<9 zQXBCBBu~jrZ$JsC&!^F4Z4pDV+ybrq0WR;?tHNH$W9L!IzzS`yU{xmmU~5L~g)mi{ z$n|PL5vnIGvIGFh7F-J^KCb(ltk+50T_J-$9iKd4X$j!3=sGM`Y_b*XT+v_;{Rd!p3I)&5`1DPEXocn1aj953b^LYpVD$83$&Nu{ zdyNXG6w0S5?7S@_s?g;755LB7TQ$yu2Nc4ud)LwgrMP~01?Pf?bt2Sv29q$Oz2i#u-VA0$WH!W+{P}`hC^zz+O@;{QG7J|Hwy)V{vmH~ znzNK42@LX%vi{Om@ddr!^9~OxRtyUG3=++V!GAV$KhMjugK*g(84?S2qc(ePfcZlJ zh2(q6t>pP`lQ1p}T6wjG%=Cvic*rGCx;UvTJ7wLS3}q1b6klD_ynq%vM15$qLTz^1 z2JHHZQP{NkBvX99y!nNet#PwhRo>osHiz&fQWIBB0a@(jkgJijc}4%A*28?hALd4? z${Sy2?oO@eWy9Z&TZ6Y)C#7U`2_&7&T(q9#eBCQO-iOfOjQ@-jixOk8-*q?ze8786 z2cL>UZz68#ZHgxHTpUQq?`6Gycp=be>OXmbR}CuUdjxu1!Wq1cB5qx#4PpVoIxos% zLiA>dzsJ{VJ9V&Pv>ga-s%Q*)!z=K{iJu_N!ZMg2la=__(z!xlUa}!33h|*;18|Y zRmf&Vs{+g!|Nf{ZJ<_d!*D{--cKt8(tUn)_M3h!IoZ_DzbxmQzBz#y_VKMdQAM|Cw zKGTq2byou4`6yj|Uj~>|)ma*Lf&I1sFxS~VjhVDjYeoOlk))Rkt?&wm8Q;S~v%-dq z<2bDxz%vjWe=>39qk^YdST*+===;)j3Hhd7b!6{@%eD#bY~UB@GtZaab%gB2%7LLcIH@a+Kw2h3$t2 z$)Ec}+4Fl)CJ0`hlGbWO941QS(OQx4#6;iiPn!Aie@}!mzW+<~4E-Iino$%O(Ksjm z?}7y^(`@8aNX1yEugmbC*z@XN3BCIEGwy#zM@d0or_j>qso?GMIJ)p}_r6{2Ydl;9 zErg`*Q;^?t`RT=Q2vIY!(X5MO4mb<9>Ir*&{6Hiw7T*MZzYEj2DuX*97jCV= zECYPldVKn{K11-IUeiOOFyZl>%u$I%}0xfODKCEHR^HY5>#(T@UFxYCxt?d`|Hqdf?*B$&-VI!NAj z0Kaz~t1tZ;YCd0Nl zWafo863y{v$L#nz@OJ+TOZd7@I$o2jbnk5n2S|xJQI5>gri7UfS!G5(uV#AU`UPy> zNJN%{70txDl|C7drH)-Eaih!6T!D%q9@>_!9gr6n!r!&jlFujpeI^8BVf(V@vNEsP zDtHz0>O7B8l7LntP{xbCG63W#AwzYN9y1Rw4pjPv?^t!GD|F@kvPih~;mOCbVPE)W z=|L%=>dOz$j;BP=xB&Z0pcU9OHhkrvw&MH#T)8G_gK(wr&zMvB_WsL*7hQqw*N2eT zTmL17;qWH2e`#T!4wh>I?*iNoLkvi|ex04X5$^I~2{e1_dF%Mq{pz9v&+#d4?vs8N z_{I9=|8$tjyj)4oSIo}u{9%-?hwnX$OYyv+>{&dGZ2DAxI>QG6g~+3D&)jh7?w^-R z^S=c-4B`kr3hHC;F-Kkg92@ZyI>0ao*&kh#$>P4S50RYsSaFE^qjsY}z#+1>E-lI=y zA-2=C_0G|(jMS}T< zj>GnivA$KKO>W)%UUiMcq?rXEuf`Wv&jm4YJj%uTfRE!)BZFF_Qo!M&9s`+ zx|qDYxmLiSOP}^GkdwtL+G9XmDN~DrIEX6TUDhiz&2T`Z56cfp3Nib9R{YCxdf4P{ zcyckbwtz6?)vXXvG2_Pc5Ssk0KQ0dxQWZAbz<&3n=qHB7AM{Z-=er&$1y!=J0hDGpa}|zq64g zgAmvWxNgcHJcu|fRX6kUopYeQyEVK7FXW`=D=mBVfBnMUivG^;=&39eRQ@I{qfz)gQJ=OCTjqH!I1> z_t@7{M>xsrm?p=P7{~WC{%=~BMn#F*|2M5`p+z*|tYYzx=>OgW`ooxA6Ri7Rp7Ytj zpV5DL&c`&Wpb*cd?A;PX?Q0|9?Gl{L;V*i9KMOn1{P$0ZVyOK0b7HHWL|e|GVGQ^S zNsL}u%{FeLV*BN5^Kmddk(R0S^vXF9*(ng)x$hU(sm;jE7Yo=}qMw~>uF<~f#d$B` z+K6lG^*t;$rkFtDSxV6z9M5B%@zcL)(YhXN_6x| zI3wXmBVGX1T3#}MH71jk^xn1rVKq%C1k{4!We~*HVN6QLTl$VEFxrK#;Ie&Cb;!#f zt@=WP3li&nAI;7QDsd_t$t-HZ5%_WPGI7r_tycR|W>yg~&Vj!YhNgHptoSDPeA9pnltH)RLk+W z+391A4k(?4=D4Wstsh1JYpIY^{V57fQWVNuW?IOxpjdB(9R20k^EED;iYfZGNyyZB zavW*m9uGEQQk7G)A)~Jl7=OEqUy~{CgTNxZ1{uKsyTF=JrFImw+xDG9?(nGkcZi+uQZywF0 zBJ>E6UJLZfG0fW*BLlaM;5tI)1Xymwt%$-ZU!7%Xx5{4x^|rCD-xRDSt$X5A`9dZ( zu)x9;-PM$QpdFro_15xfTa>^3;(UQ>t9Ha0O4WAHAGZ95@8F{j{s-Ubz|+RL5Sfv5 z|1*ho;HAd)YuhrMzOq6>MMGFtUBV9W($II@zouiHpX( z9)HnKgru*gFD!$C9@DcN7g4mwkbIH3h_8p2l8p#qEFwGg@lu_w98Fp)n7!?@5Vg4yGG? z;#NayG}cMN-rgntQp!qhRYC|O*Z%0qnfcwHN{jO@ThSstMq8&m(v(9X3kmVCMBWe8 znPinXPifRvC0cp&xE4x%G76yihOyZ^Bqf~ZW-D5$qNfonEoE30qUOOMD(`S==DB%@ z82Vw|L{j@@NMfF5_yeFXyiJM$U^U={I26Yw5@1L$ak#*&%5E+My$wJSx8R+N4E?-u z2ntr|F3lt{&s2Wy4ygkXH>^pLIvOODT!a_sw`$Q60l=zn* z@Fg8lpYiKIGr2H;#e>@#G4m*u5Q@WZob7cA*@Yp_-{bW7ao#9@A_8FPx>bx@KTi>K zo7&oYZ(jdX)BbeD94 zG{Z172oe&KN=PFm&CnnnGjvLaf`l|99U_PfrKFUc(dU1j_q^vi*Tn}u^5fq3-fOS5 z_PtpOMAUivO_G-2K*kZBhpd2^sfLohL}D{4twyZzMxYy&M&*W6!;(u)SoEZP*~}*~ zLxgQB!H_j_=PtREKl`9F??DZDNXr@hS#Q8u4}lli*xis(EKWWHsq+pKNyhI9m&^yJ zU(6YIk6#n`Y5H}$|M~Ca8pPnglWX4HgQeo6HIPf!7BWXC8n;imVN-!&NHb0peIEF) zidVA^4Ym4DD|VCeK(HROZc(q{vxY^7x17{zN^xt%rIO?V&;@sM!Zt-k;$X?W75BI>Xp>} zeNbKj5l?;&d6SW+2-C556ZHfiex{jt&+iXloBKIbK)X3)$@w%`*z}QVY~0V*UpnOc zzmNm+nr{__&HNHO;Kje>_I*8_qEbgGzp< zk5=!e*1P1iymu~N2Ov;KdwU(?fzKWYW|D?8#${vU1+XkqQz=`#eG$d%5Cg#wDa#^K zI4a#~D5u&xba|m=#lU+{^p?nnU*%1vdC!uG7{`^Pd$tlLmUnUSTgI7444i}aLsTc?iV zWrRZUT|v7W=^IfBOmg0r4#cwQ18yRN6cHP&s2CF5Sy zwUs3aZ2s?IcQ%D5ld~Yy92^~%yfXlk+*fyMES*jILQ)d zz^H*J2X53xu0pP4_#T>?PCwZ)vBrUoYutK6W0)?QBy9XapeRwjpQw-u|K&E$8pSf_ zM^H7S`bj@s3*$>s^S~>((KqUvKlUB2U!}-H({(Vn#c^W z%$0x(c3!7nQOAFN-qQ1-=FG?qovvy_SIz!p{_-W0;`WBkBPjX4C~P~ITXF7r@l+I9 z=+VR$)sL8@M)ZDeH!hJ`d`K#;{iDSx*s{&asbx2r(9Glml(OJhgNo0yxn8nDk3HABLksSUm;@OF$@m9R(Rqj10^ zN<>xOFt}VMVfeSCu%KjIYoB&06n1@j`tOS*6YnY@1oH<@l&=YcW$#mD?II)!@i5yz zsPHARH7ujfEErUvGGLo3)KF9GKxIsda4}lX9V&%+rZ`7mg>>|rC_js7+YUEt7-Kl&V9s~b;=~b zt25d@5l3){%|J0f_Ex~JCBlrV*o@P&(H85bz9W&)uwW3s!KemKhUiRj?i0~0aqLfD z-e&uxQgc$~V2u1G)-|S4mSE>6?OuQPe(^Z$K_drXwaXw#vSpG#DNqY%pXwVBQu~|) z0%m3vS8m&SeqS$(;m*uaR^rS`U>^06l3E2%ERhuZjT>?WV+mg8wxiN^!p?_d*k->< zz+3@9sq+dJvHriDO_E{yYx87nIFloeMt#fT_fzhr|L>S+#u98#WY zUxoQnX$St$~|M@Osd!P^QWp+^Doy~JMW4U11&uuH$IkR z*&&R-P8KF;zg6`SU3%);^k`oO?+m|6=&u+7KdkE!AwNO?LMUTNA7CWWeR@4;uR%U* ztasYHHdUEl?&t)}*95$^^+6z6FP)qV%^2hPC=SIi=~(EV6b$EESjfpQxW+ z^eYk8Ov=3|vLC5_O4;Y%8)4UMW0S>!vxig?fOFfV^-CD}{Go8c2gT*dfPB*5e-IBd z$7zmC1T7e2&~d5-v<&sS2ui&ZG1{TQhSELY2$d`JKuYJr>Ev$e<1GXP`!xfbpix0sX#xbF{dL zTIRe5ytGyr9MdJUg4q8NS`USkFUy%C+yY~P3{m1ml=6vDA+aP^u?6N17m+Nj$7gM7 zAsg;@HE=~~%b)i#C1Lg#VJXA9rp@3IT7$}q`y*VHXj;CwHCs!j;4i5^tybMU4E=IfP`L-bi-ps$7zLiGKTq2RQ{ zAeC%l6Qx@K3B-?SeFe!J?E6(#EN0#IQW9sf2cecUwHpD9pfVKjoVzDy_D`)<_8dqy z@A$1N%VohSY%^nu(pdt9SyHL;+gyY_UMTI#2j8RkKlp=$;MY3GME<_{FG8$F0Kcv`3JPId+a7K%x zwq~T0U6FjJAH8R*$`$oDGZ4UQ2d7AupkVpW@|h^Da%(jqjn|R*TOI^ONEuqczv@_y9_A?ywI`wG^u(d`+V6LNyhOwy&0jgs2UaN9D7Df$1A z<%Ot&R6-nk*p0#PNTUX$tHdAS`-_p(mWoCq*OR#pzf} zBvl8#1+7LM9EvGJj`PLt!@N8E!D~N3$0GUqOm|mcz6hX945H7D2U$U>20xvzt~xeW z^ETpI88PLnKb5O7D)%Vg8TT!evk#1%iXp9UsnZs0KJ0M~cj;@k+GjDXPP&kIhw4Ur z_Uw|#JPaZNYH%~=i*HhNjk5dR`18b|_qtmc)yd$y2>OS0dMuOIc^Dx(R%J$vVXm-D zD=M8_k!f1)-}!;gS}!W0Pr7mfX&i9RoLCGyFtm*Fje4GJgK*dztwc$iR0Exum&E6+ zpIPO}K;vCuH4j?#{KH}sO9EhyqO6fHGlj_S;9P%E5+f=Qgf`u}G9fGHu!5bsrX%tpM$r0=f&r7vEAvAk z`c;qdH~j*nFOCfk3+SMNMvjq}jrWs@Y$Y1&AKHCGddAr-SifHsSXt_aLNpVO;fECG z_*uQu8VzH&8Y#7df~@8ROSDTM)P!;!J`2Qgz;fKj*jX<)(q}QvEtXq^VF|y4YL3H! zk8m5br_ZXf(|(ev@peLqa^}0v?V`#t=-5txZs1SJ$6~%>x1zVP^A%6!?Lq_&Duf-e zBb}P0f!QE-@-ZNcs3|JbwYq(&Z>@yzRg<&NBbx9EoJYB8n|L8E!>h}%g<%9FQQFrg zZ#Wfk0sEC5NycDF-`vrtTUAEa0T=e=|Cas+4X^Z;DOzuY0fZPTww^%l7n^i5h|dgB z@h|~n4&KyP)_;(mt~0Y_%8Ew4P2P`VIiB~wyLUZP8H5lvIc8fgc?l}Rl;!a89~PV% z)v%S(fB*O*B8tpcHD38_^r_6Auu^|pSghf)b?oe97JXESMGj^U{>er6N(W{ajIv|h z2lJW0udPcgryWop4dzp_qGXj)lgh#;Id-TNCg(Tw$$77M3s)oDoCE~lh--gHcc05P zcCzUytw(K_wof<;4#GaOQ<_&g8QgU0$-&lY9`Wf{GHK(TeOGGoewyD*{Y9+Fmr5%$ z>Af#k#6x=JBwc*vngbX7=egVk<+n+ybtvV!T%u;3MNF=YjkxCY_lN3_RerqKd4K)m zZJ2OkrAGkzo}i+5;R@^^MYrOldBN%Hf_NBA2hZG*vr13xIFsGv{1DLUnmFan&rHoY z{(5a9BlL%xM&p=Y3z+d}c?;#jtG}Ysk-+r1x$xZtGbW~rG?{S^F;B4ez*rh93lbjm zPEzOD2G3YfBbLcJ+|$Lp-Uq39SngW&$_Q@PfuT9y?9BcU4#sYb56sp)GU>@Tr=q2X zgT->mu=s(bY&~5vG?qZrQ{Q=3ZuX%Z)ttG53IS>Z$w&@XSw(0_HF4Hlo!)Z66Toi* z!)$*E!=hqqF2`Ka6t0jrA`?q25!umlEP!~gkD%nhJl<0cGUu3#(?Rvou^UX>n-y-% z^T1R+v9uTBniou_lGk|)%yr7@p@`GU-HlkI`^2@BTI>5EpJVj?@Bo3WIbzGuIWP2ke1B(T=7aJo;qirj1GOKrJ!93nnZ!MnZl#+G%JF8Q< z2)y8g(W!#5QKI;K8ggTWl-&MP*A82Z{3p12edXyFs0E~xhY)AG4Fn?KE$K>nX6?& zE@B}5l>oGP~Mz@H%k#~0aJHx{41azrKlx!{t$y;zQGhs;}Q*wAkIGlrwK zO8YbT6WuJ!i;BGSZVSX&hAEmnCTO-++i^0eiuSwHl`#iiRuhep3;BlsPEnfeLBlMg zx`-Dr&heu@hK$-nPg%FY}^p0=iHdX!O62bZ8)tq_}E5{>sf2mkoPX9IoKVk5mH0vE##=%R=?fH0hq)fRwY z9B`1O{nsXgNDU$v#J3@)^Z6m~uOpcYv$V(QbvJr4(!`qV{lD<81J@1(pU&wx_b>Ez^$=6WXrk*e|x1{}8C~BoHbomvF76r*TJ( z&y9aus4@VL6yF?wn_=c z`>N5~xB>+*HGEDV1$NxEDYh=Z)5ga3I~wQ&9XNAWi3^HjuX5U&kGiTmN^OJuR(}~j zT4!hz^=N5c@SDqK>82zL{H0E99iQ(Fe#^fpv;()>hRbl(VTx|cqizl?MWMq&ii(R{_CDIMVwWeGX!IBx8=aCx2N$B8VIKc*v{Z z8cfsSimo~1gpPRJ&-*{<-LQ5e>cI8hD@Yx&nUW<3BOX>ZqVwUzPrxU3U>=WFcPDTdd&-gqu^%6PX#sEt$mM(!}k`b1Vh*65lPq2 z*&uJZW?}T-_%ZX?AXe`$#zt0L;w{x)ySr~|E&ZI_CrBumd@py=<{r{^bG4q-Qc4&; z7xNdZY>2kYO8mnr9GI^2onaWF0BqvVAcKcxsYK5=u2$5W$Z29Sh$Ah}Pi^8txOsWl zM(UngXNG-@)v-|IF)dcqvt87uG26w~A|0hm$rdVu_v@rtOFo%u=^{l8p0Q#Sya-{J zy-~|3=vp4@l!IbTZXbtN%6#vMxAbei&t)c`^*2}9bOB5jHa2X{RWZC1*F``W0p#FH z8|2qdEe7TWyYIg9o`Y6JS%%@CD@Fuj!7t1U>@j64q=)HSy2HYK$Sv&6eva=Y$G!HbwEzeht_Je{NUF$Rtz;ZhJ zEv5_JJ9+F7X=N&U9dD{&yiur}Bi;|-p;&e~6P*uBFt3R!K@(W5&U@Qm!&h+RC=w2a zr^h+(C}qY5Ym(2OsD|lFutJnda-ViYGHyaOP3KW3)>Bi0v%Q&gi9m`1?B%7)LQ;gs zv8K7^3eyu}sO`Gk#eX~n+*wK`2gslRTb!o8iiA4{!en9-&+ap2t! z^Kput?4%#J{vNEgu%z;*zP0s`UBV{=Rl14WptJOo zr8I8B#L%s+*Bz7BG?9L_7qaWY_JY#%Rdd;RUx{xh=okG2nr=3>Ltc=f&+w+Hv`e4Wy3a(N=H8L#nR+O|Zmf|iRV8G)I+|Hiv*lp9CjVCW4meuw% zp)#l+o3XT&R%XNy2WT{{rQd1#{Wc#h8Ez$E$Rz8e={Gq7Kq9d)8A^P%i zehDYTVpOX`%~Z&)s9n`PyZqzHBPc+T-eqW9PiG>G>@kWTJ$Ma?)oLY(XR`b7e1oKU^gpR)OarW3{9uJ z>0c)`Xh`ty(-Gjluz75LGjMG6Dq&>(_1vbx+=!5uqM2&DhGPx;+q8U#ozoMTN|gcY(6Nwkc&+ICE`c z=#d@0Yu}0%9Kuw?jwtcX97Hqpc->v2Zn?r5EdNua>Ur`8=^_=TE_UpbVmnCk*O9~3 zvOuI8@9(NTdedE-f+uRp1sPQF#C?Mm>TB+kW8lcveH0jAO9M*uMGt^us6xk1#2it( zF3T0)1a{MHiKJAV3Nvt?;(Jm88*5lelMg?0nNs+Cw&`9RW749+Ac$LETA5^1*j6mlsVQ% zdqg4enY66s9Xq7@8k|xy9Zr8Cjk3dCRBJ_bqnz1HM^OpelrP_!adPv~V#*@u4M(BXtycM+NOZ%c z^6St-%*!Cchoj8U0fD;+^vINmU>`9xOlB1-lUV{48N@vwGyN%j_aH$jlp2OedqFi> zWV>_TU594seD2cr+$g1T5FKdJ?_K^^F_7dlXZkVf27@}#2-vUd@||&4(zk`*Qshfv z@W%**M8Pxu0wF-GD4(R4js!~=o=Iiit`qZ<4Yn6)f#?k^cnSSl07PP!LV>`m?aZB} zu5^*a2sbjNsOkz1g15HfU{(I+^rXLq{hFffM)Y4{zXUpSWTtodm)3d&hMdiQXip0b z`a@KL<9Q(8)x+qu|L!(Pc3RUk=!54YjZq3KDM`*XSri@h_s9*3j{&>Al(@2v0=5aF-ABwSpFnU4l1QlIRw=1Lr;@G{>@svOv!rG0i-E zSn&E+A_+!8zWEIfgPY||!T&OaZwxdDOUcfP2@hXW;Z^&@JGujdK43)&G77e=cP zy&g~*Ll%lfLIg?^XQCE=%Qt47+ZSQ92 z57~ypZ>@KXP>xc&rrrg#Lp67QfxL=x|7()*<1&F_g|2i#=TN5`dFI(~D|apvbT2Nm zm&m$Q6vv8m(-vNOiyPINcTQi=So2}P6x?wj0JuE3{BRpyv#0?8U4X50&U|lss)V&h z<1ZS&yfPEYf`fBslhtps!qv6x@VI%OA^RLIHL#@B47s#EL|q7v*CaGJ>t$yxKk1*Z zHg)%F&YXYe)OZ=8SuT@9siqd_DX19nsR{b=a*TL7b)zTemNOQ7?A4fLHe|x_3Zfq= zR2*cvQ`)&4PmN;){nB-ZBe7J%`huM7#c|-(de>+}R^NKto&{d!#-s42=S@f*Zj0JS ze;(z<-9P>mx;%PIJ!Z;gA@}3^A5j9*Ixz_xdZpCb-*9%0egf%y%8 z_|%Vf(o^HX-CqNoSWFGyY#TKFm~)cSZ7LL97S5t;bi*jO8-*HiA<+d>S%rH^}o9Z|^&8NUI88Gksvp+%w69etY z)eM)_l5Y7*ch$|YK$=5A2Wxyx{DN)RFYvh!j>(kzP3V$FA{Re6in)-y7h?Y7K$6vh zP=OOGGMXI!E;SQ4aOD=niugg9dm9s^TRlom&zpfPTF;Brgs3LO?>@ICo z6tpGCQ^Y(L_1#qS{SN*T_HH)ya*!|lRSiaZfC(9ca9e|4?aO6DY+-L0Qwpcb<|~%z z`lnPxao@aV{9#>6w%noilC*Ev7HTj)4fEv*c_AnRfEHaXS%Yt30hFuL#!!f(PP@;*neWPuAc@wGqBxGfWYdYX0ZX=qA{V z%H^2V5Fb-(Odg$9CAhpGH>s(iqH?(GbcLZD2*i=_F=B%YYrXI{;W|uXCHt2CV2zb$ z8&52P95st=>QMIrKic?dF)Il%F8^e0U`x_oL}%kKp_s-@*Uy7pO7~w{pLEMCx$e7u zf7xEhWg!>gMir8Clkkv5-=3Cx5UlF?by)qVeH^#E+5{2!#5h#WCM=aR0ycj%tE03Y zSq+B9c|3v;nPKGyB6)(&{krDrMXmo_3l=CdBrotk7kWvtHiQUhSSo4-psE&HYEHE2!K-3v~{FDiNs2-)#4nX)07#1Ga zF1J0^WGG>Gv=MfDykVOcDeE)rQG7&%SiV-eDP2X65nj0`(fe?57Ljd9-~%0VtMCo* zq!o7Xc>kBe#%3&iSS(j2+XUm=wac5|vmyX)3q3jZKEFz%-DC$Hxi1?a~V9jknl z`jsCm;p1nN8<`dDPK}pYN82QJ-iVMOK?i^WdsvJ(%e5(#636$%ud&ol$3=-92JqXy zqx~Y9usEs7_|*|Qnc?`dJa%*JI8V(_bpew3^}{u1VP~*~-Cr~k#wuh5AXe|uMn6-h z$Br4JTen>A3M*XF8v(Ay;+@N-rrJQDmmTVpT zD4lV$*|Fz|e^^#^-sn17YMW{6t?IWMdI+eEECA~Hj z<(R33L20KK%4T4rM!@TPAy83=rDB;BVUC+mFmp({O&g)N7@@!*REbPy$p{-2FYzr_Em zaT#>yn*VmOe%0oA`)Z93^CfZCp$qT~DNls|FAo1I0Hhe=zE+W)gt=Gi3CZi{o;I%% zSYaEqs~vle&4jB8gE5YHW>#0{S{50*N>eZ|N1SKbYJ&vHWufUM?bi5C%EUQkMoO7Z z;DG@hzN0|~f~D-+%uZRYR0I$GQbIg+nu#0w(G**3ZrT1vnPEa3w80tLYoKygetJc^ zY2o_QD*p)!nH=LL)I^FsCXS}sGTBj=S~-K z>~YP=5TsqFy@O#n>nt7k_Rsuk>sG5o9LzHxwu(yxrTs_6M?hZYUOG~T jN-uByR z{e=SSu8m50gN=8~>e|1UtbaQx?;9Qge|ym6{-&Q4@MORh$lW%Ej!ASt6)A?{)XSJ8 z_zN~2H{l1b1pPJ*zKQ6xxQ~e(*5crOU3b4M*^?D|o`?JT`swT559kk9?CPYv`w(xr z!$%5aV0Yz&bN}zR0K#S;K^(1f)tj(54Bp0qm2u_jus!1WZBsthvg{3X%1~UUF=1>G zjerkN^IUG#wj%};P7mEYeB|der4t)o2UXL|NKQ-J04m@KP1URo1b|#4cd4LZ<%F(R zgc`03T!$Do|J*zqZb&k-g31FxS!BbH2PWLi6V;ZP;wIQ8Ar}fGoQkCpV&(3>m~F~3 zm@|fo@hPn#5|M#a*7Z5boVxT#fXwB|68GoZjzqVlNja!ik@^o4^7&Ky|*Hm5BVmT#f2mUb?j?Nrp0N(1DVNncP z1LmiSkjna71IBv}~kyyLH%kSpzGkCfB(%#GVl>KYXTbHZbpi4EIO%DKy` zXcS=SVv|~zair2c>+2%myW4Yv_Ys6M;?|_+`jV;%MuEMr%{P{9S@Evu28XNvT5MOD3Q zE~b44imLNk4z13&WA2v?1cY-E7+b3wPnbX1QRkC1>#DC7F2$bkt1pAHjs2$Uy?J?e z4u8nCIRF{d&`crgnZqb)6)Ou>^s;?vvtKLdYUYe$fNlw^Y0MAU$x{%?CuL7{^SxLT zoTLZSjt)-@p>8~|qsSK4WrY;jE%PD~-Flmp6b@?atx?)NW9&4ckx7q_F}BSF6bmrV zPb?NxX|4NYYMUJXQ$$}V0>#us>TUjj-wN;*$OF*n>>nL?)9OHN*8|tz!_&TM=BVvD z7vBmrtc)$D|UI|Wtu_#)HEJ>_m%TAYuJND`jI!A;~r)ph1-1%%!o zC1m@la8_#x#o*&=h|+@I`nol*W}usK8{Gvqf<$#Mz1CGC!mkOrtf4R9{~Wjk@0mHz z>gGP~j#}Fc-0JH;Vzc6jdsV*IA>^Ia`2I%#RW)eoXSYnF_6_)#)ItSsy%X&Gd>nCs7sc5+b&ek#cdRworvktTnOU@ zMZ+n&;YXVb&&UkEttS>4m6|19Qqhftjt8N&la<|jP*1HV!!Un#F7nkp$qI2pXD6Dm zWVY^TDX`?+0bwUmaZGA^Q=v>r;S6ZYg(4q@v1Juo2BMoIXNXP+_^qsq%2p}52V)yV zX)sHw6Y0aW4gbe~9Nyw=gAy0#fGey@30wadJ^1Chr zW=9;EqXLmt1obk&n~SjJR2b@U8L1!hJG}u?STqN$fG7%7gP~SuulrpzL$8eYwF7&0 z#TuU=646-;YReI z?JJ&-OFZwFp{sQIBQ%O$wRkx*8~r7jIL%TGy`6dOM%hR$6iuG}E@H}cbGZD* zA}eduifTh71hkYd1l)<4xo}HweZP2_ew~wsoWW-}KtGs7iHUZi$c%1g)#+K&skfy? z+rm5)3oW~Be(UoKDm0RH4-hl2pWcqIpWk|Q_^5TymW-I(XYNfEoA~9pl$(~jYqOcU z$TZzf4~sxo>Xf(2^eb&%cHjCL*UO+h6R+8FH6b3iu*M|`wYtt1hp^OL_jt?bozvx! z^ro)`3lFJfPrM7fbKf0?$nWs+%c^XCWJgPE6!)=*I7EL@dp16%D_<8)?(=SyCZp*? z&)%~ikdVuZt57ZD>w3K%zMzqhv8F3ZvY?M3E7%A)^hm+^P82)Uk#&ebRZ{NtPr#di zg0^%-6QPcxf}UZ`i3*@5Skn|rj%%0Qz>4Q3F7-}bi{u==p(+ZH&`#rbL(6!Ubxg&0 zmfut#VniYWec*H>Yp{mL zo$-7${-QOYy%U(NSTguqJc16Pr}Ss&HqRv*`UI-mQTT`(icMc7uK20s?J)u}8n5n( zhnl94Ha*8L4?pQLa|!+T&FCwG9bSP%e5U(pz_efd?yO-HykL`mw;rOjn22V+z$sIV z9oqa)HLA!PwLzvW3Ds^L65cEt3bDp@wS00;BVgZ_U!ODvv`=dYu$+jp&kEZB!Gw*j zA%z%PEimaMQR*+BU1dp(YXi#yX+~4os}j~0Fe72tt|gKa#0Dv7c0LJ+d31xqY^Egd zWl?|3^b`yeMqGK{y`9nYDr~iZPO&?|7@#z{f~xt#^$3 zm}>W(>acbG-uEq1XYC%>G_}nS$?KkgaWCrdymzZE+xhju`zz1CiL4)dF1pbHdyVbU zD_Bb=^z%G*T`CCv>##g|*SJ>v@6H%2DH44R`NkOVThfIe;q_05DX zjc;USfwNymEI5#fAO&H4tG&TaRU>}e=b_!|4qJBT3Pf_DF!lh9UdvlHJ(DNH(xH6hG%y@6bquGg1Yrj&K* z_jS>(RCgg~7|Jar2t9uZT4q1ez$&w7ZZl4!`oYK3JdoB~H{OdA(L=p2DlRFr@(Q94 zpOHnsu7^}XAPX5X*l)Cbl_`umgW1jCX9aIF&*L@jfcaGUsXgO zQ~(X7*}f)T6lxvn+x;rh@u67u-u=+beNPT5{H4t54MiIgP`j~04@LY`5FKf39m^?M1i zrmwdaN4>wRV@6Zhp6jJamjvMhhI#ZB`WtUk5p*gGBJ%&2nWQ2itgJfWjjX8Hv>X&$ zx#=WAW8LEH(;wzJxwUm~S$9=Lsd~+rF0>TGGs#=UaQTK48?#D^?eKLlC(CEn4Q z>@I?PZrz$yc8v5DK8B29<~wugXsQZZacoi&o`l3tw9t{(=Ar zI((#~n2e$26p`qb-f5Pnq_qYeF)a`Z1$4OXq1?WxTtUwTmqZanwKCb?jI&*lqxUCE zjD-#WN<{5!A-bfzH)@BAq@}IdlBdG8sjA`2o~^3}kNm+m-mjBLP5D>O9N4E#Zr%RE zP^NRq-DZ*sPckX39_?(g|AJ8CP=^0{h)#K8*wguhB9I)L?p>@x0CYko!Wc6BXTKY&W*QDC-1=~K zXn(;S$>IKUkbz*sj>BUrwNGdW*yJF_u)rA?t+$0x#yKc=qD%ZAnHn z3W8yQFqz9LS@ifu(uT~+%)gV4R4HLF} zkZ=K2xHKQ8xCoDyY?7H@T{Neyks0O%P0IXmNkCl-pnKlM5f5Ih*NX2AL)VR@>jk<} z4QCYy0B>z7rOnrI*U=q?3SXHgmAVEV#JQEPfb6@zc{Z>agM+G94a}9)LZS!Vg-|t#8w)R1r2;YwWfU%|eyg%z0b_?-qIkYT`^aU?$+N z{NBg}g@x-4%Lh6AjR}kt3|v1yb6a8#=R7Vm8>UY22;3S>sqyaos-nInW6`hN0m=@y zMDCMGsdJUHL9nqqT+>;8%?`U=kZ;3Vg||UHl}Nv(dD2Be%op10*%t(k0K}at#%t(`5E1K4Q&NZ$GaU9#joVDAe$4&= zW2FiA;LR1^7@k!q;~y$Ih}=|{2>o^e_B=u&;nbuc$lS=b_t~-ef$e`G0bclbB!YNv z_@E^wtlusbVa|VCT24^Q%t+Uv{qZc#GCIBKGmO=2hKYJk8_b8pK9ELU9WUWyhgCJ8 zVT7<3BX@UldteAh8-;_cmWMxAdc{V7v!048$I-1QNAYjYWropZoMC5%*?47!wX;HC zaI`#o-CT*G&8AaxVrc2)xY^TiarS_v%^Ap`0Pt69gL~9BQPgB==(fY?CEu+lh^SU? z^zu<39`7k`R=>-5y>YQqpBISiRdSQUI){p-)~(Wbk}aZpz`<6C`nsO+HV75`oLztH z%}Z+=vjgAZ!^U>^ze5(6{$-4gzI4X%1BD|TI+QGUFdb48xnsitc8iAl3aL7Px)q3Qr0g^0g9jJHU z8gB=QB829EtD-50Kqt>#CU(y(JmRj)q=Xm8b^*eH3W$9)&mlwvcvYNPnVP)gB|_?~ z+@n3hkRewz{}BGj5K`m4ui6%&s76w_qE!Yud^gTYb8YgYQo20oCZx~a=?ym+Fd^Rc(#+%c3Iw@X9R^-10IxY!!h%kuOw@<0+ zX0&5IUL_~H1ImDR7n(t|mNRp~W@)wEOo@BwC!8DbsT&?Vk!WH%+%e1y6ExfnOnzRM z@lLnZmCiG2#}3FK3*btth0DS}eS!OR<#Z}~)%JAcJ$3&JOr&~&V&&H2qijFFY8*>q zE=wi!Pv{?sc~3%m&ktL@Xg&R(P2bl(uTEMC+Fj}W_06vtxGHVgR>?Z8fp&?$=5`zY zoA1yxsU)y!X7%{4d-TVryu!;1{;AMYYP6^0VmAbuVk~J7${8 zggppWtC~F*LrNL0±t)^$>@mtZknrN8~-E0~DK@L?cQV@LEPGfX(#zD2hI;2<87hFz$jfD?#*5)w*jFYFs5=v}?!*ZL93DN53ygbd<$- zT3B89tc zkZA?orJbg#6*c2V@eSeEV9dbDN8`_hGs9==PD+?!q5{I?Sf12G*tYKM?y@+&o;~kg zx{Z^w^^2p@YBT>b-%IN!=JubWAE{Qv0jU)WFhyXA#~sa#-877Vo2&|{y{xw3fsik~ zBjA=~adNamL?oYvUIE&H&lMP z!CZ@BQ%uin{3E;ldTCO`obfb?L~V9kt4vgGK3?RhfDvbyx_u=D}}{^Z5K%Te2G3>Cld#rlWfS}UKtd@(KMjU*=Cm=oa6uRGl9 zf6b)&dVU0a^KqJ3q35q3_)I>i%jX}Yy+{b7+c~Ss#7i5`L+ERohW-(h3*LWpzc9lI z1!vW|II{l;==)|muHB%mn_36jvrMg$zbQ(YkrRl53G91hmGp!^y+f=Ea(;M z#;6$lx8WkW#(LNF4t@m^;(&|D2L+(<8|jew#%E+w_&IYOA#GF)+yn4+!}mm9)%^41 z*j?<8e?`9QB32tQDdfZCO``&V$+pdDptSI4C?A@5x=zh{xA!}si)veLV@lA-N@_Q; z5w|S8-hcTz*u8b)A1`kU6ieeCxQ-Bc)_D5=Vc0bKCb$#!e2_v~tvy z+D58|{}0!}T-0B-aBi+G2(7!LZ}c_rF)MUt%(^EIOdl@G1LYSsI+04PK}|nqwRAZ= zK9h`d6Cb;Lz37~)G-!fcq_Nyk7!b}yn{_^+Ti`4Hz}0e#tqjCk*G(idADv3duQbvH{aXh7I^98td%GjSmBTn)MGbY=+_LuJ|hl81!AT z8b>^nA-B?k&tV+P!{kBFOwx>NP!(umOQJM`DUh}H2TR_$#N1D#jPozAmq%)A zE?GvVq*`}#EYEAuIh!uur1c#l=IP-Ne7wK1&jX-aLN#nB65}b50fhNs>o6uz$^WDn z$Y5%tfgTJlmNc{I2)NX9-`jkK#$b=%H!tPgbw0h5t zvqG;pN5|5?rlRlYHEzN&m)A+3{@9NyPsmd{-^i~oldgC453eUDW9|>icP;K)jB1Uh zX?PV6h~vYAy*BNcV4%)JDlt9Gh#@FfP=AWuXLPy6gM*`D;+Q@yGDdNBz_B_|GsbOU z>?7cA&(=Kn)Pm_s4Qqn}V6(7W_%Lf6XnGUG74zB`{kKP7{d^P?WJ6wD@&r8~_vm!! z=X(PVK6(7651roB;babok2)--b~;#&5*~K!u@T6e!grSjZG(c;pXe3Pk6Z_2$$xPw zhc|o;sGCa;&Wu@Y!2T5zW{>#y!BRqiTu8l4gGD0+BM9?RxFr~Oy84w7CtP#S?6!Im z;HeGpfvab8J~z+ha$3B#!vc1U5vc|$*BJ$MfZ5Sj=52VGzbIDLENOsKYPD(&e)-P> z^h^NShZM6usaV$5htUzg6D`z58ng7yZ_siZUY=%Ij-j>gV}@Br)DAfFkncb=O^P2r zn@p!z%>SQ?&!cL4?Kf(ORiMxMUgIK@H!I1-X7m4N^tC`6eWPXe{y(Cvz$@s$edX5e z^0bH1MdNSdgu`R;Uvc~*K~X_ySJQNx>qdfyNXzTTP~Y$7mo|6HkY@&>*GD1x4xg9> zFsI}wUSAH?x2<3Fyq$SZ`_-DqBP-l>)}vj*N- zwdxr7SB}JRn4utEm)-38-}$7xJxkJGF6B*9Kjtxq^m!o25ilU;=J9VHjrNJkD=?>X`ePWo9WIhPMmn&G>pz?k%p1r*B9^I>J&~e^C@qldmz(uUMhk zuaXwJ&g0SpwxTk{NcmC@5~r}B=n{`b&8vADl;lg#61@YteP3bH_S{H!xBtNPGdZ{J z$I_gH#IW?h)RIC`k))Wrs{v5b|2|x8Y=R@iazEE=8LbinT z>_56H6nouW25hpn)@C9bGu*e?W|)r4tCu~)zlc7UF@1z=qzfM$nImU~p$AJn|8%`S zA7-2sGaEO1hXXc{VorX#n|*TP9!Rbj4Gc%W7yCoAk895Vs$)rR)Un^}<~IHj%qIlt z^zQ5Az;ayZheLGh&o9h-E0ebPDAGK4FuvvVfYgtomzh=P1@1mppF*O)OXQoA9>GD} zCv1ptFuQ329)zrz333*LI5pRy^OPm!=kl7?s#KM)C0G!3PV_IE3;)HC=-*`c(WPGF zjshx&UP^7boD)*&G28{gJ-6L5$Bk~C5VWZI)|N5ZUkh-{>Lrsa>@Rk9%MVI>O4qgD zl&5-Vmfsmv*^qAzFU9+=QF3PxL{Ueei>kKU?`G1GGEVM4J)3zcxns5*lDB0PVn^>U zcT$t^hdI8Bsilxvd0YRJO&-=Kof*qpF%g?Iq# zh}$xnq5i=oC6^P&%e0=Ze7g>DKf-8<2jgBrJR9>qgawwXjr9#EE`4vNxVs?Ek%sG` z`mRsD3*A*-L@W+-?xZzEy_n~HTuOXl@wn^7+Zh)|7re5Kx$^b(x#jWzbe}9gI zqA^|f`Zp|o@3Z=(%U{n){VJ$}t|$7PtE5QpWAI?id7gDl)D-jK>^!eh!T6=x19|gt z4GGO<(e%6*2lm}uRwxK>|JdrtpJ)T&Hvi7Mh^S5C5uhrwDMPDOH}-*9odyi-@>yV1 z*^Uk?{6;MapWtW{>C0iCt2|8kSL7b`bI7&^hbr>|Z-K&MaIV@H6HC{P*Hi7PIiM$G z1-l+c0wsGRWd&xr_EfUPHrcwuUlcdWyLty)!aaGp4L%X?m+;8jwk*xG*1yu}AxQjl zbOB)mHYVkogODe)W=VPd%MW;rQH=oPe^Lj+$4w#uxK#;wwLju|*CC#lY3kVrWFut& zNZP3fcpkFl`?;n|9oC#@2>Tf5B7&MafCYBM1nWyzT)GFRdi$X~C{Ew?{?wB6_6H;Z z+7XS)ApyYwCS)9A*%h8}x@pn)ZQZ!NGwxq$1d}t3mP(?DdcP+YUu#lK=%#DQ&sUIM zIfmC!>AV1FPQw@4*lH}=cGH(@BRj&q7494sv^W#La(vtx=)bi9fCFLjmIEGuKLkRE ziBkR*ig9U7iy!?KU|2aEDpMn=BxrcN?04ERxiP@kcOGz}cOjj<^n(p^fUnB;Ah#s= zT0YS~fu0Fu&Z(Un#+U_W?*REYa>oFf8AZCjqXz(sWcU?P3n;UIc}>0!6xq)9KzsV= zacs?`(#l7gI>Zz=FUd6LG{^C6NQ?)QjBUWr2aJtNb^r{Le?LH_9s{0GuY&9#dj#1F zkiq$>!qirDR6ma8_VJa8Y-&6GkLP+DXJ^$J{7;Kb2M_EUTGDHvs>t#t{nr$Kt4)BM zjLbTOa;W!3`eYGCiu{SyqR(_U1b{$&EMwmBbxJGp@pnUh6wq}OY%k5j;gp654P_=%abNAx&q*y2G*$;OLGB-uy!zIC`n?Y*X6r%B|YfNwf|zYd_jV zo~==S>>Tj+Mt6BLKZriK$t2nHP=BF=tyX{W01x`19~e9Oa|~!F(k;(M@S#!q1r~sD zn&k!n2xq9CLHv2JxIwLKaa6kWXRMfkerr8L4z%6tndv>-{lhkAjLqkSTSg7=ZpEse zF{>$UH-9b!exS>@i0y@ih@h0>PtGmw~P`*9T6wEB}$TQIAj;3 zJOW^zx1YDd*|q`fl?B%QYo2|UFm^Y`!!1Lj7J~amN=e_Z#mya7UBg@|fnC7Kzq1|M z?VS?%#XQnB`W1PBIB7^)ZZoes%!F5L36!W%8_2H5h6f{;M^=N2GRMPTCAb4BR-c~q z>*nc83vtY`p0hUPiL=MGlZpQ*A6f+_BkM=rg)klQ3A|KnIF;*Fqap4P#oI?XNpw{R z0HH9wI}+#ive+~-POCKTW}b}{RZ%D=sGL8#`xy)X)_46!A|97l2*c5xq=9b$jP?OM zfd@m1Qa1N^6D+HNq8+s#^|@Q8!U+&Of1ZRD17H>6`ePMJ_Ori`_;RT?3@aI;GN{{w zeKXY7T0{_z8Q$}p&Fg{$&=f^Ua^0Wx^!ZEX#r~e^=g~Lywu5tw`3Z< z2s^fXk#6JNMxP_iQ;cv($`}#;u2w?SYvq}NqYw{MX z>w-MC5*)Ta7GmmttDn1~TF`+cRT&b0EkXE)Px}Lf`ri<@|A&EQdg%WFVoArpsNY{W zo}T$nhlcxU%MQ<04(LWx^QAcdDuFeN`t3skE<$Rhvy{g(BYMq2&tq71X7>4{Ytwy} zlCtL-{Ynn>fc9}T$E13865h1ABg-P-_w730=)z_rqt^dny8ow;H6th)@=eqa_ET;s zWd0#}1Bi6!p0tu?^!1Z$d%^0Tjz4EVReM75%$=Ge3WJ6ooQPx!o3*|#fN%sy6`>z@0=`NKkfGj6x#j3@DG`=2!v zeco9ivzMB{{-)YnK0)ybkz8|oxpJ`P_us=!5U|EgJW>Ln46Z_Iv%kMBd~Z0ZQyRlf zO4qaT+1WQ06*fK3grf!ERL?s58OYdIrnz9Ug;}zx7s7Ce%+$wf=j01cX+gWeXZ==! zD*;BFPyj+xZbGOPKb=FoEY{`hhD1aJ4LLXUZORtalWuCF;H%P7oXW@uJA2#*ps`fmtQnsr`K+$s6!XoggysRtzOwI* zBS{vz4!Py>A-2|Q)@$4A3TUzoqOr&xaI^H~l17=# z7KPzdIfYw25F+_NmvrWJKyO-FHpFAn%my3u3Vw4`2}g6n(R3;UFM%K@02PI#=o}S@ z*C7Vr?nxWlHddFtc4gB4-p1h8c=F5-o}N8A^_%}6OzL^|;HT%XeLUoB_&`TEX=Lbcj_%^#q; zT8=hNPt!{LP?z^%d{tL5OpcMvffu5t5QeWBl$2~B{hw^z(#|f+f(86%M_Y9i_ zX2IM`zF@mhd;d7jkn(}Xh>8~(Vp1>qgFRVp^jjz>;;ntjR(eHxO?pGRpM}HxEFmi_ z>dmX_pT^{wmZqMQmXhAzGTagqGyV=iH|Z-tFp+%g2+|Gz18RiV23p2_sj1X#xg#&b?2b)0h{Fpg~}g?ul`ut^*m1TM>!^Cv(Ft z4;Y;{e2Yb#jp}T$>kxDH#lpNjI=~Jy3;AzI5>5Zx+wK3~di$fB-rmDdJ5&7UgNB5* zs>dZfGZcsS@$z>*0!YTS<-O%MZdXBqg(F2*!(+_4XR~wCUpJmOu6kEsy$>jv2!ysRyy$y-F2}~hG-8Rr(U+ivBxc9@a zATW}8i1WQTDo5%-)VhX|gI)=mz@C|oOfR|l_jWpu9gKNrGL?0?Q_WK|6~~mWDL+p$n|uz4+sN!n1`OQP&RqGS0NBLE~Dw=n?8%iYwM>pI&$>0afyk_992 za7c!duuO3v54%l51I2!PST}}$k4sX=ByEBGYP)oz>ki&hS6)%m|qy8~0ryzN?ikA)dxU zJZ9^+~4y~fwyrwuZRbg5&rKR(B6hm{f-66_+v5V0f)SA})%JxG8l>;K<8&@`3jEr{W7i z*8GISY{1n9g5wOQ8ubb9CUo)wYSKUJcHk~UIlj}ZOig@>>e7+5H{b#Hnv-)Om0T-H zZ-;rxp35T*<_Z8-P!%I!r^U-;!te{%Eo+@rJnU5)mzF6mP2zQUPQ#!>*#oW}``|)1 zytCHUs{dqrIpkJqO)8`Qu#CZyfZPwE{#}rhCndg@DJ@@u1lw9_8}EXi3ZQ}AA40OA zV)u$1dFZrdKG{w-*c}{PkrJZ6WEr;X!;5PjmmdU&talWS5f3pDebBgBWvpd}GNf{5 z8pr$NURpoSpsHI}Zt%pc>zVA=@{!{a{PzfFm|CNM?AssL_`JORe>U-3{|_$Md%93a zpX+pb!_;vUV@Zm_{5DfX6LCO3+4x}VV*dMbs{eU{PsqlfrE5g7Cb9G-nsU2=Cj74pQFXY^COsH>RAJf|H5zH_SP3k zl3Z{usfa(h4MexZ8jMO3r^-mIz#;N(9@tl%+^A|FcTjK|=i9vX_7mr}OSU`$!i<1O z`LUwFNWQ)4!`~n@LiV}LiGaryRu6=l>4~eu+FGp%@M>HQ)Zs^Ts-+r8D4DxWZKWM| zLlQ)vK77W3k6hw_%?0=%bNh1CixI(-QbybGh3;Yz%fCI;5f4wO#IS10=#$*7Hnlnq z>>JnlFW8jyMQse5a5%(-I8-5X6C+JH^BE_VJa-O?08kA= z3X^*73M2>vsZBo$Q+a_eCU-LKsw2+UY+u{_O`l^lXdqT zwI@!58?(ki<{pu`vXY~!`DT0g9eh)H?Y_U7F`QCfzHaZWm}}ZYkBX6_3Mj8q)(j`Q zyHRY=%O+TH)m4AR0JdLDVh(8Hmg#Gz=Q4QUH+Nnz>*AO^HW73C3PN!g-MUP-y~PZ* zAVZd7$`6A`PTzf(kZ6T+hw18dC)RxG_aXn&I9|VX{;5Ctv(6xxEq|q8Ma&AH6K)Vx zJB!;ygD(HCm6K2h*v|{uZ&0-wyFl#`oEH_N}1rW%jVIfLy19Ae{JnW|h zlD_h_AulGB8a_P`0zrXZ)PW$v;f_b#4WN`Mwvl)WkuB~qoxoZLwoD$NQWe0e8VzZ) z$!3nTgp>nq6RVvq2(5Y`iod98BK|Ogtz_d^m@WeRJ3mXqLU;@S-B=hC{XR*CkAM%L zC|HNoFa0k&v2JDQdAyyYn$3j##;Z2z+Bx+{Nb-KLoSryf5bw}zA=|x-_rDiO#tg53 zX0${cg3F7wvp~7(v*^ibxVU4`ur(l6yPe_3+>?Fu^;&1PH_3>k<)Sp<_Wcd(-TZ65 z3lTZRYc(^Nm+B+x!u3ytSJ#a$T<5HKRyr|1Aun6>yztf*hsx%G$VdGRO|DmHdLRHh>8m@A`6#0>5xcI3G*G?~ zG&!jpC~Pd>Aa;)lcJwQXr;*Y^v-lAla@qYSZ{&|_Y2Gu>&{c9aFz+VH)CWNQ|F0<1 zb!_A_z7Zg%MeXK&6b~YuvHvr@1+WIo49e+|qx9W*ZJ>`M-owO=4W3?h9Td;(8*Y1@ zg`|vKx+0RU?FILYJ`$SXbe!>DYF_D~jt+OO=g3g<94h6TEz+hZ6XQ#vU{`!iv~$ED z<89`5h>_m4iw9(?GI#gE+p=YMG!*q5C$#6lhfw0Vrd*P77 ze7T$+@d+X@is36lq*v@K0+quAm9hyDGe>Tkj`V2qcS-9-?w7owU25{K(O~2TcQxV~ zlGODSen15VhPrSI3wbQ|9UYGJn^gq2ny?yE2ZUT=Z~g}Dt}(e-6nhcME*9>?He%ts z5gBG)6cL^o+X~XZ$gVv0zmg|!a-Q=&4M|?^nIgP1;3wJJOh3@j^@t+LaDK<^$}N+Irq{bi508r`(RKFp4J*ErVPpuC12m6h@dNr;Y6Ka)9_AY~F5u^>Z#;YUP zdI>J_QchXwEP&#eGr`f*xtufHb%>%29r|@VAH_}0zZg*;9@FXRw@SZwzP~{x60X;y zhiIhB0L(tJ2G14<3Hdq6&C2WmWSKNM{%@8EC{x6~)4)CjkP|oe%GtUD4fk%Kzv=w{ zE>qe5G76ROG0hsHL-pF2^T8-w!|Kr*;?$j~+vx|dhWN6jCAzk*r;YIZE~;pn=6shd zb~Gt*t~nzEulG2orz=Zja}dzQ7QZ9SUH6u&s!>08*RrE)vLCLael8hiJFv;?y28Y> zQU(%;-~9)EMTFjBQ2*u%oOmkZk+)t!IL7a^jOi`}n>+%}1fqT!7t`11ZRH%WL;Za} z6719s0p81~1p4{x6$eZHxsyjan8y&LM=;WV9^9iMKuNbOxaD?Q`MLVrJ?ewci5)&^N%!d%ZKi8+$#r(n{B z(YHEAH|Og*OV_WvY)@V7p8GUcPrmxM)*Zi<0rD*}=)N(&UWY8~i52USChNQ2q(r=+ z;9eIjMu|o0Lr`XEj(IZ8_A2aI$ptC*CTu*Kw^Hl5cvoYz%(XLDk z9?E1Z^e&f-!#KYuGkX2JD-v)}sEX^|=pFm31kt^Fw_`7qldo$+eDfO`{h_OV1+6>3 z>UMEq#qxgddNV3GWl~fE*{s>N$i7X-p%TLmvcMhU(k~xvrj=sJ=Mq#Q4C$5)b9~M` zG?5_6q8upNrdz!<#T6g3#u5);w?@~>iz_izTMiFG{T&<7Jq)lP~zy%0~2oEx?@0 zjQ|db@Z^&527t?_q{6Fefc!d#o_A@bLeOVv#bWY@gR|X{h>L6As4uXM{ik@+ke$CM zG1TU(`>xf*Rsq<@Q#&72aF;~EN^I7j0XdAx;vCplReAdiN&KLqXT`L*F?NDCQF8#g zkCC`#^KY_b!UL;7Uy`GqH0q>YH``*Jb(vaG4zB-tlPI;?A?qzNAo8oT&FDAI&P->@ z!L@jJdFHV4_29L4$wlwX*NY{d?(0W`!`F*4+qN_u)(W$XuTP%S=v^O9Jiof2y|-~* zLr&I~AJ$eD#49WB2n#T(&k5UkiZG=^(LB&m1+MHU2E7kdbqm0m0QKu)N3OAkz}Y?I z-gAd)R~SBiJ|U8!I3d1#+l2C!7^_|OsMqW)!6O|EzRLE`UlB=DF*CaXai$OpTu$`E zG0)~edSg`0H0GBE4}9A(IR*bG9}9a^Go-XG9?`AOOms)nVrE?OG`V>`eD~yyw(?A* zj%@i;1VP{Jz7L0wxot7gILh!A&fld>nn9y)SDSWy_DEa|qJJ~tY)#Lti-piL2Hf_0 z@2gx%ez_u6eGs?OH^a(k?PkIlL0sC>z7w?nKg*RQxxf)(;cWVW!O@mPB_}DTRLw{- z0-nH^LL2#Jr-ps~u~4=~K3oV6yLLoC`tOw56LaQ*&>mK12OPxa3`#FJszACtqxmo zim?V4z;&C_>YD87d)#)&i^)+4TRTrF#Y>ehovghWwtl*@&vJD$5v~pCd@l@D6J*x` zkYH;}8_zVG8(;##*fMjIa&{L}krqR-GHa}AK*{kKUgY6iRlDsZkFB>~v3?7RoO4~2m-kXNZkJ_1m z6cC(vWcTnZh`L}`IHe*PSyWx;+3M*PZPH` z(hB}bqh(91O~O~Yo*c&qVwA{!PZSS+wqzUW`rfjtAvw}~YW3L^pA##4nM!n*R7uMB z_%K}Sv{kCRH}rUbS%kkW=3|DJk9(Z1O@5CtYrkl6cH&@?dvq)}Hle-A8zz@}4Ub#U z)Np&JcL}~v6dCexfJ?g!)k*qtJcYP4>FCRgrHXm!eG#8RT^}Ha zLg8+s)Q)%yPEBLg1znit+y3=@z`tL9SkJH74E$(j@W@!JqZ!r=&cJ(|pk(CPEuM(c0dc(%n>n^KAj>Ji?{nir+4Q zNQL+0n2n{qovZLIP2zwxrc5IiSCK?6nLd?$(yXQMBc5d{gQ%DyCWv)Z5ylxBVE);V zV!jd!)GZk^l6V?1L?2087M;0H^ePIkyI+}5oWiXs04MNO%I6PBE%(u{{<_qS@qiN$ zeptv2ALZ{DL9Sb+Mta5~NqT^F5<$~4d8I+orQzdLs8*b%V<6Om77qsJRzHqQBT|IR z#f)oLwlG`O$*OL~6cRHGU2Ti$wPu#yjd~gMN%u+i5-T+lBT~1u+gM9FIoTV8Dw}u8Uud~ZlRVn8P3yx!MB0` z3w#p=z&DMM|4oq37qpLVvimca*064I-pr-){vz4ze-R;I6~zsr#s$8Q;{( z8U_xH*WCDVZgk(P%%TN2!_^)#51n!_O8cVxhPe7Ue{6`Cu?My*e#lCpjU~fuB>F(L z6h2tdpYA`;%b@D3T-+Y+-()Jy7MzkxR7~t08M8IGF~dqdeIQ_#{B!sVn`3{px4|}+ zF!?q|q{MX6_REurt;%VijVpSRd1n201GhylC3e@3Cv|XtzsugGEY6rYdGwQ~G0l`} zd_&CRaT(TP_5^6H?qQ3aK~-N~aH}R1Y?2to6r8y^QdElnu5=R@Q09YKzwn6!aB54H zR5UMoGCrx@ZgKBB$Tek8kwH;A>1wh23Y{Nx&;sWEKWk*E9(InQf@gi>)Ph_6HQLhl z`?6Y@xY`B1JDjR&t64bnwKG<89$>Ogx#)zWkExr&Ww1cdNC(ZSsj9YnJ&VbUk!~KG5Q|*vfhgPqxex{U3s&lH)nW1Y44Tg z@+YUMllrZ{7P+QC%g%e4Sa{k(=C@*48*4MvZm)~~-El<8@xW}z<}8T9>OF9!uJ8QP zdTrCsnI3eQ_gR4W>k{v+oDD^jUw*&XeRFN7n90zyT@{W$AND{s=RC4((Oc}=*x^@h zAsdkYvH*mdM#3AVHT=h@HrLA^)MeU}A3=7LJovLMG_s#kupMRdSiFccA>>>F%BrTT zC3n|-oB|zo<>@{$N_;KzdQn7*s|Lbm;8cgBgZ1HFnTmXr4M_r^IPdRsFrUcFu;hf~ zZfM`?KNZvXCD|`Wi3t3yGjXfgUl-kUujrNr(u?8Or1N~;Gg9IlL`{e`M)B$cn=tQ+ zxpZ0vk(=kqfg4a^d920D81=hsReLVmwCQ~xx!3nN*mGh6+o9={8>xvQ9{AbyLk0IBrIv0GPw_2e>)*W zg4i=hHF~?XUmgL;D(np6!v;l|a!VH4+6)tqK5`jg` z@97__9niTP>B#?F29aU~Bs0%Dy}5tv*Y{U{eJw~dU_w`Cb4q?;6k+>&^2&L_heUoa zQoqW3d=pR3m<_X1ikBV(d&G18iGpeEfPil;Ir{jzDg`P=SprK-a9*Ir2%Z^oe%lne;$i*j^?XbfEkD z-uXqMZ#`lxR&r;xsp5%)62M<+1rH7aoG!XK6?({+-ebPMdO zKJ}hvf|s(v9pe+CPjh=RHoit%GH|5iYC=vTy0|o>e= z&voSeA1|@CC-C6otvx7vgO9gby)L2FlfQBe zx_j92<;iXwi?UPLB!PhJ6Bas*Q#kja51Aj!phC4*0IjRz0eP{f1tx^T%JI!{>**|t zhY5Z|X}>z%9;1nae8gF4&>20T{>=_968~N56AqdGMFyM*oZ_y17zdvpxJT^54g>GT z`V7o3Bmwz|jRuJ~-)GN>g^mypZ_$5w!2+X?7oPu<)XWtc9E+?}d*>9UiEN5mVRZ8c zthu_lSDp8SKk=v*<}ev_yf1FzJwklt(m7jGeb$@OA;X$(<(+-~Dfo?5UpC9k7wVYf zTAEztJLU%rTf6ojtNKGGa@DvmOq?hzW8>Rgt2MYsM3 z;X1C9-Rx_7T`A>wsV*l#UfY8-=Y#Du4J$$!`rd z_bLw7ZP0mqeD)RuV+GH=rBFo$5;@Jw-EX$MF1!ydBVCTLIa7d;!GP>QeU_%As*!wj9`GP9r49K*mZX5AD^BOiYeR@StTQaG(2zc`PpSS- zZhbczz}V3GWfSpn3#Sd`s241bz(~82s5|%i*AuI!G;b4aK0R;isjA8~%=Ig7GAMY~ z%MjCp^DE5;{=9Pdxs$Bg+jt909PM$8r}r&O{bF*gG5HCfLV$dHK{d0gFA~hd+cy`? z-R@49UGdqeJJ!cHE~~ipn5`<0Nr|O)^joi~sp4_}7+5*s?y{q4~Qg=Z^RT zTt~e*^SFP1BjnzlI^bZ6oj+J<{VWi#tA z*HC>%`#J*T7&rV|)n6g=w&z9-7lKK?m`ZZFTA^^iGajNBU&|M3)=MUU)e_6N$*z}Z zs~f^DOD69Lc&lmQkZ0d)Q;TTskCV%r=&Nas>L$cTX+o|v^~IDRq|6W>W`(wmivFha zEM#Jm=OZOZL{Cl-7(GIfodc5c)_QDhFr3LyDxMkq1yD`}x8lng+rP>YmTa?3dNEIE znGFq$&aNp`QD{@zqWILq%_-vGLUPKgi3rr{fZEwal)fDW)bxff3>L(9LzKbl24(L} z6NH%Cd1UJdzVU;g8oI$tw~uQ)$)C75Q-Pq6QSSEDH(=N8<@6C%z$5LFZq9zE5?u7g z$EU7YdWM>2v?@hngv0iq;?RQybE^>6P5Wa6nU426R+0(uv41otR=SU?a%7&LCp8=| zZb}>qPtX->zEsC402h{^%pY(;O$Ibe#$GUcfZ6d?e{`-nd+W8P=8^EG1!8kxn)vqP zZakcd?Sk-VbwO8+X)bM1k?9e=!#ZGJM}EY9h0~FU%HZ`+OBg8M_wmbO{k*Msq|1(w zOHtbJo3QFX{3RetJ;YPD=4Cifsl1;mifIS9lg-IJcG7lU{?-A{1Jb%9(vli@UsB8Q z(UKERiMzyr;!D*Kx|8m`&gT6YNCh4pX)L`kwp-8*zKAnc%|ZJ1bc(?u9}`==dL-3x zY0yh9Yv{Aa(8#`BNYuCEVbcol+M!M3I{gx4QJYPZ*=~}IS0!r;`<+D6qUe~C zT8MPi9m+~GZ5jH_0!GK-?Q`e9Ek6M3X(qFtotI*CbMLoUAZ)|u_(2bJn|0>0@R-3w zAQ2@yzubgytbRICjyvL~!Q`7>e*SSQQU^FJ?g#~cs1?2vyKQVCUW@uCla@B~m3sL}kD4H41KFLg0 zWVP)PN5vsiuulMGfvjPbLmt0}7;Ar(E$I?B`Bd_} z4g|7PLnUze>|NQY`}~MPR9Fr_%4fWya2`WEK4UK8yYO{KA2ujwHki# zKXpr;br%IwYR;e0h!-uP-iT1vi`x=_oeC{1{j|*)*m#QUJ(2E zCP(Y|7A@o6!)1fKy^;^M)WLU~`CJe*K)Uwxv3VNuP>K;*WoJ?#eK)c^zx z`CU1*c~9pKXGuQQNxShDp|(+>+?;CBMLF9`Dao|A=^j-I!q3}?uj_h`EQ?DYKKp5t z*MLc?Q)&sB#WV>S;u-Nf zGsMVG2U-@_*=kjvBXu4;dbM#P#-?Y{A8w4qm>ttupD9-U$YZ@!ekGZFn@auiOtHbP z?@w8!h?3D03Ao5Lt(r+%!H0;;SYV@*lEXe(WskD!D^v-omuB(B*-*XVX^%|T&Rs`# zf#fNfVNo|hDtZbU_gKpDtUa3bdq>;XR@yc`M#z$^3()x_hn;W>l*itY}Hr z1Ty4g`MJ_u?2A9cneE$K{M%!bUdlk)sE9w~o&j*G#sR-bfUg8R;rQ%{NRw5e%aHps zvk$@H<)4@|2h&aP>w1t76Vd`f?GpkZT~@vl^ihrn^RwAfY;QPS6)1F-=fd|>JBW07 zfu(n>=PnxID-C+g6n#?ml)4TY%)A5J)uZbCj8$~ zG7iL3@8nY}tu@ggO#05^i1&@zpzi2xw;ncd_W(QRWGUV>5Ygs#a2e)D$XVcPng7&m ze>LGmv>SimH2_B5QRnU@y9L0KQkMfVZ_i%R^)cRFbsb93{cIDjUZossL(YQk69+S~ zKwhNRFBb1>j56w26ciDjFHGZBc<#nQe^<^HMd~y!MmKfN>G`ZY5j4`1J4^e$HsFdt z67m%&YjdlmT`}*i16N4*9({`Xm4LaIEcw$Wq8%cq`D3rGz`wKS`rh`CM#^6%R{)zf zE5Y8_8^Ud9oK7d9Qd!UVJNMEf>I}p9>WAzR8}h-IL|LP48$cxo&UGcme$6$;PF@r) z394AD+J7>uI;<7T0pl$q?t)lCgR$z%uMdWSNXqqf@wmyjA5?2G+C9ZnIZgWm`-%_!w2TNrj+ z0*?!|f*aJQ$QQSVLQe@g<;ProlMhN1K=B@sgL_!;RpY$X!Kx10Y>RI&8ujPrWAm$t z;s8r5DKB}wZmqwRAq;*+gu?e4q#|a#N;IqrCzUIJTR=Jel&k%*Udtp_hL;lXz-h&R z^S)I;^R%9zmo4z6;XFqu970``-~*U@M+Z`sdoI|-d47EKeerQ+EpN_}=?N1G^ByBA~yHv8Zg$>%XV z+XMY)tlJCr_i9g4z4wOuFI~b@4{xiW8&Z9}#P_jC9?!kC=&12|uL`1ansW6~&Pb0! zo=iXA7EDvbE<(CXpb4>mwa6GV3?)$Pt-C%dPSlx8babs_pr~!%({SbX``qTyHvbK` zL3HqM#U&vL*y@Hh79`%yIT{U^P8mt_y_jU(XjY!Fztd2o*t>u1Pr&%)g2foMTZP$8 zhFa8Ym5}I~c&;!0l|lVe86UR<(fsicA0@I@f9p2-AVdJ3;7JC zvSZCKgp2ST?R?-3lfE6cCHfh!lG%MWF?SmOGMuEtY@m;5Hko(2Pqt7y%!=wvJT!#1 zh=xZ4ai@XcIyIS4RVL2Ef->u99#W(d>D*q>I#eC5$&!SF*2c+3IBtISMe64R4|r2T zyUj^G_v_=Ac4Y)EjHLdJs}PzrO7^CN{slJffx~_Gam-_A&fkaGx`3zD)RVXHw$)b` ztJzT`P+V%FkQo-nR&1^JT4oYNT-Kw9X$t?&)?ww^fL;v1Q_~7se(|CdB%ir2&s_`g z41H!fn1lg`n)88H9V#||(%wz9rrU^wQ@2CH!(Dd3+EgiB5Y{lY>A8*X>k-ip3Uuf0 z%ltRHiN@gxnD5fMuF2=qG+V7hC~zu++U-kJ4VzSIJh#z6DzjZIQqAbiSr}&(5SNMV zdlp8s)Oz12U3v;7ce0>1byPM{I2-ML$$W9M8AmF#K2xd=vGmrYS`i62(MSNf#tV0E zJH%SZ6~sMC*!)ckLA<6(P5ESS;yyb+!#GI=+~i7RiOlm`vHo2&tJAcNC~)CJgqd+>8c3 zGaU8pqhh!WpirLMYv5Nq(&_w241FbGr7C7iD zUfFu4s(Sh4{HYLD1G#gM7aABYnW^FwwaL4v9W#Lp^#oEiq+=-LIU<1X^}?VzRzq}Y zsjyt8>^#z8M(^sF+lKdry{6 z_+I9=3EOM>mzNg8?|2~gNK*x2szKtmS&{dPOUXNoQrz?R601>|1!hx4sNG-97tuFDm#TfVk9)Ym^&kM$*a-TIoX zv{_(?CW&|6D(eHl`YhMj!~R*wK1yN|(ekJ@gSq!gS%EnetwtSiGJEFB{R2SG^{;om zxX0d{y*4spfbDwK9K2anT>X|-bzx?7wvDjD6uFT`})t;>chO-!4 zr#ok;1g`U`YR8r_EGh_hYDpTm%6^ttf}t^UkP=u!2_wZhE(rrh1w*g-Vw&a;+f{yy z6BIwK)}42Le=GDhsgP$m+JZ)Z@O()zfHrf)Tpme#lU#fkGXxi-$u@+I`}$SOro(oX zPo2VTzc902!zn!)$4ej~fw#`L>7CEgqvMLSIIf6DDs!Gk7M{@=S8PS2h=Lv?Q%+Bx zjwa9@9-z&_Xq!VeR`#O5RM^lq#%&P)4|(E^P3hR|)EL5$+@bLEkZcxr0mJo?wBYc$X#}q^R@oX$MFyy9^Kw zt)H;lLhLe!#&t4~PnJfJ-UkXFQ{5~g<4Po8+*kHL%Cz<6W!>)u&1=M=yP6Ppv}XF+ z*DIC@t8#(YF;yGe!-v26X!n0kBhYUZihtE?Z$&8%YW<4CgVBAs?Sk_;g`t82*1a`O zfs9YVE(~|WuSDMQmRVh3(UGkUlm~PMw-OMq?h|t-d738JVlLcTLflh=V!TLW0$H|K z!Kq%ix(FAP^Ih%&yX2?nh4m{-7y@RLa~b$5DHi#xg<0=kGJ+@{UghUgKHKjsyS2Zc zx}DIiGz%Fv*|_IQT}}Wu#k@2E%!)+%rHu_QRhKuJhyYW?Jx&y*oG}HUS48~upae4p zSb|aOgsjJYdI<>gi?IT*k0T$cA$+5DZz%u|(!IuV`a9m)z4U_O)B3p=N?xIhg)A17 z%&Lnq$Q39P4j@Oi&=#j17cMEJQC{MUe@hl8lpA>8>92$6P`7;^^h)=qi zvmJlntr`fu>AV`1mb%ovUy@(9W?W#Li+haK_S*!Uq_3+ekj#$tHCgA#`8%28v&wpc z3Is(gyPg}elm@coayh%5rRwqi^WwJfdf33DOm97CX%&_1!K}sb%j15#n9`e*a2Ipn zy-W&l4?@G+asoOQxfwPKkvEUhU=y6Bmmqc?4Gk*ShEVTMeHaRVKBoy zdDZAkP#CU!m#@z!4pGlTp}%5={bS*fuWrD;2L#lHf_IT{as<2(S{CmG`9_3;RQ-S# zfUND>7oL8f4R2;lxX%eolS)YxTSQ#Z5u~cdd@;yo^9kA-ygD^5H5nFM6}+ize~mwi zI2h}{Iz+Zze+~^)X%r=h$zt1I8t((QGy_VQLyBjFJvZJEMHFYsQ z=`-Mg!Y@d$z#}3PUBgC}eMPX##Ob0pco<=bddlhgK|)^@5M=yHITyr^c{OZ}QdekY z`l#nG6I3;a%c-S+^$7jO?@5IfU+Uf?}~#+MJN zneL2w^~6y}hJx`l&wRyxhIZz0*-BPAr_tcWWy^m#kH3h0zZ4C*FDX7|2F^re z6{=wKw6j1mjGNwi2=lWS)u9m{cfpKgM6pAVGIcO%SAUO2?S6Hj6JsE>Caxahwea+a zwRn$$uu2%6f^8&UtvOm}Ue?n3&_PQ%;)!2g_D1`D36DRBlt4NQz=>a+G7b$z* zB~dSy_wJtE7h(>Z82I|!W}`C}_9Mt=+vyP^(T~f=p#Kz>={Nz^0O0&mXPK|sj!G$s zS*EXzTiW9hRe;jj3K-aF^}t1;YQdk^8L8O=RMFgcdSR4BGj!RbaMAH{=|F8rCJA{S zBw;0tLx)R1_gNaK@SNN=<@v%m3d!-cqJO4D8A`*XWAIuDO2e#E`0}Ajd3$@3{6`Z% z;8wbm&tB+tQJK;ksjnIXCLnMU4s*7cyM ziOYv?L5TuLmzd*_WD&AFb3*NT=3CYTSrC*yI}6~g)-lX+T1_Z29YH=Sy2ZMjwc34I zi??6!Jy5s^g*+%k2T@=qhP%=L&32F1=NRek1m3IE#HpTkRZ zBOqg@4Drh4jc83{s3#6YtBvWmc>3gP>58evT-kEt=zRu$Qosp&)hxZM@V##I6LAB5 zE~wZt`?2?ZnJ!4Wwre-e0gb_Kdmv%iHKE!}Em^<7s-IW!{aL>qlf%sMP_;V45#sZ~ zqa*uN*9#}<-e0~+b zdeWKb?dK=sCZm_@&92v8B-fIXN$D>K3Rlq18)rM6q^v&y`9!qi9m96wdkM6Ph5io- zjec(KF`SVI^|ThvhyFz?=$>PTvbJn3Drf$j8&qzyA;uF_1fEjKJ6)xF3StYbM0Z&6 z?|e25+QiM#_NX=gp|)hAV~nBa^3-ADJe}X6!}7a|torb(KOfD=>(PI}RRBKgkLvQy zR?zORcMjg2zY2=il{J*n>hjzWy##X$<0d&C&20(cGQ{^U_2>I&?RQxaDyoO*yX4B{ zhvqbQmv{&h*|Ow6A!-YVvhRAgdhn^PB@}7Yp=SBpU*?tXVs>+DmTP7_g>FVfxVA^^ zWIMsuUq&9Y7cpx1{k=^?4TK3CJx#z5E_w3bESV=fKe|HIjbPSS9x|GmrHx_7f6jj8 zr%gEw-xCS2X%D40q(I3d_H(!$D71f|NDeTo5Cs`y1W?T~coR!(IF-*%swXuZtquaE1~klrCurnV~yG8Wibn zkQx{Sq??(ch6X7SrF)PRl#oVRdR=(m&-;Auw$|FV@7Mf=ne)2NH z1FRcb{i20AzPTL0@<@YPfD$HR7@MwR`uT!*+Au@OJdmU5z6|FgEBwWe&qkpx!?9j) z;gZsE^=c(nFhgX*|%@{o#LCF@W#Q0<)&WqdZfJkW@mMmhN2`GZ6>i*}R) z=eEldRhx!9qz082YMm{q(AJwDZsqR8v>}7WG6RnF*)%IohZ6iU=8={7&%HoFWRP5h zbeRcu&@s&$D%QXxUusXTTF(sJ(m-}_KZ77NALzxEvzIo22!S$I0wbXgH(3F(ey!}C z@H;1_dr%WB*wS4I0}lpOC!@}(!cZKm9Ze+SRdx)A$jeOwuMd~UuezaAEhW8L`#S+T zNhp(h+`2+h&voBevS$T;)VbMq82(eQR_|O1N`w%^@B}F^O_Gh&jDpZQ6?*JQ2 z2o~52&yrfa`x&_~ZonToWRA01dBeAY6h5ZFawer+8G?=p?REh5o8hD`y*3wBpTF1F z<}^!Q{DesX|85hsvO{oFoxu9j%hk4T=KC4`t>lFfI{~RzKB6uZ?<2!yj)yM9daiCl zz}-s4Dhg3xWMEiQit-mhm#~ni<>CxC3q)^$r9S<`_CR;mk$3M#B9d_eN#|pe9iP#G z6+8S+Tvi`Jvxm$#Sm_-J?+H`)f2|40ga+>`Vs9Z2OAHI=at>%hp3s|4-pq4?ri`VA zoAFNWlp_?LC1krXaag%lCC8c_ndlrAH$k0`3fCWr{9Feg1?^k8H`4Ygd#-!dEMXdJcx$U)S`71L$ z+LxK{pS+gl<5L|+kxUdfzgl@a&Vnblp^L7B5+V&B$jMe}`vEvf>w5xNkL?eA`lI(>HJ2&#reR*v{A{ zavw+m;y&AwLhwKkXQGnlVcgKokeEU;tnqNr6uTgzSJpStTfMw~Rd+;A(-eOv6a0nW zILzix-ple|XOGv@3wASRy1&iVJe4rko#23o)ZEhfh2w7Y>$^GAnj*KpjZT%;ao@%7 ztHq&C$$#-yn`;O5!!ub!uZ#=L3U61|ljgEn@vw=l=%#~kqryV`((%)1pkA3xhb$5r zE5zc6n+|Rz?$q{%mG1@)ccMRXY{e+GHRqMtV z(_6!gV1ARozByIXuk3RA>E#98%UZRWCE?@Ur;8tS_2TMO$63s^_doxa9#OicN2cld z0iPcGMfK$VK=#?F=OI#p_G9NwfFF7LXElrbCZFE-{fO)zAu@DyjqCKks{zsaB84ar zRhjPfI5?O&^rcUf7o=*Pfb7eB%hKpR1x*TXJ2|sr$`Lcf-DMKjHV9VX0dTLH8yGRY{HK3@l`-E7`5Zp{rkY}KKwMIa0=%@yLG?qBD zzV4yhslSTbF7K+~?oIM}6a!w_17{7O4!(q~uLF8BT3w__$IBp|ijf;plx6A8lh**Z zO9D~*TLWM8UTQnpfcj16c$!6b8PA#14s(S(dg|K>WwKS#;U3B$9WW_bRNdy+yxA`q zV_~5nT>|`XZiKS-ba@vtN9YZmiq>wi!|hejI??`$MBTq`8-Z8&@)=`1RK_31dYk9w z8xE3Qlo1Fhlu(_#myFRt3t-KO4^o5Ih&a2<%ss=n1D*DdkmP z{Q`VnfSXyY6bJlMTW?mzn1}j?iC`M!?=xQ!ZIT2nKChgnW)85kz^;3{{A%$RMZ#)- zME~A=4gKxn$H!CD=kiaxqDE{!5?^hEnmjo8^Igzi#Z*0U(@Urdhzdqo@*?ysR|&Jd=# zNr4ltK%$t+2rvafIBP2nR4WwP?>%_qp^WVt64uTmKg5@f5BtTzcDIFOY7?YH?m=7p zW4rb+%(Lzl)lbre44G@&{%!?yxQ=X<%wQOYnvp|{RUzS3QgEsp&H(_Y>B}5K$wT-E zQJ;lCt)WT;5A}RXi)Dds*Ru1c+O6aE8K0@(E?wR#-||h9&}@=&PZiZM-m69dbuf%u zd5ZPstjG2pCD@P*#_iXiBgxkGYcZkI~OSatm_;?i|Z*cMp=7d(DdEL{Kx zv9vg*WU_>A7@r>Ab%O#CX1e9;Da0EJOes%lcn4nZQ-Vj$q5q7l6BB{uKm|G8(^|RI zy0wtA3tr!{|Ih%yGr{vOr&IU;!RgQoi~WbwsR?d(<{KY{@lyQ=^W^#rY{M z5Bc7Sv!vVN_b6h|lwSE1C>#bWANW7x4}EO|L=h^%P-7@%_x+&P#U`J!Sd2@ZdfqSW zzWM->^oMW#+YaU_jDMus1=tikDvkrHV$F(_Uz!VQBbG;xx6i%s-{S*N?G@X2!29$pYo_T3Y=x`_9y3eMso^Ms# z3XG=d*x5iU#GHA}+TGk28kluyOx~D4o)sq?OZDr_f8N?|w4(q7_QdXX`VBlZ{JWgE z=up}kwx|@Ce~6RBJu@d0MTC}d>k|7>wqfs&6i8MVl+-wkqijK*6&4>a?GC{PT-u>U zlx(92=1vDRj!^5A$CIVfX$FCpHIK;haB-m>4Ys=p4~GO+5z2RI($K`e1y&>Mm%*Qr zC(v1(O2Yo3JflOc;%U)pK@Q!E+s8*w_dzzba_Ldw5w&6r0uKLiK=4rHkoW@{t(m_X zt$np^q}ttkmWiHGgbx_{ z$fY!08<{dR+2hWBU3s5XnmZj|=RYal2`UQtvlVcBU_GKflXaf+o23)>!wyYCjXCNK z&ZQ>s`YOmxS}4M8dBXeS{b$KiW2j8T=2V1U)!X3pyIR|P7EI3mb*+oJLNYVtX3Kxu zv&@LZ!`s7#IJe7~kCw&OL6&`$b+FQbya=5rYG}9C9}rp z=*AB7tFyiQGi+etKkx{!X12S^1x9=AHb^8%3=I<16q?i~DJ-;jw6^UJZ2fo#^bTz> zo6O5ai3KbzSNQpP&FXK=-CpBm2>Y;k0IXTPH@dFpRa=~(MZhK~kKUUza9~Th;iyyF zdk3Y7e(Ii2`}x;I$mDVdmjX>370MrA5_I*4HwdmktU>IGNQPsvAmYQ9%uYH1uwNe! zJGV`2-CZ1rx&b5Y*J?B^-zzM08|3|EPDR5Nr%)d zG&sifmOim;4HY!j5%Z}xZ{}9abFIOI3k2CFUNsOqJ9Z0DA`Laz;goKTNJllReYBzA zrhkXgOR$=K@AwsXLoR5t3W(hMDB&$9Ni)z#Dnm|?hcHL6yaCC$9d>q27YCW6P3|P) zg>dlsSl6e=>fcX5v6Sc0=*_V-|48Q1O`}OPzS|OB_U}pClmaB*#?5?|+i#2Olwi?H zk`GE6v0#cra22v(a21LMgM}#3BzD4&ktKk(1eb8@PF3_r(sc>9W21NXoZ{S{-0bA# zfJDGS%`ewVX-?2FyH7k&sO^>-(v{*>`+`BgKUyY-Hv8*g&F_0r9=f=vo|4Lo?&~s# z7rt;pKbc6_hfQxCWjFq49|J#PqQAu5h&+&6>7q(A)8qr>_1>Dg;H`IbLu3{4xtfzWu^aXRu4~w3ja3m81D3R}1v>X1vn}+8>33 z8tVz(P0==*B+8Y`8ouBKWwWFoegaE@;2WQ*+<-nYhIMGola$Qn7J(Ea>#cVysx|)* z4&m)p(hG(a872<-jE*!xUw{6p+K3Q6E(BUToB4h}kOIKCq;Bv?0rYBu$}UGLyu#T4hb{i?W{fZzVF)H^MV(m?d3W zZ1el`{8PH-ed@ipB6gbRzf%3qe$$NIxhLX=_-fggeHgJEuvck#ISqJL&q6#d)vuh- zXACd+W#B5rK?R`-M5gchIh-i@l<7Uq0(6ElBI-b{acpIy++{!~JIrg^lCM*Itw|KX zHuDZkpELbh$ap#CDwQ2|Hpi3Y=5@fbqj<`+{o8ML#ivo|Oq+Y=nV;$D_mAsz^ZCJ? zrzRK2VxDmwkqQ^a&R)VZ|MWr95|Pfc%SW~~#tcaumi{u0{C*_Yk)NHwn^PwwjiwV) zuF-4hbtQ&_id3{rqFu0IYPE@tVBTOikf1{&rB3l^)B4!~kA96>g{%scu&k)C&%fPE zlXBt=sH?kH)Z<)g`M(6x=ub9lBWUcW=nL}$o>_t#m^B1h^e=uER4B}GproZrx7h^N zI@1Ou>9qX~-6(aex@VB@k0KyyH24$(b|2o3ppw)Li|l4_IiR|cVgR`JKC4vmT4Pze30``R>t z^GUfpLxjPu?Ywyu*Kfaw!~kytYgmgE`s^ZPn)zR4m&fJFVseC+TiftOB_Y7^7!o-v z9yp`=J9ib~;de499;n~<*3&%%eOG|ujA0nCjwk4GTdqLUg4$LA6kh$`!&l_%pgm)>S z{9?pymJamZwbfv_%e>WScf&7AanI^5(UU$>jD4kX zXr_Gv%KylSso#<}QGokuA>Y4d&;N3|S+_61|FC7 z+aVI;dS8mQb+?XNr?dU>dx|QRkWcSU1Rn|T#7Ntqjrw@Hqn^FK0Z^L0cgFGV@aPfi z>pn6bN5P?+IGx~TTIk6~A{fNUYp>ps^Is4d@Tp$wfNGgv7;-@7d|o)m!qJ~IwKI#Z zN*d)+$oHRxC8|_x#05|#zw**`fXZ+;sM38xB+TFMP7jiM5f=`9P>z5|jM&9y#IfW$aMih$A0XWD`jHIJfq8sfxs_`HCU} zS-DP>UX&i_3kHu=vzR2hwHs!L#(Av-!w|{`F~^ClLtkj6saukX#;{%4X&wW;Kz@XW z$k;~^1Uy~7+>Yu)`D2kY+bL3B@o7u@e>XMYqmmbZ^=o-~BAi|t0!y)kEh9cEv%JxN z|MYorjavq3P}Ci?L8BJry5lLFXmhCK>bKh+W6CN!QI>6D5j;0642yh0;VwU!Xon54 zU|%P^ST$`K3)qU6?7s4v;19_DeH6tBl~`l+YS6l@m1>>7dMVYEcy-9gOj|*;3^h7vup!9i7 z50hp{7l!aal`xEbilOqegtCcBmJw$jC=@x$N}!!<0=1T@o%lt@7y;d-fP?T_e^nB_ z`Ihkl$lBD=uhA8ruOj>H^rf@zFYrDof`9Z#X^yZ5_`?l5;BZ6{YftpFDhK3&o{$v; z_k&E}3PK4sZV~xNdqBZFIoE^}4ni=Jz~Yn9lOXnclo1cUvgosNQVz+Wqm3D?``Pr} zje>mnRA9>vH9;7xPBD>~GNu2@8_Gy4C~BenZ0Q{e!Z(|qlj{S!GOyXm@k4T?9cIK! zpyFo!Aw$jOq>lm}=aNcwYr{4#Q<|^Qa|685ak?XOg*CpLcF@UFtg;Fnt~VBZI)J%+ z5rdceathzu0-D(&Zx2W-?#8T{(wYG7Z-_6R#1ABQHRirr<@X*w)_XwA z*Voq-aX=tYjFJ24&pwUN5@6*({?E$c3t2*5YtFtJe~Y_@Z>w2dbgnrXGA^=Y{;JwC zJLs=qW*6tp8@lChuO`mERo`qeIT{CgTtdMi z93D8kvFQ$o7&W09O7wA$tU$%5=PjnuFrg6u!I53I3$;b8=mp7N6yS~6*+|#G9dmH+ ze-};&?0FRlGsL3h^Tv+LcKjY|3jQ(H3tU>dwA$(+3o>RGFvvC|JDV=QyL4oF}!?kOGL8I25K z1Wg3iqjrro2e#4%Gk?dWA2McJxiuROPahB|b=XOTVA=G`KF(akD*uGQu_e)nUZ+i` zAJ4r&5WCDEI7oGhAX;yk7gx?Ts>L`*33k994jeswGG&+^p)d#J4f>~OY$Du@tXX=1 zzz(Ke2iRcNEomn!Z1TgbBxnC{#D8bxcqu+2_t6K5ZS0*0T(i3#e!2}h0H#AOCB}|nYoIfX$CbAHe{lwypv6TwIJc`$_jy( zdq0)^j*`x2|7+%`$36T1vU0?WwSN1n4b#Wp28o|#u>Phd8kUl98@tm3y*1hWgm9$r z?PHy?(-KCz_6xCqhRgczr&`P!oX4ijja-uWBBUeh?}D)4EN{flryCGkX%bu!1RtS|O+;yte&@Y2r4 zfF-lq6E6>Y*Us|vl0gXK2i#zX5*ug2a4hQ|*GXPF+^_gSGL?t5!c4!;We+;+wG4d_`oiY&6s%DwbNGVdeF6qwXX^r%=t09x+R*9^vQ94PL%NoE;Lv=#;)im>s7eis%9vIVJwD9T4g zHE!|X-5N)0VuEcs``QQF1uGv}&~po+4tyQyMUghxgL~FD(n53qJX zJK(*hilyN>%^9o9kV--wH@I73N#?~S7Vt`T%5LC{ZySgtCB4bxGN>vb-vXh;N5H_2 zo`ke`4~o~vTbVJ$JCa)Wv~P!26NwDrF>eM&c% znCmv*%}_VHNuZ1ao;N;;PZj(v)AUZ<>uliUsyFHHmHNrb-96^yzm# z&6FU|_db2?rDwI)W`dQZ$=E-tFMG|RIT-b0i|uKH*P7q^O1Zj@JUP5)@=O2Y-a}RB zd65mlKbs(>3=nP{*G=orQV+y~)y(S%WT#`z{;{zW=D>>5a96qI%P zeU{mB*$u4)l??v8;Kn545qKCjoaLs>d+d87*4O553 zA2fVv3TBKA&oTXmQmG$-0U5Y?u7r7TJ=pZh=6nO-!zUQC^MJo_e@26XnMpVh2ukcJ z=0qgUjvGR``;y0+S&|Q;-!Wr=o2B1e zBy`L7iT%bhf0V;v{>D^=zbBVZ7eo(Kiv%uy?Yg~n*bIS7Dd+7u6%AY3th6>v=YZfZSyreW2Ju<#!cpozsk&+LLSh@Rp)d z`+6Q0Wf>laSXkwi><$QLy%O`0HdfSHxxYb^jW>=GfLXNbtwXDZnm89>`t@lJZPr80_8ml=U$*UTA71hc z&kSK4lwj)g`6{rVny;j+z{&~>tuF}h#g<1xo>_*)koQr-K@2c0dKpacb51x^La~Ym zuk_DPA~=XJ_^v_^i4#J%dJh)FVPQs_+W6R8I-tXla1fgkWXx$}MxR=w;Cbvs;}3vy zw_%GagP!no=vaW>=)hS0<2*#*Kc^dVtu@hWOY3}XstAl~GF7ZEqLdUqe9~Z@ElvNz z+lBnXK(P$=9wQ)&H9A|BdYz(MNtLMb8O&?14Ft%R=yl=Qw=1cxO-iN`eoeM1}jo$b9W7jF!OG%wpX_`6+<`2LXPVKi_L>uQKEQ%+DSa z)QynfQk0oo{dw!UtSEwMyKezP$WS57gGZpfp3m3_P+O5OSLvT#3O~J|1vo0OP;>ZW z2vvPzA%={41et`jgLw(xQys9Q5kfuJ4$bB=bGriKK{2D*qk@`k**~01KrCp@>pMFd z7l7fXMf-%%{&$+U2oSg&>ipDCG{BFD zrU~B>6LSUvbb*1y>K5InPd~2=O2IT$NbXY(J_R0YP^5xu&{O{h7T8XXf+r)ExBDId zw~_s_(U(?i4lJrDyJfd3IZqc^HW8SAvDIOQD>w09&E{9eAmQm;e&3OC)a|p1mA?|C zgZ}?5LGl75NYgq0LDc_W<1M?(Br!}53zk^tZd5tC?VRQM0~@B85_mOqynP{KLUYAf zsv596m0LyiHY6|-=6q3kB^KbFfAx*-$8Y-8SU)pP#tzCns3Wsp5eN*^t`3%K-(}m9 zs{v~PW2~8iAiX&6ig+4$Pjtv$2<0ylqgzGlfO~&NpD@~&i|V_ri0s`bFvA85~pb#F-Z4aaPWmRB{ zO29y`++!yOhYDz7;Q8dsEN+|xkP;|GF%tPsuEHR!k-U!^S$Xr_By-orN0uNwA&3$z ztQ)3G&qd4u;lYc^aOc4jD~~KuxWZ*uUOE0v&h-kDW|MO_ zqMqm-_SkJ%m`wbi_2&h=M#w2l400bL|vp3fBW+76ldQ!WSrg zccfz4TEOW@q%}($p1%CLM=p!<9hQf1Y^(3qTS_?~a@&Q#UhBK=yu$|T$J`F3AEc*X z%x1N-Ld|G9@OFU+ItYCwJzWob0EfyLI6c+-{wvl%+}Dcy4}CV&ko*++2g3f%3`^)E zvS@Ay^t#9d){W*i(l#evC`xlc^a;0*ldZRZ{&#vY@=3NT+|!Kv9S!@=_Zv6`c{w^k zwZ=K3U)c1u?#b3f=iLY~(kIa7{h5q%M}DHn1HFD{ZN?0RaFlV_)b@(`??hi0j@c;JCQxQBsVw#riyoX&2U%IJeIFE5fPEGP~P|EVGCEG|a-L z7JJD)wRb}oZ(uCL+e0aGn0>g5i&=DyNnn=)uPZKoC`w(@76@L5MC?6yzN%g(!E?E8 z=zM(jBfgxrP5X8v?8!usb`A2b9Sp!(rmO%Q530#YUw{I_zIo|yDWA3&45+BUKq|0% zxLmJp_mFU+z6kyvdl$`ZCKz+^b)1FWFWG8Yt!F*-5_hW#$hns_G0jAJ`_bo@Fv!w=5X4=f{RK4B|0LL_@-5FeC*??+Sk>j z_(@Qo5Vq6T_l8F4?j6i^@=hxPP>UHUuDpBeqm|!Tw*kL)mYEogx#@S~lC%;GcXwrL zQzQ#3nBd+2Em!?wf+FfG9Gw8CXaqnEFh9^66~p?XqXdqpS@8KC+>cE^cPf&>3BQc) zgY;H}9!P;(UrPv9Vo1h|2dqQo&>&z=D=_I%v4gd?NoUL0!7%jB<7lRYj9uc3a50Zl z^L!iktuJAfYf_KpQD1~Bl_v%AuvMqOUXlYPom%)v*eOK2LeQt3{WOVgwr(1ssNbTp zgshss5qn#R+6YiET5*n`p@n+Tp-7zv`caMpwQ#7(l1UfvWw*#Gt97+A z{2uhy95~M33-QrHSd!^+JuCyXK#SnRYR1A<-X684<=tb6qukA$e^zMVu)LwjDK~Lg zbfW#g7=ABzM2q@wOn&CY9dQ#1#(r*7hYVAA-VN$h0;sg|(ZD)pl|a z^dmY|b-F~~{1zORwu!sHWS-37W$^h$=CojC_RwcWYbHlv45Z*mmcGC3kf=TN?T>SOFCxu8qVgMt=))T)?d+rri-J~7?TiwvZ>(tWsgoJ!ys{G7E zaUqp$4mrbhCZ8lmz3VWwo_K2Ff+9T^4rx_?R@O2*{YSV=?e9W)Kc-yerA>n4ljND8ne z4UH0Z*cIEC&wp$kdr)@wCEETcJ__bEPI8?t`jtY|0U!0X8+r+}ut)Jc;?Oldt1mJ* zoYeIa^{{F@o7%ZhKCYi{F*%3h-pAJgtVVarS@fX_fau8KO{sxQIaLps%}fw!g9uHm zC*`oBYL$W%f`5`Mg`p8u<+JEx?> zm%RwNJ&rQ5-za5-#+#p5&4OoOueW=khh_r&ZJ!MB_XH2{y6_C!Eh_I$0+D8vwej3`u&s~1 zOc?N_6++#9#rP_kWcz~cdS*ZBERApaF_hKUa`=%-sJy&E^WwSU)oYaF0@cu>GAv|J zejjh*e)TEMM4PR2)iq72hL2;CQ=wjv<4E$P+uQcTOqFr7y!1q%BT_fY=rAc;^=ggJ zd81ULIJUA-xGYMC>_@cZ{{2I+KJHU3iflFb%meC5)yf; zn&jmRv2LT#Gv>eS&Gt`dd4A5(i$F-GvTHPG4;S1w=|Fs<5>S(`iMD=F;y5vKUjAsD zI7_-J+L}J~(R27q{aG$EOh*(+3b&>O2kPP!w)Hf;edL)LP*YKmz4FjUni84Xl{>mn zsaqc1D`-X0AR8@*_lc5p>}h9R-A%O^Cf1dKqd*LwfKR(}fFUf4wK9MRsFJ%h_nWME zEOO`R1O0nT2WUH(hAJlbtvs1t658d2c=SvUfq8~(D)oRA6|1-I~1GVN;Z`5)VJJa=o! zbn>1xy=A2sSyu-}VuJ@GFCinn#$HAPD)Wmh+72A3kaMk>TCyO^@EbdbE@39+qd?(% zj`-6Yx?8k@WNApPicxn?ZD>8ixW_XW170EqVuX!GIWh_E`_-EqbU~{QhnEFK;WX$5C_xH!YTq4&9cr@h3uDkWKPr{*C7IWH*Wp;1Fc`X7|%d*I| zuo`&vwQC7M5>ZyndWcDa2^foVD#-xjR>YzDw0ZdHNszk#N?h;D8Zy?;sajtWHFohR~)IE_ge->Y>Q|CCiP$}rd zJVoFAJut0fUWs%pNMhmX4{I{q_-TOVah}}7nfdb_ttMv%!8P^U_`Wycmoy0=xg46} zOxAa4;MnRCeF?Ij-T*^9IMC(@G}3dENlcM^ccqt>nCO2B67mac%Fgk}yBD5o&GHlG z`baCO*NVF^tj&t0Y{S2+u=}2Q!arP;+(#I8?wn2{=REx#@`-Lm47xU$Jq*f{JNm^ zAN>2X@4ujQYY*`Ov#g_WlJL5lp>;e&ga{9)I>YM*Hl9233z`(@$3zktbOlEn>~(<& zs7ubdUu{C-zf#LeWUAaHZQ@0J@9Gm$wVxabjiadr5*>56^mDBgQVGta9q?IUPzA)2 zSE|YYxFR}PR5e|u&h3Kj&<>VRHhdH}#u57n_r6zFu8q(=4a(YiSz!vsof533(^NSwC-rGv|b4S2`rUaHvzP8 z=opvNL2bSai=Hhs7+44BsZTl=vofZg0A2G}x z&Xj9b3mYUmodEnY@TUjy*@bqYj+=Ocm#);mkMb1(HhJLiMa(vNh_iJ0sp=i? z?|!^R<`#V~fMq`AONg(tVj(@WC)R0aqf=tBtH9=P(4E%k++nYcde{z6mxjAB!FwgL zGc?rD%#_l}JRvlZn%D${lM=M*|Jq5jK~Qdjvp?}|X>f!VRj8KK>Otm)t0}(9Y`CEZ zzO2zNQnA5C?Vv3M`CMqjC+Qwe-IMtlYv$wMLF9Q%=Xd{79*fcKwto3dEA<9C z6%1XPcwcijNo89sKdLGAG{@qYr!j@;;n9RaWS{r-U}BTQVjyPSSte7NY}y}X&YzGW zGjuJP5{);WUK-anNPB(ko2A-c9)V0>vtf7N;!iNaoSarF=L_fN{n=RHScue^>$mck z+bp0#c?=u_kB8I^kA!z~` z-?+@D?$Oe8^j@h#4v}a<5I9n4)kZx+B|K~74WBmj4x*ab;u$RnB9BFl386)Nw}pFo z)Yu4gPk6i-luWeNj)KkVESgEYXNTTAVYKQoXxvo>^$&zwziC6lU#RTlq+fDZ0K$=X zOz- zM82sXxWIIP%Dz22wHf9}>eP+Wt=it$8xT2}lyH`GF#H#s0X}{|CVK2TTKweO;1f~& z+u-e4U?%3G{^o2lzm^P()3p|I4TeHQ?82_Y`s#=TCVXA)w2b{I8LV4E_Fu#}p3dF$ z=oB;3j}1|6JKwlOzhu>xSQ7-lu9@XOAI*QnxQEqZGcGHm4LF%5`z3jm_J=v&sF0F| z`LTIrSPscUT3L2{fSSNJNX8JBM(X+k`f3Y0m`0=c3m~AKs&x;V zjHKXauM2+=)DHb+g~y4;@fk)m0h5I=yUUN)JwZw6H;MpES8qsg3u+^lh_-E%gO8>~ z!}s3vVq*o+6h&rQk(Pzeq>^4jUy;csBX!}ttIP?h9m4%QRF8t+F~C~z#BV;KOb4BL zLuoMb47B>UGVw-8(q`GR;o}MhG+yKw(1A^_y03n9heY&z$wctfb7=Wb$atG!%88Di zwMQ?qElW2$Yk>H(NlrR5tHDS`&Qm6;GLs0a+JdvzbE`a%$QrXcdEtwHmLD@IN{U?F zYIQ~0pv4jsh3qs}iKb8I#nTzEb{3hatlCJ}cpB-2GB)}hCq7Dr6Wqg8$!)Gj>r$lZ z#_x)s67OPSrH6Ze67OiubWE1JxF`=pi`ITyVDk)!ruaVf)a%vaJ4U4LgP=7MT1HM4 z5ya8RMb~8JzbZ{STpzhWr&If+?HZMLpNcwT11$y|)qa^=oK$rIX&C>NVp~H0100`E z@wsj&Yh2q`HJa2QL%?uWr?>#r0_*g5O4`YlYai`=a-{WK25Xd0Ulx(~eXk>iHhT?* z`(hKy%P;fWcBK3#iFE-!=wj{Wpx@=|UB+{c?)CdtpOjScZV+|Ebw_BcM8O{V%L))$ zApC0>WJ-}n1`GaeA%HZZG=^2bk(UFt_Tbkx%R5h)??Ur?l|=O24hnzgBCFaD%xRhS zM0XwW!`%m|hDK7n5iCtP$5Vl3g*---XuU|)E)zn}AF+0tso}zS9i+E?`~ZMJQDL|z zMiLAgld0eQX-EAgWc{{<-&khOQ^jz{us#tTtU z?I0#V1DnqxJDcU5jbs!A6(Di7LGF@%_KD@Iwp@>Isfq5IQJL)LtLCR>kT0 zNx{be5wv88H#TgjTxa|T$YBlG2N*7%l=yvmtnL0y?636l9}JO-A_vzBX z9ndd}A^Lbccq}sC9a;x;GXX=qpRDebQGcK5+3DVj#E1Hc=FDIDxkqO!mP43BKxj=x zTz}259^nOru<4PVmUQ!(mOJ7wx?vTMH3E{h4@8a@e;m8ThU*9+Icnd#O<(rY`mcSv z;Cx1`3%UBeQ|~t=f#8Q@n?;ZdL%#RD@^5E^%4|EzP+;;vifW<#Z{Ojw>S4oK!nD6> zUWQvCX!%xjGhQwgD0(H(ma5+5K#k82wDY;pazG@D=DBqL8b;%YaY@=wOC1l9CsV6r z1c{8s3}h5%n}!NLATb-8@%L9fb%;npRg)+F&v@{JAG{@GGkHiM-u3Ze3v5uG)0*E0 zfTGvUZg?{Kw68pj@A+mEZ%%agHa<%X&;E|bt5FYjID+5_SM2)kgPqC*bm*#yBc4Ld zbexq!BIYdQODKp1Lp_dYJuK_>HltTMEV+TQK%@UW;@sh$k#>a>+QAMe!42Id`Uvgn zyrICW`1NsFOyXv=0Ly3Pj2y`5hWSvJIh-&xUhZSwVBxJFj(E^tD+=}UfY`)NAoZuW zhxel&k2@xxH`2@ojn&>8K}O_VCu0anM7ZxT0aM>wO;^Sdm4D=ea`@9VH^qlETd!Yc z!dMjib4#`1a&HTZvceYEA!6g`#^IH8i%Pz0{yJ1=N!t$=B0cAjoPB~H3f98BQs~yA z1F4=4an5a0_(;Fc^YCXAAj$j@u&Qk0_1{z^Mlwncd0jolq^CQ); zJ(51HL$GjJJlS7v|C-py5XWyl>|N^;M#@wZiF>tiO6)fBU)MX`LR_mg%6mj7{;mhK zjUNBA9z0uMA%b;E*Ef}_9+)WhKo2t79{zagyYe%Rq||OUB-*90v5HFE!3u?~-IKcf zQTu(!sA>3QNAv2wXYg+ka`}&+U!wY?ZUvC<&}dWA$XswnvXd1Hh!XXXnA666<%@O` zwueIla;$0-`gQB8q2IXm%kr!aKhbeQVzXQv15R91rypK#2Mk2-;v)H6`v92bV(Hh< z`9k>ES?4d7KQfk1fwbvP`Mq8pT^!gws-%+ep{7`RqWf=SHlM)`VwBEQ>w0TN(RmSf zyDJ4&Eyz>9VhQQ%`rcih;+PqKHhtAF%yhl2Ekv?Z9#saLxyw6EIPFVQQ1qy%0NCCO zk9{+rDl&j(Z;Hc3M#tH;XjRLW!N9th<9$RLb$Q7hkP>GBSO~)W+`6=T9oRT*W70v{ zk=>0HBP3c%9Bc|r+Qf%!5ne0h{etZiY%x9f}D(*}Q9tK#X^xeO_eP296v55YaEF(tYu zOPox8$woq}cJ((S5f1-QAHXDRfnQQjJJZn8$!9e3nO;BxfiwO4?1)3|~RAm)i+ z_9FAy2l=DkylvGla$WVZumY1DT+ZYz8=J$1tPnS(S zVgk+r%r1Sl8+6XSZ&Lz%{MW1EvHe$)JTIg?q^jM@jzxNu*`Ksn%zjp|LU#(6Qw zl^f74;^NkWqpR6ID<@d(vwZs48Pm#5$RWjLX*Smo%A64RGzs=tHO>1 zd!s6^hk?U}H1$2hRIQuPX!r`17nu^w8jO|zT+6o9rPm?=j{;0P`wWmScgpmD3{oYO)s$O~7&h0ugB+G@=sf(twj2P{{P za}FohnIvuol5X2g&Fz`mxDX$T`~HIvNm)?73FsK_l#F%efPMV^YM27mBg!Q8@t_jS zEPo6~Y}DWvW#}`%Q%d5C$gIBnQa9o0oc36aRo&s@)!B+O5P#+ayhGlfuf1FVFNHYZ zsVI^B^5Nj&_)sAAJstted#h(Hxa}hHwn-(%g7{XIdA%9>XUt-Chek8ku@Q?=Kx~A& zc=kZn`V@t2l8c1Yoy$vql&+AiQcE;R>-LaJs@pIXMOw6sj0RHsD(&g)aT=-w7W%9y zdTSPsK7W9Syt0MnuzBmg&LwJzG`y-X6nM3&|LaT-wOjcwiYbq^t7l!j=t4#k1 z54+YDOI*X}KH*NLQj^#4Ir+%{hR-ud=qdg_{@m=96>Ih(9X}Z=Is!4>yjPFv0wa_g zRSNRISnO~IG>XlgO}rE*MZ(zhTP}db_Psx5f8IVc?DLyc;?YZ+UVM7cm^$YbK!B z*4KB~RIX^%NvQMoly%BgZh?Yb;FR1m>dfIqi^|TsucRUrL&NO?Uq1r)httTSw0Q_E zPylKa&CwCz4b_XZ>GIZh&q-UEq!XZ+Jqt~B20DHqMae^0y^akAg+7!YD23iJ(=jQ^Eq_ z8l0suxJ4kcfF<-Fm$YG2WH9w%j5taqncs?8sm&5*(u#dn`^?;$Qvr38S<8j1WU#>6 z3#Tvl{Qsfry`$j_zqjA$6Nw-(Q4_rl36YTKy+@51M(^_ozt_ zMDLvlVbrKW@ILbWmG`W3&Og?&mWAP-``P>2*Zy1?Zx|cI{@k8GN_a=*qO*qxKb(0| zVBaF=*h?eUy$E%uT57&FWm)szq~bV|e?}(s-z>d$V@g|GVoSh+gqI7b2H=zkHL5rT zVb{lT(6RHE?-1yL5JRk))BqnQ!#-1mmJ*X@EAOwqYQpp?vV;u>%}_>YYkGdxZ$UY} z+``i;HxK1+r{llHq+7Xl;s9OO7%dO@bG^%HfrA>fkzb$dYtfCLP=S&Oax!CqK3Tu% zs~5N&C0QS?F|Bxf4^Nn>_DQC%N3*=C!OxOWYLkE;D>Z=2C)4iqe>k4oF`^dE-PVN& z$%R?Y?N>l^TR}~@{%L1W&Pt3y;Gy+qLeIIuq&nXNVmI{Es5T~{Y83Wk_>#%pMR)kX zUV;||GwvY;lm;#pb5=98>Z)TxYbLYciT^9g@n z2op&ivhB|^6f9TW?moaQVvFJ`xFrtMX&)WOO23NSuOvkYjBz!BUguduG4tk-2z&73 zW~xjkk6UV_ogQ_mtTGkaZATYG#fE|xZ@FSH%~JU#`IsPE=zVtOU8{p+1k-oORrj!2{ zxpO`C|AXB5|3={WO5G^V_D_9F@R{y~@s9+$mjg_?l!(i&*13-NQ4x9`hP*er#nxiN z>C97;b+3jOy-12}SCwJAhpm^@lIY6^fARfMq7#gSh7qL|YExg!xpBc!$zL`tiY!<6 zKX;U9%YE#9qu`KGGxO~%rUIq(*;DF(il23wjjkR2Ce5(NiSU5*Qa7u_^TmzT z4kit?JBPn^LAjaN5W;X`40a{QAq~U{HBM; zNl5f;KE$vK0fe%A;M9#Yg{mdmiK3ITgLyM1u;c(W*-IP%#H`nd4AJT~!oe!#& zz2Bg*SHyJknC{rE#fR~wmxqYU6m;cdK^{JQf?85fe0$hqnI~d0&*yAD-eX~Q$lP-O z%*FddVtQR%U>NYqQ;^3dZm1_woG{6Yr!`5^vt#6^8cjm4YZ|Nf8UVcEdm_A53f35q zeU>RSy1rXKZKHZ!0?vWjZX!DE>Iw_<+=N0LJEdHhKQWgZ*Zv?urr(~mzTy2RuD{didg&j_ zv^QSA^XOpL$!8=iJ$O8yQk-^`TPfi3^rAP>2l4Xy4m_=~hd~;(sz#dDb+oJO$4dPU z0&}e8v1Oo54ndwHa<`*fdWc#BiASpjN0ehGOw0rwQT%TaVAW+*-r+sXieTA-fDo$k zeB(|#mlBHBktd;i2s3F$mXG8gQWprTkCTev{=Eemr4sVXV7CH$cJy4?EU;MQ|L)eN zkBcwB_DeG&F9%S6C5x^e9bq`%Xo$YDTKz_N(KbtPgSFY{OI(Vr^-N9Mg4Ok1w6uO* zYAIfcX8gTZqg=A!d{ojsM9T%2&fjOlV1XyFXUX>#1ei6%y~qI&R|-yVSn?b{QEluU z%*-*AZ-j8(tKNAf39O?)4yh}M0SH5y?{A@&5HgG*z0Ut|5)vqL3|gnU4h$IT$aFXj z{udP^F8#l#82BF7yRMCE0%y?ibmqYH_V4D~?@VhAhPM1{SGcFfaw2 zj5*xSx$3iNJ`oRWt|T!0I~(XcXq6F4j%Jq7xB#$lGS(W_Bp`po zjkf^-8R7HlF@bzTC_hEUs)JP|>08I!`|v8q0`CqsbQH%I&!6zWmGMpn*Y*$AP$2yx zW+=tu+v}r>gi^xM(YruAN5O&Ore=I85E+mx!jTI;;-=!cwBYVknz`7r-Zr~(2pBm|6{Uzm$YR}FZ-Evre_?I$Cp<2402U8>*r0U^_9 zfnO9@6yavO*t@4^)IE-r#m<%YhVAeB?)#iEkPWK{t)@r7QtbQO7Rh}OxnRwpo*Rm0 zcK@y@qw9e}p6`|CuWcq5!*L9$NS_B*LbF_(Oyi0cPt~<(NnJSzAm!H;7>BW&$`N5N zpN-is#!et8Z5HS%3z*df;Q&|(d+i$EssDet#*NYA|Cejrg3>nKRB-E~(0#Su z@BJz#fzd6P`{XEIHgj}kLH5I+AsqTnEb#b7<4><6EemoY6FW%qO{_l3R>^b#?sg*O zj9@m)50kX(9&0`_6OH)HEqFZ;wJ4Xgpm2e{(b|W|J8!);D@S&9KxE2DLBlVL2cvm5 zHrEFufR&$t(#k3_iZKZRR*>y2!rqkx{Z(pFVeMjvz}Q2+{vo*E_zhY>91?p<*(sbDQ^Jy7iskHla@Xz;BbC^eB?$p-(F}HGgttx%8liNg@#Iu541`FUE@h$U;aPf^7pY*m! zX}sbkO1=124rr!AnDrdK(AZg?*NKLVKtmpa;--^ll7`Lc9X+d0;po!u(Xn2EgY=Ye zcxOpzdGy&|5?XLPWxTVTSSsyq_tmN7plEg1659FiPH9xlzIAka$ZWN4`3Spl%ai$k z*<78Q&f5{1fKC0Y6wFf$SWG&^b^@f#md0h9(E*2Z;DwkTgKBV(h7a}V-O#+``+?r4 zdf%YtN>PSZUrRy)BjsvF+|T*0wwA&e_A&)~Ab+;`*d=3B-ijyXsm~xHa2W5)V1oUk_DtRSVD!pG!%~ z3g~j_5BGz(ZS+2hTX{9O#DxzVj_{Be$}Ynmu%KUmxQG7z>b>!s0#&VIzdsdn1p;2> zM{c0;$%ggEUTP#i5n)557x6s$RCkK)!z5$Gq2+Zv$|oO7|X#VO~+v)QRva#{ld zkg=43=|U@jO~9L{kODxkZ5eCN-*ladFyo-MhQ^kAe7+2gw$cEuy@6cLV%mq>*?jW5Le=a3jJAB~F*9u}l*EEEQM3=V1e<6DO^0 z0@$U?9MzJ1{r2k2)`Myw!YkuZJ}iy7f(JkPY5h~Jo3R#ceHotZ@My7 z1;L&FL?$1i zeaMa=XbYymi3-xt*Q?uNBbLm#<31($sF%TN2AZIS!sx~VH?3&e5H(&*p+HGV$( zuYlkrM*ep8h&nTLp#C#k&MEntv^@9|x)V8uapS8G-y_B|{3!$)yN@EbpwuhYWiQh8 zagZX80Ow@(t+*v&(b=U63tdaHCVShaCj zd>JrV?sUC$zPj}0lJ<3DzY){b4&VlYMq)hlBiT$h(h3sWi-l`9q%@8XucN@ z*5HMnW;VC+1BBg zaXvAgjB41E2dwwa)|Lyd+L~ggXiq`kzkH zr-K#k3U~sW3_h&q4$iT}C07A%CW3`ydi+OBL%r*wmD|66hRg+C?2l+AV!q#X`Ab<- zH$pKPGa|s>7hJw@d9)rFc+i)5^{V3@F}#E;N{ry$9*LH!x^_2{{k>R*G+MKBL$1R*DCtxx2?-RA?Yw zlw&b_G*E|wjtu7rXHtJqbtU?t{54T=Egz)&L7Qli^lGFCsA&Zu5xZ48+hIWaX6o{n z%awKzyS)1K0_M`~uadlNvP}N7+A)Ru-1<$T&e zqD=MfJf&U@4Lb-c%INz|>BsF(U3AWf9<+4axv#BfphqH+%a>sGz`Hj4(_$lOtiuzK zh*w#d#0$R%ZPZ#y>PSLQU)JUamK4&Y zOqUC#P-%fcMwuVCB2(f$SqXRgREu=ujcMm@1|Lm2e^O|H07(6nGnHAN|5D)wo?LIj zib(ve16G(noeP0rpE9chcXwWg*HmSHZB4y~vuOTx)Zb+*6}PE+a07~`fugV~?XxBgv+}ADWk1$i4=_^P`jnXVY)r3`?u2l; z>z?W31Til2Vm%Ld%r~y?tuM14j*! zGC#hI2{Zn?q5#jkdDfewgoECuOrf-%A)R{L16zf(02Wov(Iu^Cyv&|398{0?w!=|L zZ48wh5SS+pkr+C`ntlQ7>LyFM8c}?nzHE6`^H`HRgwKY zG;{UmiMS_+smukItObXAF>pI0BRs;=b>a$V`dKOM$3qPzjszYq1=e6Fq`|lcbxGh) zQuXY;m2iv-R4X?MN%t7PA+AKbu7CmH#^=Y~BD!UT=|{oddF0TWalT*Oc;h`bxbke< zTr0~9l^-QqL4}R$L@`>kzJRuPORRU!S83M-=Yt0at%h0GUfWRh$Nyozb)eyUvA(K5 z3;}kvCXgK=y(5PHU`lKk3P#aC{?L9#j_jko4i~EbbJ|@pF2EF`egof{+#6pa^q%KU z8+Z$fPD!vzw{*W&sO6U(4$1=ER&2rH;h-uRa~K5Fi1{qw4ASKz^weIzB*xVsBlmdAMsPyH-ZKvZ+&Qd zcnHRx^%`38=wRYLnI^NEMub9P)3gAv1u@P*dcR753#Ntj4n{yVgowT7$i^`F>cbf} z=8f}pk@I!eyV_37d*uO_mSMylD)g)LJtbs1*q6odOyZ~_;V{IZ^ zHDgb|F4{0ueCW(MD`ju5Y|?2)t2|==Swzzxw8msTK^F=Up-7bDai-*=mo6nJq{QCc z>UotQBJy3n#44OE7G;Z;thkATUR;*$st;~+N~tL{(@s~oSA4ddwi6W`i;*u={5 z)JFO(TB%~B`BKO0e_x~%Yy6XMelrazP^fyj(Pdq9hy3Z0+=T0t-^Gm$swnO!{`G4a z`DvMua5hk++f%Pg3e`?Zy5Hi64`^Thc1{js7?d^+%G?{LB|K<&EL;Bq)yH#Cq&r-A zjDw~K0e=%x|EOxd#ksNW1#MN>q|N`z{0FET;FS@dmXia&#p|t9Ysx=ALriM$&!0uL zUv_&L2>&uzJiqL%2|f?m`Bk_rx!$__*KwQ|pPV$SWsu?pA%4K$Q;OZivx(nK+XB7- zYHBvuZXX|rOp5FpE{n1}nh(6n4$ezD|GTAiNkW>zgjs(z!2@+AQVM>7bPOAL(X^x` za|gk2&UCZfNWKOF@|N=0Q-RqH$#;}QNRsrFXlUk*a{KYi!jPG@4id74Y#3LVH`$|i z{l8BxY=`TruZ?0M?H?DH`*B`r^L8myJVzyKe~nvuZI)jHe=75c96p@^U{(Zd=+DF6 z=qNR+9L$6w#$|~^_$F}B&1|;U7urj(7d<9VH16Qf$`(p5B)@H+md$ah4XzGIB;t8V03PgI&`_pT z83ts2PS?B7yf@A%aFUvnBQ`;L;3pV9W~(`1JV+E<1&zLEnzW-(^TX+oH=%}Ef7z>s zbr>>Om0ulZE7-kxivpW9&)7K_Y{o0kLy3LJdz*z@Ur-KnQcf7JTZ4EbJFL>fywckD z=vB?g3GhG$65pf(t&86|j8-px91WM6-R|HEEIxSdl>3?KsmVrv;@yNX=i@oZ1dlnI z)4&O1ylNf+i!1)2-(sJj1NJ(Az;C`&h7}qZ1wOq60?|5X6#X+?G-o))YCh=5>u47MX?N9nc=>=G+_#5 zA`GF&uVn$tA{$A-uPA4^mqlqw+~=Z!Z+<^u)olTdz!$54_!=QSpOe5lGv;C%KR7Y% zvt9~Uv*~T~r)^aHWNZsXsHKqsZG;BdxxLd?B0{>cjTWr*^YA$n@57fo~{`e$y4I}B}Q}-IU z8MayZAKp7^Qyut)#mhYZBBap*1c>B+`v@1q<9i9V~t`J4hSm_U`j#;7cC)TdpEbw9mC0J_%2h$uEy!ag-E{Id4 znyONT1oA}&U1>r-95*I(bUTR#FTHm+$X=c#$UglJp=y#dZ6fKiyal(MFhjqnM`^ym zi%=CHe?<9t-fiW@9E;`gr|Skx?(&7K98MuDhIiU{VqW-C4bMt-32^S~>O%P0m-AbX@V#D@-(~sV+CMdr<~H4hd%X-TZPWJ~V=pkeyzSo|Gy>B??P=CbD(j$F!>l{%aP+>KGOvE>1M9UgV_v z#}JsJYr5wh;O*#5iB!ushQ3fVBl`mkpUG*%9F?<oV2Da$n&6lMlTteR=aaf9O}ugFYk{%;mY5v$Luo2fC&t> z)Z3)0ldu`3crCeT8l6L?Ixc->=I09v!>BBhWTd5w=>yOrik3vGU)a#W8lZJFygK8410S;p z_~SyQJdDFf!L%2k=HJfidE|MD^^33E!Wy+|?kcnA>JKbu?L_j4NQxb7N*d+6>s7U%p?%k6txxXiRAdT3FXp?iHpc(vo)mNrOwohYY z>%hQIdjTU%Tl|^4`sbvt6aMK^Pm)=*@frj_`dj=3X-D*SoNb6{?Ys|1{n74xrk{`H z=8Gpr5><)Qk=I(J2jTkFfF<3DnxH0+gOrND4F~(Kc0`0&SI`1czQxp|&$C*BSrYy_ z2#gfsOhO6&QwNC(6-^{4O^1zV;$~c&b&MWbwMHV;1?-XGHjo(l^B}B&z6TOlp*Y4o zN6FW;M~RI3X3DNd*PYqrF=fa1{> zcean}T%D3dq>;|glDoNJmFM4mkbmVV!y7qu!K*{FwBLb?v3H`^T+nU$-WUc^r2A;j z6$57^s>G;5zaQ(z-VVk2StoTOV=0?tSn*boE|ss5(WEAqzANdi7RAgxJ*O-REKWqgSe33t2^M{i zkuckP!;nn%nzTR$Z_FE3{S8^b?EE4XG}K^Xzta2tVFj7aEZ>`LAK&?w747o7_S)sa zEnCh57+tN@62?bI5So5t=7h9V(qwm~Z#uVsO4sE=s5gKbi8#8Xfjl4ym&&Hc#l#djQ>qzRtlXEn2OxjCKZSe5Fq^J$%St4+1g1lio*N?osVPNl#tgR?8{}fSAT%&%(*%1@7KPldra}1IV_yLFF2*}3sD7bKPnEn;>f}`9@~4}usETh1Zv{){?fysa@eEYVB*%bK7(9(qBvF^p$~w^N zxL*XSAWVQ#knNa%Gv)6_-)brHdnjE)C$|%o&9<|#nRK30bwK%{nwD^(VR?=8pBmd8 z88l7!kn_xQ2hNhi^!Ibv&@6G*{a_Qdnlo{tsDdu(FdRnt+bhu-;$YfZK|yEv%3S>N z#ZJ+df}CA0srMX9o10$7tX``G?QBk_%tf!m#NJVKqtK(v%5Z_R1EiS`-SGGl_}sh* z%8<=$QSdT`M#BxkY}c}Q&=BFVbn&=e{nsku2n#BFZ+dL@h$Nl0T&fI@>QH3!3Hyd}k9%e1?iAL>8^y8qj{tp2&+PD6IOo?Yl zJq?VHusw#~*Mgsj;`%{H5I(YZRQYG$`7tgATA8XB?=9KShOAUBUi21ykhM1~)|Gcy z<%IuS%ll>&MlT+RfCg4ypVRvH{{G!nWutncCy6?*J+oh|PhdAH1gCb4-#@sPE5oIw z2CFhZu38-#dCrKwp4`0QFuQwuv~lqZYv9IQh|3*3TfpQ2gb8sn{>=r~6v7FQC0l2E zg%{5jPINzl=NR2-{T-rf3LiAK`7fSMmjGl(askm9iubhm)l7Qt(qC&IHSUaKickhw z(T3v!?N1?>ML-D1*=jw&X$$mOeKX8w_DcS%7>%=5TX2$l!VfAJZi1@mDcZZU5k9Gj zPqJjHdTC1{3HH00;q15Gthziy{YuE)$}<<#$eqjMdczk_^c>f=ald@=PRhf{%D47G zA5N<_y@pB$LE=1Jz?h;=+VoXsbtwT5uU2%rBh)?Mm{^L}_80NvCd{7HA+DjPX*Ha@ zLuFYHpvp$*Xk*h%+D;=u{f#78Tkh|g$>HbtD;68U0)PZ;CcNNM@TrH#DHw(2GK)6HD0n5?M*;r+*Q zm!gHc#_1jNP8tCb??frFz-EYt^;38D`!(tzPE1<`2!Hxq^yk{%uABo+fYgrzR+?zy z?}bJubv)tFuY7ldUy*`nAPeZWstccs+~PL0+P|ZEqwkPP zTPofVerG2sRjP8T>zxbL^9(i<{%)8=ZI#M*{b?&zlX~lYiQw6S{5Ya#)f|y9-Sb9O zgfL8b((so$1>#fwW*qi?wC$={g(LuAXK80$d*HCdxT)xpt6nD3fZgC4!#rD?m_vTk z0T;tBtW9oV@Yz}-RM4oz%PLN|Ya1{wA`kKG4a{#7u1a+`tKghiu8Z|PJvRT1ZlG--%cx!}SQW3ED2pGXPy}NZXsU_YI^UDn6tQKgK#Ua_W}09J)E~Jp%+Ue2kL`9nXbl*TEH;-f%|$%` zOy!WF84gk~A@3GG)LENTJB9e>=3zBu5hr+G{;1^F&5QLN{YDf5st#BzzweKKy|@p^ z3rGEzU{wHwte&werm^xn3Cm>bCqJ9)?8NcPiCx{PvzptcaJxU?pZpU|F@^kI^lScS zO=QG)&1St;gZ69|V_U#g`pn-%tLd{-=^Ist+Voz&t7`vf@bKi*jm3T-)9N{}8F8Iy z^@*}{V&}p-8Gu~c)?8jUDy@yBW`%Hd+g7>``{W>f_}g{~$``yHDrEeU++S9C9`Qu1 z%PEwV>`ABvjz{R9JPy3B~DH8V55{(J{w9*k(Gmtpj8+JeM zR+twNCx9(pd}Y0xAxhIH-A_7BJJS8hios^pMKD%$BY4w2t1sb;$GOj+^je2yp2=Nr zFb~-?M_h$oc>eg{B>_9q;=!YAQQxz(A1Oe)Zt%*ZeZ@#b`z9Gv#=c&W7M zL*8i4MAut@wbQZ_uavdFKJ-m$v{Cj|q&C3_M?jJ^rjys*fBeooB_jiL{j$-*zJK*d#qfJRnO3ls_7L zuz`-NZ!Wq}UzicJHh3(@Q>-`ANn=1V!p-5rAcHB1fKg^4v;4xo#5)qe%Oju^*e)ED zL1JGhxKj65#H-ToGU+hq5ujOgX%@0zN2i5iie%h2V?ntwuBe}=pI}C z7uZVXaCK^YTx3V^ER_2Z+Hxktrob3(FJ#-e(b*Mqvg6e3Y=QX~mS{}T@SEr=WA1VY zZU3fakB>&>FmYx-And)oY!jV9@V!qf&vy%Z*ibk!YkECqi$I%q04TGcZrHQep^wF- z9^(XbGqb|yCp$ZyN<8O1!-}o{fk>-@7*M9r=k0#f(->VfyYqhhXRNyY_?;`CFIr*!dR*Il&Au(SoOy&zcoF9^r7I;D8;z4 zZ%Tak1lA_*nbTbeB%r`^5iXbZ4+kU_4&w3N?h0kzzo4Lk7T@nCsK^|rc(DT7JEtSZ zG5Z){ug+GzXcC;zhU|Lu4((6AiW!+|aud{teuXEN^s;=Pe&88~Fzn zt)hdWhPD~rK?t;L3(k(y^T2XuZBIlh*n@@k6?Y_f;Om<+GIH z_Z&ntfPVB%-BKNfq;SAw3JI++Ts$l_*dJ-O<49ZR7_fahOe$%fppsj~tZ$?=*&EK^ zAU2G1m9J^SABGf1!mg-(E>lZ3tywc;r`Ns0vcUNBPWYi+SwVW&N0wkql z7$%+50ASm6ttaHWWomk)A_H2x92bh&!3*)ny-G)n(`-BG7GX?iU{f)A>;h2PB7HOe z$kUch=?3pebmRu{=I+zSA`9*Y4$d(qLOciks7FBGXu1e;JbU+UaXZivYLV%rTC{Y2 zeC{pT1`pVMuC!oiHVb}`H`PfvU7}2NG8mMOJ9m7E=pZ*}+g6)>9#c_U9djdd(aZW` zxow~9Rk_H;B ziWj##?B`NNAV($TS6){uygSpqFE|JR2|;=1*yIw9(cOz>Tn*g2wu1bg$J@O_Ban6P zMUtg?g3GU2kI+Y{s9Y2m{ zmr3y=NDiGA+E&?X(vJ&04|v3&9@hv=wr9XFwHHZ{`PbKd(hj)B4XdV{%c@!dnJwTSVv&H8zc zqmup*hW_e;ZG+nV>kj<}sK)AEk?IpLhQN>MI60LSuz9LyJLLpX46F2+EIzPXj)e7= z4xI%=z;yQcKR|--yV1?+lR#=^Q7|fT4t+ZUe~Khb+$ON9*hICwtdc4F6GN5@#y#+{ zqJKXMzsZveRtPd?PxOloR=-7Q^SfQM6jGJ@68^&>Lj5DBzK7)l(6Y*%mn{1IK&zKy zD>zsTj0KqW<7Iwa{tj3!Yhg%vB8PFPdeuZn2=wtx`u(ysq-BfAiKY~O$eiyg;3T^- zUOzzNTnvi=bcPQUe>!HVQ*esIA9|;E`pKSne)6x|wq(gUHZ~x{blw_Xtn;6wb8r}t zeiSj5Thz;0W1;H& za~&gUbjhzQGN5A|pCV&d599=;1=MmuS0k14NhTzlH;Cta(OPdb0#=D$+1 z($;qdPEgX-d;Ex~!yGHd9^CzRh3topjnn$#tug6v;f9Zr%Me z2+y5)1=OAGxv)=tx;_>BAcps*g44ZCmx)n*(ZY;tBA))`5yC>50fcf1iG|Jv;Cc!K zFdih6#g(s?b=cy643cB1*o>eZ`)whFPr5t9MMQMh8z;@BU~aZ5PL@aT&u z_31{V*PXz(nW_|x!20`so%C%g0AM4j&r5{5@q-F$kzd^KGC7Rp15uQiVr|K2f8}L7 zVjT3DV+7`77=v(F1(bWgjqw_W7zof9UtGLbsi7q-A< z!C6q%>eeI$!EZXH*6&KuUKE7t7^j?_TIPTH=;Esixc$R(fMM1AwHTIe$uq2i4nn~yr*bFkF`8uw zufabt8Kd`1)kWg4Qz#AX#u$H7eM&5fAs2eHqXfXw8#U;{HwcEv!>Mz9@Q1?;Jjy+c z8c9b>fF&e%S+B_YNZX@fnTL0L8yzBGyOJFIZ!&qZne+vxchdzH)rN6QlAOMS!ir4x zANrY;OfLQ+!%qfIInC<>^$6)k_HV7)-iX0_UXyu|aFOLLFFE;SNui}cA01fqTwCxo z@JWx#SZIKFPS)mdY%&7MNuSI*whmac?jDtNHZ}mhdW8!zH>6{w99>$*fW>0kWzR=D zv8>M%*slJks2}pqNJeFXBJm9af z#%<@=SW2+PD6M=Bmhc9!hn{Eh-strv?)g>cW5t?kU%JMU%%?cB4Pl%UxXq%`jEb(& zUeVn#oZ00Whyg*U@UEi#FUiU4^b`kxxtHsEXtjA@2y|=MtUq636&A~&rBrK771BTP z6Tr<7Y&;jW+?2O`^e_H0f2}dUelrYb2!szjTks$4vG*0^XoK9hafIw&H@pU4o_cMD%@pprVT+x5_ z{r2YigF^M=F~RsOU~-T%`8ukRtCyDG@!f}XmQ(j-OYvx7kEvOB$uwjB#F?(4Yrk^$ zd0G((iYW*`12y}-8NKnA%zC~0TGLzxwHN*2CNC}!dOWIAIwLjSD|Q>dFx9r;JgbKM zZ*Pi7pc+ZLhR)A`nJ5n?aFagKf}5tB4$f|57%#*~{)8trVSnuUWTL`Vu9`a5Z0~r(CD10|q zB<`^skDdX$942>*BH}UeDwh`}K(aJis0zvY_G)q^tx_t~c09xu>?L!uCBccb9Z&!A zdc$nvA@5IhIUXYG;G8I;A>v_`Z7@9Fx!128j7_#nJ>`G-&SF03$!0qpxdA|Brv$iu z{oWb7x+ju-26#ElRQ1~4Mn$+<>?&rlRAl@ldwQ{y2djl<-u-=H+w-ENEU}KYz!Vv! z4wmD&c-1CSnDvFDf>-1_5USKtvK9XNVNB&1rV2=uayRsshmQ@LTu%Jd`}uJNf@cf;?0hf?sxlB!=c_p&60{aSceW8vyPw6y=->)}znpy;r# z%A2S|p!Ei<$-IL!jtSf3$uI?yXF!zhRZ}e{2?b^-Y3sTz%S@K)LvdnyBRq&W;px|B z-{MD?)OS>r<2X_AN}Z%{!HSA(e%<1KbY+txi9Oeh8jee&MeN`b4AHqM2VeakG#Zg9 ziRA>2Sh`I<&zB}vH~Y7efsZ5;_Pt00Epg{?Zk+ilN|nv@)5UcdKV%OT^vS1?>SuPy zHD=F+mhIUZQ*iY6)RBE=_CzWV!(XC0ss+{7`po-`O@beYH1NK*k2<&P+HXFRlgvGzWG)VQWBHc1sR(rOl&XD_^7iO^msk6)1DvZn2@ZB4M~heqGCL z(6OLV@{6~utES47n!rMVBNu|7z5ktnIzdt?=^4MHjlpd+srG$_*T>alx^HZC4o$VG z(e9>7x41>Q0;q%?ofjJEhamwtLjm0;_h8$Cmw)`|bD@A>t32Er0jvH4htvK7xCvu; zgqhO>S*?6=5r>8T_B&FcB{L8C<23Uu8O1xp+1TxQ5QnY4 z#oYk`MH{R4n&rikSXQ~a)|QV8Ni$?C?P~hWMaB!uH1X81tX#ure zfTiE6y2J! z1hYH_ph5E$d@TR9D%7)b=6}8qpRojO(c=Ngx#yPM`)`FZ-+r9e1j57YevP(XwtV{Z zV5B#|y^pV6yW(G7LN}@%34-;5G0Wq?#@h(v#q64UotP{r!jhdGGaGnngPkAGj8zES zGik~C8;X3mn#5!D*qniwl#pfh_8+BA)Rh+g2NXylgTE{UOWou4!?3_A1_fn^KX_oN z)od)6{b}!R!cPY+bw&0_tx8053kr>R$QR9D+477Z5kXx%A&@0tg%1qlYJzq{@cVy9zZTZEb;{4lY>R z4@~W@c4sagSd%x1DscXe0Qxrf@~SW5IA7QW=V$QZn{iKLRb!7F-ST}lo|5{{HjdVgd}VMYts+>H=|kHT3&1YW^XB|{~N z$+)lVnXMZJ4w)NbN7(1@3LM#Wxs~1RxIqh_INiq6Ka}2}HwUXEo9&v*mynuzxKn4Q zv3*d|Nok^ZVHq{&H`^ zSSH@{UBNe9SKQo6xAqYB8%!-dK)c{rywPoM?r<>lff5X2I_ITjCX{JMSX8%|>S1-- z`}}1KE!V?5yWoGsugbeJJb z*zaXXB~mFwf=~t$3|=N{`6>7`o-|=;Y873YZ_fTa#;2xNafoxDu=|}y_zd`_N2uyE ziptjzn9a#8doNE%kRxtrUMH}Xh3SH5BItI{rP(a?@u_N0?DI?u;2bLiFl#;m@j6It z-Jqo#pcNZ<;|3tXlfo&m3ab@*QZ2ni%9UC2t1QUD6ej086~up=fD+<+JKd0SLi(XT z^&CUC$34Oo?0;m9`*>CB&Xmvf?N{*CInptF-r>qUA_bVOnqE%N9r;lQ;@81wMa&*Dl18uZnX1qfdP1n-JY zkDIFFuNLT1&K}8-r47^qDJFW0AIIPq-b+oaRM?!e+oQmu2;2;tOSIQ3<2C1X$x|jD zsapR&B~{Q>`IkLiNQcp`N*My|Hg5lruQo=WLwO7CU(2UvAn%^zzx&tmI_fiuoQxvtSV)pW-fhfn)sMy!{{ z@Ag`V@!iI7StT92b>c%lxh{R(A75FUC>#$gG5Q6^w^G(`i4BDOg^=_ZPylL30(wrE z?vs>wTYC7fuO$nXqqGV^gNTjWi&^tCcYr{Nl=*&^y&jUbsb$oidY?3T)2o$F;XHof zUTxPc;Z{YT^A34acI-vba82HE+pegjfnZeo#TV;cV0MCPZv}>6dZ&@-z3fJ>^ptAj zG{L1(O7YayNV)p`ok4?74Jw@_2lt|%N9YZS?@5yp-=sN1dWSjXjMOa~npHA`dAfYx z-`m13Fh<|ydsK02tLw`P9%`*hT;UYqK)i64R26PjVr5goW!8?VTjLDLU!%!~dslb8 zJJiCGQ&J%8DaI5LeL>Ew8-G3{(g?)Pl^e%g8BZ>aV@!>EN2H#p5(!voGdW0A-D?I+ zGL+6_5$_3&#itJBY-Mh*u1wjtMT4J!Q5PaCEM0HC#OU zv*v;K57u5R-40ym{fbu9Gc5RtY&@B33>4U;50nrCS+i9jzvoa;v%{TEoi(~yx`rj= z4N(7nf@mYx%*Od+zJnd;fN%7Kh2Pae4_jKf~e! z3HDU$xiUJvl#M_Mr8EM!pO-7%Q=%dqgB^B4)!x(=6#ESRu_$RwBW%?H>PRfg^Wo0m zjiY{O4|&Ql9-xefqIk?eIi}x>otsFecdv(Alm+7~gcTdp%NyigcoioT^B=Tgv^ctI z2G~Ef6wdhrO~#TZe|6&5w+X-eqdgYVhKsGr^M0Kb!Bm<{C_eOC!Xm{+;Sdt+JM>yc zh7Mv{#UKNO-{zL*-aN|g_sVINub@|Z&S$6E@b;-js*#>0hM$f`De#FcA$3m`#Kqtf zs=!wR+~{fxAg7HqmmuR6+-tBg=oB>XDK975f{Z{Zeo&N<(?swip{o_Dn~Bg+nXp_2 zkuu1eXh$FI3@(a7q1qUrhjUIU29k!Z_f$n|1&JuU|ASqYlK&PCZ<@7Q`RmnFQs_qI zzx}p9p8l5TLtX#WRuIUK-E!;dTx%aIN%4E|m0`$@lGa_64X#w>r2NcvRbb&nCTsTc|eQ8;9JlwW*CNa>}7@8`pnxxDfFO zrTQS!K{@0@|AOKDbrJmm9Q4#R; zewm#QTw7D5O$3Mp^~D>tSSmJ$vE;YGj7_e|$cB7*0A=c?~d)&zl3(|mqQ z8H=3(kDg8L4U|KPU-WbUxnZ4eF2@Jy+i`hEcc0``N+f$=x%td9W{({neHDz-X0G;6 z`Y!tZTWKtxjzx|`YrR1~JJ&uX2ULQj4~tadS>-V~8M%~Pt(b;1YFU{h?x6=)jDJ+| z-J0B_e`Nr`yL$P@eUtU)dExfyWcyT2Q`@Z*5l`737bKE%Tm0rp23Xz|eiSs%bB`GX z-xstiG2ZCkzTWFe&ICXA7U5%+V^THOge{`+>u(kwC~H#P(EBt5ckN+y?O^r0MD_CZ*Xt+C z;hXj@9i&=)O2bYl@Z^Qh^9gkS9xeUiR(NZ1_4o_X=DTU$LoARHx`W+@-jQ8AlHqft z(Yz*A908p|0@ZTX7sIkIMk=Gu$%?W1cRw7-;nNn#aB#8T%r~*Ui)5;idC$sLvG<@! zLQeeJYL24rFNrchF@kmMUHEGzQlq!rIC!_~_q_^7e+)6y($me^)Ut9rY0#x5j%5+_ zaod5J>X#T&)$xg8j_@?Mu^$`uLnaqIg+GBvlH3%Tjb7zRz`sV z;2%k5z%s0paveAbl^Fi$J1?p(WM{Jho(1|*!062z`A2=lNv1ZgJe zp1F(|ThyF}cRrN<9(*XS`i8&Qps9p8DihZ}kk6pl9bCK)EY00He~(IWLlo}iDdg(B ziBq-{%x1H;ZHc+2@?j;m<8@pnN;(rwngI>lkDppJM=sJ*8=pn)dv0rWmcHlP2o`7k z(0wSdz8f*=cG3v^n%4JHtRkkG*Q7)((v+E+I&heIR2s8CvX(39%d2R6 z{~Gv42Dom5pU6GucA>A3S6pev;N76t;$$+%c9$@W8qOVt8WoQjfy2P&R1{2F)z5K0 zmU-?s!IsiP(&L1b2#VOcCnFyTw?|k~0|uFf6KZ?aiAtr*Kk1vr@UsdpujUmyz4Uv= zxS@XIR{MecUxl$^{L@cdq&Tt;s2>B*eQ7xm8qnN=6oZeSo6vVuhal>wtZbH#2#wu& z8SloLEuR<^@0F1_rdQ(`G?+af%d=)PtvisKB-Vggk);lbXM}&ttA3Ve@1DUftOm4C z4(Yx5j%Zd93TWT?zZoh}-h0el_kT#|I?h_5DCjKw_sI*uqUV!hmJ2vHsA_o|P3o3Z z0iIR%i-j?8`1!kwYZN8=N&>bCF+$%xXY*0?VJgKFOtf+SG%R51@+tm8(Sw%nBSi`Q zGgH#|%?P03c~^)n|GGo_vpyFp=py`9r>r@RJKm&OQanZUV)Y+QqM5D7%kj!^ zNvpc;_LRe6?R+JW1m=QAkwG5!IWz+Ak~b|q|pf( zW}ti#9Xr9ts#v2HRrbB(lTe(#ftA+GabKmOT=%;FUN)QYGi8Q5usDwgcPwzGZeEE? zxc`$D@URF8TPPb%T}uy}zBv{JeeTOjTM*ZEa%8}Mpt6Cc#Ndv~e_{?wpR-DUNm=ES z>X1#3&0_90m0=WLL_60@vEdq-dsEnVS~d8obkTp1sZJ% z1dIDwhek*x)DamC5t7jwEQ1z|Hl&2?jU?F36sNVc92Ca5`a+3T6!<{6eO?FE>(^|}8irbM5fT@UHvFP3&xVPbqK17X8dbVXc zOLpU*l^jR%tZ)!-Y7Xr-nvB(jRAe@J(T+X8h%GElx!rVrb5y6PN#*(-7-SDGAU(@WlvEM8)I-8KY7mVf#{6Hhi z{DCrBi?-yxb_>?2*xjpAI=m0;kef&<9aUtM_k65&c?!91yaqI#S6S;NMP|bYhqn>X z;!|I$xsvCig40)HPWv`;0LYn;5Rxv=RAoN9?$Qmz4IMlx8z&ydXO`IfrIOiqB<;sN zshQ5FmF*u|Svo>Ml`SCM`1P}8u?%gj+mn#b0x$Kq^F=Mf7*vu{(dpqV;m^egt#-Oc z8|sN&o!`{!*D(|>!X{*}jh23nxAi>?EXlZS0suUKHNI^RCGplA7hq1eJ-@~c`4rkPVRbMyjkeR{>A2RgvyjXgf)c6HhT>_Heev*xU#PP3d8jy#()CglLFA;4KgBr2oq|$aFY2bb!!EH{zvrH^J`Nb>{ zqU;J=4%90F>ky|^oldH=hN)wfAoz@4y_x|$O?F17=HI)FR0LQb7mJ8$HgX{A6P;r- z0ZWiw0j`!R5J-b+m*VYEW>&kFAg~vSTn_fV5@5uxPdLqFzlv_ZXTf8Qa8^~j0(+Vg z7Z5LpW(Q2wM|514_}i?qm4s~by0Jnf7uOAw?{u%0i+lD0tt1LftD-B71n$4aP%}^p zv2W8wW~EkJdlG_3qwYjWZPwhc9UMu?>VrrQ+J+hxt7!of#a4s1Qs5s4 z#qwF^Q5z%TWYPQsVDWTq21qWnlEYsiL7E@uxP)9ftCbsrS>7HU=q6Vs@D=aKI=wRI zF{bYhHHka%UBtd`_vcsTOTFeqDtw7r#$PL^hER$@=`~&`P3{ zedy>7`Fmis$7gB59EeaH7p5{{C7wj+n8xg-*z!M!*nV=4^Ow{zSfkcb*1YudHL4GX zAQJU~KJG`8%MLMgHsA4ag=X6B6=vCKsM&7g+iLxO{y_Gp&^vvvd+*xlGD{Wd!bSP| zC4yxNgXAvkn=4H&zm4k1zQ1gGkiz=##^#T27Ujl?rPSkJ_P^>SaF-MMcgO#SDG}1% z6UHL;AFM=JR&z`Lf9$8sORRqKtSM)c{~!CQGhXC1CGgPU10FifK4N!W0n;CQ$nkrR zS%^2RGn@(T zYdc{r9yC__4>j34f>hi0CO(qhNf9u%jQQs--32)t4r*;AIQSntW^io+#=?BUhbjX1 zz|L|XR8ZDe!H4sgx~C{$vBS;;{@wF13}m9W8#;BkSm3;sXe>63T#R&%?Eo)?pV|@n z4~uY|;zdzYVx$sJd3+xIex9d*3-~3+3{q=(3v?3O6vd{s98irTa8bum(05-?l(Roz z*Yv&)aHKV&#$@W3VAYlHFS`nFJM3{9<=Vui*HnExJw_j@s<~jj_>of7f zL5&I%G(+(0oMwSTd9_$56ZWlb1i`(t)CE=Yb6E3f>5|zHpMYyfB=KXcR^)8?eFmk* zC`mTEpLUeldw|1Xy#~~g5j;<I8>NJ?I`k)Uh#Zkt&2+cEN!Po+Fo`vlRqRl14uhkRnE<}z34-GbaFS6h z9Q=M@=koP7ACZ!Q&W;103wC|k>22)^6FyFODOVOg7recwyJxcfr%Ju{H}CC5#lbP_ znCnNhgpK9(t~8O~zU-a!=~!KF`<)?a*Mv*SW=Eh8qy|X_r})f*Y`=osZP-Rfpz09~ z`F)g&B-kN5CStO);s;ndovuQxOG~n9ut*PGKh8|%AMv=Giq!=~%Ce+%!ab#ROoNZ? zUXj%4^pl5w?^RtI8H9c+71Y=#{Q8KNoFM5rfgHtblIjJ%s$9HC-Sg{!cPi&hLO}u# zz9DJDGk|D3X`~L}%TcJDz%GniIc#ndVv4hiyeje*oBK|;yM@I*4PhazgxOM>2w%KE zRL!x^&*&c-C+W+vjUjlF`mNN~;^9KXo(azOh-bz^>mbA7`OqEVZ*C({F%BKDd3R$h zuxfn+K&oMNqNtwPtAkp%b$EmZ3=lC9j=g7s0yVeU*5u*V3)@KZ#2Nn)*--WvI&aCud5@Eqvu7Z%vFgAz zNg;GQbiOoLdn4mHY0ui6ck8=>t5U|Fp`bmKsnxcH z1d0|qX#P_&PL&yLB^PHH@|Jodg*S{Rw^6U%A!Fd~ToW_z3-5-&`T?iBdt{L(i9Hs) zSKXixw|grtOqE|FQJMpgf;?~?64uoB08y#-gOkxDVm4&sQs9r*QUuU7_aVt2rm?Xx z;z>EI{F{Z#Z;FiVkqjF}{|Aa7mam~K){uS)z3z1Q{73fc-8Z=bwM3_3sk?-KLSU2| zsOGt$&PL{MZeMdJ&0qUa=bkm@0W{r@ajF}hxTvO(ZXka;^X}eX+Q!Rji-Ltu%Lmsh z2fQcigs>GX`xU^Yp-|*m&D{UGu8Nhq4eQhzg(qT#ZKIA$(JK^rl=Yq#l1CboYVV7qE)acNu3@mFh z9P0Md1t}Sp4Kpwv1;Lx=yJv(Y(8%Xt6_n(}>#rTdZ(?IuqjO?b!s}9P$#Q_`UQW&y zZ=!j(-wGz2QnKDHzuqmam65{iaD%Rf^$Tnt18{P^`zTZ4(7bHF+5QT+ai3tu{xBHw6n*U`1 zfEsz~U+O$j@gX3i!r%H{*yGG0UNS(zK=F*U8?N*2c7BCp5L2vMm$F6z%HmV}0B;Pa zduu^eN)ctI3V*4=aF=|R`#d%WJ5FRJ=LDOYZM#x~n&-FTXz)`hVlu_bI-TRP4k{`H=v)v0p3}?uDV$sN#_Ib<1FxC~arNqDXL!3zA z1YhHT<(wd{nfvQ9kKe$pq6>rkW==iN)f=~5R%YX#;${SdzfQGIQ(>k>kv%EElb$9m z>rZ{TUGn^UsLWT#oa$%Ibda~~U=+a~0uJ@QN4Peeuk7L1G?~Y^w#?$yyz)=60KDSL z-!}LE8Ed88|Btbjezx_KRxe)JP2WrVK z%EJ14&n&m%W}j6i!Z^$mouQOdbe~b5HH|xHM&Wm2qnM^sx1_`)kMR0HerlVh)yX3= z%xP~!@$h-x9-Sw80Mvvx#<)Y95UiqgE#+$Bxx~q++h3ItE+f#fw`Vl?f<#hUSS3Y1 zXs5gAL&8irbfTXnPf#ix+MLD~&zk>srj7*f$6O z4eT6S&yg!oOHh_OBC38b?QTVVgoT)VH!LfE3cNbC+Q0Tgc|t6m!A^G6HDJ#H+z(Uv zC&Q0ZVHLD=C%7n2y8f8r`G{_ArDAW+o0kLrCQVP%xZ=`2J3)Iy901v?3&o~3I~VzZ zN*r+is*tS_Y7f{lL;EC;UQ!w-I~TyZ?HH8PhZ0@*s*j$a9;Hl5JS9RP{Iw{HeuqC@ z$o@ShvFu;<`s}P=`1n6Ur}uwRA^)7ONA~VX7O~>P|J6AciCdBXFO`#t8QXqq+NHuh zxCZdVeep53WAvbMd%K!|2nm4$d)A)`19LeYKS(Tpm9awcb2Wmz@9w`~b0?c;#PZR^wOUXa(hs zyBxKO)iV@dVdbzx%T>ZnB&%H5r98{LY-a%XrF}k%-I%{2OW%h7d-kITNGRS9JW3Rs z;oukjQ5xYf!#BY`g4oYU$3e3b>e1|?WSXD*%IMA(#x6jBLiBbkR{x)V@6Xjddh_Lc zq}oy3O`+H`gWX#WxN5;vuOEA?Vxs&=XqC1*w;wn9bHl3Hg+p-v(jPc%&9Qcttk1To z+*DEPZfG1vF_hUS9L-BO+g;7a*;dMq%$5Es{!47)P-RX#Rvn;EbPBl`UxJVls3(Y2 z8|+u(=ky`obW!*jtqscBovMWro7{~pWGnXUyWbdiN({1IdNArk$*9;tSm(wuZ#m$j zQo9)UMb`G&De4{5{g~%uPLZasRn0LtVA2$~y`Iby)oS(6#pZ!-4@I>3_>tkkVZ}9yA@MpS=UJE39wFCMP>c~V&K;DAlVevC_BP;+>aol4Wa!cZo-zJnmFoRl)S;{yg7% zE>GR8E}V8|hJkC944lA&yVF`KdCFJ@u3_z{_`cWI3Jh%k`< zcek4L04yTy$X()jkhb|QLH7si?{(KM-y~kRx7hRkEp-sVuIbL2a>g+I8DgZUn~|x_ zEf<0-jyk0`u?Bp`ItY5=9P7MQ*dTJ?tKB7A3RfYn!bXx!_9JMdV?Tp_z}YR(e8%ss z66vdzL&8RSwki2n-tWBMik^0rv@ZNkmNl6NWE(8ldp)-e=F8z{=X+e(aGLvpiM+VekRW*58hWl>pF^0*40ezo0wHsew9 zRMUWn=Zp%KeT=D&nGFe-Nj_i~ryznYq@Z;_4FHDA4s6En5CiBrzkKYHFn{T6IMiAh-*8?QfCL0_YQ->`-CVlHOn3Sl5ZUUxvXS9 zBwGV7_W4huizfe#`mQbxGm$6Z8LieEM=RnUg)&16WRYK0HVC||EnL4|0ri~t$cl#X zhq0c1Dzgie(-&gkS3k|&GG8e^#2BUsqe_U0u8g0(s4~;EQiZ@oHX$!4RRm0Nf#o<~ z{EVKT@1y#eJ)mD;995hlb?Rot+_ty&)yg;ru(RF^+vim3)3pZ#$8TUPcimZWr*uZ= zY~u&WOQOWj;V@}S*M>(Pe#bN&9QU#eiAbT#XTcxpf*XH}ccc5J`=a!q$s>6-uU#gz z&Zw34ar$fblzfceyq5@@n(Dde-MdFo3(|Y}Ru#EBpfe_h%tYJ`vsL#A0R#QD`q%ub>K%2?`8sCkI zEDp#u@St)fPfY2&&Y{#(Mw9V0xntLa-@hsEzV|(L>@vovm>jna1|EeYQXP^BH5ovZ zo%h((gZ30&QMkzVs3?m^`pVY#v9{JK@8-K>dy&%bv$GvWgwhl~J9x5r`^iUX-E*M4m#nha=DLT`2kc6*#*m8dla_Xb zh~Y3QHUr;XBiH)`AtgO#8|6CEQ;B|;(5_lQV}c`kM_bLKHKDElF#u6<^L4Z*TXa}( z-Dnv_GpCq*zxt=VzgGKuc1`hI#lfft*JAZ~IFXvFsO^v!%MnWzCE$A*aP1g$bvN|# zo)}Q8JWXteNLNWHNeC#9XzrUgn{i{$h>@WBmZedXr8hVlR3RC)3h(#6^U|lvG|{2J zp0RK9!G<#ZTVKIgD#DZ`tu2U9hSAbdvWBj~NS`JgrYRw4^Aw>!oi+w_{HYQrGgIi; zMTzhY5*jBW!nDuVepKZqX|H(t%knwMAovrzsGq$0G~}5<<8LxJtUI>T$2(u(7W*2B z&;op`YLT@WCD7H;nJ`zRHB1O7YV&?Wc(`XNHF2`i3b2PfszfHrr_(@ap6OEv{8cb` z(vmFa=x|GZp5lyhxL!vq5uF4YZ%Z&D?bCmi!1)FtvTu2c^lpgKQfR*;el?P*SRJ;`-hAs1!xJGe!DB8W7`y~d5OKu9ue8a)9y=;*d zh|)U=$^@?tU&>6w{lsVok19{aBTv~2WPLwhebfJBE!1}d$k>$o@9M6hOA8O+ch+A6 zd(DHfJSWg1eszQ3$!vFCCot;)8|fQy0C2}?f?56ZL9qD5QPJmm$1^s4DZo@&vf+*1 z9pR||J@`e zh)Jg`b41_s=;fz#Jk=&#K-9~rDNFHRWXb;&Qh7r2xbT2IujdzQH~oE`aGA4rTAewc z?Dyh&S`{LGhC}1MTZVP2sWv!_U3LoZgxh3e5e@lX`U}8_f{5J|jF5z?G%EP?-3lMz z*7BtwKu9jNRxJ{ljAEB2;@E!ni1suo$Oyea;1X0Zfkub{zb?G~8^&~tf6F}%IKA&e zV1YlJ3LNHdRu}1kE+cgXAl727#cshJE_Wlad<&jaLCIgox0ibDtr8a+v1cODZ(o#D zCZ@cZyG9(`#mp6gntcS{&#Ho1^LPwlHtnk2 zhz)5H(3wX5o>*MZNV*eEig{W3T6Safdc38*UL35RGR9J?k2>!CGin3c3pWpoSlDsu zUj*b5eev(mYWTHu+cd`!M|&SulO>x-7{TKfKhy8OO>nd72{NO=yUl-{?<+0x|V*cObq zS?OL zlqk7Jw$U86Jk7oJ{d}p}3$gC=%CNvRav~yj+bA6ymF9SN>=)5*euUs+Gw`q86OTM+ zWWezf0ASVmBT(XWJax1McZYR*`c++4iD5E^gsy#yH|3tq##36Y6fAQEE)?-Tmh}Az z2N2cgfPx%*rRzlJl{*cGaa!I?9n8zI_`||jC`h3;`!b;S9 zeUDB4e#oRYXd~p^!&z+paU+4JublsEX$IB3aye2uZSlx#&lY9uL%6bG{^0WDbQlXh`RHgcq{I=o;rGs1H0Hf3Pd%A_17fH_BdC{MwWA|l`9#Q4M{pJys%vQTfUBLj+ zn*X+irtK^5wPYd@^ZvHkLFD>>Xx7dok%J_k*H6$(Y|0;;Wh-Rm9-Ww+!PeP5PI(kd zI1JLbio+Uz(f2#BQ9VB7!HISR_j~IT@VI?Xr@TS7;B$?DCa|w`rz z2^S`2k0;vQC+UHkXj!_U>y>er?Jl%o$Zugw_ML19kof*O&p3Mz`@cq2$|lw7sP}F4 zUp1GGO14)Ad}Avuv&QvTsP`TJ=M6VZ6C^z@D|g;t(kqSr-^S7O{}`S+As`wx zR$H*2@KQbe+&3)%&F(D)+IZRsnlspV+>Lk5qf(vD(5t_UO{`>!(-r*a=8s6LRzE5E zGz9;JuM@wq2(iTo4W-Kg_EMY0gWFCYaFRvaa*!}2J?$E!96*hQs==TfLeX#@qN#o&kWCM8abN$(MP*6N`_ku1)vGs zziL-~`2xt>Ta~6F5`ZqE+UR3i(DzPh9{De3=(sAg1MuME`$-b-VpJN@%0R6kbBSa%7?1XU3$iUHC$Kbx|V2hS`ld0K`H`fW=7MSVb$rD8*B`=L4)FI zpDD!!+kNYaV~oz;v?j^2X=<}RW4HcPRoHpCBiA*ty`1uN=i%wU;Kdt7+-fV$7xRs! zPg}Y4p5)2EsW(r>+&cLS-%4{qWIAMDlbo3@pM-aMethiP9WxBMkNQFjx8&1*LmvA= zG`=Zno-l4t?=41Ki)Q$>vT@7u%(67`Sw51NQrX-OHz}k@?qbeqMSgIAto4=zbwH0M4Pkz_0D&G z8xORD?9Ii!4Ru}GbwSu3Ve)bz7FS(yQFeBel*lVeJ+X(t!W~U=88)Uw;QbLEn`ROkdDvHia>uS~N-ZCxl1zc3*|Hzj&U!P~`~$SV5y4fdW_SXq z$>C>(LS64u_qyp>=Z@hc(0Y7#;12;kms0H_=O;?D z$ClG?&&i^fK<^nA{PtGx1+egvsA-ZLE(5S1yV|t>Z?<7p112wkS6rFGQYtXe9l@V#4#dTjo;#HHgy{W|#oe&s3|l z{4DYv=x<(#;xHS0mi|x$rS?HfML+^#@TBn=C>1nAjOQi7qU&iVpHO1jTtJ%BuIbKa z{BT&HTpLYQA$LjFwnrlDw0-XU%~d_}2>Pq{zr)sSOiv%)A4uWbNjCt2W!Y0&OK^9= zz6puxF^qgZgCKX-fX-<%n4ZW*8r?UNEO6KwD|xA32WN-rv;Al+er)>DB2(FgLrS8D z!`m_|5*W1J7YzIm+s9I0`OB4rUvu#Mhb!q!hBZiH_GX^NYa61_n~^>w)s(<%S}e%u zXcnJ^#C7+Uf!3vN6@Ot}TgtWlDzy_wGW%}X=M^84YU{}&4=A1A^LS?>tPxq&Z?_4ifOqBo#wfJxW3er#;fZj$@0SM( z-#h#Wxm8{#Z667;D%CUCB!HeK*}g3M5xvZ#w>8G&Ee#epKoMLt0m`@c0;S~ACUW=V z2OA+}GQejg2SBX{kfIdqP#4QxXi%aLn8atq=jI!|-xyQVfh5aTzVLmuO5+cB*LDS4 zQkzI=hSSb<)lPV!#3r)NQBkKFDZhEHwlKNeGV%jJYn2i*y@J}XG&(B#s>#!|V;&^* z1hJb{fxiBEXkUW(vI56Y{#a-RSho}dMv4r~dU!xqZ7lO!SO}+?B0B0-ft~i3L|C>6(f1$uQsI#ify+rfg=k(nTWx-J>TMl-XhR(J`5%DgNSS#wfao*Q=d(`&g=#RE2Po-^qSdQ)63o<=jKi(B$6M-bsoh^rB&fKXgVOR3lh2l z%zJl#v;5=g4=3E2{B$eqGAuFcFShnNC?NUidJ$_zrAk3mjn+7a6SOW#O=4by%k0V(!$nZJiBORK-1Ji5(+d86z+FTHFr$3H_Qm$* zJ*BW}DmnWt$Tjw*h{Pu3dR#1?5?N#nF29Siczt9zDaB{(pU$UTt~C^4)d`tU3!+ie zFykKNGZ$zWeAnT}e^1EzM`q1w2r#1qy({7ifSK}!`n+oCFO|9{jhzjp0J%mqKXC@+ zWls04LK@r&scEgk=-)yi9A=8|j^Qx#1lYsy(6=4-t-K{*I5Z8|ot8kExJ^Oj0I#oE zpRPP{4$Wo@GEtu=?$t|xT@ctwn3QQCxPUuM4B+$$*)N(rli)MiKI(VR9A}qp`vHFr zZ1c@j06^~l?2;;hKA;b~*DH!#Vkny=kD^*rr*l`BxtZ+(0kxEP(<=pYl_X>N!p5*2 z*bOPjxj`oY08NwPr4$x%zz_vc?(_QyB5-`XR+#%)lUpmeV1%6NP8~1c?e9ta_28SM z+Jarw4=wbPfU%=vt{Qi!*%E%+JeS`|rAqeol%>V1Pl!t(7s4%yK^w7-i-#T1hug&q z5O-ve8%K4n0Jc(HVQh{4qwj9^PHYC}Yr!vv&@M1Rr6Z!W>bba&!K1BURBAfHxY~}{ zTtWUF|0jnBKX{YX>*S|8{Bs$P&|5+6W67!`Xxcjae3|XeQCfgsyHEW$e#dJw?DyOp zfd8!O{u@S?{%06@VkzPq1r?xm{bz10rYyQ`IeF-5i)fhaa7&VGI9Xj}QMP%2Eyq?& z>J^T+8{@n{p~NNNZ!(gB5>+_7pZy}JNSjhh60`{sZ7S983~g;DGJ5MDTt}EOFLU)_ zxv5saj;eQ{@N!`y3w*Wk(tS&@UcLf^74~BIEEeX8|GC~X=4Y1?-rWl+a-VrO<8=9I z{15>jFzu)}t}@k2C|_&Rku3n1@*)@!@FJs&rTb_@k`Tc>0QnBR8tt=X$3rD&qBt^9 za@lu*x1fBeh2xy~Usr)RPM3}vMuhLlSBd3np!Am_^7iK2lUv7-a%1kty~qy{Rx2O9 zwivM+)HVbuG2vl>r+MJ1Yoh)$SnyGsQ-v=w=?9F8JM%IOo6s5F>B24QM8J zPkv5nfFm^%BSm!0mx+d#v77l@ z3KftDlt7p7ahx^$KO<3-Xczlbk*8 z`qC9`2^(X3^pVDzIj%mrp_s1b1Y@fvc)67{sCs}j0Y0OM^jXA)TR`hoCXr(vnZZzq zTTku2AHHK`6f{!SCattc*}JBHTkI@ zel|L^rVxOp78;&h8TDumZ>B6715~8I)4y3c*M*#kwvPYcQ{4WKhy*GD@F_Ae|BFw7 z{hu}|K2k@h4`@(Ia^K8o~YLD;cyoVJMa= z0m|QCTo+}u1x42qy{s2?b;q7Pi{)cz)q}t&Sd&YF1O*>13fUzR@^IviH1|;)E6W`% z&6&$No;l=*jjWHGC)Z{J-80HXHxn@0w?ZO{f?m@;<3QaNMR@3z&(*J+z6*BXCR9__ zt0qR$yD7L69V?YV?xIucqAb8l@8FOFr1nBYDF^-FrEo%$?V`{jiJ)rrZ>EI^;{^cU z*T)P@$R5UrP8*)j`r<60k#^DDV&wm_Neoy+vbT;aHY3SH8Zf};u~jIJtNUQu^(^Qz zBo{btechT=E^m-(P(A@{w|nUx?hbJYbo?2IS&*1O8pg&Hk%|T~a1Xm@n9>nI?}|=i zU(qA_dz*E@m$yx{bJIlLH`b?Vyk^tif@_5nfS=UzcKK~>d_G06GIQ%{(^&o&<+e;{ zwo`J-)Ufkc`IOyw#?B^+7^te$yb^3hJbP~495Fs8FIrhbTx1t%#GMWuuQ#Z-BdS7b z0MS^Za+?>dVqig>)vHhongR<|D(Zy@)@K>k11x%@+7i1r0U6_)>f_=HQ>9li+6_;1 z<=7>hvTX_9gy2?N=*n#r$8N3(K3!fTZ!SRemUq{XrC5plel$X^FhUdTzkDW%inv5y)br z0vZTIk9&p8u$=yrZNt1d!Tbz+;b_)-t@!I?_MFvk9l)Fb#wr55$+f41jWNKSERgH> zv+;<)i!%1m&HE?IuoTVG`nO-%74LYcq8E6uqU&EkS&F~+gUD_egFnC6y!CA&c5?XG-(@<` zx=a*V+CV&xQ)T+Ea3c?t!)2B)-j0c;#LRuGih6+S9KdFl7y~s;@2GSJ_AbwJjRK-l zD^%+HYH(@!_@Kg-MbO?VBkdmWQk6M?@%@8A+f-GP>k9e}R)j+eUxb86nm|;Ad=RcOYTlfRQz27{Dk8-e zCB)`B60u$l>pBiu^VOYv-K+iU&iC2X2PZ5WrnK(le zEGhce+Nr4Y@l6p|g0_!rhM#Hnoc0#68Sqe9c}85BED%dpM@JusOtDg_rSoL{Uybt4e?E?X;zF$l1O{4nMw`{8ac_=(nA} z>@_?8_6Qa9i3f{wSs#mn!(cRyaj#M*QfCQZ#1W+Q^jDhfm`ljLs)hG*xpnKuZ8%4r3?Te4vKkVhGw7=jsTUX}2gIS2N(It~!Bo zD6yl1of$k(;^o>8*_jSp7*%YT0@7X4-)WE=(aRFV5BZYJiDA!kzxsNXVP5hEJ5qmF ztUhjUrNyWJ+R>s^(Jgg_fUb6#IvA~&l-}+zy9-RX>o+Y&x|h?WRM$+eJ4B~Uu>{|; z>vd*qPn@n5Jk{+Lz^i-HklY>-)zkhWXtQu?{x@^OY>wRSCS*44aRs(jA%8x#X&;{t zcun&3xWouRvB)nil6B-*+KKkqg?CGSvZ0{+V}g5LOK>_{fX2$7Z;Wa zqI^vOjMjve<1AIvUZD6Q>s}7aJK0yNHYpsb98QmtAA5KKmKx4COcL%~djRXefd9=xfx+J2U~2 z&_$d_aFG-26i_ah69T0-ywI<$jej)11Sf5-WGT0aSmf@S^tR*yR!fN?sJhWgQ=6xlXdd$6#CU|yWRyktnc;Dsg>pT| z$|&&8aOMUJq@e(UK(13b?_|3{?eR<54Qf6U&CXq1dmoJe=;H}gA!NSfebQ@K)AzSg z^X7$-md%7FxI-yDpCA4Fllkn`6>4{P-pKz_Zd!{QwR?!34L&gJ{?phjXazp|6<%0& zo)G~pkPK3JjoNx+H7WKMCBQ_IKF_@EIEwTzKC4@;-FdlkVDs@n758}sP3}8S#-AS8 z^V2LR3n}VeAW7{`t-z@m)eS%h8hiCiorFdG{qg=QTY_?XhfSJD+6RC?^ciA6oqIQv>QA$mPo4` zvsp92btRsLaIy=HzMpovG^pF%N?dPOkH6(rh4}-)5!T)|4ym9qsM=Zi*Qc^eu#TX) zqb|6Is^z?Ei-Q15rpFr7yB2?>=R4=H>}(@myj787wRargu27g64#Q^%s{!~6RHQlLswiV|r|;ddH-DNWO^ z^Zq+Pf;;rmCr#7b{}lGRaVy1#T05*`|0VFXWCN;XKRinc6w-4NT!4v*x@p2*^|(^B zQGgb`0Fr<8+{6xnb52c^5|pjNjAqnTx&#cCw`Z>0r{*_H!+o+sFwA=r3y9*dND3-U zWh^`H8%8{!J=PZcCNHT><@e_OsYaFDLmVe%}@^=*##7$>9sMv z2GMcB?pP0-$$JSffDGN#`QuaKt90*G@k6Z-;(GqjB@?eEGQY}M&k8P5hvv>EiWhqy z)(yhD5`9hZP&|4b+c`Rr1l?%aqNq!RY!T2Rxa09~!zEszvx>`j%PMz}XK_;MVYy2w z;})dUYJRWCVz7-|@x>Aj;P^|2qmk9eCXf&X*E!-}lNHoV&=Y&PdA^f`1OUZ9Nm)vDWa&&tGjPJbZffsj}u~XYz&ZTzum&qX>3O zw%Xs;FPWp=b>`>gI%%lnH{n6UC*rEr$4p+&fIQddGa+Wz7z?qFKf)h=?+C~P{_BM4 zahp+jV&U96>5dHD&qJcZqI;HugWJ};i|K9KVoyu;gcZ32j<+}L`MxM*;HD!uDEQN` z@IWu&P}TarAr7a`QMB%tqA{drhHa-6r`c=tZo0ESO05|wh+BTsfN=M>N-)m9QEeEO z!TBJHd}Y7#2p{xknhSfY9ri{^i1lZgX@o_&0`zbAY|HD`BP-b$SfL{)3tg|X>Uj7@ zp?;v#QRvl6c9ch`ZS!_w@cm+%)&griA zHPKipHH{5xSBR_mbc00o>22=}tPX)yHyhCHF*1L)>wzRi5g#a9S zd~ERaE@HCjyv*OG+?l(Pbx2Atl5uulUs&Cv__**c22z4R6 z*W#7UsaL_f+2h=~%@t;-_?JlV0Fp7Lw2Z3QZeFc-*UE(=C-Xc)mcJ@>&S4+RO;ZQI z_5=5kQB!;Cs&C4v-a)n?Hce!e`i9}~RUon~R{zPKL`~bfECFs*aC*#wVNDI$V0x7w ziizemLVtOlsB{iR%q_5BxD6wQN%b&o{a_#a)Cf&QRl9Nb=eP80Cyf{qc*`qMrlP@#Jnmn6J1?*+^|R~wX;c_GI4 zyh!AIX8FOikji1iB}at9dfIruN;StHhA`jy+MPq=w@r2ReD?&+c2862pGK{Xy9+0U zsith10wm^)f*0#Mzkkj8-!|0OM!5S=wtidB4e8B@Et-1)sRwVqA~3?d`S?yZ;ojNA zi!+Fb&yuS0enc0?d2^n!&1}2ln(SKtPepXP&pgZrtn;gT{ZqO7i_L=(s6kC%se-(h za`l%7ib?ks2Z-k(Lb{F=3I7jOZypb2A9n#~U$d`u6JsAl%1-t@yO}|@Y%#VFLiQzF zLdKSTHw$Ah_E0HCvW_)NA!V)XTfLY2exCPv-~W94V?N_|{jPJ)_nh<1g21g%v|hcv zyFwa_Ye>F(9XAP(0Xv=z_EqF3gG`j|Nq*D5rxV@GNO^Z@iQ}zN*71^tsoA=JFJP|t z%M%@R6^CpTL}{45^#wXClA;Ho;lTSn3!%GU&(4P2nQV@$mGG(YaKn9@g5nQ~UB2JY`iHT40T>;d_t_>{8j(X?9lJr=JCX5H>rE0j2`B<`Hx> zqgr^)ABL5v8AdIAsA!oqBEkvi9Laj5pr=e{0eLsmC-;c59I-X$c73MfSrFfLJwfhg z6>IXD0zSqGuh{Gu<59yAbB(lXm##%eIb0n$&%BT9HXuY7%NUbczX2XlC8?oI>QZtl5xBi&h~UJ2Cw(D!6K zraz3{4Ip{Xo8d{^iNth>R!>-Sk$u$MX4ysq-xb!BiCSwt0FujlV8!GElV4&`(t=CM zdpNk&6PyQO$!2cf@+oVr^MB8z4tkeLR8vjPP-fj6+bQGHZx_Em)fqC&t;#lwLw;T` zk;SSM6>Z%nFEFIU#$G$y(wSY`Eb*6EK~*lX&_} z2=<)KJX|`p_d63cB`+c9)78(u^0wsfq=nSJpv@2)T02aG%{y$WeF@qss1-ALqYWq;nO?*tnSy>73_f>Y&I+pXYa|p zeU^I^#{1^(kc?o($l&nq%Nop&f4IY>v~7w%iJbVXhIIx?t&TV_{x%A+J$dt^s*s-L zNFhJ`+qc{lhUz`hb*^%D@y!jzE!We6W6?u-N8AmL7(K1huo5H?4h|}&$V8=8t;MdW zbgw=!iaJ?fRMGOQ=Cl5-+zDYk^h*# z?~RauY{)=CrAuYp9~m3+T1o)G3xKXNrVPmP?(2Z~-8(K~Ry}#(Ws&hDp36&XLBOoe zHo4OJB0c!;_TYAVSI&jruKi{{T`ko&F_L@t%W!OB^V)$<`}erX#g#c+?|TMr(QWa(BF)s00(DakQoIzy zrXIzmeTDNCIQUM4>kl5O9;J1K5zD&`ev0jlR9#rNwQLEFzKyZJzuoBI7>$}${`sL9 zR%-ngn-$S#!QW+}hdC`?q>L#>-qsh}k1yC8!BuVM5mrHuA>9$VvPfO=<~}YniWiDc zk_;iM`}(&I24SNHd@Y*TlwZ^;^n*yPS;_lTF3McC#FeXs@co3(Ci3uU`IIZW^O~$8 zz%yoCsx;@ac>3$RBF#KPb7p$!dH1 zBJT>4x`lTi=PZJ(P@lX2pgG9$lV#5+YHzh&!QiiPOr#sRX02%^BHmdu!DdyRrjR?NL~J;)u9LVQH6$Jk^!a{?Q! zq@Y~h>8p>VgSIiuh#Gx+W+6i&NT4+#k)!_8(*#OR{)%2YKKp)$TGdBowEXrb58}@u zum?hOzU}iXrQ1(G8Jii{jN~pnv<(fuUOiP7oRc7ke`?2q>1b}50v3k9xM-??@_)Il zu>We_W)j$l=VRK@%e|qo8dZ~9#X=d?9w9I94agCmB%D~PIP?6_-{TRp9s`YpZQKy}fi=#(pS#^W zvbV|eAyaj^ZapV}uxdvb6Ycbahv!Z!LDr;ZB(kSv#|P_7p9!cIs{)(ldjuZ+^6^yM z?$$U6AY{g@+4lV7o+v?H!BHEg!c8@awYS47Ch4Q*s(-^=QSraa=GKxqF&k$~EPlaD^jTSCSPB;dcE(GtIoDVqySw6 zk+57<)l{4$`DuIjFlPOLP0HJkXCUW&{*H)nNvU`5kyMp9xXILi8D)P zK3m=S8#$)MY+JMDsyo-{G+N4*@BJFS(!s`mr1G=z!8$rYhtJnSRoA9zrOkxvE*(<~ zEg7Y)Q-ZPbRE!vO+raaBp0;Pkdx?p~@tL9Za|Ry_hh$@rH@Vw&g^BcT&UE*kI^w&9 zLAjr(`?Z#Sosn0`_SRPt(vx=@!a*iA)A~|xHrwN36aZ1mw5j^<; zbK}L=NJ|5eaP_Niw@41A)n}8i4Qa9L1TiM;WGa;W@VTlJbcPH((36sq(pYEvuw&!f zjR%(OOAFXK&He=&SM(N1{x&03)o4r4*&l({+Z-|W{L=6>JWAppKP*e{n}_dMEO6*SVwKwp; zUELp2j-3QTF#PHLJf?)i>7K{6%mGBZP9x)4hD*uo4oLkhbTmVXeh%IWP&g%0ugxDn zj_Qy8tjvdukAOYYUuqAtz9A*-){;^_La_;!HY6p}P`lDz8IWemV!zPk=ZJk!_>lSJ zsrksS3ZRhSnV2!>`}hL`t4xfoat6j?0NM?Nmj7v{xq@CvD7(^o@pyrAY~%3h2i92v z6DB=Z8`YHK(K5Z%C)OF6i8_soHCVTiArT&cfDV(F<~McVq<1!MuKbe}%g%;+$Jq!ze0gTV$gfy(1D`@hz(%n=tXJSott(=`8909OVAqDI7 zT}wp)>2&Z z*EczQB)rK~iT=hiL%}}iX*rQ5>T(>*6SJ=uYxf_BtmryLPQ7+sd4Fy`GR+nm!EzXV zpX^j7lBh&+*xxA=4^CzRC;3rBaDQA9XhYd(BfOn1B*qxBH$*$0`QlByR5aJRb--40 zVzKAk8hb3wY<&Rt?Yg(|YQlBoN{-FEW3*P@FbK)$$VU~oa#ItYJzIPlACp)plVsBj0#R2*IK@}>S1Du8M?F_3LLTsV*qYf%{;2;9{QLWD$%WGMV?R?;u}qXngwm6u8&$YQJ{WUi z++eL^W#lveD!Vmxpl2E8X`6)-bX;>ah-KGEd&(||a?q^3nGB6>epz9cK>g;+OY=F& zLC8^YXVxIc=;Iye6gj6{q}pNYon-mm;t3G8hdBenN6You)i-6idiRjfGLw0cboJ|< z=gq2rBK4daY^kPv$PUsbJ;vBr5S~(^k17Cx%UEcAC|+OjY!qtIrgx^l16vYu)!Nbv zA6*^`{#eGw7yG2YHyHd7y{3$OoiEM2EpnKpLR%eEa0F>xiNmw;3wqcT5KKl#x_N({ zy%A=bS7%tK@|_!gI%3oqAN;ehd;!s{*vk1}Y%!roN9*}syj_fZubdgjb{~!J7pCdR zJ#^rYPgzOOldp347-5nxT|1icwxs<$ehw1&*^#(}D-EZLdvCda`ONo>2~a*}Lm5@OQWU*@_-^F<749i!_-(k?H57@IIIM_6423LI!+7l>w za`NS6WMtmc#1g0LqN(avEf})1P&|hsq&<6t%!J5IxA;~MNm4!PZ@fyA$x;l^2KWY) zG80vxBoe`zYRnTp>i*zlR0_IwfYh&jn+y=e$F z8-?r2x~Sod8o%H{--!g8xc5=y{80&qh7UZygpxcJeQ7Rvov!1R8B(*)aPwmqXF@@_ zM-_7?PV~ij(qraB7w;c{AwjG}2;* z3SVA4T`z76nTnZry)RB}7FEzZ)ZFE1AJJK)_K4aW1j|372!K@CLR!hwpa3LM^$%bt z4dfd0&Op_!j;k#TThu%Il-UUdK=2y%&MztvRsKbG>`8WOIjcyKxaPC?68ml~Dk%b_ zMxuSA_kwBb>V5KR3X)9pu|T&%{U-Au0c`V}&L_P1BE8qavU%>4oY{!JtJrf3)VTFk zkgaF74-IL? zjZa`(LdJ~Ln<{_A_!V6d>PjL+T}vV(jQtHPCXRrRApgZKI8ud$=Y~2h8fnc^y0_6( zi`ov#NzeAlnZcn5pHff91qrq|$U0^c&A}m>VeXIdCL5un-yqC4H#Ya<8{^-Q@OV6w zN;{o0y(@cIIAFf?CPvA@@-TV@TU&0u&c3LUWCfu(X8%;fw<}(2HfK)YtoV-8J_fML z4)ctZB(l<%XoRf+4U`b5Il_BuDL6R}ko4ibZ)eD=hnTLw0edL>7JGOv$lBii;CnT+ z-S@HeoB$6!d4b1>Ny=y2*`@-Knm4%`tk0{vAo-}@d4_mXO}toOOS)mpRGmuoNOohc zUUsMCuokW$#%WDLIbAH=oT2MiO!}6>$Ve-%=-(Q9MQmHZ830;;wnlz=(9`tl!lx@l zUeq|ac`jl?%j)3!`+gs?TeQCt-oQDC#JwGk~Z5% zN25O0gJKrU=*lhnmFSLx2;zYC6wiaM_W`g`*@WpAg6XFI44j!>|7$5RJUNCqDmt|DRxzS7DaYKo7p{GCu2QD z%>jt7j|rpY)6vJgCpHxhRU16j)O%70f=JT?!cHECOmRJ__+V1Niove-EO&YG6uyw0Ey4e(JQtr zcr(xBQg4ZrBX~#OGGXpfk+I*o5K!0Fe|US0))VX^Ao>Pio!4}_SjK1xh^$}vda8gX zRRF!fbGTyElO>;Ay|$_TnxZwOIXnq^_pFaV+hM>Y1GA-c*KoToT22hc$84Skh#%ze z+b*=Th9i8O9)bKTUfOHnI+|CX;I5)u8{(5UKF(*NYX#@za;_*jX;})Ul2NV{zg`fI zJ!9uv!PRD|r%(c0s40(!m4auee{vp13!fxwK7GqOy5P#lRqDj<%qW@)o+bNcFYniS ziU6b=o##YsJRP*f2QVN=K9EWJK^=NCKwJ=| z+`?*y8J@=2c%c9}(}|B1+70JZv2606tXJ>sZWN}5zWvl4wlasczcyqkH1yPv`W@zgyov{OBnWw9)Pq-Us%m zYt@4Xpvk#-r+;4F_}ao!a*G%+F5`+Sz(KSRTbg+FHbc2ET6lt4i|r4=2OBG5T>C)A zIj)@?BAx`@mP~=R0vp(ADCy*Pf2^b*eKY@{Y=Uo*Sk%r_YnxP?_PZUo zIJ<;-`q+Rf;4&?jBMokzZk)&WKzVf1^1NER>>sWAIx0#z&h|@pR=- zHLt@r?;ENa=LDm%>&?I%h=KlJ=*+x1A@yP}v8uz#0J@_6xp@lWJtpHLd<4dH;LyaU32)dnp&A*KePL-hb~x0-_Li0)Q-OYN&!Hi;)wby%NdPOP$ZnQ%$N`PoRdENN zgAl6C(D}}Yr{o17*mEsaY{wb03zM@saVm%Cxu%1($d_mV!-*7iNGnGAN?)O`>~4OE zZcg7q0v--$K0ycsY_iG*Y&Jb3{qsb_7a=RIE{kM*Wjec>Q4Eo(0o`$s?TO z{}9198}CqYlBBOdFdoH=-{~)qVgZQtC#lNC*;gB)ewYcYBHM$HDm2zx<}U@`L68#- z^kx52al{QiY)8(;TShgL$W6>pw_4}9{I96utY>Y9IXd~4|IT$n}*)YYAI{xqj^OABRU9q<$ z#{hoOuz3#dS;oOuag}r{F2tX+)hP*5O|;vwgJD>>8|rP@izFmS!aYwpI~9TjI+N@U z0!tVi^r(~re~B=)r*P(Ymf5)tK*XTYuZ;>NQ-`exkh?12>YxCK)Q(WzP^phu2I_8F zH=y~0U_mAv1W2M?i&X|{vQd2$%?RTb+LV$Hczam-uMS?S4S0tBsPxF0s7nmO3q9ro zc?8K1ft#~)e*(vVeu>^kCo*HSYqEltrOVm#iz1vMB_91sU~AoB+JKyrQm7jnrqVs( zM(O(T&)Tluqbg@a0stO=BSc7{hOwz^35eCNW$qBLU66rYjC=o;IzvH}BtAwAAH)AY zR-_MawH8lBB>c>qEao@NiRkFzEHuLaV|Q{iLda$dllxF=p3L`IGrwg%OOb;_F!nHd zzN}I+a=#CLHowH{)j0*ivXUAz+kL3wZCi-owqU17qviJDEJI2!-<0b4_@NXx z578joF&A(hOshk)4>@1v%P?qY2o0Pp43ao8R#o>UCi+m`&s?C)R_#jRbb7$(__}*v zPrFp?0QrC;CPb>|APJ^&>=h?L3R7#7J05z`cCh9b>E%BPg8@@UN}s4nAIFnETnqJ6 ztDjp(5~MEby+FP76`X2+pN5m0>Y1KenDQUXI@@lTzoi z%n}f{=s0xSfsg%(lZU&5!>FExHu!QF{{DIIwA5m^`bm?*r#;&&qQV${UCD!bOXyPg zIzYY%w`Pu^dU52;rxHiGnFih|-c&06C!Lspbp-(e(j zqx@O0_ZDXLQQDbU#&`5W8hvo8`QEp#EoKksorSRP{6{s$z_jbCvy@Xrw1YxH`}Qjy zOZjxD3HQAE1b$9q7+!Jck!9WN;bKX({OnVp1J{9%ZB~CRv(L^GEVy1&tAYCrvoM1 z;Z{y_E5Ir}O((rr;%>y3fj6}yo`09quZ_h8b#?Fm(I+BJ@NYiSZ>p1C0ZPoHy&*Hy z*GQq!1ol<%ZN{;b{Vi;bo9--?ZcsU-)vUQdXUy=BPEP^qn-+T6x7~U$xA3>H*$S-<~cr;yEaR-f?txTIbU zJ|KkyN0lSmth6iWHhX0lOHzuz>hf0pxjP1~k>MRu}U5aJ8Ne?hN^B^0B_hctFyNO z8ludmtc1Hp?!2+YlSR55#m` zW#d#gBUDHk8#+$(xJ(v~OQZKsPNv8riq^dbtCf5jYc4)-QjaFudVJ+gFS*cUeb(~f zAAZu-NR_bEULfn;aV{)D`ivP3_3cvATusq(3q5>~jT;xc6_BdVM)Cg|9^U{og?K5B zs=owcr3bLx$FHMKY)`MI-F}lDd7?B+$Su-woQ1#D97PZ5)Mu4LQWi>Y1Z`-e#+9T& zdtvSKboT+nS!a_*gZ%Yo8V0mhbH@vx5{$ch$&=4lVxMear4EzD62gxpV=Lo0mI4 z_PHuQ(-lfh0y-!~J9xj_dKEl8(E>cH1Q};?`b0YfMdOmVH;GPt%4gzZ%207JSpN*E z1OI*0!)z370C9W8#32*)$x$?tdF@3h!wP=d8+LgLIbwvFv4SY!v6c^b>U#_%%ovlq z?0NnHu*S~YFnmlWq#21O-P9IOh-N~`ltY^DuWzk!1vt_YPJUW=!|^0f zHgc|7D>&5NhRV0XooLaCv}yup$EJoowimPZD1L|*Y4otS$#32kk7*o9tR{tmZ){=Z znLI#k4a#bF)_ZQ#kXtQRzvsg9SsGvn)Yi8ke2#+R6sN4Z6nm_OzpXcA*v5wVC29^6!NfzDxvLKvKuDdq{?|d-gKWJ3lWf zFy*LY_%F2Uw?JJDsx-Z z`$eG>aUfyt-;tDnPIOq|zZ#}L-<7cq7zzn`Dh9Ik?Yr%U+yJAs`$+v= zQUAmx!5so;C6+w>aMPF&Ej}z}ik~1$rET9PmkOEilf%BY~K-Wf{4G$w@=%%c! z_-tcT8c(Nr6IhaLsnVnyyR2{@5exZSpMd(fRjqUiW!lDF+FwMNJ-}^1K}Ij_W7aO% zesRp9Jm#0P!(sGXwfQ@7(S}u3W-KhD#Od{M6`(~xOj=6|e zrZCa>%;wt72{I`8k!f?ciXxkt#7)J9NW$PI$sh2Sy0+Lb1Kax98djtFxf;f)MW_q) z9mt>bm&>+F*t^l@=HGnLa&CCE56S_5X;xa%tqEv(Z*n9<+9kKH8*yP8wTOg0zcyihWA||U zI}!ExxrhS~pf9ksZw&m%4T-N63t%^}|Nj#Q&Tb85Yv9J1NQ)M-Hr|_PoW@Hr^NIcv zkGgodbHhHg=O`#miuA=juS`tF@ztVAJ}3kUeZ8JwP!bLFLkw$+YjT>)o0bQ?&v7P? zyoR^6c6i5x`I_&fJl{J%L%9g}#O4i#k;TAtkE6oJa=S==j0tsbsIAL=PlbOZzu%%| zW4I!{)yxu;g{YJuq4B<$tMrAeBAZG)FTT-aqb45=QPhB7%r+I`9GVDSc`Y7d0@*jA zM^${l!3EbCEOQ62qml1kTpru95iViCtBa5X%43SP+mV5|qU}Vxe%Z;gLbf}HUH>tJ z7wRc{R#6Y`-Nw;%1xWHv%x-xCFKy14Zrx^(|7z!bB6@#c8#i8iQUv75giguwQz8IE z*c97e?Pd^JbAyu*I|@iR=(up|s|>CSrdhN&r|3B`JV@E43@s3NT<}Vc;~}l$HICSn z6PKP(GC9x~zgh@MaZCo%l~eSuy2x*FS=1wjEOs9xkjk>5@NmRbcoREv!$x=w>@(3R z!9--p=Be23xW$7HzLI1ItBi9QB{vRujAlPEgp-|`b2PmCi5JP3}})#q;TXBRK7@O-8_`+IYn z$5-e7SVzl&g)UA7So&}f=Ov){VJfcceArLY*B<8Ru-#mYlzIXGBd%r*Aj=BU3EmH& zQ6^GgxXXk+#B#(uphlw~_MA`qg*Zm-KJ0FAo3Rt6&Isqt>E`W={s(xlGf=>P4JbH} zaj66Praai41DTxzGjB`k>O+g)3wpO^ssGE;f~Nr5#d z6Xta7gc(v6e%5xwoao2kO73wh@=wl@f)sTrxpu2aD_0;p;!4++V8`C60oYYS=-T0V zlBMda2}|TPQ~&BuSn7M%V*Kp7I`!L+*Zduy$;3&$UI;26F!3H;%0|rF3^)>huiRK0OD~m~vGi&GC)?j_`1&T4Bir6A!x7jd ze2=2ea46KES%w@o6*ALmFVMEwy{Vv=(|vyYac9d0wKw-;QExfKK@m6l&fXeu!>KNpP`{2Mbz9Y(ER()Qfm-e9K)gqUi9+L~?Ir zL{tu}$??ya@`BpaBinOXr3-VM^Sklb(inal2p~$r5`lOhRKEw}u|*zeg~=wP{$a>H zYfe7~Hqor%ObnH$1q=;ZW=@o{Ck8KPBaMGMWBAT@7FIe#Lzw*uH`?Ju2E z#5B&zpf#{jeZlb=TWhGdS8xw*|7h;Wn&b+)W>Tkvfu?U9}3jA-%?lX;ad>a zi@ZSp3jEpP3=I?+g(R_`3z(yXyM*f2bBGt`rLt&L+U}E;ANY6oP z=fj*+RYn47B7cMSJc*^AN)Zs zT^D%7G9 zB<3j17FthI2@=bVm-+lw+~U0w6{&bZt6K+2A?**YefWM!cMeOLPwmxypiAw7qS&3; zNqvDyOAbCd$4-_zEI2xZ1t`Lr_Di{JryTqRf=V6!TQ z7{-xr<&L2%s34U{j*3QVdTZcZG7fH-Y-_{3_JU%)d|HZHsOhzPk+7@r58-kqhS+kN zSfaEU%+KyR-bi4c`6HKNA2O4!g{lQm&2-bR1S$T9A>#o@*>3mJ*)u}h`*VJNI~xE zhg3O1L~N6j z03DG=fS7bSF3#+5reekfd^Oe>mcusKLFOp4upoQ!LK6P_2K}@(YBp3~1*m_4vX zoP+OJkvZzdOp20yQ&yd`kkB6nKL(Xt6B9&D?v}X|wT9uq%jLk7&%VqZNH}D{@J;kq z-GIzqadXHj=ul}eMF*nkb7rW%)fmIM|G8FbmPJDMxkuoK`!@4Y!+I3^vbFn|Z{}41 zSa1fMz6em8l%mD}GUQoBj+maE`XW{waQvFi?h$IEmFAc5OAW!?Wfh3S_}=8P3L9m2<4TCtOR z^Yz5p#Sjz5rND$uHrHKRBJA8OiN$!Elpr`bFF_RUDLBL+h)pOQB1uqFf`_+i7Tc5Bwax9ngS z?f_hRrMJk1jd59SPL@3L&tu#f0iw4;y~SQmBt?x+s1akTePr-2(hNTxn$J6EkQ=C58$(b zfcrX@_R?}(A*v_c%2gLQ=?zq?V5+&m+~;2fNV4wZAK%Z0?GHZbanu3RwruP7IVt1Q zqLpk_z6NOL`&I5$(XUPQvf1j^dXQ=AV%7UiGH93LPME^iz)+qUd?dXk%~4bk|c*9=yC+&`zbsaINn^gZyy1KzIXnMpt$b zDsNl|yV^wE6@G%zy~5)Pbf{_E-hfC&?cGm}^L1~T2!oFv>Ma~1Zb;lu?bV|gi5vk* z>yqPztY-};xk{o{5jsv+NI|xe7k6I&Y`q~ARYQ1}6XaAJR;HKHYii!=OmCrf50Hpw zffrJx`}Hh#vDq?jvR0%G=5f{|;PDnLrpXRfT^D-2CCSEAD?#R07G_03hD_Vsv-47A zQm3%*Pmq+MRC&wn%~D55yKrfIqw;K;R{TSXHKdiS%K{1IIc;0F*K0w}zv_Y`>X$N5 z#dLq87h}@)VYdO7u_2^CNvJdlayK3vP)|Qd8w(saPs06Ig1mpRu@Xz>R1JD9pDi=3;2fkH;Y{VH2}1Y1JUa)U(#ve^jY#gFn!UGvEAmB z%Q*Kve@lAW&o2EPx3c#T&MEk}CC-8#j!0__x2k5hZqiflU9o|Nlks9;;`ecPHI&nt zy)UA!M${s^v~XOUX!$9db^=#}WUH3H>D;k?#qU8!!>B_MpfddwrA|Z9FaDU*4v=*P zZ}T0f6Q-S^El&EFYLy%}zBPXv%G7-@4wYj7lca(9F7DKwyV3K*G3IyG;$xRhrXNo4 zs**>pBLo0&+tNbqO-lTs_U(UMVhsPe#1O`e3v4PNH(h!0l^#z6 zq3tfC?t!}ctoXz57`?wcTo#aIzrWwv8#mInt{e3uQ>LO4=9&q)(8!3Iivh=Mm0;LVG6`E=XXl@txAn-q(wC^5|R*$ zxK7!a`~KMe)L{CTWhQFMynaKPy&XB9DmMU3vGuKS_-NLrTZfaE^puM!PjJw%^`Ru$ zho2eSViKhDeAtpnbS+(nMbjM8qXpvwjJ;%B#lIRW&r$;CYV^uvqBGkpA&(-fHkQ4` z9Jhh831vc=HLzSvnzd&YmAayVt1zj6(T0)|M<-ab_J?DrcjXJel$}X8s=3aU z`}ob+$L{Akt!!?Ht^m)$TQm|uL%kRuaaH&E*K^<8)YMVvl;OoEGsc-i^JVfJu?Vx9 zJT-WVE?ilbWa%O6IxQZ~tKpImT_7}Lc~ai&%$hgGufy*(iHha>q&Z%b^k-dCg7m&m zn)T4~pG-HYhAon%wI_Z#5C1qQVoa;s@=P`fP&ORYeehQjC}SvV^wpXxIxbDYb!C*|n~(wTgi4^M)Imh+ z+I|KdA^%fy<=6ef&)e9iYQ_u1P^)N@VnX37$q$O%t9f>Y!s{dB8J+5uJDOvE8**ON zKQrKFoG}_OS!m&q{34kvV(a4S78FBW`aQdnVx+CUgTS;YO$PdHy zdDru(RD0pO@iBWZUG!q7)q+&4V+|4l>G&n}Av71w1@1ScUabpB2zeG)hlLM*(Cnto zf)GxlZy97M=mHKcw8Lj32CBP`BJbDgt2gr~KP9;)z=;y8$kn%eJx8ArTg0KlRB!8` z8L$Ay&)<7tM%;wt2uC^vK$`>PYDOl&XDxemM{v3%rI8V2s+{pS zs=kP8I$TsN-sAU})10eh&Szo9!pO3F{{KkS2%<*evHK&7)DIem16L&6Fq;RHN#)zx z9bElEJ!nK77_7k>A2TFhek(LZHX{__VA`Vp8TlQyQsMTk}VynsgBGz3@)md0?bBX zw+kHDs$7GwlgNSdtzoR|kvA+S;j_d1?F71LTCuad4rg=9_3Rc&!gd>!a>P>CQYgrs zz-sSydS?Yy^3^&dP3y3@hcQ$ot?kivP=-45qIAe|Xb1 z4RuI!Bg0GZ%!ja-hpx~0oR)Y#B(ttpK>!kEZxqFCfhfsvv$wYQ*w89?{~Po&!AUqs z{Jn%AoBC1D4JaI1#u>l$9zU5zcx^KHI`Q>*EB(_%D=iz!Pl%itb59?BnHMtkRfVp+ zzZ1aZqq`?kQ}Vwl|4E9wt6ndY0$F}qJP*54ZGjnYdU7r3mfKr#@jr4Y5FF?vB&w+c zyAM4YsY$jw_+5ACR~T)lgX*!Wx9ETX3xTsGXtb-}fTR-w*W7WG+I?V2HuUE!z?PNd zh0%v80T7N4-)RJ&P*mFk3mCCjKx=Ari5r9v=WSnCR-`}sCy8@%vGQ_uyg+>~E{oXT z+BqfjsKb#pJm`$;a!eVaRA{)_s}O9m)W-97BYZlnNBTQ+k_=?Ln%Ar;%N`hh^J1^M zsnpp1CSK6BWpKQC8Tp zjk*h(kbS#n!*#%J6Z=2Ti%c7;OtV*=uRs}&t<}Hz0E3(OZNQ07`y16c)b}NaJ?>q9+Ht3|&># zz$Q$Sp01Tpz*3g^w51K`M*n%5%OI;+U*HvT>v7AaTNTU6G?VN^wAMN-MKNtB6M(06Wl5etHW$BV@cM$Gm*%~ zzr#gI5S*7CP>1o5xkzP=H^w?BDf;=qquoE=2g==oXVc`*DW`+a9(o4r=0rc_hmCz7 z(eKp6q?ly~p!qD>U2+ueYLXP^2szu&kg>>ZCS)$9-d^kGz(+@J0hJ|jo@>m5v@cUG zQEh%UO1u8lpw6zP3(~$p+TeBqwGDtYjeO=^07TI|71Ec&HZ&w&+zKa0y$*eSI_es8 znq%NRL#9f1*)-QGx`vl%u#R|Bv2Bq)6qqH?SAO#nzS(rHwTAS&uK!IpCd{MbWIf$GWu~8Iku!3%Ns8MAhJgM?1Ch+5SHVKlj zxxsfpg#gI=tGv6EQ#tiN1qXi~|HG-C4?A3Ddj}RHJEZ!iQ!9hY1D8teuz{j$QYqnM64bXT2&6& zv1ITAwIl5~zR=$91xgMw(C&F9bSbzHOYV>U_?$lU2IGVS<&o6nr!88NNHMz3xv`z* z1||=~TIW2S<%JNkFAM&)8_+L2j_2NGys-05lLrw|4Nf4NA&Bvy;kf34C z#*GcCP0K3VsxyTXtDmn*P97gqSzrJ&NIGzo4Kmt9A#MW}{8CQ(5R!N#P%`zM=>@H2 zhcg|}juCGPSlbR)-uWDS_5Cga7$UofkFTyFwH)$zgKdAvtRh>8?OUFgeh~`}iQR|3 zmp_4@{jS=Qf^BY30!JDP1mS>B8asNrX0Vv64~5nDn^?~FzwD9^_f73}PzM%b!0t{L z7y|8?=$o$>PIfRaEIOG@`pl0y$DNJ~f~9Ro8kbV>;-2uO!Y zHw+*x&CJjx-O?a2q=-@~AxNs!eeiw%_udb8t^19|Hx@I`bI#d&|8|FgIeV=h(i$J6vUP+H(kGDj`AA^Vw1aa!|Uo-67!Ddwq=Q~gi42{J49d- zPSW#qy2{M-8A~!Its5*pA?Cq@Fbr!1d?R8!c-5bkX)_8}D_Kj~0bP__)Ftw3GC> zGL53zg0IFblFn*wV+{UUNz_M{T3qz9wORyJ+^i7y09oob-nU_LZ%~gvXRdM!%~vf1 z4#bQ64~ZzuMgwl_z}ccTK9Xed*5~~jpWKaB^>{}4v~@8xrVEcf`ZGHwi?%>nS^Th4 z8eELp7pgC3KelMCb95eF-w*{2W^!x(=~4417XMq162)PPM~=o-qvT_M^^!wEFQXw9Nj|lN!Ey zVrGo)?fx??nC})6gyt!5ivnmj9PoFJ$RWZSgR+se*M2$p&R1noh*1)e_y8)jjU|t~ zF}i&=vCkt3pLGa!R!eY<`#~fgSmbh^IO(fYnE^M~`C(W!aR6YQxFT>b_7NUkPUp^T7^ph zv>!w26JKb(Xwy!)MRLou2~k84Z0xBM6aS!Ma;3BOJNE%b0`B_E2or5BIMCibk+W)c z7dU5?`hRua`rNsAUvd?-E-RJO>1b0z)86P_62Mpf6i}Xe0SIPLOm}R40&mJK(_|FC zZJZ~}@`2ipI6N$f;4Elk3iL4pU^V36-W_EVv5%voPLqe-GDNR$9dU+-^PB!i2CQ+w zQ+5DPJlb%xUe4OBt$t+J+9M04Y1Fg*2Vvf>kA2m|Aa+6|ZacF?(NRl(C|e_2Ig+%#UEI1JYJU zAInW2SQ#S-V?sfYqpWVltv9x>2h>vNxY11ExbC?4XHQQ9lQBATCnEy$DHyixH!u0^ zkV9^uTCLZT)aWILHj{poukhK#<-yADYaOF+>SCpJ+u~o3@SBV z7q=0t0WXIsHH7PIDs2iz!^il>`nqE7H^mteX*@PE4VG8%O;F&YyDz7ja5V@@etnlRW)g z2B$rv_YUMS9Q(BR?-cv3rKL3m`4dS}8Q)796;0W~HmMQma_*jRymh+RlSQ0DI5jL`kn$(D3{5>J zL*G6#YHAgsO9%h+EAKMy^g|F5@P3qsufH++g+18h(ec;A=v*?3!kah{rx)-@T?|xK(qIq%?7ZR!N*-wD0}kGySWUKFc%f| zD%d-MG@uN+FQ<%MdWxVW?ohp{R^ zj2jTv{Y6T=QtivUuu5(dFTL~Hr&M1iPg)FD$7b@Iw_!yeUd#@V}zVuiAcpI~(!Wq)UEVWodoS3qQ-oa&qFg`rmnnH~dVQY~B*4 z7SOY_^|()ZKP%9=R>k_Pz9BESTMj~c{_ZN|xVK%4ZJS)rG$;qJ7j4KYbkJWzt;a_O zMGd$G)ddjLGDNBv@Z>V!4jPu9FswVgNV?a+AA2bh?tmQBWK0gyy~Pn_9&K2^%~Rjg zn(@0*L;^7nHIC7PFg}|rzl%sg`U$F*qCV(WYpP)Y^p;Eo3JkxS1`jm{p-bPI@@RMK z%Y!aHOa{qZ;Q56rgB72splr2#&~Wy z?UGT|Jv(ywyA7Xn-jfoE6RYVGcsbc| zoFQ=h_b4mbPK#+lsu)6zBHA5gHjoTUOMASn(wgEAV^mfvY0MwaBH2uPk{jHZC4t4V zK9dc}J9#MD1`rCW0 zOD^&f&gVrQ=Kz5a=CFK|pIqP~+SRC8;+lUdd~}QB*^jeqt_@9!j_*>q8WL7Ia4V&{ zLZrXm3GsYo+5;x)4&mY;AnIN7;y^>Z|zxN9M%{o0n9y+<@XY(E`1R1&`*e>faN zZ=%!yH7MXVBY-we%K=ORQXWBzyBubfCnDr&0|P8dVSb|3l(ck1G#?gB=Dg2KHQ<)o zbL;?3rmJEKzos+Wfx3{! z+=4?j0C$@r z7e@*xq{=E=oCtWjPV0B)t%40WMB;(=v|6%R%P5DE+N#HnbF}IHrV@e(LTH$3n${j4 zKP&Nv<-%v1Pmfd$Jw|D{WaQT? z9qGuC(hEi;zo%i-Tp8y^gZ&(no5CAOg+60l{ua}^zl+|f<&NbwV7wd(?U0HtfWUTJv8100Qa_z}rS zOGo=`MWwfX@p2ipWN9k|M-iq-KAvHS=8ZXHgc2FXT53zhWB4(~$kZeWGt~Z2$)G^5 zu2rJhRYMs}#>&Bi)~awVIx+7g@`eJc0v3eMsVAu2rs+qS&+9XRMC=KO5PLvqDb}>a zX4G6^FDPIMa|?YI$fn&b2KZQ&0r>ERq(S9cBwjN_mq?qZYQHD{kOKiq56w$M1FwJn z*d(+;B}vdMn!{wEgZuj*Kn0s&bA{;>xDAh`DlGv+s^$=El&8h@@;kIM@a-xe3Z^my z!EX?74nESIK_;Y=iGksZ|NBq%E~{oK2(W z`epK}Te{OzGFg62I9D@~j_4!}CIaVaGr+ST3sb^J7xp4$RY=nwwUm{%+puq%?fRD9 z3FCzjU|H`}SkjBb6sFbJ-zbzE^Cq98M@_IHonnX+FBVv*X!>sIzZEJg;cUU^N65`g6J1<5yMr z(H4y)EvQ#KUuO7CfItAj5G-jn?ry=W+-`dF?%Oh?O;%n1Nc(_$pEb;d3t<`9M!JZK z`_6hCX}`Y;7U9G%)g1@s4L}O(`YZB`Do)fD{x=~Cjs;8!9#~BI_tXZh@)(vJh^E>G zq%nUs0XPjbRBU_%7lK$&-KpO}?m+CnaIZgw6VX%I3#O_rQw~#3PRD}#P7C!|1_rmkz3z@^hHA>45e1L4SKSrr(Cwia;oFsoppuZa2b z#;}rR7&eMiRO?p42X}lLJGIb2C}c#q^V`>zPm_Y+@(MarZRTE-;s!7SJ*aB7VQoq- zhvzz4pNenHe&YWlTK%~Ff1CZ7fgsb0Lqti)$#Uw0TKSzkm4axko}0TWEIo=@kQs;$ znh77jo57=Kff&A0m)GRwnj99L7vmQ6|2|NuTr#K~c>L0-h!K=0{qVfyi*@_q+wX6? zeXo~YaAfpyZme3D)=z^|a-I)=pm&d37DX(O;J++!EWpM2Z4Fqllxel9exl8WMMV;g zW-&vZ5e_BYp&Ia!b}OerM-3&YrOfej%IS}t#&yuq24*RFGim_Zh7!{j_Zl@UMrl=| z>Tm)JALIRAEdM5t<8kCMNrk=FkuP)+l2I2TUkNN-xOuN{IFtK>-dISXJ~4}G)aivo z@d2+VjPPAd#g-#6W2;r-yhk4m6&4b?8qa>5;=$kvW8^giuCw#{P6jousptL+`I?C; zK+ml8FyNQjOR+e~%IhVV8Meub{ENqUo=fn!FY(#-ls+M5J^Vk48tm~W z>}!)RsMJFojO$1!!WZyjo&iQf_HH?w*ffj*o}{W^W6`wDo2Il{Ylf*2&OweLxn#_o zg674h^D77MZ0i`^ofm{!t!di%#WX1lgnQ14Frr>aG|fkb1tnwX7WB(W%)PC2mIWb@QDLCT|XT z8%m8XL+fCMS}U2D?_7~mxWE)kvpe}o{8`d-JH!TRVXyRlzhJzrln6vlI4|$=TYiws zPw78n*M(DeUPirIZAS*4_a&y*gAqVpO>L`jxRZX0gdXG(1<)bDCM|h}b!$z1p`5QA zkbco)LcQXecmxEH6qqp~{wXyuy%@W+`q>P?p|BFTqXvu${aJ4Nr8yS$^TF9gs~PsP z_*?a>uO-Vr9#EtGJ9f|8gRXnFQgsJua>+wpZVjAd9YlfaYbyiQ*6qMuy;Jh8iHkd4*{0^gj`R?b zPIeYHe)(5*r*GvMDdmxlY8{HU615N94GPy1MPmRy<2n~Wr&*Cog(Gvxig z6t#lewK2sY4fsP%p$O}NN5Awf1T*2UJ8}6Ahs6P_gEQ$1rlMz`-6|hj0iDpmeU>=| zt9f%USP|E$R9g_2BM2ZbQtsw2k%oXsk#sa=TFOX=3!vS*=`FV^$wS&P-&E1})s6&z z|4=N`eVQcR{k8(sRgvuSxeF@*e8xzqo&fGSFx)Pk6t!kNXDN#YtTd+a>%)s)yL~jLjOrrE_C0gnSq?LBz$_xXeJh zDsDe9Lpa?8qe5OVoL*Zl$$pAaq*VsV4E~3Mx8yItgdSsz$Kv;+LL-6LRJlUz{0&}PEZhK4H)*I`oT}XbU`TAYVlNSUD2)} z!b{C{?Ora1~xVzz~FRZZuc9DOMA%WdR&!PsxV2d9~o_4kd#UAiU}^Wm*P1O?p5W z_-l^}7Tx~(5pz~n^l-}~!9<{S{tRgrLx_y15CE;n*;-DSz-cOUp5@9|97bO z<2ab$Gyr&bw4@nPsCncy4ei;#ucC~&N%c8zv0d<|ma=^kQFD7z;~&Eo(Lop6O$6vy zPIen8|0l}L#UoDMZjeZcp`unRidf?k$5w0d9*NYL0*$I!vt`EzIyK;~fcfVMLIHn* zYh5az)y{k+5=Tbe3YzsGygk`!wYM+Hau3uS8;bW^P z+v>8eUI{8_2UCB@^lz&nIi7{vzj#Ij*!M_t#LM;-q`1*Je^@+`LU_msa;b9tIxSzo zxOeYaGq>QqKxmMQ%+p+h#;KUfkN7}la=Z626#3+qAQU-^qSurCdBg72fHiMfiS(l+ z|0!P9%{aTZmA~uH4!smYO|g3pA)=KruyC7tS6Gu~V+=d4?(c9N9?oq#QP^5r36ISj zZEcKo0w=t21#rmpq=!c8tN#GV5_S0PEw;qvyYMIUx)Erv&%@10=wm6=Z`l&k|DwVG zn5ii4jC$R4ylEkpOF{2kEw+2;=F^HENl@q>)rD$P;Nz!Tx9Y9g=XLWIP z$Rz`ZA0jE`;1m$oG+ex>_=%c24ys$@);aM0QAH2!zmxjtIo+9pCGqchr2(gKZP7X~ zs2lVvE$x5?tQjZpVLMTlZ*o`e;sT4AdGT&=;;UP^T|uOZDtwjfak$g?e*`{8eapTa zd$@f^1sn5I1h_n_NStUY8{>jmH4AfTK9=FfN;m?H2fQBFDwXRcRCdVgDb2unU_T)} z{2Yj0^)NY1i#Tf7k8OE%{y@PkcIVak@m{V139c@cCE$eg2}h1B=+;KpvA~}w_o>rJ zI||+>?GlQt_{s!@b<2>r8J(&_B9Sx&`^e3MpvG=^4-X>Q>E(B3nC|1o4uia_91o-c=v8f$oq1 zfy;HEBsD%9S)5bsfK<&D0#1BuPGFPqDAPCL*f^mHj#$iwp}%z0nCC)5irD1sSnJ=) zzq4OxTpmS-ndQ9)E;ZTi7-5cHIxRQ}ex^+*LsCGzBgX~=a}Q7y=m|@_3pC)@3IY2J ztb^XGesilo32_b*`IFMP9>l1QHt|2u`u>uSDKT0nMgIJYj^u=)p*3WwQM96_5C8fWo#`24gCfxB@G6Bw$ zYoBS*XuBb;Q=S6F7QL`A)`F3eCNwiuYY02$o5(4xaP0b`P?S_X{_dm7CqSL93oySwuNyGQu}BXEn^+pRbl3#vm*QvNcEJmI z_2$1rw>N-zhi$!AFn7xH-YAJUcH1wWnD)eOSUjqdq`bl8@DW8hd;fj)u72JjX(a{o zqI#CYKOP9Npr(E9;{Cd#+A%^zn)!z#xKL1=1)mw7$yilrsuj}>l<|P!wf9m$u^+%m zRJ0ObZWRJLn$MOmNWbK+IvCI{=^@Rz<}<-H1jE}1LLUH@aDqQ5TekScV#Vpx&Z_as znU}%QOn?S$w32RV5GuO;(I#Rgd8r@e7&O=rZ&+LLYPBJ`g3DSm^hsE6dzePnZDj~I1%C~Li;=^GuN~7D&Z#{>qVDfTC zbA4DpD>Qv(bwy-k(=zZhg5Pw4ReoGlIKSxx^Ns{M!PIl>15QTDRhSp`G1jvG5@K&} zXe9AmHFv{dexS+OAc;BN8V^`2wEQfFx>7qtaMf}4C1Zv>n~SY!t+ zqBdgoCzF00E-5-d_ilz(E!bz*)LCdkmHt6+JJKb1AzySvF48tT>eEzV!3~WXa2>bo`KEIbsfT)g=r-Zp*_|vP1|pnpV0#5CGnW>0xhNk;@5d z9LK2-C9~49RJES2Ow)WUha=NNn*sQcB1V}N2)tu>zGM};n*7s5x=pEm@8`F*90=#P z4XjRGme>fUU^fM8InCk;hyrMCGLn+`v(!Q<*ouHx%ct>Pz|%j&YA?uE>Kom#){MV| z;AUfqBgb^*n;7InL)>(KPZIQTx^*bD(R^v}fbcW@2PJQh(J@OT!0Txr}!uIX@aN{;p+rxxBn(O8|-cVDu`*MF+Y*l_X!F9x(*|$WR;O z$J`%b0eXCi$!%S{%bZyFqvT2HzLTq*|6W9fwfdFZI~xQ%sE=r zoO)^B+vh2^zy#f)i7g}ogE1)6)Qv? z1U29>K+|u^kJhzvia`@@!HC)~++AKNNWzdfFyig<$M~g&r_?p{w&$l)n3ft|nbW?H zi?p6FZ+mf~>JM`zuaT}DY0S{qg4Af8Z=HlJ>A8Q+CdsEp#>a`|95CkmW#wv$BQU18 z83cfw%-&As4p1C7NqF!F_h^L8U>Rln;SYAZGskQB+jeJ7$Ca!ZK!&&w!qw%fX=pcd zlnAPNzF|Gvyp=nfqjyqO5%ZSeX#v5M1XjJnbovpsv8u`3tAD#Fw(sGI7$8(aQ2xK~ zcK%iO6NAZHR8_szRhDnx3f~(^Cpyr6yLx!5?MU$-o}E9oJG)wLl_<{~f$y{0K0Wk+ z{rZy;sk!5)y~h0&*K3cnpsnoKIqvuoh+)-=Q=Q8fW z>wxVtpd0?39H^_RDO`w4r_n61|8pxc#zyrtG&r4yNOm*f)5pN2g>|y<O6zoulg(Sb7F&z<}nUfKK^ zcQ$=TJ3o$gKh>}GSbs`odA=CpG0$BA&;3C0vAogn1J5H+>mlyOeU&C~ij(JzVs>C- zvp-d(!<|gt7YaOfHhxwqHn%f*7amVEfZeq&ji0o0SUzw2(ZNXGzfZ7uNUhK|NL;Tn zND3|#!pWulxX}7V8YmrY;g5p7a{53x(yPbbG5jI;-+dDBKK=wQl-v6G(yiUAE!XeQ zEwcVRw!M?dm=Za}CW!!E?!dp81Mx>4QANDgfSe!|Nw_iXffqJ;7N_G`gbs6`H=Y4R?_Em+NqACpPJY^oWVjfJ8%Qds9 z-v1*(SUr{t&USNRaehoeQZtci>h_f0{1;ng%}A&A9Xj9Zp4GlTyfzK=wrwH2>lcq{ zPb~P3KPcBgcpQFI1+i)1aH~(^C{Egx5pFSZ*hi}?C*v>mT z9(~zk$VzV5ZOAp5HFaPfCKivG^2LZ-4w%3D{7a1}(kmm2!)#6JTx$y~)Nr#y>?n+7P?~FnviiUa{SuIcnz9 z!@vmnGw6Ou!1>B?@(PEdhBF~DYT4LOLf3?02VW(^45%;u(Q*KFy<#E|T*TBTQ%my( zhiBTX@zH80gNy}@F@aMQ0FPPcwelnh4_bMgWJmD>xcdds1|5G}cO0?%7ml8L2B$k* zd$>$QqTcIqvRcEaz~ni%MqS^~TLZRN6rCPjGCf4%N>wy0WG_VVZVlq-s(~#-s$z6X zT7G%%YyJaFi>B2?ZLUOM>Rg}o^ zo4+o*qo^x((^I0m%A!MXJqT@Of+Uk|CVE1NGZ5rm*1_FM@l#FJWZUL3Z$)(>Dtnyc znaYb2kH>A>>xw6yE;CDqsxQ2c*jCg0kx^CqDlg>YvI@4pqMxR}^$@8)IUp=x3Dwo@ zsGi8J1zZIkFU6|aQI&U*JeU!L9NoA~BR_fLR2g{5Nnbh07D!N3^zcM;@d7 z@?HHp#!DarI!(;nA5c<=<{M3%}w~FVRB*T=YLZF{*%%zgO}X`(>!SQ#!Nl1%aX0^77fk4WtM6xwBg0cx;pu0kF(i zy-cDj_KPNy9r7NmOZR_9K$}GBq%3SkzVrwTL(TYU9c?Bbg#A)A0YMYi#l%_yeW`e1 zT545)SrgWCb|#;n_+Jn-T@FXDUsciB(*34BBAdlT0Mp(im2C$#*Gp@Vcel#y zn*`y?qv=t?T8jR9aIm6FD2Guv{=>n#v7JUMFMt}KR%uYJJWRrjJbs1HdT#4v{)JCZ z`O`-KvO%4)o}Wkb6Be{XW2cS37BTGUToYQxPIj$ z`b|?H)2r&p`%{1B;g`!Z+-9@$#ohfb9}V{iz`7IP{ny|3w_U7EW-H(epCOtQ1tnVX zRpjWdXcK6!zo=blkB<3Hl8n-2t$bID$o7t^662ZjM#=TKjp>Wz4#;=6gsr)>tV{r@ zX*GZc3+)4V^H0VyMFR1c$FxJCE4@FA|`!s?NV<>AiYNvT4Hhaws?a^`N73oykQ&SwZ) z*A_qJKEv>C+7B-y#KTu|`a))&HBc|ibJ7aD^VrPEjdb?L-sehGJN(1PPTG0=vu%n! znh-9h(lBkypu?{E>&?5#uMyIVHpXFJ)g2&C zY@-SX$jw!MWtii68Rq_j7pwl{J6%N;=Ttrs`%tRSgItFOF4yyWN?fr{E1G<;A69)I zgMN+uI4ER7lK3TsS~g_}Y>qGB9O47M3rj`OyZ4JB%v#T-!*tYAQ9FM$a|S9_BVJ(7 z9(>T}(h>V1w-$ZwLe{eVJJ}^ z#ipZM_%OVxM#J@EW%}Nye_^97JBf;p5(DXLk zOdNf_mjb+VKC+H~0~&z|uex?%zeyL?@Pv`F@$wkIXTP;{C%N3{C3) z3C;=^210D+o9slDBfbaITb0>8LJL(+(w+g%RxTG>BvYKk<%@v838vTyxXe z+k8ym<8(dm$MKd0O_}dCpU$=tO@Fkj?Fs%#gp)-vLEI;k%40ek{n*P-;+;)nMw2mA z9jMp?4(N4v)%hhX2;a^cTa}v_o*7=hr3hIXk4Xqx@&!70zr9Tea(zGq*Lyf_fw!G7 zqdr#_zk64_x(!2nD9db7;#23BK3`VnZ`QR~We&`%^kEW2;)awv_>>_vCnpko%%zac4%y;(+yS@VGh)bR z0k_JK?y`79I$}#-MJm$nqxsnCw_)cU#p0V(-iNz$|FQl)Jp5gnB&Ajb7PVeDF5dmU z#P>LntIi-G>*w)mg}hPryPMZNJ${o~4YV6ACA)|}{cd{b4FZH#x5<64Q({$8_fZwK zivvr?Qrx5Tq8}fUleov(^(ykg-ZG|UWdUz`Ew`6-B*#>hPQVOw8v6ajq4Gxw=GrQ0 z5X>MS&3_BO}_ov75i^K520L$2z-|Ru6g(q9dlxB=p>_I_D!}{ga zPW3&{`|Bq4%xZNh{>UP|TYdu5De37&wU+K@MuZZCehPYVb}*La97jISKz|$=?Ig_S zR}MTcD%72=`$*Y%&~ENCOdLFd5}iP`nlW`*Oyk^I%^OjpUSJF>paGn(QS9nt_N94S1Qig(JAzu;w(8F5YlrIm zYx>GWh01+hGnoh96oOXeuSg-I_y-XNLq^(a8D4mFXc}ROew3WVHcZYz0~S{j`N%m9 zWqQVG@+P>uHG={T=kz966)DJezP}TIL;DE$#RD`7Cg&_>=PKfVqWmj>I?oXg{pV7L zx%1TjAJMjZCA`}yzwM-c)A{P?{8ye@!?Z&*{QTFC1vi=RL_?wnOR^Pz$bSxW%qi7P zdmkk{(I7Nq<3+ew8VlT~;I=AxN5UKy{#SSd(yMdVE!C7b3B&!M;9&68n*vTjvel{9 z9Tf~p13uj*`U}gSg4Fw*`#hZkae~Nk^k+~BTjv{|PQomNDj(oJ$F|#{pq!af?&Kn( z(HT#YJ9pcBa0U||;9l6+b3yqo=5j8M`wgq+cnL}{!H8~8f2;#k2&nT z$z9i(JZ1!%DNjuX_nQ4^f#u^-BhW5Ez3>oMSAf{#nq*tP4h%E2%o68lMR{z>$Hy4% zd#GIsVF6h*fwBiQowofY=|pu7lVtv*UPRR-;)Bc#Kk+u}eRnSw)p;pX_;L$RD|6<4 z4CQ&Jzlq$|vPT_IP+&+OCS~n+)#O%T-vD*S4}r7X(cNktv|I@idORo7iReL`M8#UCSL_vvIB|!QPKTs%ToN98BYjr4C97^!0w5)O+x^v&emW zmQgUG<0RpHdN|;cL6Apq5f3FBsz_Pz-4k#DayW6M;L~*AAJZ2&fQHs^T`5lK-8JAr z8gN6kx6C((HcU&Y3W~gQ zX8B2ASZf|_1KjEE3PxZw;AZoqUTSrs$;6MSZ4PC~%VIv_rhq!knv*dFjxMk9xCJ{B zyl|5-aszr~hF@}9qpIhUG3^wRryY_(4%_#<#+>e?AZdjk#P-X&X~N4M6xP@>3~S-} zix((K?(T}570Q{`a0}wpQl>m{cVAk)*=(4VPTR-JG-hVb>pb~YKW!$HB5in+2Lj{Xzb(;qTt z@?HAE<5uawpZ)AM(QZjf!9DX8s`hu$BCz%+sGb#}$EJOT`0HJkBJJq;f|h0aEQm{D^rfWxWqqG=HpN1nm|OqbgX!_09U&VWnZ1KdsYGDQ_s zGK{>WG%c37)gs#+k4PLwNHO`++N3C%*vffKGJciBj;WYYxYcb?l zDDcjF*X?VL;scWU^8PEs9`er~n19gQ+Z}EZCdj*9eUhSEl)c)XCwDR@`=bc|Ic|85 zIL%dxpBe=N`C?Vh*C+%%)y(}fWp?H)fCw~cW5UzDNx=EKl!JVRIwsJHg$Ig6@%>30JWd2fE{ zesPdK!JZzAgP`H>wVf5O)zm@DWYq!Fsd=I+a2+<7IFBP;`YH9eLU>eZf zT%yJns&Z(%s_2L8);Om_FkL&?|3zWarnn6Y3NJC{xaIvkoK0}1qq|{!|N>>f7M}S2^UY;ZU%K{APb(7 zg|*^I#(WNzQk!%+(*%s)ceYbCw+VqTWT|%C!BE`D*K*N1hpr)*ouNw=o`c8`OJ4Zj z?x>uS>#KH7`H9oZnwwigW%!JK507 zRJ}6rAyCqHe{jpP7qsUvvcc`{cs5yhJfZP^d1)kO<`XtQ-NMgAAvoZu4dA~_U%gMm z_d7E;pFa$Lps`u^){xKSq#9*^0{bn#*?TOdDrCffz7F>``|%c>wZ3AJ0AtA%Gm{Gm znR-qU>o5@djM@@8>trZY8C=EH`nf6OM59eylgMChEBpQD7BQgEhuC6g>}Zdn)5yt( zx7u(UCR0uv+ii%CE$qzEOr2R7_!50_WXmV8WMTEh9U ze-$RRqgd4|M{Wlqe*`v}dN%vxH==cS>yhF2=%;p+zu`6-+@z%FCZE2rm|$6*oCzH) zEVJdo&tAxs7MG@~DmABCf12}_=T$1-KH~RVz69Nts`HTQKZlI)&n?LpqT}{2h(*E} z<#8+`oMdjtP~ge{A0<&%obL}y{1{u3(1FQS->DQwNDcrJp=uq;(JbJ^>6^LIcpyO#@i0+Actrexo8r5$4X2tTo6aigg>kiZmy9tK}C}Q*xnNP7|j68b!fM zC3vQYLJ`3oMQa01_F>%+eEES14<0B#Zh44^J=n$&K-cbdwIs_09-I-0yUg|Ka%R^|b2BzZukuM{pgQSMi=q$_IK!>1fn z!>a*r?FLp>W@Bgx%|Ogc%|2f`*>|?E6wC&xDH$}(6!_`8s<}#?r*;Zbz2>D*bqWTT zz#KmxYyoVyk}qu)heo(lVDB^irTS(`0pPIdJMCeUrX%QO?l-d%C{ShZx7EjuKHO{z z1UZ%I+5@Gp!3_3Yh5@}nZ@MFzj2-t%f>6V=wZhAaC@B@p$VxFc;$03n8{%FjrT~1~ z4m-0))3u~fR$y|~KovovZ6lO0Aie|T;zqr~rQ*(he?C~wHt7*{@ldByF{)QGp8BP@ z3g|BJ4i>?Vk~>L{fc}c@ z{b+mgulQ@j0CY*=WH=xM$2wZ!GpM`m0-7Vu?)TO6`V4TxlsNDJTz}W#oFW|^zFV`9 zezf=!aMNbAQC=AtS1uq^(SXb2cPG`8m}v?+7dh}ijb;_-8V;FMLTRYAip#mEiXoc< z)McoOrP4ntROMsOOGL5fC8DrTvAD?0O#GT6yNrz6V1Rqb(c@Da0VhSAwYi`>M zm}*v{pqO4V9@oYan>U`IejudSPG<))>e??dZN$T(w6oE0VxeI6zpR))cXuB|o#Osi zXSFdQ(gEHXfVluxzl1evvaTcArN9wDPN3^i-MS6)6)oKf^Rpgy-oqdO7so1F(bmm8&X!*ulu z*UW*jg68uEgAEx4Th5a6UrvP@ulOwMt+xIdy<6bmn)rVzVmnw>UljG^g<&EjK)cI7 zt-xDA{2M%~{xw*Ys0e{uNj?ArWCeXR!f`1D(UsALVzB!^2Ty~rBh*#DLBARQC~~hc zs>{bAdJsF@)cks{#~RWGch%S%)xLpi7XKevL0c|>V`A6fb@tQq4mcpKjZZ?9O5FG~ zh*vGy-5;*knBG%^9H?kt2~w|in}n%Ae^@LGmWO;Yf0JvTx(=Ur__Y__V(3rx_<}bgS*~@hRCX~U>Zh#>88V=h+-A&9oN;6+D2V<=(Z(Um zr#cydPt=et%kYbVN2AhpIw|U&GJl*vQg7n9q=Xz2{{D z9}El*eC&Z@TS&Xs)boB#d?LmdTv}krq7L=bPNP30(U^ExxY#@qaiw3nORx#~&|=!4 z_6g&XH)&``=F*N%FMdN5e)2^H}0@X7B*g zAU4J!j;l9pkVyxc4XG4svjv|kH@wLypYRV-v`_2PPnXRHFDHY9hcL_sySa9KFBlCF zMNS6+8=>kK(D?^?haKX5Y#V=7O^Xc>u}2m4yJO0_Uq>jeEdPb-X77O%vSA|5(kFIp|lz+6!3m$=55^`j(M9Rwy(RQHNK>2I@MoWZMBdhE@M;} zN03VS@vr#$k;*nXi6U>qKtju&Voh6(IedvZ(a~zJmv8vqP|^Ju{_c@;o*=5#vS(AF zA2{3O{k13gVNa@dQHt{i1~3>Kt%8+>-=JJJU$ln>ZSe4>Ns`65_ph85KlXql)cO}UiluO-Y z{I-O7?n`O?FI}I>VjdP*p%EIEuDyir;E&FO&Pn?ct7x$pH8S&=(nKOwudS4E1xw6? z2AO_ua*qAl7H{jop7Gr7!zHt9h&*T^YtpX02?*_W`a1Q>j+lxK z-Em}5_2;j;H9-f`vg>T-MfTwAGfH$6w)3u{jE}fZ6_coL0TT$O!rq%B>`JWXA!;D# z-@T}N^_IpK?*$X`fy1~KqhSejWe$^7KS(&fRJ%Dk|AL^Z4u^}TVkWVF(KYFd`)^A~ zs)Jjz&K!1Z0h51ytw_%P_+CPd>D8a8U%`Ig)gZroCe0}Z)C!+Gpko-YC2olC2YdM0 zh1mel?$c`C-)%tbw;JRXvoH|yVa|={99I7VnZr+k(DWSiltEQZ)j;}s+5B*QcI~C8 zW>{$`3Hf4a&AC9VfjrzmB(Rw*tY1T4wNMLIqy;-msqStvYs(7SofsPU_2OLZdY0Ej zZN=zN-24ZlHd1ZcQI#X@)#IjM7S7I8qt2hF4q zStectNc;`cJVvt~qG-X;JhoWZ=-P$1|1)74y1;rh_Q;8J?@Ik~9Fz-;5F=uK8+BM!ZOMdUTPfj0@uHrt z^l0{z8v6T2E|2k>f+=m}PD@@E6`9^TM!<0st2s{p@T|iAurmd zEgVihk_|YH{9!USUH4~#QyozNX$?k@1k*uaKo)SGsATtYg?LXDvdds0b4HUweuJr( zbWWWa-=^q5nsI4q>61)N8b<^&c*<5t8ezF?XedL%BNwg4a1Rfq2*OK^*3YU#x-jWF zh`qFwQYao|)Z-vr@0F`V8CTuPpOp_8PH5kx8>V*Lvc7|d<>hAnmj%%4dPz=Jb!%sn z{rFM2nl`ux&L%bCca)4{9L#7%{!xQ8fZBS{3;lpw$s1o?Mw6o<#DR7YpXjQCc@o9- z;3k?;M;A!_Q=r$h#M$oCrDZYOt*XJ;=YzEW!s5VY-cjQnetf0xKxj4X;j@r*t8lA7 zPeu)BK6PKkJCfUe28HfQ8JT}Y1bVyv_f;VcRjnZ)yVRSmQ2@U1$G{iPd*9C6u71|1 z{fK9R0nF0c$OqJ}W>b*?O6LqFEH!W;X3onVVHRj+A?G6#z+d(n*KnIn9z`-gh-nN1 zTWA5&WYD8mRHG@r6_DMf&&2+p7hAdpdbm{}qaijd6dTHX$F_wI*D;CMmA2Nkvw+We3&ip-2YQK2}f10XvnfW%Ngw4###f@jJLi zEKsaUdJ@`5m4V3<`ghzG@Ca9u8;Y_EsH{Ru#c12)x)rvJI7CdwZHr+{MD|~H-$xvP=Kdc z!5ASIP-g(<24{Hu^}TbHLPyW~0}>wonh&1AuLyME(x6%Ca9m=c4r@ z_{OI&hHY)C{IrsuJ~rMV1Qg3f#sFV=2ZzUw>8kuik(f(A7*g*@qL$RA^snG02uL3q zQR@f1E`NujB-fUrLZVr;1BypNLg_@5zY`k)19d-^%U+4W;%g5@6dYL7!;qf+d;^1J zFI{9%OoTB#k5nLT)x#F#m*W9`6U;gmGl4jv795c0B^90KUy)Llg6cVqnHfI%(nhLn z%M}j6F=Ybd6z3a=_RzNE#5CEn1J?g3=3KD7~iEKaY@?8kvnNV1K( z(2z_|on|eOYJ-8=*EG0M>%kimQ1+L@%Let1HG;LH+wXAE6vuG!Y6Qx&2J!qeuPQTYFO_FkJA267f6^#5?lF2bOr8GA36r}+ zM&TexGn^F`?;S9E_OSjL;OR;oGXm0lrM_MI3(<1$l}{m}O_)u3R}|k*3DPQnAbQon zVlb0UsTbt-Q_!tVQ1c4<zm+_p0JgNTPonAi5G6_ERJDTDfz)zGZ5*;(-m7B)n(?G%Z(ckogA7CJ@*<2z3 zX|&7-wU&AFj@ah`VTr9>uTDqZYBgVdR1l?-Pp;M$$y$tmSuM%*D zgM@(8-Bpxc94rIS{Ut;5Z(Mi z*wEe{-t8pm>83nYGmY9cnGEjAG&qOnz52x12{gf>Gm1gnFwB=zy_qXR$L+gTyxQHp z*D5>BmOL3j5B+9o?YfYeiE)Uy-l5%Z&89idnC$$3a5g{gc^~M^uX>z;Jo_CjdU?ZpQBSChBy0Y^80Ar42S7GK=S+PrR-0Z ziqF$L(O`XO63?&B$G?l*A~B5=w?K^=my%PL+20(TQU4i84k&e!RIpxM%p)3cr{V!( z!ly%m-BItM>nv7iX6mmn8y4cs2LrKaF&P~Fv@1Q3r>6Ew1Nci+1dhaGj$)eOm|#SY zEPx&~*xK|V1;>$OgofET+~o|)5h>tZo^0@12T@XP`aLV#hS ze+Jw{j?Hz+^E&B|tlYh(sv1l97e73_n(C_Z8WG}4_Qr??_cnuQKU(#QyAXTs1GAs^f;P%a*VKmsS+hU^Xiow8aD|PvADdn*Vf=Rcuvqncj4!{&ngR?TUh` zj<0J8vwch@P9^)2w>%m0Vo`X30Ox6F=@dco zgoTBjQG8$qVYmS`h8tDD4Y{p}GwS@Pn;}YVpf4)irk-!g4VUg9NrXeW=45;ugw`X! z)aGsGShv^K5Ik6PRr-ssHkhfkdXO^g2=s0yOXs( zTxW2t`WGo|g!wflgvIzi>YN8~l_{ZJ?O~{*_}xjqy#|J&LaUZTUh=33N;M|(&Al=s z*<7ZEK`QF5&5&HZ1k<(2v&h=k${M#N5O^8>I z)V+QFI6N4u-{F>&!0e~OD{iZ4N#)k6fK`Rxt^1sK6&$heMQroF=scU#s=>yzjpg2e z+EScAwYKdFowIFWYZ*t``f1^UKd40Zj_Ddc1#Rj#0NUA?s%fYla^5#~-AEjQJ*tf> zp}wK3J`8!L_c=pY$wVcBP|F~%1OdIxJR9$12OSD1Wa3w)5$Zmc^nM5cXjPs9p-X`m zF(3VeMEmHIFAf5oekG)M6|iy~Mw9&`NYEifpZ=xVQRU9U0d2c+y*!FplNz7OWicBx z{h4o4ALgrS7*MRmw>%1~?K7*Ptb5fw&0MQt1M=PY5;Vm5E)6f`1gn8W<4;!At>BDl z!XRzY1CKvB3_f2lalF~=+#*KHUv~v2EtAiNTxx8OuGdz$=Jk~OtVOQXQN19xAm48F zTJSxY+xR}!(3AWXlyvT&jD-96w|#p<&}&;A(D;ec{Q`Vqg5o#`l7eA(K|LY6j%7kK;>LeSA3&Z3Y{lA3o-3Wy(QHAZkESYXm;9J zViK=tbSlP_ZNRMn&MDUPVmq3-A(MxG5bKbfB+{~;_M+*_O|dyWYD~B#)-(?;kk8Tr zdX`J2qapDzHR&660n?8cDkRE#!Y-ttu|{4iZ9jbQP_+urRPZ|5WK7wJK4qZygY{m1 z#9%2Y6z~dYh%-{;N+2lkvzTjCL+;A;sO3g`m=Yl_ldxU__YN__wLU=H=31i%1Uryg z?-Cnqy!S`B9Vrqc`U9R>O1mPG|IYB`t@QwAQX z_bYAdNNU>l`G~m#ico@ke%6x^6rqTaEwSh>BpZMdirbyM;$8u2^ETA_q|g{{=4n6$!5w0a`yDQ4g4v62dy;*gI>SxX=W?*svPT1Z!x~bd^|r z!RtAFW{tItI>g?8SR^GJqOV2bTTXv2FAc@3Mz`7hQ(ekneX7%tb@b(=dQ$63mvfKX z3$=0R>Ae>^H2-ln7w|E&>nqj3Y2873E5Gg8yqfvSheb>eXTx62{%7I1D#OI*&=Bf} z%M|wFTrd9uGO~qp!7}~G8K820y~1A90>Bu?L8{sbC}~va^-_0 zzf1mGH5F1%I^^X-P&X`mdqu)qU`c66wZcXEjsN_nJ<--8($sb%h5tgJ8lN_ao`+)A*u=rNNuJYZ%}PIwbB1-_l}j_B zmfofOgjr91r+8UUxnp6hEWltM6IC+NFqj6iY^y8TRWL9uE7h1B2tH=bYz_CA{*b?7 z*pR7tNm@FXmw)14J^PwC!xJ+rwrZDQ`WMBbY>@kXKe?1^=?>}fTlsY8bfm>usZ z%)Ygd;75I(n~++V*4V(6D^HPA>~Z9gI)kCZVFB~svgpc(uSNn)OiWu3B3|34o94)1 zvhx5<*gk0@^c3Fwyh1`64=R|fm3OnAHZQDLDSp|%mYvrKbYZT%|F;Xv7qR=d3sYXz z!i9vVKll2K(=KtHH-pQgC*`isE{}{K$X2bHFXiRl#;Tx|)nODbJ}b z!((A!HLe=@P=wCx0YURE*-72E2I3k>y?Kh7!BC?Tl+8Xb*6l@o56A^Hv|2#xnEl_Sf(3CI#sl>_ zIEG?&l%jnf)`qyCKE%M^wp14zx1&t;iZs|K4)a+B(7+-+~ls;=iRnF z4DaJVzpT%jH6|Gtbl=*l$?iD(tXDH}qt4j$rK^40P-X67DY{d(revZvb$5vYHZ)xt zrc-iOD@UhD@$tR5K_{>D#0wdY2ys2hTi>DXwAiMjFL6^uNYG1oo#~t&!Dy}Pj~53J z`RvH_C}H}2MRnwhxv|;hTty2q^vwfpoE`2Yz{<62G2OS3UAT;|DILkEKb~HAQz|u6 zqer38rFL3X9$wfe2+-r7WC=e5oNtV1E9(gXEMlb^B?q!!c9H{RL>EP=RG;^_%5sn3T1e@nA|Qpux6#Ejn)(yu)gp)c47z ze)H%4U?i@ABbW&sxE(r^uL?v9(HsxF&V&p}0pMO~5p-+s!-gS1wrSq(% zvVuWyyzU}J97XO6uYao60&GmyT>I&%vOSuDdMNM|N}~JAqWn(Prpi)a21rl}c@fw) z9V&|o>tcu zI^#wP1hM1`2kKt6bz&qZKSh^R&bZZH+29d9juqNyykxBH!P z9KKWA=X}L z?QR0>u$nvTE`Xmd)q|gh66#>5Y}}_EJaWq2kkIA@cOtLDN90!rmY81^D}i|@bwnSZ zITWK=fjc3LD1a2$u^7W%8l$Uq4fc{Gg}={&vH8oSk%$q@{sHE;%9Y9{g_3@g|avQ*9*OH>EBn+^$Khj3Ss#e&Ie937&K zyn4h^lxBG-9i7=BE=ov#TX>I#mt<;~`;(+!x?-qL{|Z8VIDuy9kKr~`a1;eA76ufD zLf`QKgKeX5e($>v?)z+;dSGICdEhgtyo(+-XHfOcX&;>v@Va9X45*1iJPXU8i~x=V zd$J0(6j-!D?>gW4djN66(ewbD-MCO2hNXuZlI<~xVU}!5ttVZ*4X`KSd_k^vJOP!P3dh96Bg&_dGFe>m)Zo4)!`X57%1y# ziAmY0DF!T%ZR5#N?wHC}hR6L?ktX0ueM-t6 zxS(|OpNz6E z#jmLCqbD|?6pBby$XqKh8r_lxN6lTlB_b-{os#ULzy7N%ard)kvg_Bf|9Ah#EcHe+ z!H;wjW0Ihi9{^AXXbO(M0S~ihc__oCREYnu(+a<+;fcb@#~v-3kH#$c7DQJLME?2} zf42=J&eW~IwpgF3YjzLzv|h!4*Ek-Yas8(OciX~^wmF^Oa=k%DLwxcU|9X@~L(;2y z((Hw#<_2+GK z`1Z-ybzSx>1S6_*!b4z00ub6%?UNf}=)u@?nQOIrt*|*zhX{<0?`*vvIyLc&$a61g zo!g(O!-Eq2;$zuzGu`JxFQb zt)nIsGokqkxMM$#&bRs5PBF&skJYw%hk>6_Ittt$;4{onai9u&n7%MjWkuePtr^b> zbh~8B#{Be@O=^^j4r08M>+*;-QHeOG|c?sss85E-l(P?F=cOa*+7wI0MqT(0H_=CZzGdm=+NJKVtDz0r-Q zgE%V0kV87kBOP(vtLT|!saQFGs7?5HJbdWH;ZoDSAK?IeIB#2FOD4NEFe_LzIPN_w zjJ#m*sG8gYT*?{1zKH)#Gs(=Q_RW9BIe;FH1wGaN345y``b!(cy}F}l+Kht1|1d_u zB8M3v>ix9o#PQcdFZ6nW6j-6CObN-jS`UcHWCquZAT*Zhfh@)(rCl_HMD+KL;}B5= z6WJzJxgi!VeLqNtl$leykCpEjlF@90by;hKX+>%T#cHU=pSl?paIqvWlmBO&?j0jT zC0@0Egz&kpw=5yEeMObck~z3PiQ1X6_4=>xg$721P2kw+J*O%VTFiqP;_8GXp!Rs1&m@~z}}{aaE4I*ps5 zxV+LLSMOtr(>I*2o~VT}HOtkEO}dRwjSVEFtbPQ1h-4+!-wyqC9uAoXnO~Pn8H!;3 zV?y+@Huedc<(;lFD{*?^>+KZR2}lf;>3swA?^8$^pR`r)sLubX*B%xI5+4C0;{0x_ zcRnyC;1k0o5?S^IN&L`MqP(DuiiK&hrodEQ2p>peo+t;IKoU^c0o&nSD_LLZm3;eY zl76&_vf4bljL@;|H*lb7c9p}ZXEYojAyQ3N~2X55})Hw@LXquhawo;n&8R z;Vk_BMu%ktp{gv#cxXVqUqLnZ{*Va}SJ0%_XQ@D{%wlMCQShfwrN;Sc(}MZR8R~`> zv&v=Rg@A{9GLa7fK1?+Dq}%s*uH$zFW>CMq&z&*py-{ zqnQsvV@wn!EkP76>6)v+f!-inm}O36`vQg-W5znyTk-#}2FwXQj1RtPl4<|2P`lLq zu!2~czJB5Ic$)wC$421@j=#G}aK~fe*t0``traID>nZfXUmpjeV3>>$W^RT*=838N zS_)39ATk9^s1BLGG_hQ@g!YH!JT=veoJyGgI^#_%DHvW+=!LzJc|}%sf#Jb2Ve13XJOlj|a~L7P^uXgAdCiDjxqjTR3;E4{p4Y0t)?@j)i-O z7cCm*)H_3UxY_8GE-yLftXF;|%NlWJbdNQ_&6lA$c9l$w;Jw|P8+GglKict9gr|g& z$AV!j3|O>4f+Ycio-H37#jNR2@Rfp3RWe^}9b}xH>QGALF|9d;_-TWuF2kFx*tRRW zE6vtC31x9N!(t_8ECHQcx6F^TEqt&!bs}6g{w=q)rEF)JIQJNx9C#($mh}@QX>Su$ zz$fo|n1EPpH8le!@VLc*9|aavw!n(n_g#lMSuv1vXW)R}R`+@e&ctCvt_Id0me3*@ zz^*o2K&!;GoQN{2|8AyrZ6W61^`&?Va|fu|qH?_fMz~tyi?{5OKWSAtt-vKl8y|W= zc9J!YWkgQNJC)AAzmEzu_WhT^b1M(5@6J{PY7A|N?eRkas8OF1%GhL7>gKFv=b6`X zT=R4!K@srO|I~^xDIu0yNO%PvufR@@MH>rWvM4g0$8ih`6k>o3fOLxCQp-Q2x|dKy za^%Hu4&L*}&spq83pV#Z8b`#ELqeCRU-JwunHpow;0>5&rn4CD-}Nm6R)5|@<9dAl zLqxerrErX0iM1eQW^7H2*zmBde~5uWaSlne$%Z1!0HA*mKNkQ@rc0|W4%xrjcSNgd zOjro^;c3(Td{7Q)^Yhb%mB6Lwh3m)S=$cdAwNb6F_pw~Tk{a~A1!luD#HOb+pZw;A zfzw%oJ0;B99>1OSk$G<8RjIs4hi@uYbmbf1i)5yXXKg;jdUTdU!0_!uNd+KTTeZ}b zN&Bz)t@>Z{o5z35Z^17AHNUYw3SGd3d_P}h`NPE7ajPyOJkiDX+do>Y?x*m6}W zxQH1n={6C3*7JwR3Kk^}mW9K~%4bsH#VWR}(T1UvC(od0a%ExXb|2vQPdeo)9PzEd z^qex~VCp9_Dqz%u6J3O3eOfAD4LiBj;uNE;!|s7U6%H9ndhq&`S01bS5iba1Gf!5@vD-Di z>EXk`r8CQF~awntAYS14Tu%6X-A@m(-~t`#t}spMTeViX6@FA&y~%$EItaIcevXaIh$?@ z{EQL&F#yP=_0+1*PgZEqE0NQbxBa=P)T^e(;b{FVb2P|6-xx3Vnh3FAAV}@86y--N zFCOL*yUA8l+8iuwagp-o45Qblf*Y1p^N&NB;9j5X{f4gPV$@C2d3W)LMwuthDc`BH zqNmU)UtdYTLCiKh{l#=Q`8X^nK0cb=!FwJi4{UF&^<^nf*>b;Omw{!7^A{0+9q@(+^ z1YV-KD@8frZ348G7z0ekDd4~{4-v5gtft+#HtOM4e!%rb^H>)H=@GCa4LmlWIzjIF; zU<3~c>lAY)!>8yClMvPTTrm(lNGSU@zv}0vg8@)A8=|uSKtJ2P@ex3D{*0yF`l%T4 zND+h))#X=z!!Vk?U55e*Uv(4GQZ&fjV^tzXZ|7$LAbFXDbC`wDZq5?IP>F7$|Mta@6PuBcN1?DjPykzu z5tYF#dQBgR?@lMbMxsk@ZGH9fx$HigHow?Ps@S#rov`i> z9Pn+OOBo9n$PVCV<(jR=uvv z|K04~1@Bcn7}^qK|8;fJM4$p8-_f;s!a)N~nV!+s00IA7UVcm8)l#D5#AQXI)c4DZ zt4-rb1C|odAe8J;DjbLyHek8xh&2F&vGtJ#LORtTF0^OF<){pjKEYzhG-3BT&^o3O ztj11iM!BFVUhna9TbYU(azJE}eka2v&KW?A7f5oCGPC$|>6WxcrCv8{>fo;+-tL#u z>@BlEJAc2?;zEDI;Fc5(nO&w9?0W6?$s34FIaH8gCg*_0KZ~(B-G!BTvv8XFE0PMN zg&>_OT_SB1b%wIVO&w{pdiZ-l=6gn1j+!={-hR-Z)GVAzPcFFA-5Z#omN*#$prj@w zI_W$H;7Dq4B>${ugcTWA-k1zty?%iq^f|4{%;Wpn&X8%gus?Vhy;=dq0@?Nv4Kox0Kg5j;qyNR-_wwtu8a?6ylN#ZXF!Eu~WbRE;+Ee zRm%KTJ4`vap_(Hki_B4=-4}X^c}5aivLOow@7eCNjDt@BE!MGj-|*C=tt4^;U^lj& zsbRjf-9(c6R*4R4rpXOll7}y0^O-=k?z-iyNe*mYe1q?bOU5+DhS!Sz2 zH|F1sWrl8d+|e6=b(N>W-x`M(!1&0dq(%H#XK#g_U6s+WDG7;>g^+8Cl_LNWyxQ>P zU{i{n&0fpXuXK=^CCeRnF6LGWhD_ZAtJIdJnZ!&4cU2CBlIWyw>@ulqM`Oeqv zJ|^v64Wf#l0N2ev+F;`|fU&ZCw+&`~*hy}!L=7PEdX^1153MoYIyX}`RzC=sG*x@K z#D`b&z@)j|5#ueh*oUX?3B_#9Rkn=^$YwJlgw798PUjKSsRgnfT^$1;=Z~(7Gh`t5R5(3#{{%0pFka@9A;#1AR^F z{c;Vq_I?OV2qyj)TFN3{4veNr%k}eB(*QRdM@!7%6TOhxnsC|JM}H?SmvXpjj$DH- z&z8B-6;0J=v8Ek#>1iUbEugRcdIig(e3J!0Va{WUbtx$@q%ojw_8uAZGg1qyodRA{ z@M-mLig9o`JStA;%9n_EHFS)==L&URCxot9ZWxckZM_&gerZ0(bN-*sfalH9rPIHi zLBjU`>I`~)e6CEdB=4`iJ4PZwH&wq(fp{0C=NssCVUr^HTJ6E>v=XC2ZtVoMttYU% zZs*_=!(i*~8}GH;XABK>ZmzqDm#+*(B@S+K!wAd3f*k59lrLo#@*NVjNZ)yl*QiOF zFMyNE1DFvuI2q5yv{Eq344YJcHV2Vd5^Og;aFLT;h)bv1d}7n5HQ>$N{}HfzDNbw> zm6Ij4_8V}ZX&S5Z&t|`Q7QRxfM4zubA3febij(tF+UIm8Ie08L$Wfq`aMznlBcD>Z zFQF;ed-v^Mx<|rP>!z6XO#8i$O9tEp@(<&v7dT(#3EnN4F@^64|3vDS%nAmCbX(XSj;#A#TS({J$fs>a z=+1?&!ky~f+a!A5IQ|_SR{GDy{5vWYj(JW#o7KwG*nk+irN7w2&ulR=#rLoH@;$fs zULsSC`#l06o|i1%0(|gO{PGMNv%S<5px4!!dG(5lpd~=$S%KCbfQup`n!Q!Z>Ri7Z z`AexQ^*2@gm|`H7N{AA0crt8T0SPvwGJIMpt|9Npfw;nDG4I$@nF2XlrVgGk(#Wvi zfy@=r@ed6%t#aExo6yFFP>B&Yh5cq`ex+q2i?J3PZc83L_r}SA)qJq!VoK8Nska`W zdv)D2Sn;aNb>~y@q$r3mVRIF1Sc`qeOZGehJ&ylt8EHx&%<$se%G@-yEn`5TfYi@l zshY$Ju6}}U6VwKGE_?eyFKZGV%*KVFuyu3Rdgh)W*f{^R0HTP%y(;&s#_?nTSNy(- zrSki{xB2?0ZItZNj|DGoy57M>@%aN6l18CV$hLe_Bs6TAN2b=i2|;WRLfQwIEMs@vMGU$-U-hqamXQUhWl^yPn43orpq|>F13Ddv5TCl*EdBBY3*VI z*7Ox6X-Dc_BkacFw*3lQv3LJ=rI_B=N9v20^uUxpXWOb~FHZX6kM`kZi_cLnpda%| z`2|^FX*HX34A#HG{mtD3=WugrrwEk~bPyZ{4CKZu~*=i+AH>YT!WR-FI)Ya~mA0 zOFX3y?jd^9YM)b(jc6ukUpzVJ^$>n;xyIAtPhA~}7lB=b%_8JVSp z!biUBRhtEH>A&mauyr}016WCoZw1O1ub=sSZc(7qHVrUx4Ewlid9};vNoZ0e zZPy-?o%O%=9X}+zUfCVd05hTREMRP-SZ&;V>348GNK|U4WUF>(4rpOiH-Y2AzIp&` zM&2=YuKav8vAPr=5tkze2`${ZHW9E}d|^l&(tgQ#_uUxV_3?PZd2}6wgfSeyp=H*|qLnE5< z8*)3T>PISTu?6vE1@Q^`IXAor2T$p~_n8I4oahBTz|5Q+^fwDTFCW$o2Rl?%A_k6Q zb-c>fZsyyF;CX*lwzkGzOx%v7o;T#R-4&l##HzrUu?p3^gP9D*XTNDPK(U}R%AT<< zTvvd;1^5-jyeOSqt^g^C_$M5f zfc`v)3yly5Q4H^0t3WnR4WzY9&hOlAIB(3AT#xT*l^%Uw#{LZ~-yKogbFKe?#{gPC z+A|#^oaLX>_WUsrKS8FKKvnU8r}%8f`yC+STGPQzj-5{6an8nxvjKB_LEgohpfT#f zm=UNU0!J9GAqszZT@+&+zt>dyT^1OM75S|UH={R|FEuTY_aypFfTrx!A05<@oI+)5 z`FBaF+?IR0SBGAi(amq0pIS?zYf@J#mgGjJ=~GFF4Ous+OZ9Ow?)#5939Y#PooTol z9(_^-t>O6_E_7E0g5fSKHEJ^7XKHvkL6cpH9wJxCZ32MUHX1B0^)Lrm{~EOy5)`M> z@pc45W(?AEYU$Z4EdyT~yRYBga!twXb||SMk@Wi6Yh8*3<>GI>7lM6h%gUctM(aPx zef>7yY-*~75leQuPhARGi*Hf_%tx5L#?0D6)>ZB zro0b~H!VIp`|SSE%)n<6vkN}xU+W9Vu|~XwnkC1M7VsePX4ND2jKn!oOqyp#?jk7& z5mlSV+%ND3@CSB2#Z|1`kTI&j)rAk9K7gXfn$#Nx_l&*xerX=>F-_DosMvHL-u8Z8 zQ@8qy79(C@zrh@OsY6yBx^$X7jNY1X`yf)kT!mrc%Gf}PnWanqu*Q_|R=1b{^$Fo{1t^MtGXJ@;I`}@F*hGY9MQ&RA z#l!^`FIE-+@HfSpo0? zMIk<&;KEbuO(uhi6u8MS8<4}5eWv65Y)c~Qd zRgZp%okkcIeN}~YgCOL8)F7c>*B@?CUE8eV+ed?Wia`z%aW=NBWXMN9fc$W3MMTK0 zc+S{-TEWf}d+Iouc}z!qh~DcaiM8ibJ#D_6r^|aiW-lS3Wk#AyZr?91>Ol}7ePS@~ zOA#%fnE2ul>YkDzE(~DdZ1dNdZKYQ;@!DSrO3yfQg*PnfS(73SPF( zyJ-6k_0CaDn{<0FE!P)>-V_V&YjT##jfk*nzXiNm_%!`nv*tc@duc{coLJ$+7A&9z zdzB?n;~QjK#b>bOG=-s<&IXM43kUUAP@1V95Q09B2FO+QlHod4KUeisle~6+K0b9@ z_Br0^9C|I*ILuxCG)P{3syp%No1-6oe~ZM5{${-Md%Nv%@Dul5m8C?BT3h;OX(NH{ zGHkAM=S;e@KCkVX>7u_5|Ck8uOMJ8c36B5&TiN*H3RaXqS3i+>jW5DQ#QzpM(3qLa)Pk9eaf<7g=B&4pVlpJY-@!`#bIjwS2( zoWgH3Nm~qS9#f9=hN0>04Fisd#{*d9!-x&}ka)WsvAZC>LqfS}mcwV#-aU5@RrGgo zHot7~RgP`Dbk^m$V!Qv+c(mt;WQ}TuCN)9V2PMMiAYEKt4n)=5XK(VGX1WOxN`|qUiHiS-;9<7^3cKQgD+{RDD@k_}o;fh&pq#=k$2K zGM|oZZlv`3diN^f7^8ASO1eL_gh85ray0jlWorhb_t|qb@T}M(fjr#fQF-iioJJ|v z(eF_wU-li}l+RN4JKg^$At$(3O|(5U5BkY|9B`Aruw)~4_RHWb6s7hAXWV9A#9rH; zm?|_xyjnY$&y&ZTn699y4iHk%AUe~tcgsi%2R(ZV8N=H{<1ujYYaGsNK*vK;n>|*lO zfnP^#l8Q%7k4>DskC)_iBWC43J|ewWm&wHSsa)#i;gjwM?dA>VclZn&vnqOnLnXhn zIWf{}E4wdsdPe>@L#12RJOIm4M!4S0QqnNliyjIu);Jpu1YiF|$}xc1d4jX2BZAXa zgUC&!O9Mi{aQi3x7&A3ev+l>ZSQMW7FR#avCWgsVM9=MHJz&4*y=L;vryrr+v@uAV zQvzuWZ0O8&;OJdCp1F^t{#50ZqxQA1+zL?uK?vHO*bhoB<|GMO@dxkY*O{n>)bpd_ zKWn#D+oio?l%G$%tG5!n;J3Z?oyUHf%Ds3stow9Ah|UO;Q|Scf;&4CLDMDJOm-2t4 zBZ&P=EGX8iGw{n39ljN|8azD;wA48CnHGlrY>?^l6LhBDoMI~Fm*9$~!8 zeq*)W6}`;s4jPsZeA+u%XO(ayM_uH~1_9jRlQ?&)8kzSic z)lxV=Vbb<Ie$yV2LCp!IhyR ze~q&reUcvTt*S?3{h;7E!tHoxB!%Bmz&k31fI!m&iIHRX0b68+5!wu%NDTz|mE+yq z9!|9IEc~W}@9JT}cWGPo>&aNv>ers0N12o%REb95!ttL&8&T=WNfcc}?4-Jz(>^Z{ z@C9%S_~r%lI@6KO#OeD$EUV!U$d!w73+)p0r$PFII^o+7v+{VvD zIX7!e+O_smwSc+l6&ykWSH95KggG9rQkF$+`G?bYr?V>)r~XoN{ws!Y6(Y-vu}xa? zI@ev+MI>s^oP=yeJO#O+o=kspz0V=;P0DDTB}jXGKHv`z_Jv`)`xrNfwX|zDN->_o`f%TO)cyt+6fQdw=3;$nxXWs=pf~y z+N6SwTcL46NZ8}7RS~;qS&#pd1@Q6m2m&d83Wo$&nVj50sr8{X$ua^?SV{{c^~Nd2 z6T|XmeB#AD{%}KG-a^^j?>ILF78Kfp>g#8YmQsY5a!7jtCX>za+06q3!)y`E zU9dp|c`(@pXKZoLXg*b?$*bg)eYZsO4Wz_-1{L7Oc3^q^O^!Y+ii<^8g4MdQjfoEr zcjm}{4CjGP%IXZBF-+0~N>1~-*vyiu^6?>M$isF~TG}WiSw_h1~sQu$hX? zi>o~{zG#2>L%ngfJv}!Nd)OK>^*w5zJWKOa9?y^t*Q1sI3E$0=CXh5J-1oD9(}YC; z^nC6=)lEf)`B`C7)ivFcOs9c5C2}-dTMXUj`zsOM!AHgmmjci=4+i^tsJ<|=C6*LY zPP_BTN1e-X7`L0zJsAoYx9H!|jdsB@#h(p@B)5Iqr;X((XMWC!y~PN#A{^Lm4@hNW z@|h^F%ia~|KPMW{ewyTL{rIIrjhkpkM<@Ll*c7Sff3))W$CbxiE+H7nwBM*>o2^b! zr}N0X%R)ZpF7a9ZwJ2EU74?srl9FV1_`vZF<(kR?L}M5V1;}6H5iXJuF8&cNLLZuT zC;j{<#iKGBpK_)KV#FieM^@Shj&+(&u~`EtkI;gnD)HK1eZ)FA zokt(uE$T|Q$-jE@{kIM#eX&W&`CJn7AI6Z`fpLa-Qc)W8~o8cR_>#y zp#<<}198ebhS8Xwk37t6kD8SOnd+uqV_-=l?gaFp2CQa-cHAkz+OskO*|bA3anSE~ zSG7^FzPs+$^(5Tmcflr73aVL}EXpv;FSg`iNc}CRt`yH`(&QRZ9pXyS7gAzaNvdHW zTVMT*&Eo89A~5t44TGGjmiVgn{(#RgLaRSCy(9X5cG@lV%$3i%F===;6jDN-S}Bd3 zx?NJ&%{pDKKw9Yjd!!$JOT-Tsmu}!6S1?#+1ZlKL1!lxb_`c((iuj^o|%$i5P#fPZnplSdJeJGyINz zy>cN$4jnkO2(&!T`x2o2@vO93zuvwdcq{QS4dT}3abfnd{awjkp#B2MzxHL&N&jz? zYeu0NDlWaK=MCgTAG3f!=Zp6j?FBR+UM3=E)|FqE@NSZS7b|x#3rq^WlNiPH#BZS< z`fbc}Qp%1SCgD;DrAAkZls4W#hPQmfI;HPV`@Q5;#q6HeNMT%!!(#H*N`wC!N!2nA zbrHjHIJW|$zZ1e@mbJB+s0Nn*%FQeHm)Hg^V0y@YRp^ekC%lKhQ*oC%U?ZuM$;NgXp;po+VK|~M}Ukh&P zs%L{~&yy7QSDHQS0;4)Mhj*fR&0Q58XgE32^4ul8I;!m{dh+E4H!BapJ6%%mXF)#` zLMG?(d|UrQlTgQkL4B?TCcw~=$*<K35TlD{BC8HaOVYYyLHd zY6ty4CS)qFpD>CAgI?d~oM>|J49GRlF+*NYiA3}M*>(%8j~^vcn?V?0rI9DeBudwf z=wSI2Uy==Lq$D_|f$y>GX7bN&I-F&fUoT+3(#>6ke<&`zhs{K>fbZ?Vi(PM~?i-56 zn-?q>ZTnR3=4@ZB6OA|giYw1rKCw=o{^?ja6PU|N3HVMP8U+dE@b2!ym;8LLjkJk+N0DkLNo{SGdRdg#^N?z=Q zddogi=bcw8JSTTX4@F}6bi>3V8kq9z?SDQg9=vHjzx&!%xy}m5N6_oSfG&X2s!%|7v~4;+YCyXlMY7x`)MvIxU`Kov;OYmnpSbCOKf;k@bkTmq)3d;SFb9UYt3^T%+ot^TmZ>%UuKfB(?Sm(5{+yEb zj#r+W-AmILE`7}R?FdN}VRv->lZWP|qS#b*iWO)4R|Rh$*zl*mL{vaAL&|HJo^=NY zgO)D!ah%1wjQ7kAA1oE`bA9fMKAXHcBuW`@+2b6|h%Ch|k1_H*|D|?gq5-^J-H*SY zOK{P2UJr*G3_(3Ac<_uXfFGs1p2?LJABJU%(LmesIQso*D(DZikRjCf2j$hb+pa;h z<&@91uQcG$%{$7LUGxo5jnj}{5Q9h>lE-e9QJ3olW9&0iLF!UHD5oyP2IxG|9l9K( z73v37XLa^#MH@!au;tz>Qh(O=ImDYnKhL(ibxw7X@OVLnwk5QYp^M);YCoCSdpQ)B zQ|qQ9vMqX~lX;TYc99);w@(=zalazUd$ly@+jaLVe5=b#4wsr%E6H5HpmO;%alcOh z92b)b5oAk{8PE$lHrQb0zHnq!*4??n{gt?!cuU-9oGH&Vfj=o(n(Pe!1zBX-Tovbf z2!MmqVyEhruS$RCTaqEHnE59lQ}+m!k6^@z7UG!CNKD?L5aPL|4}NUXEz0Fr7}l6K zY*?|a^7>)qG?4i)x*qmK-pFb;6^RL__zxU6t0m?nuV|OLx?JG)9=#yXqrHg%{I@An zI0n1jHC@KvyXBY@xf>M@^`$nLp4?`Eu{tkOHZ15cye^f_)hWspU!R#$K}0-ii<3db zX*Od`k1Qynrxqd(2)xj1DfqdA6SOgF;Ptqc771F)%4RSojODaKiiQpthyoQ+TUTTm zezjgzWI^GAweU7~KF~J{ZA9#|h3A?aza@@N?KAYRnU2})Ar!o=zcalj1u)LPmF09P zcsPM{bT%>WFZqk%A?|6{PZ7GWzD2nFV$kN8+NRyw}>u5aNcMYLDJxWYk z;YT>h&4y8`e`I~I6aByf$X%WvnzRF9^1!NJS=Tkm_*fiLfUe>>&h zUALCt`q|Fx<3w~{vbd&Va)0XSdUzczi;2O(MfA7((I8H%)G zCXe#rW8kYNfHu9CAnP}H1d!~YZaun`q9VG{f#S$FV3DhZfin8&4;&X8ml5PkL zi`>h1tMLgOL%oafEggB7 zeEJfm-U-EgCIKj+`Lq;|{vEjM- zBPr(UH>@ajBp011GfCA9L&eMCjKUcRzWfQUw9Z9O-TMoh;<$#;mhR9i9qc|gX05@CMYj^W7b=yNE?@7Kox|j0`YIDXPYkH+1BT|P zdH$^(o3p>4yDdbG=6G#Ph_Ga?4gBU70~0@|1zgh=R~TSo_+Fx_+nb$BBc@kt+hM-0~w$)9BdE(R3~FR2Oh1X$y$C#E@g2 zUAIAnuvs3|^f(d4b0k*@S`_@CfpW_EWu@@C4cwUlKD}N(xd_O;p4)9dypW2LdO=FD z#jLn-p4C^LGWnV@ZvD{=Z2J+UBZBPKX~!}G7kBXc3`cdc@6qQ<79%bNOF&mqB|~hR z$3T+CVR^?#cHny_>^OO@2aS~DBOGy-+4?}6_Lsx#2v*%}7mnGPKnHOt)w^xuQHpQH zAzP`)?uzXDi7y47icWa0ZqWu-?vtC(^Xj)NqThyXdJ2n*-SE1^ zVktHzWLPevIhKFK&XkBI*>4V2I?h=T7|TL^P@z@&1L?Oa>kt#+$D$+%^sL1=K|K@Y zJ29_onVI-I2HwH@=OEZ)%D)<`=YBIo#y8|58BO2rDtL5}PLrwH12@mg-kO=}8b$W5 z1Smg=X{vJ8;V5_;RnGMQ2W#84;p%2Hf2VjeZy?pgz^VJt^+3omn>k|hYIzfCeC#vWLfi+E!eSPBM zBtdYE%vu>m`EvNWTc~Hg*iYKOk0NtYcqwFh+Aj72eUA1)zPD{PG6-8`l}6GcL+|KU z3>OIrZcyZ+ze~OmytB5Q+(ai8djh8qnTqLZ3yC%>ISopzFjAGSB{zfj#p`Wi@v9IL zKgYk}$C47*LGwyR86?`>GU`_P5d0eB8iICQ?PcM-9$5G0w(?l<&=XXh*gibZP4qNB zDu1a|`>O+3uX*Z!6f@NG;Au>y+`oGx6(4xgQd-nitUFJ6lZqA!LL2qIr*ewAj63Ro z=5GC4-&;Sl{i&h6KDN2=nRTkBfbz$(%&WhOix%Qc1k8PvewL{!H*LBj_FU{@m~c<040s3*h7XTQEA(ki=65upu*#c2v!$h|NdTX zt&F4$z@QLNNH1#N=AyT9KcV3btFB}+ZPG~pGs}hq zZE9SG%Tv}hQ0i~ZG~#pMkcU3}Z0`>_pMTm3q-)~bJ_Ts#*w8&MT!L-HGn~lZEmq z;P3duQMu%+!FbD(#yy6*TZVe6DWunq={>hFL!@8`+b2~*(aOMgbW}utZS*&ggS`%V zPhD-nTv;)CY^DMp`($pGeo2T_YGQXNbzgLT9}{zn2|3)5VnRrE2BnOns|}p0yDI_G zi=ljgEB^#@18JzdwPi=>?oOc?Nggq>wc8^8Q$bIJO7*ztV!30keb1(=@8(j>)4Z-T zDIgmXyejda_h-w0n^Tb}=WSvfkEo#-;-avjrbuuz&YAdZ+J^O$N|+|m>&=3_w?WC& ziKjvQ6S9H9r$NQh#YuM-F|0TP~G};zGsl3axBU5Fe5W zlB6l|cvvv>j|+J((iNCkQ~iuda{t0$?F^`nv!Av18D zePJ=%n}9B{WFNmb+)D1uV#D*yfBM!3g~SCKGig1)?EVh*Bve8$H`COiAyiFm$eT2z zP{BmXM~7`19txtmDwX@c(W5%p{a@a^yiT%uJ~}9YGYlUK%G=P&0a9n!Lc%vE(iwci z&sIwQs2x={p5ZA3DQ#_PKNNwO@VM9Gf(HElY1*V-omK;K%vTdXyVaBL3YGZnW=^Xq zwLek$Mtb|4XJ=8PMyOHUx_{b^OhwH2>5y*QiCOmzLV4Ja6oYtWm|7x@Gf z0&pXkYn$z-s;4KyDD9yPE)N}%)4>0v+FT~ceAJeZAJf^|PGy{OLAgsnZzh%GNAbgS z2+E2np9XK3zhH!gKQA()kgLTPObC$-eA`7b;G2Y!FY`{Hak-=0Kssd2L={Wdixn2} zi74L1Wt;9}-cdYZ!`0P)e^laiVSn54vP$nxDs2;kuwG-6 zm2{Z#1>^5W$6^L2G1Rb`S4vzXS}7)`o4z4>(aLZ~txrmM#KrU@gRolc-DaZTcpyaK4+JDw*P`y5fWVW z$CF5(x>#GN!k2UoLNtl4ITr{5yQm{X6k7hish(AT*j5KA{XzR{kXlUwt7W)2#m+-@ zHIuFiE80!iBA+q|A|!<(QC&hg6jL_sWfD4pt7QfC{4W(o3X}=%FlZkO4FvniLxHlT zo2G5Z7NFC6_TkrlVc{o%5RlO1;a0+(51r*>H6oGorjB&%M(Um#l5+Q1By$!gZ^<=g z66$?MCVt*kd-c@d)l`rx#boHI+X6PhzHcJ3(e;~%X&mkIemCdN(Ck@BxBF??=I5Vk zT7Yv+ogRHq#U08p_9ud+7CFJHhO_LIZWK61N#m-jn%YAo`D0$!P^&cLIFFRI?RDKv87@eq`Bvy-yO{Nh0AkD!?_un?mf$*ec@(94be>-EmJ^D(w!I7;G~;7_?Kl^?c)n0oWeD$%lpT97gvpK*yw0kJ zR|4!ic(&@tM6L?U`_;}KZ3*qzWSaz4-4ykF7HC*2D6Fl{Am2U=6yZoyhPI6LFmE7_ zjC`E8E~1M1`x5(;N-^c(w1zziI$(5*0OiZ4{m4|opfZ>hafJXbbnGL&9a_UO!MYBo zZofHiG^2r4amW&^(w?t7mQ_u9zo8xrCfbT|EO{7t7)JjFvmCbhgOK^TO;q#1 zqcnG@O68)c616n0*D(omQO;{{8f2XFh7?)vrptv*ynnS29#cX6AC8NI4vobi_mrc% zGSG;axJN<9n9KHEu2P8w)B;4K66*04H<$@v(!?3DFlf|h+Bm{hHwoB2JiTQoq>9^u zrmFxeS<=|~NNeybY-1g35fc{qrDaA*Pya|#pv<`c7HRBvb;L6F<*Ic?!e^-iKKet- zEt#|YA#wAK|Mw%-`~MR|K;WTRMegG4dfxe0b1{^Zj$jjF(k z(N2AU)MFq>i|~(^cC~7j#`My=nRViYbD^cLFAzRgIO^a{O#a(knbwU>1w3}F;L*M(wvV<$fK^D>xpj$#uJ$Q?&&WS$w zrkd0dLm*O8gj*Gckvx0RRdZ>uobXVE*Z<+0BgyCYDW9=vI;0j&a*o-K^8?RXfcA(- zN7fq0^h%=z-$b31$!t@+9q=R4kc|err!kN@KAr?cuM4tT!ay*7B=i{VVqH-6mt+fu zLgjAa({DtD8~61=E$(SaDEiBv)a@BxIYTYaM!K4t&TUM^PN*;THgP0JKf30kGp0Dp!8M} zqGti7HLhE*OfUL3V_w)0)b}%fUwkj0H4BT|Z-%gE7#?4j*)5|%Qs#~cQEUvT5v8Zu zxfHXkc~76hSe#1T=67#)Oh7QNHWk=T^hn5Be|17=$4_=<58@TM@H}N? z8jS5ygd6XoKtn#3>5`SxCHTS0-N4~R1qK>R&-dWnFLWMLH)dxDkh`iwL6HGODUC%V zDXZrU#l(tC^$Fn9s&Z%LKHyN`R+Oiuq3?xi?S!%nL3cwZ#tWW{u(8g8V2$FmX!QAM zKYeMP_x@>QK8S3@AL^F;8Q`4wkm!^46{-X@G`KJt;Xsj*(RYI(HP?$WI120DZhvI_ zzI`QXme9ap(UV|=HGX8`i^lzPbgY-!Q(8Jit814W>z+iP`t_jYVV|SQ|MseM5^>Qb z{nXeP#=vjl{zRmOl=9nND`($KN5 zpn8jw4aBBz_v;)!=ZgAsD)bAM0yN0}daV-UJ!dQJ*<-<*qkR>EhxUQnQf_%$gR^$IK+T$5bamhv8UUqKRP>vLgKf6kw@ zkS|%{s$-yIQg(XN&M}>(O2lBF#1!`UHz_L>2nIh>SMwp#Eu|0Z7Rt!Ozb-*JD44v5 z2m9Y`slUwUZ3?-i`tfSQa5}yHPw-35^&fZ}_Kcxhiy{AfVGMKte#N7Reh`|^8h{V2 zd&keh(=VEc4bH;1l%czkJ1Hf6L%RBkd9cnYttav;s{VZ1JNi?46EirxQx8a@r^B;F zM4e-oSzisU_vcrz4TUk!<+^83=W^EVyBCxPWlg#N^eXm%`1kK#WnWJa?H?XKi0RK7 zZtUZp5faJ~>my*u1q7`9U1&4yJ+Wa$k(S|QbF;1KBq*7hn8bFn0hz8)z<3%4!qv?r z1R{?<3mzz$NEbCfJE)8Tu~%2(+Tg)1Oz#QKD-(A zO{HUS@O5lL4$B4&?f~YP%UdNNI$|a>{`x~x)!mmeqMi6AzbhTN1VccZ+3(LdmvAxO z07X2R`#(z0;mwz?4od*jU{*^)MH7-{MxOL3%!4<1HYTI}benhi4$Ol8u;(uIF+mDl zKKAx1W}>0m{l@ZRUt&;9vgLUzZLDJaoJ z()%&kX!fZ8@t8vSuD5pWWF>P3 zhgGqwmeqD&ihUDQox;P;;NMaljjm&PZ@&a}9bvm5>Ci{ix%fG1V$VV`Y1-!(5~JxKtp7BQYBj&tD?Mo!!{c}d#^cri-3P>d2nvD8X*=z_V}CJdcuE9U zhKYcS4N3}S0-fes*-LOzv++;WyvEo=Zz{|~Z*HyU*Capw`)zmKz!e0{BS|P)1jTh5 zOQZWHob(4atB%2NDBwbMb0R z7WjPKc}6;${LPL4`sRfO2fDa2_q4awd+3+{t)A%BJ?U`zAN9n4*<9-1>PchAN|v|Z z{P`E>oBg@D{^*8(Bw8K#qxY8|D3&!s6TYsna+7P;QM&lp2yioWAJ;7CLia{g9n0GTs$%XIyeD-SVZ>xJgO z3v%AYshL5^rlNW|W2L$erq7mC>Z08#Mk~s*a-P_bDOF~7SWJeqeD#NN5@rmE*0u;`Rw zhK6N&xw1X*`loSq(GQMjU-pEXbPUL$vkL{hNZ4ZS|E>>D#_E^;W^^`X7n#sPXopR0 z5e<4mg!BpmeGgV+Ve8!e{n=-WO>`ZQYWI-AC-idiJztbw%S`Y*7>t?jLlJTLgaS2w z?x%%B=eA`7g}_ulBHOv46sTI~OMr9RE@!JoFidb{wCg^}tv} z+(Xi?m*}_*VxG)d**)6Ik!B;M@aNunM$TpQ*@hwuRH%UuhwSf+N;J6Ib80%nJ`zI*~`zEP-*%MP9sjaTynl&)`=VC1N9Cv7?9@{KPooEJ03P!soMunJ^ zTkWnZ-!C7e2AR~Y5*3*&JB7K;*!IP|Bu3rxF}*Qs4Y!~DW$`j@HLmutI3@GX=-ATj zk}(OMT9atV*%fHdQIa+|7$<+_RI(Kk{0apawv_$`S#BEOOoBKJdr)A;#387p5r|8V zpLi__3Z(y-F@P$?{R ztFERWy!4bPTAD8OCem?aC;&!UsVbXhxQ#W3CtvBlcm1WHvlt~`?v4-JFY58HDwQ|^ zLp56rVFDRGsxApVcOyIow0q7$0~P!BRTX)BnB{?z_cW5j{(Kk?-hr(|vCPJ=3%ND+ z+$mpI8#`ksmCj2t>f7mWz>fV=G3N+?u%k+S4&9p8FUKue?Yx0);*2L0aT#ePyM4UC zlm3hQr@UgujYKWuxq=y8bNq4Uk-x(!7djER3VgGtO!9%a$9{c_NC_`j43=F|MfdI- zr?OHWGv7ZznhvwwNf?uz7VX}NexkQ&ZS6}n(h^wjntFX8Zke4RadAo+c(uN5Z4S-M zo~2q=B|+thFqqt3ZBwK49{)kyQZ@4ILL|3j;QGL7ElnKGe&o<&A7%sKkcrTmWu!Bk zW}#!0N7<`(mr)MnI+H8qDlW_XD5H@dw*IE9>OO1j%R{lFu^~oO)H?)a38bpNItC}t zN?z>vKFkhHfM2g+gJ4Yo$?aN-M)qru$WlHyY=!oq&6M+ry&|#ckn0DXQ~%_`#%6|; zr-EF$RYTr~aynvT_Ih~toi!a99aHl71iYrI=3Z=V-%v#X-1)!Q{C~N0f%55&f+>a8 zb`(DW$i^h^y#(Vys}9y7uu7^17`_9tl&ZSo-!!~mI!R;KI%x4>%cpeuNY#eq z@eDeqU+@zrnV+X^W>EUxy{@haHIY}TobuVxoiu6(C6x=>ebmuVDYjt4;?}MZulh+2 z5uDE_+Gta~Bl!aClP)Qxo;fg>FUhNQ*t;xWypfc==rwS6|A)tK-~-_Ha#{M_K>-~z zj6WJuB8q_iolH+b{tn$K+lll@=Fa8+*_$n@&wVV=hiWgb$;*{d(CRXF##)%h%=VSf zH2toJTS6c(S3A<@=eIfD&@~togwUnK_#G)?WegOPy(%3hD)j3@r}uh+vHe0`$-)pG ztU++*i9K*Ry$$c4CoK;nWe5>dq0?;I(f@2yGsnrinIY-SZs&8h3uZ)hTx^+ttgo@jbdt5Wo2A0dBXIuV?QlY~0;Yu>h; z9aR-bjK!DGc^d9hdL{Vt*PjF%4sF3pwQ1;DS+&p7?&WdnDB}bgu3~`7bj}ELpfE<{@Q#K;K7CA&Uo{0>4QPmjqxGB7M38NSW`M zEJ_6$g=a~oV_|vKPeM=yB3)FSo<@lq@Zi`8a>Ajxvri>eRXH0Yib|FEbSwvu0LPuf{jI=Vu)_CoaA~nz@5wJZowuy5c?-Pj}96TrcBZY zDtSl|NW-ZadBX`s#ETe4L2H(Trty7csLw)+C&F55G;D0a;nU~Pa&*n$@6Wirui0RQ0EQ?UqA&fbY6(Ib<7P#J;YZdx27|Z*VWPV}(xy<|Bc{+~ zKf~psV4TpOUw5q3gMmfJFwZQ`8VJfqn@Xt+cE$*KRW$zLy{@YC%Jz@jGxZDO5vb1- zRjy@lgQ~9S2e(`%9bRgVt=|~WaYS(bA%?Dr&b4c%*1Gus1Y!^ctScLVmS6YXDfio} zY0s7EaaAZFEY7aS4^p3^(qf(&IhK5VN{IGf;t()hx);u_aQp38kiKD`-2HwNT{;YB za$UxqL0DjE|NP}Er2^0ccv7XOqedGbKgLSp(ArWs#Ta)g>R380HJ4bPQ4yZY1mUtO z0Apd#s7{U~m21`P);aOmUEQm1d5Nk>Q;{?2WId4WBfPA1fUX<_PYoX3v|k%hX+){_ zApMvf1B`GXP{>FsfYv^R{sIHmmZ?WwhTEx?<>-7G-y|??Zm#&Hy%c5pvAgreosTDvqg*e`AqQ@8h3W9RIj=-UZ3+hn1ZyG6kFuNtx#?4?z=R3+Vv&R3IjM zxEUL3WOBLk>)Z;N`WBpFRj&0^(ZeqjFZ`~$Zdv8IS8qlxSK-5U<}&5i#sKayviFju~!u2cy#Pun}E(6@FTDO7@0r4g_zidkgYbW~e_ zn4OUD`V&@^0NdKN#q^5l$~L77*>#V0(!q*yaUBcb3FU%;X1vLWgvbj_K9+p8h7 zm?;#4nsHvUA5>AMXXnc3j(7nEW3kI1EGY8cO0Stwpo*~OzgqvdpN{CpVALY52-y30 zaVh@sc_EO>d<=koB3G1CMm9;APXF%t%Dv0Xg5SW=Ar0LRwSHC4+FWH7mc!JDi|Qf) z2&$K{KdKZxnPM5~$6|>JF$<@oIG-*UY*$+JL^4+>!_MoH6Q+_E_SkfXD&LJOcP5Ir=d*Lpx#>jue5gwyw4dAqIw0Wwx^ zu+MBV>s`2yYjuFY%`9t^CUrQ-$-}B5^C){|?)AZ9z%t6Hd&I4&=WAd)E*0AOhre?R z94!x4T25^Y%NVMq)+L=3=dJ0lbX&3JR^j3b#tRSjuLC%fybW=>0l1nnE{w5sOcc0o1C$`Yk~|4jGq0K+_WuqDC(BlVn7BWn4}&;zZB%A%yDp7Q z(BO~2kbhydqSxqwGK+jcJlcJD%b#%)p})aTHnIPB9q4~g z^Kjz9xs)?Gr)#UDN0b)4OR5I@AO*+#tD{OksJvZqGJHM^os&4~_qck0XPvBSY|oey z)}!Kr0yyLcyt1OBSty;=Lz`(?)b;XiBQTM-$k_mjItnEZF6`RgBd$uN3F{Ol6w8VX zGed7ty{p2l+znK4_0Jxor`O)jc!o3mm;H^5mQ$m9J8q`W?B8Dx1vXXO3TrakRiS2{ z-(LS`grQE5=5-G;Jz`M?c$jAU#F>I2X&A&+IX54k>$Y=yk4Psu6~(hIjw146rI9K!s&NY2{E155!_gh7yE-4$a$yEVq{h~i7KpWP|T zx(&K%G;C14(7n*VxIwzE-|KSmiRC5m4|w)|6#9UpX0fD=)Y~MPZ-J+gmjj)1G)JjA zVPVy)Dx@AD8YA_bK3U8<^C?TuoXLQI^yl0^G}hz7|I3=91R&z+z=Q+*J(&Ux3RLSf z=3$jZcv~r>PIM6P?^ogY)ws}N?9*xT>~Nh*U=TtynZgd;7`VGLK#9`nL4v_2ZziPT z;85bXN@rQEE%4boT2r7wTj7END}h!TXE=H)EyPbZkEtDfSMDmD zEG}T41?lQ>GEcixq!ni81(QR|^!SH(v!MW)`}#Qg>BkN){Y&kdW6_fa3-4s>wMGGz zgNDXGiC+U-1%}@9`A=3JZY?gqf1|Ts)JsMdQ%tsg+K87zIB>RY4b z3gVBZPZ(aEA>9IrolR*ve9|htuZ~M2OniVrfI3Rk#fx@8=0NAwTe>KAyxirWv}e*C zZ)jzG_b(Q3I{uN+rsdNlTO>mav$xSExNklXno%z|ZqIJGXj=lVOQxQB%-h)McO@4$ z&1{fFbauo}7JO|8ENQwwP@|;jy4zUq5k?jxqv``SdVmaSVPM@1I)NZ(25lz1~izZGpKprm;6Jmd&;hr=n%O55QRla8$`)YY%(d$GU~8IMAKk!Z?5BMiU-Nq^3ZJl(JF zU^P+*FI4p=kS}kCeKLc7Kgc-hyD}fm1#0e}0S$hpZ?+#?=%x`XcEiPqDTsR5P7oQV zF0)biY1^(Z-Ib~(2nf8Gzy}7Y>NGmYF%2b1g$qp5gtV#`!~CEGb5gG3sQ6YTpI8g+9qDC?wusu*jA26}n^^e!%pygAU zqn9;3L*rfblH|?V*_~t-k+1SiF7OOvnb0Sx8h6aIS(BlarsGw#!@Kwz;>s{lV0`TM zXWjMjNP~J`^nzH%?aK+pg#*Lfg8cA>k(i$uM?$1ZwU_rqQ@nlA7faVGDm=w#esBNg zJa81F-zq7*VI)i)$h!HXYxexdw4USwWEhiKN?8Zpq+!3`ZTHn}@dR%fhg!ion@7R9 z6G=+aZ{-a##%XC@yy?%B>2$!pNvwBh9vrzY88R5*K5K?YRE5XM;?Yp#4LxE6hYcM<`7n@>@%rm)62JC0-fIcpBA&ZWFKrOala$A@{7Vt_Jb$n^?k#6$oo`joM0xqYKDhRY_9Xv3BcL~nBx1ovUg^yBHad2G=F#W-ps{QyB_ENt| z7k|CDyl8Kf+AFzlE5+qJYD#N=m~(7XAWs>rp2#gP3gQ^ zs6zRFF1*xDI1zBQ-h1E7{M)!|+{phGjo)3Vk5Wj9;b+5bYee8@4Z{=vKg&o4-itZ* zrWMbd+kx?ux@L+y=Qi6&JkDI_>(#i{EX4r2SA|4@?V{JK04q)!LQ(akYWw}qkB>{= z)tJ0%w8}?S@f=ijW>1nT!&K7ys+Tr9wg?B#Qq(n&PO*Q&BD0yH3E`31}jYaKrDSo34WaZv7 z#Y*H}jvuJvUj(_rxBemMmq)cu0DxoCS{zRCQ}HWxritAgEcW=NGOo(X&Az?XHq`rK zqOHx1DYyUX0;BMGelFov_kj(=jo!I$+}Q^A14$u;bp?^>@4)Q;$JkqiMg4C5!!p!R zl0z$9LnGZFNH;Td3R03vNe(UDEg;Pd9g=H3{aeLs^vgOMDdp3(AdTBXPl!Xj^TH{fg zR18GxNK{U{@%JE;e_xUR`CxKjOCKTyXX7F6fxXVn*9hrSSMAAM=yO@QV01fP^~pP? zB`y46FP>`Ip}Hjf-(CP6!~qyVlX~axm75Q`<$_u@dly2e7bqp(56M$&7g0GQ@3w)NOa^tfPNS0~4uUWK zlj4!(j4Z*7QoVM8r>}b4CM04fA4Zl?&Ji=Dzm#9*ket(iTnhnE215zTAC609-U%kD z5BBE(bon)p%6&Osi-yJ*knn=ExMfpu5g1qw+Vq81j!FZAakP4yL;k=4csRXF5GP2W zmj8~eu%b_qct+W`48U!nF%Ub=W;RO09@dE^w}Py|mHRg**m2Yoqk)PFmD*o1ZoH4` zbhEWZv4<)5iIblHINROUk`Z>=n?E<+pF#ANDt@Q1Od2H~Up0#_*teKIIyTj zeF+?y3h|{gKD7(WX}LV3d^Mv0(Uj9?!;=lWpgP`2$LwE{PAxbqab^*PZudm}7Sh#Y z3mJVnr(5LF`O)`TOpWu9sHR`j>nWM)4^5=vb4yIk5GyDl6d~`fOZcE7)lZf( zRyQ{JMy{alsduEg2mY&81_Ldcu(l%J+n0rzpScf-LvnsrFAYG-4_zW=*U+3_NR1Q* zzK_XB4}zflNdD>N3*Bp~FH%XRuYPK*;QiyWUEqcUhFPXpcJE#Y6pt`(a6Jnz2>JOe zs;2wjVseT9D<3D=^rr4V{XBKb^AV89$(>#VFW^J*fd)LD;IO+|r zdLrXpU}E!fh`hz)hZV79yoVSdwNmy&+-qNa{K@)PJC6kq2g07rSL-DsLhW=y4q>>Q zH21#iWFcUl82OwaR(Y>bRe*M2pCZa;A>5zFjih}#Y}qpsVi5O1pB=5PSnY~attyN0 zg(l*-Jx8_+%teC4T{6S%G9 z(B@HVHd4L4G7SJzp3`?_aE6EvzVlP=RM&^e=yXA{^Z^np>Gk@eU-FGdAe|@ zTSLE}sVAyeW1nZ~?!EzY$~4qHCx>aBfHIUZ84^ku>wzM^80Qljv)tb z(QI!Sg?05+elVFZP$|nmz4Nf$Ca_XLsFzry8qeRomB=3<@7JKhonwi#Ke=b_+fkyz zp;ER3xIW=X3aB1I_u%4r;x%-}UFEXnAfMSOYhgTXmJW4)>AJH%-k$RE(JPl`j`l&a zQ{UYYX!|So(ob{H9q-D)pj|pQR1R0wbR*k1w1fNiI^~Z5Hb@GWSPmt1C^~OBs|p(2 zL%w?sK@!{YRnTDZ0Vv$-54bmF;kgQ*=Dyq~-rog&CL+Nip$7w7F$J zB+5BX4A=O3Pee%Go$S%;asCK(mh^Rabr!YgcaTRj6@eYf&F!aeEtkH%&W&{UCqf9G z*}Ym2o!25%9ao=Pl#@&L7qu_9vAGhSx5Bl~82R$z2rN%aW8iRS|FMR27U%(~VR zC=NtQ6)tF5pFK*t7Xih02KQ+7pS}P2XP64}KzqnzrsmcbFy$utk5)mI0-*D<8hjXq z)4Gb_7ur4RFpwV}3_#Dg5r~G0+?jT;<%33LGCHr?EiBwEBQRtEOSfV?o-DMINwMbc8>HQA}%P7^YQ!gP9kX7 z@0XY?=)&da*mF|R`5CSh@Knu|gd)WEe7!Ek5aiCP!^UJ@kWhkhPK@5BEk#^>?-c(t zQh(FA5Mr4U9pumQgLFI7ZX1gYy4eF>-Rp-eu*jR^Mu)K1A4PuI3N=D_7*OD&z2{z- zPete>kF^w1&L2oCrCnwz-yIn!U>wwhUSsxpqTDO`CE!}~TUxwfCYqdSu9Nk{8y`tq zuD8Wqq*~LTW)xO`Ch$yZM5JFIAbu)x<_ekEG>bpqF=GF(dZv@WOA(DZ#+Io&ikZPs zVk73uA`K-B$?x7%CvKYV6&1spH^qofwf@oM{TU6n3!9!4^<+a+x`2|`T-8?*ikA;} zA(j3fc}FgQAa^HO8>Ka{mYHxsS0wYoEjR-HwCxIMk zcAUg+bI!TwJ{^4dP$8P^_L#9fbc97WX|}G9qe_$0k<)wB4K?}G_MkEL_L@V|fTUM2 z0_{B`f;ZDF~*bq&m(H^w~e zNpNT``V{ctb+Ijo<%GrK%kk)RwE4vjx#S+<-k#5I)jsk4OavPAKxoHjGItCUK|_k7 zvH>lFBKr#0^b#9+ZJu?3;kAhSojQp!y_yjYI{?Fydy9#N8>@Ep4IyzhSi}Bv1kj6} z*NKb@!S|9|J1S6X@Cqb=xrk%SZV>1VaX(?-Y2$&@9;UtNPR z3Q(1*YjBZIlJR13bw(EM@`_!6Ub!@aUrUXuG;S=>)}1orcZ9JQZ_D7b?2ut-s#Cn| z{#xjML$%anBcb-yC-u{gyAq>Cv)%7YRfVp9hJ;xI2A+9EKgcCKstIiGXB-Fkz2Uto z&L1&8+c1`Y%+OtKQbx7AWA%VkLYemff-Q>JQU_->G@nHlgTp}=bZYW`y%oh*COUbm zUcc(C^4B9By-mp3c?@aIiz))e-x^NC?Cda>tRlMu0GVYdT~q zrAG%H&}P*|ydPS)e;328FMfT`Y5!L6DX`ybh=#}Nr-3AL^x=8WCxZiGC-YMa{4zFo zR$oH+kt5STlTmzFyom=5lA9>$!mKv;l|KmBgp$B_O*@cjkS-BNgHV_dXh%4L5CAKa zg38Ir|2DVefbqaiMxu(4Rz)5tF06&%SEs|XL2LyiVC)=@aUxeZ2d$YiWv-0wB(cUx zCX=Rz^eB8T>LemfAzQ(q6`);((D6+W8FfzCzq-qz=`GAZb;V{?Gmof}V=MC?Jb9Cm z%&d7Shw?U+SyNG8qzqV3iLz?gr!y$yKa8oKDWfCTIvAykp#SjX21{>)D*8D2T%}Ll z^q^tkiPSD!?yxJ^*9l_`f)}jSyQ|*)Hrz<*p$@`KW^Ravw*4V|N2Tk_Uc>r+A8WV- zXJSgf<9=4QosAj*U5Vw%3H^ZVSVQfO=a;d(`4HRCpp@i3QEe(+sklC9LXY2`{xVm! zzzwUhquz!J{p%PVrLm;|UiE{TFVWn9zE3#Br4FEA-PT-zfW|I%Z zw6a#_9_DfY0^E@}b~63(p^-DL-Q=ne-oIMuRebS{G;Y5K9EX>JJ_kQ>O_|)M3ON5t z)rD*RYX4TyXn9i55Nqus>Ee{h_S|$ts;2@zOPibZMbdG`i+U|dnv~{@nA#9OyroLcUKYAoz!g-(@1KVh~w@W z%5Xs7&1+N$h~`CGBe#V3Wa)=$Ykr~cg(H*iLY+(k9iM(psOwAVqb#2lYP!S0(K7vA zJ6&M{jc{a@qqB*D0UUj;Ua4GSy1B2{o=#3{E|-^k%9w9CIVoxf*vwwL50g&6YIJrF zda+bwwYnyN9MiGi^~3V3f1 z)-BggC_koM*$#|YMS`8)nbIBj1NN~nA3hydE^RcvHxWj38pJcq`UkSfYHpdGTFahd zIdm@PK;4_FXTf(OW$=C4`j2VL*MQ9*2{eQcjEA7HpKJ=HV^F$x6U8~SKTmBA-Ilhh zFs_iw#DI4!dU>8y`y()5={RRs@?-hAPDf;9%0hnkxw}02a?1v%#A?y0)<>(!K- z@Ef@oMMsV{2f}!7+R$sv=`>{An3|<&#t>74ZKswuwgRsrgX#Oq-dm0&R4qg@mVa;a z-G0@P8n|_>^E8vU<7U^%^qJhB0{X_K%TMxY>@6+qd;SJfsz=Q?`UgihM&C6H#nT8b zkfPFPOt*j~fD7O+Py1FewLc_zVIIa4gT9OECI16%JS{Z!Tp)fjO#V< z#lMMW)w_Ch?du?st?~;m9?g1yN;Ht_iem_2+AbR|qE=Mgpf!>Ut%C03= z?qO`|&0OZtcocdad4-YEare8`W!Wz`VgVR741}MW$7?`$D{r>bfH*j^`GJCBv)nSa zV~rjUy9f1V^8LyIGLD}=RhkCG!|vmJBerjABMRnyHxv`lqD(X57GxBa)h8V4GM{BoR{KAv16r?USBb!kf9toiWAxM!l0h`u#PaKFqj+jU@lxfzKZV=Y)(xg{3=9tuTp|7_VrYUSKn`5J#ynn3SK% z3v3sx~N!!?83JQqhhkZpILlearG;pqj7*eul!DD60 zMRSX84=W(#0CWK5k7c|to%;NC(f(Y%eyRS)K79JJ*>zQ2QSc?4FR0@qCR6fTq6^P- zHu*_YdAq~PS_}887-%CE%3|)pQuT~sA@=#ylz4!(HOl~){@xx9HYpYihlIOya}C0- zOjGh9;GozYWwV9Z{=0EjGrz#dNx&s8QEEpXl_3cm)O~xX;UxF6Wq^B2H)Zt8Dstnik*mc;dD{V*W!=tW0VMvtkT0jD|KPZcnXRzMfzs+R3JiY!=&^ePHwU8Sb=8 zF`4+I>ha@B;kgf5Ao{@m<8NZE4O`V8w-S4DcJ#9zaef4_KlS^6G_sI1kx^gyBvSbf z&NRfa{pUMQzWStai}&@r(lgOotfz_6;T@!fn5f1bVshqmHD$l-vjsbBtlHl7}qGL z=`7Wrjv;MOfID3ic<+w%hQZKj=bl`@hm0L=v~ZFiprwK?K)K$&(N|qcT3|4)C8+9U zBD(=wlmr{#;{nb$#$DOkOguuP-;6ROBbaAxBUuD&n{9jEX?$h+6$CM5?}W9tKt9hy zL;qvfgOtzu|K0oLQk^R1(Wf&C*R5+g{v2i#Y(4&{`g{|#=bbsxB}H-G*7!3l!LR-k z^DjJ9@KRJ)C%tf}Nb6AayCkD7LwZ^H=jcD!5@$E)^n>|fDE>HQg{-BxWHJN^Ip90C zcKkm+drJKj!jy13z|gJ71sKLI-#RGz_uff=1==nsa$lOt+k0#_5lP4}YnYqCU!mIM zZ#U52eI!7ETPjd)NYw@AR>cgCQQoR}3=F=c%kx666Z2BlU#|&TQCi+6mrHY5p0Ca4cSnh@t ztGU_`Kgku$AdlO1!Y~c2Gg{}ei6=HOFE1Kty{-SQztwna%Frk|Xd;%;fdrohnvVYI zc=vw)pNvfiXzCNQKCb)UNk_Vqj^JV z_PhL3@0v%a&-S5wvCD5)bb3iv|*_GzjgqkIr+2N#1icH=P`!hILQ$E6B%9_jIc>JHB9#Cb_!S2+j zxnX_eK-wtVJ;_D7T@pfFi#5v=djz!fmvYLU=ncqd3pUw$))?MUMH*(QJpr`X9R4w} z{)|K_orJENqqj3yWOx2E;RU<=*#v-M$l$d(#7$|QuISqS@jc`3DDVD<1){T zOXwB$?k8`lPyEw@0ehH*ZhDX!aOG%@Mj+BC@IQJ7q%}^!?QpAu@gk)iYE%@pM*idl zm0xl>J@QFkQaI_adYgzuNYPW@7V!ztJ`XyuE@yE37U-4)XiW?W1o?vGs`k^Ed8WTI ztL6EE$c_BHz-LE%oy&{LcOERv!;oZ?B?O)PVy7*2ZJZ+?S}Xw8f^id;yMUtz>Q6NC9l_xtRQuYQ>7iUC)A?BNY=>IlUV32^#!Y zcv)FXU+GQEvCf;_EPia;W!|~1T2dRR#Fz{4vEuvX#SDK~C_{RzH9lP|Rk->bp|huR zqAn)DevxZT_o!n1G_JnI&~YnH7qL?L!h`5#Jv}qIjlfH31Kf-h?yBf|23#KRS&n-i zKt7kgsT*sSsw4A>LF3P6UUuiu&BflErNs! zn?v7p(2%NqQJ<3{Kf0*@84Ob*p~@g;fh^8`rMd%^SYOvIz*DRN{;0R^tr4`~X3WC8mqC>#+%^(2IjVA`Ietrc3+(U{T&ovOwnt)7gGxbRfTj1S;ZnJpq7E1NguMw zET>O{Y-BFr_n@0^zZ#Hzxq1i z-~O-KKdkedPO}j297$#KN-QsYQ)J18T5@zP7Rx^NWS_zP zD-zPIh?*8BDgP<~fZC3Efy@mksz?nSm_e|0T`dZ8=33B0JENzmB71KNlB~zKT!V3o zDb?9gW(vD+xuVgU@|>TDTfuh+gZCh$T2+aWk>N*2y*7VnHmaznNH1l~ZThlre`Sx@ zkqt!V+APVB1$}DGxo|Q$?NAu}zAX^}J+IToB`E`4A}+c7oRyj~nQHYyaOJAna#@K{ z&+N-ARBJw5C2$rfO8N48S=rR0s)+((!aR}cKq4VGHs=7F-SwzA>=d-E7ShSfQ;@uN zf-eU%Qc}wu6-2HzN;~eQ8aM|@Q@_;Z*1${Wdz$o1FZZ6dUtNmuP02Ub+owkz+Gud$ z-d5kye+=7n|KAPUGMq~?`d-?w;m`kG{Wqy{)uPUf96%h^`H@jS#S`cCm%A$vqCds& zBlPXFQbX_Q+G*~5r_k}$`PP2uk>WLmH3ENlRZbVqOIRsLtIzqCemeVl`}h_Usnk+G zk5^%1O<~{24))(_)M}v*?84tRAft%Fz~J3rkCduYE8gqL35aZ2#HjtV-nvLJns_pW z^uT`;Uk;Ew9A_QtYaz>5-aM0O-;IFy_KYMh;m0XG$pB}WjQo_zZ}A~x_=N8*U&{0w zFYSi=d%`Kn^@P)dE+csC@x;Q63Gx+Kw&z}cGZYWkyq|?4v|s+AhnBUorub+%pqfCEv0NpM-RPM>gb46+bm18vT7qT5ywDe;> zf;!#x<9UR`m{h+RvCeO1?Q&ep^uPL#pG|w`T#}Rg5=b>s@b>t>g?s~$L=M4DP*HhI zQ7`I5W>}2NFMx1ueUUVq)1q7Mw0!W00mcc?X91_*Ba0SMd^sF^cPeU>p0h2bF5nkS zH8-r<-0Z-Vx>Yv;!~s97)PSicTxl3Q&O>X5vRF&=!zK2)RIZR7yce+&qj=l&)e`vMy!mk zXE6fw25_~pOshQDg8klFoDy8E$K{#~V@U21-M8W zO4}f#LKhqgE~=KnaIAP++l%!|`Vt~ENk|%}*{dwaZcybEYYXz`)LtSljpbm+rt-7? zG5hPcLQ{6S&$ako15tieFelPp-kYqotw0&2B{oTUSH{~t`_%9vb>}8>yV1~b=)c4X zqzj4!AW6D@OpQkXQ5Hbr@U^q0pZC7KUNN@B{K@7kF^_RtYuXg`QS38}+wV43@k&3~ zTnx#vKaRf?-|-)-EV z*QO{|NEI&orey}vd?NDLV0bynaR?Sr9Hccd+W~QBtPbrx=BMIbB_CtP$&m* zJ%8WhEZwpv-C}N#5N9ZPo`M1#O_?2pw8ZuX2V7~Q2%4L^KLwql|dcgUN zf_B@|WqWMN@Tsd4o(8bI)~`W`2O>3pwX$OJgHaA8+I#0k$M!*4r0a!B)T$o8J`d|W z7z#v|NflP4Xit>}B406yFFBdGbL9aq;CWe}%~LS44zBZ*B1bUl%PGh%JeA~>&Fr|` zeYba761nT(4)xpmo%+20AveH?!#jnED@dMI!QLg5@6iutH&OWmxSJ~ZVMq6j%@L6h z5nWTN+S&wu0#*R1&aoBUp+5nx%YB;6Y%|{p*QV@DW)!5$0DRQ7i+>tlBL#rwnHFxC zA@AQZOC~xI^p{mPw~Re?z^9XLgIXM}q)cV7@ac;me8&OJA58ZK`FO~Ou#ve#C#X}a zZ5|(xvgBv#t|wPHclboS87LtVBA~auf^CIR=3Z@}eR3Ygixrt`7vI6CMG6u05k}is z82;4Ud!)?7UQtQvzoS>>iri8LbYL?~**uL*`|3}e3zFZ1UZ#&9eONAjyl^TL&UYP^lg^%rzCZ{bEOeAAn-*!=T8t7D5V{1=%pU<(3=M$L=v48=VP z8sU5=LD8wA9H**77$Dow1^3KbD6$b<^h`7X9b^f4Oi~YnYpbHYjQ;r`D@dtT(B$*0 z6B$E8mw61X6yNHti@SS4v+D9z?Zk>Uho2FU58?quiVUR@!=OJme?F=BD>aSX4V&&u z8S%&_Lysx;75?<@r z>sq0|AJkK!*P`P@(9=)GrRU0KDA$fU6Fon9Pi_6og9{77zI{3oFxQd!pkYDM7={1A z(I*s3A1(UbvZ!xmDLuDzY2xd2?)halycm`G~ok&s{0qP%_*Uqa5A z$86O|f{q{%`aFiF{1uE>^9H0Nx3Z-=ckpK{G=qP}wxsKArj@p{gQUua`*hLQ^NY%~ zpPY-LCT1tb4*wTTVj+ULG8x(tY~RF8v?6CE3FYb=N!L4MCVH8j3eK}#3bgWgJuEi0 zO`oh8>|(Bn%UK?3i}_)=`ywZlSPz!m18rSlmPoKNL8m(M4x^=0DP;FsfMgGxbUp|J zyz~_sNK{Sph3R|s%JSJ9t53-TsU&U*Ml>9ot-`mga{o)w({{l8H>48iRg83*aY=K; z6OmrsyBnTL18v`N_Z^t5 zO|23Rs5demHS^$z3<@6Fbp3R@lwtO)0d*D#?F(o`8p`?UojD*Ec_41IFRdZQnLx;S zK38oT`tPr?d+1sW zP>;Np)!18oRr$6$U4hs;tqpvlyof1D-z^Fj3-%wJMR-$YII8Ob2?U4dckXc*^XwX zUy=P4+f>;-Qdg~O{g3|Zf0Rlv;<&&FXuzrK?!i(T#hrdlvwuNcBE*UfHSU5-b!*@^ zW%O6S#z!h#jE7aN0{TgKqB+c(p<9>v7E26EIYq8qF3(LF9DQRglUxhFFk^lzA`FR* z*;Is*8?-M=e+B=!YxW-b?acG0-&GcZB`eB$}hH%DEoXv9d-P+DonMs4!N2{!&90QH{Uv`leV5{g!mqQ$V{`>O>s ziQ1Yc55|yVPfZc6Uf7Zju|I@c&MK7Pf!Ps_PS3l(VV`E6T@}^4lcf+jvL7cS79t5)p*mgTb zbt|+-_yzZ0z$F8<_harqes2dIpl*8`jF@*?uN91t9AI}9LSaM21KY6983v?s1+zo` z=P&C^$NUuksM%ISQMBvRvSPxCt!H6e7!}d_gq2e>U%G4aMGW2(0y9UJPVdepoV(4z zLndRmk|TpomK6~Otx-Uo$=aU5%k-bRlHKz8dkrEsGKJtv&plm3HXua#BSzU^Lhphk z`EeDAn3=UxAsm34br2Q!+{qN@GRJp&aB({4an4lZpj|3p1@Dv7sqBhEK-a*+Z6OSm zPe<`>hB<9NB*c=TfkR9WCavD9e9c*HMxi0ytIR7a6VIv{p?P5bLOEy)n)*u4Oo4zp zt1VbQIbNp*j1?x2ikWuZiTCB%|5mA_YomRK3@fvtLJAd;;zTcUv^^wq@W_XZrWh}- zwkqf9-nhRu%@{5{1`y3pQ755f?J*UQ%H(L$*A>@!Nif zEh8B(^e}UiSHwmAi-$cA+7=4C)kWMgLrCD>RXVRy7oy8s}bv`hEF|eUvS-5YY z@{=MUbMsN#$vNFR?-3R1xviCuOSrrlyXG6BhXMHs;@7VW#e5a$!?5phlT*g}5h|*w z^kvKqGOPUiTjZG*{=9|i7x(sntxeJPsp#_$cS22f&uOexkXt^GqU+dF<-S32Zo@Ef z`M|2*X4fCUB;{TyuCKzY$C76J-hK=WrJ_tsS#rObDV`4gMJd`XCiZ5}Gobd+I}S;w z{RLvh8CW}1ZxVKFFv}afSp=jh<=&t4jZ{v!iXnA$g;EsZ4v(iSVFePcMfQZc}KkUR8eJ2Odm^b;{8r!z{eR5fvF?E$c%_h+|SY<%Jg=lU8Ga z;wqd2iBcHT{WiYKAqkF!G0kqB^>-|>2e?EuXFu2OPNYw1+}eF?(@icj?i~|sgj3=% zDeO#If~6J~1x#E!DDT)DvB4gld^267>hypQqzeUfHuJgnMUV1{Lyw20upJ}^{HQ&* zt-7AvUr(J}O&@-3jsk$N@ybRMRTE$s^wZFQ*n4q4JM_Ask~#%{L;*N6?|9vJ&MOD` z2g8C$f3SCd2@of{-BvpDreG44$7TdU#?zfsQSNRZ=mBj zmKfTuZ%uBWm5~r*Tk))Jcd%&Cue$l*dB6un1pG;TtfgmGFxk^CqIMTM@LK=)WLfMb zjq*kFp3gwH>U-{Y;v+Xb^p6ocyG`FRT1jA5-Y&VO;v2lTqkTV*Lnn_4j0L`Z=o{Ih zGgw^pszWDne!;C&!Szj1%eih>F3`rYD3;ym{{>U~{>1FRm|D+Ysa*oSJ?0BEW~A4g z{<8J2s28ksh3>*-Yz#{nr&6d=EGAL zzh6A%1BIlIiJMhCl-3#>Ki+S{_g_b`P@zaLYz`5b{+U|qL5i`IfgJ%X4?j-VSC7&( zDyXDO3tLuN*;ovi9;JMor>Xi9-r#NN)kNmo>Qq~QiTM)^+l9tukxt=Iv{On8NSRj$ zA{D(mWHFa>QeAqrl}j|1=Sx5y()4E2WTzO=N?3(|<7B(-5QeqNRJx#iO(Z0vW?Sx! zu64qNj5ftZFR=$70|Pw+gF2*$K1bL!=ZQ9CN`khy#UT3HA$nMrcdOY|qJPYbGpe&E zGKa=AHnwG?K%HTDZ^rK)&(q8dM|S&W96kv$X`RWf{=khE5=cM*kd+1kVJEAD!MNHv?Cp=CY8JXWDa>dgofL%CICn)pUNL4>5J*?t zaAM`Yb1a&Qm7z973j^DI9hCw)9_0UHoIq5=Ki9T5tIuDq^I1&-0?~ z0BO_QhFmwE%RSi{a1-9}SX!Bq*W*2VC`} z_{R3Sr5}@xJ!+oW|2g`txiwRR-x=S`L6%m?et+BDOUqcUW&9mK#?HB?)9;)L{O=Ay zA)RFe@gFA3%rkxH%(wt0?msZQ?B`zKd(fz}uQ4rKZGpH!2yLVaiHjOQuMfFQXjQ60 zoS>~nGW<>;HvJKQpw56B>wQ+=<$8wBJj|%c>q79;9CMwC9yES4@))A`4aLN~`rt>w z4Le#fedkfs{oY;fb`$eF2K0b>$q1VxGP3Q z7jC*L#5VD^coJv%c;5O1i2XN8P>}xsGGv5fJ83#*9Ox=8N<14~GNlG0am3B;0-mYV zjs?x1XsAs05guPWv-7OWXvPv>V)6WxR8g)%MNpa@37Rb1zr$c;Cn`G*2+06#X-cS; zS3f83j%$Hb?MXL-D`9z)-vqWaMshNaA^I85-Bo8gUoLf@w0Wg3vTg|mUD!m(7po(G zif*K|_iKLwuV0o2uvU+Pjz13M;wePqzezkUp330bQj zd{w&F7uo*6IU+@LFM$Ub$*D@6ok)Olz_3leRW2KHf6rai#NX2E$;(Vo-fCL; z{(0D9cU0!kqZXumw$lbip0vv-(q49OnZ=aZc-HlB21e-S9Fs-1UaZ77QsOANScg}< zf^>;uzq0b>EJC@N*^28h5|O1<2wY~kMVDo4CR{o!+Il>J;X%q*n2!SzjVJLJKE@bu zZPjTdujakP`GI7_Yt{q>NdK@|=kMev9bZJv{_UD-xkmLRiQiDHiP~K-*QS9LFX`U& zDlPa^^=zB-PFE7lNxDq8uRX<|T~H(7Kaa=Xum~g6=Wm3WN$p2k3EUCx+T#L@(<=9vqXXNN;iq{Vy3 z%<(Ew;%Wt0^)I^BM?%u1>qXHt){BN|uh0w2^;{3%uPPH*#3*;_lUOZ;zP9iD=I9ps z#`hI!7DP%L=LW(6zxf&U1nbEi;Tl;?gD-1QQzW(!Onm^`XV8`hYme6OS$9OYr>w}n zBzBNq{ZR+*fcV<|rQN_&NaWMZ32tFUZB<}73E<3Qh|-m06EJ}Ah0%BZq!tFSzk12c z{1JNtq5`L%1@Bv?gEGo7`AE}$lfx-mtR;dRI-{KoK2tWuk_jxU;X);#&01r{uy=s z?JQeranf3n9v??R^>nqs{ni2!>-IkUg+AP2z}NVNtT5IRKQZ<{CM=X2$Y zXl|$9io}Vk2!^*Gth9LW&Wg!(3UCim2TIm_!3AFcy5)+MfN*RDqGCHrc?VU6XS8|^ zgc`AVR-p|qBe+9+nk=5(bjCgXusTUQkPB1r)u(WSD{gCz?3Ji-Oy~U(GyaLS)xY83g4PZGBfWLz62PR5va;clZUNx#)?D(So zy?oZr9{7`nc-jG5ESX>SAsK*$z-8H5{YjE=n_s@PJl3+5+GF#NmyO&aaD5BcK+Nc{ z^yTk_$*FX<)|Ss5@{BEl-;gM{5YG(fSC;kBEucPsUOXzp8D-hjboKdD=(6Kw_eCy- z)R56PM-LNf^~sd~^I@^j6l-se%@Y+42c_CBOl z&jb|t*R~hGyA|1cQ-lS3M;jQBl)2sHxW}nYl8v2!tt!(wo5|^Vp>MD?`<^^wF$2RU zwmjZKk@JPi=T3*%df`3Ue7Z7hQrp+n^Fjr!ogTJa>tAnm${sq%PN}9M1v#Z_n||MYCyNeiEoKHX}4^GeyWFVx^bO6+rCoJ@)dp#RJ9ftClb~ z2P#d(JNE)jsH-=?$_A@}g&;vZ14|HwxeGB+52k?e1DV!gR_NN0b8S?y>eeiawv5n* z3Vz=qL0cAELCn^wUFP_K`=7IXHw=IC^S??uWD21~b8Cdr8+^iGPsyM`KFAj}!t;l3 zUbqqqV~Y@#k|Uk!#ht2@;N2E!C<)S+yVhi|CfHCAt^YGj7&5COz77CpvA>ijzH`59UqKd)KF!{&RhBaAp z1H$8zqsv-L-YSpVx&C8~$Lc>cXM5wq-4Li%ykhSqUtu}+1^oG%SJf%f_KXqQWz8Pf zQ0be#hU-P~;N|nHxkwLF*d!vVVf~5av1=bD z?A9@wu%fG_p+eP7EZ86+Ps%$N`D*yxOVM|s zBRLwr!`aR4-piWI^@$UMp2#hIv*OX|{dGZKDKZ!hHm~0k+AgArCW0mrE>^DX&Ylb~ zc9|u;k&)$7!0NHGxuKxPX&eTWh1Xz<#Mj*ky7(}?l<)yEnGQ8sPP&~$=+(r`3o*9> z&*dxUT0HzXB}pi9zjo|~8aIsY{EGS#gggr}0%XGCS_!Y@!hwm<|H{>p zx{~_ibZd{9Jv;w#U-LeD&oY7b+cF6@H2v7^fhv?1G8^m)HXKzmS4BR+F6!A?oob~9 zCKMjTS9H%^5a;p!+sby4*#Bgk|B3SKr!aZVI--HS_qmhu92dx;!$oP!l4om|DoJ_u zl~DZyX$q%|`~p*gtu+=zw`}b@N%`Hy0E7`r7XydtR*Mc!bu+{xL>Vn$Z{2}zkZKCF zj<@EAXhOf8+j3rC4A=eqb(XQ09EiiOu|c*giiP3~ywX2*p2){sqVye%Tw% zp72fwI?R|r+mD`*z1M@N-}+q3y8f+|JF^K&Cxe>Xof&w3-5wdS*r0Q0_xq1^QU|lW z97~B=H_0MFE=xIFeT8&|lCBK;0K!iHPH2L*7AQ7#F0$0DU)j;}JZoBr=&X^b-Ba0B zUXK{%E_mK}IHyc6iQu@%RA5+sWht1OMd0^>cakyMbMX&ehaXrLz3vO*jFL|A=PvYE zHgkoJf=4bl>5to1b|vYmH-N%abBUfWhsluFW0oK<5;W&Kt$hO#7#IMSy48(cIO*Bc)vAk@}r`LyL-{+1KzrJ zDx~R_{I3hZg$9Vqd&87C8F~fhRAr%&QnQh*rJS_jLNQDMTfL!G#~a)z&abC6N-)F_ z`kiV!S!ag@gdTV21nP&rfRxWUXg;8Kf%h_9qCj0C!`I^sKXUzte$%@oMg9$~b^>y> z!nJYRJMs_UfP02+O43^eMs_8jm%_@pXJe-1^R-~wgMv>8h$}`UxjcfWc4?gA8A!P6s4&SD(vOzYT zUbFDZZ?tbAkZFTGtiS2K?n!a2ta_o!D=(_I0`AqrqZ_!0E1I?@~zWpy6q7C#!# zNn6o;BYS>HW_|b44MMuED!}f7;6Uyws9_~QdJLl1Lfj@IniS*p6CsDp$7&NYLa=g8 zC&0&`zi0CA4h^ngzGM%jH-J25aQVc%=dD+E3M z;R%iE%o+FyVT4@l@DbR6^-dzj+G3Gu`S4rshXt4i4`5pZSLih90eR`Fp8%;FQ{Imj zZ$$UNshn|>o|5Jn*>@*0JjpV7H_)25RmBQ8{4}dq4!%h87JNqN^ZltQPbEhgHdfdK zUkh-fA6&!85s&x)GYAfXeBuJM>Q-U@DDto^CBM?qy7XY+3*cV!T^6j``()HvQ_Es@ z)QJ#tBW&6+%j`3Q9MQXEegN^FtkSzscJ0j`3Z(>4XAB&wBltho4w|6-sZ~Tn*mbg- z{Vkw2s968$RNs7*4zL+sesGKn&MiaDJL5mx4ZuU_yQ0voTcS)NumunV+Z^bl;NIlv z=gjt)<|at1>rkIg23CFK3lt(Bbd`v8IXz~(!tBEGmK3kmJqKg&`QJc2RfnxeAdAh*2k zOiM8eFCURnK|25K1uzmKxq$WKVAcboVdREZXxczclgCX48~<<=Y%Xw+f!T^*0y;@_ z=F<`B8dP}+DMQAhN^l?$s&b}P_F2~4ouGQk<9&GI_m3!fny@D^FcAbPN51)UtJbWB87?c#Jix;)wY+z(s|#p@*xuCMCO9bnIscVA z;N3~o*D0Fzt825*6)(~*(9EJ_w-w|(D=7g7PJ+Z_Fc-5In&p%z^xG;PI?6JYZtqy^ zc?!`Jz)F!+-Z8`M5{ZZ%W&&*JGgkcuADJs^>`so07hCG$k53XOJ#U+{F7gJFX`9|j z0i8Fs?vb@bCmA3XMH3!Sp2VfBpYqM(8i0-YEv}^%3LUKxv2WeZ~=$&Ryd^4p+yI>@zh3dp5_`MaY zd;wcmgdVQ>pCJ$dU?{4y%@{+{CkI`(R+|7IXAU8?S1ODgoTBYs3trocQ z&8hP3_C!q$@JhQXbBo^Ps4MOD{5d_vzmn+{JtXey<&6SOF#DI=`JM%L4+AP zq;cqw6e$IyL68zb+HVcleciX$`#jJ0J@5AY^#^X-T5Hbz*y}hWDRQ?|rLd2M#}%9f z&xRqqjtrPcy&63~sxiMZGcX(xWB!C%=NM_MGtMF~zz=rn%8zCX&cs-eHdTIcB8xDX z8d5W^nkAkLq^~Ee*RWAmi&6{(X%K z;n5eYNEPI{C?A?!@m-Wu?V^iu@PnL=dSH~~Hx9MU`gUjc_S4O;U&E8eS)e8Jw|3LH zB6M$1weB^Ry5#-#JK085Z{U{^(UsfiYkl2dkxYl9CpizSCLm|uW=kLWiSBEcU;ABg zT%_-wLxjr&M$6xON!Uh2YIK%7J*oe~fQ;m`xEh?gEPF^u176O(N&BIKn zKcQWoq(;W6;=4d+EA^n9lqFYX_JPeU)?yEu;=5&@PK_EM15WjWhdx9G#aG@=1Ss;# znRv4mx(>^c4_+U{Hw98Q*vyD=zhpH1Jrq6RoO9Z7o4R2`6!VRdr9v74?4FQjDr{s0 zr;#Ar*hi8xB~d!5yg(45gv#lqN_mh+9&`M1C#*=9mkl`PZ#JNqbw|~hp(XteAf_8X z+Z}V<+gSvWb^EhZA#tg%%Ig2X&`?Y?t7H&EYs*9Tb#7dwI9@c<$Pi8)MmLH-i0FxGEIC{2VRDLh1UIN!X^}-&D5pTwEWl4%~(F<-AaN-Ok_=O{p(z zV4f6HA6O|$3USN*SSl8=4|$QPqVX0u4Hx;|+l8vPHBe$mjX4$lJtbWk;hch0l@mlx zz?o;jrgIEkEqb3ksdN%bOz&P}-@2>cvG)a}-`H?@ijszu;su+Cy4?i-2SJsAC?$<= z>FHUqx5tjeb9i_s&XEV4k;iZI?Lj_#7DHHKUqz63XdkZIIRE zo?Q=*aZ4&ILUj3EdEb|2F2~PFSw^%7$EQK+&IlAzz4T8}u{~W~sU7(#Sypx9_+{V@ z*`SBRmzcBon4z0I)Hkq9RldE-DH0I!)HjRBg2!tt8NZ8XAv#rB`&3uJoZO`FH^95U z3k95kd*eNCx`~kE(ik;-%WV#FVz5@_>q+89!p@vr9#vu*Byd(Nz$$h)Dzxhfo59*N znUAYKD|gJO8MkJhc9=1#i8F|&6{-9iCb8B|KlxDyjl^$GdH0pt@cs=(?Zoo1*UrP5 zCyOi<--+L)N%9jRhT@gm=%R}Z<=~uXQ$T1SyRa)V$6JM4O(W{x8eZ2BE;3`<9$&ip z>?vFOUc*=ft1=~kh*ThWa!LL4fTkvV3r`wHpyt{5tifSmIl5csphXCTD2*BqlI+sl zX26xb!{GVF%egQLo@XdW@Bz^mwE@3VGOGIAC)J&lcJ089J#iY`04rTvMmg#}Y+p#l z)Htvn!Mce*?u!Oo#*$?1T)A0vq4cZ;}q8PD5Yy^bFOR?IGO6M*mW}yXYRcHgJ zXH|a!U*I55UIk7A=R9L~FRzC_?_SA&kxUk;3^Ben)hxhsd+x%V+(;HP&hf{iEujRt z(Fv&}L7^qB0aFGyA>-+Il9y`O-Iqu_Or$J?Qwt8RNaJ_s znYxGWYs9(>23I(=hs*`(ba{_uzH=g;heURC7idWgG36m|TFX_^KL88ca z^Un0pcw{J>ueuw3M3Jc5I9oGe((O^5XzCDF00_rBq&$Ood$n*C;k#v7FN%l-k<4ao zV!244RCtwe@DF%!{6ruAgEZSB;BLPx9wrSZ@2S8kr{*P-6}tO1i)L8jz3Ej@~% zsho}edP7hUTMaSfiXigNEcj6MQ}0B(dtka)RGbK7hAw;8JsK1c; ziADqJ(Ex^j7F?x&qa+6Z0}T_b}qw>fNnR!2+^dHk|=1I>#rb%sU4LhxE1~5swyzL zW270J1lJ=Q+-Ccgki6D@rL&LZ&0T)btJbFRpQE@9ke|qn#mR>AIr9UUkywUo*^h$( z9>~+yjT|jU+AvmaHISmW^n5TgFzMkP=Z>qfR1`s$Cp_Ml`Zi9U=AMC@GH^sxrq!4| zOck(CoJE#o9Qwd)2yLFYf7q%&VITtC$l6>ltw7aAjYxQgrCqAEN?9pGwvYEdS*};E|AA zdq)oYqASiI)F3#U$f6<~mz4s*J)9*NFjt#f%z{4NWL8j{2n2luh-7{+Gy?8kPcV># z(YMJbX@AtE`Hms+n>~jju8t!@Kk9$GaHl0 zw>?AF{?wZn%Up8#z}=bN)zZ#!@v!rOtz@M%EppJlG|p-x;F+Cjm&c9UqE9qXjwGPg zPlmt0F-=^U4Y0M``lWEet_DToo1@LcY-3W4wT7VRyEku?QIL=f5{{P?najO~4%s?U zc+u8e86zAN-~Iy!#{+JF-@gy`)jRCJXb_8tn$>HU9vy`MX6OU1=(LRwa`QC-C(Jr( zzN>B99oBNq%b5dO>J>jeK8C}9AK^z9o`BUBHv^53FyQ<~fM+Pfv;~pzV|9-dg_~w4 zb*^J-_&f30R0!a< z+>DP}X^yP@{i$!*aql(6t$ceC`vZA=4|KMbQ?j@0m?c2Id04G=1bM%ezgRb5ev$Sn zTY3q%|Ch5Y;P9uj zOn&^Qvs_5yGA^ILzf`aWdjyOF(MU`L-C@^yz0V!sQA@KV?2Jey^L%UG%(qyY0Y9UN zi5KEKt8E|Txt%r(mTs6%Dl@GoT#mEL`QQNhw#|B^My~2aCDdO3E&n-6^UqT`2r`gl zesIuHCi&hNQj!!c8Q>Nd!z;vq{+StAawVCfRZ40uU!Zx&6!)o-cB*YP)XWD}P^7$msBWJ*6^XX3vfEnRmfsC}PNZq6ozXnNN;gRwAH}b|w&bQxu zBuyuxDDoPMc9jml6dY7JZ$EwEbY61%XFBtCGpxa07G8Zy}=_ud^PPCjY~gTk%*~-+&|{xa zT*i`lk$)+z0{KQdb6A2|J2A^760W%Of?c};7CBcl3E;%-Yv89s6^=)JCL4vm0u5&h z`-Ob;QN+k;w)u#p{2YBmzU#L5hn3~+>X`6F;Qu_5H+E&aK<`-ApUmPLcp8Y`9|kP( z$aTsJCxYJCmP+7CkYXZhXOg%5>)knU=vPj*rkA24Ho~a??})dPtUhh0nWF zwbYU6M!w|PY=kwXMQRh|yG0@dF&3oVN?y+l5jvg5`&QP7Jsd!6JX8Ku&+Gha_NVi_ zoAk@SEPR^*yS$<>zl1~T}jNq&dCUj^IV{OALrUaFqi+ z?G6@(RtWBJ{rmXEDkU^vfL%OO`0%Uajp+)5$+!eDc;+l(*oCWe#`Ol_hrX{u&@4tR zv_73D$DJjdZQQ18&3c8iCEx&FwCWp?Dz7~Nje z;i6fWfOH@iABd-Uwxsd*d!O2$U1g0fzkbuCOJf#T3 zz2_pxJyUk>&!uAE*#y&F5pQcWg^+?3dgdCEhZ9+2a({!c;rKJ)H#?tlwzl5RcA`jr8>Z}?r4rXOE0 z>a&d=== z1r{EpdyiprUTRij4OMB={2v>HOERy~BinX7a~thv{gbW+83%q4@-*2Anp2*V^h;@a z^jVxbR5BHdc_mfi<2S*6%n40|13WW*A+wRhtk81%oSSfs78J9lt{kVs(#>x~DEOVl z#}%)Apz&+u;E`T#4h``}+!#|eH#?H7&d<16RWzphax$7nA0STJkIesvA35~j_>pHq-x~S>v2K2mdH%Zv&D)+0tVT^79hAv_;zBjGh3BaG zuHpS=QD`W?kjnz55icg!6uP{P>ueZp3#ex6tO*K`IvNTO3khKL4lri8dAH$RLDYHQ zgGk)nXgODMXdb9zPGu}C`g4}Us8T3iksvBwQ6GWl29xW!fh%a&3k}#Zg(LmsUc(>0 ziV`fNgniXHU`P+z7hhF=qap`*T@eYoE)OZFyV|qRV zTs1siqv*Ch=#n60CHQZbd`SQKXt<#by)MthzBH&Z8i@GR>~jnpb%Jih&a;C7P+@!%XI>%T2WSGR__z|6VI zM})yo>wc_VOBXIB$HF1!ryK4oDSyJD-NJO#zu<5hCSvjJFF5>}Yw(M;_2iNu3K4o; zp58&3mPWZ;Xf#&s-~pX0na}sB%>(+5L=&Mi6T$Z9(4pGGDjzg1g&;!P zx%^tGnYIVudo8M8lY3klwQ$0UjK&CROzZWn-~p6oQ(N}WDa5!8@;DEFv+wst6NUJ- z6u&y%17c()MChHo28M!fX4-`hSlfKfHe^tz%yDMIr0&ieeFRn=LP=rnAauFlc~AJ4 z=_|ZqG|M*fVq(ilbG~+WSCQ%;&ib3ah|2t+BfTw7Aautg09Tq1LITuLNd0rBSN!c(_ebI1|vi+;KJB!3E+&Gsce!h8lIZ8#Fq z>eOYoSxDLua`5cpmpnwkatbO_*j+G!0e4Ucak>xR3B~+iomaw{?=#%_`XN`RgRLa4 zVF9@2Dgn*o9VdNTn^)cYj0bO&NFf1iA}HUCkB6?sfgyBhjX-@UvvwxQsuPDMnmNf8 z;I5oczYxb#njTSPH2=&7NsdF79WZAw-99zr@mDW5G6_I&Sti37R0o7`xzMsM%ANtq zE51&g-5!kNKfK9T)faGAmGeIj9VcNO%WAn?yR)r*j_9w|r~XY}d7sLAz7l1UZtzV` zd_udny??mHLWJr48dyw^{b`VcETyr?x%tAu3)!A8%2dCRVX1_OkOkOv$IA+-!l+VH zZ1zZh^d9p#O0v0KelE*g&J<4a5dpxa{W6=m6GiVnU^KQ2g7;3_Vg+3&$DfO^YEopv zNssCpzU4dq??H3v@L09U-7bJTTDF2dI*u4m^q&~387IomvE zO#+p*4av$65x?}7ShdjgUgVnXQJP;Pq7WRaQYhv8DU>$LeNU>XuuKeBXts$HNonx- zGg`-ICia}9U#0y6kF9$=q(rK4!~gVnuaO`D(RA(>ZAq;kf3(P9y7y!&w?3lOkZFVV zl;GRWHKMoT#B7eKP3#R(cdVsQrdj&v{MIBm8-$A&E(N(O08G$5vkp%7X)PY_KU9m3 zQQX#?5?^ObcyIk7eVM7#dehi{Yb_iu-`)-?ARh6t^6mYNnU4O){m%k4c2<6>5VKQ0e0o^Hg`@9-w& z0x>v$$+!wFVNISmwD%WFK(|_S84LjsZP5F#7xo-^EErdF#W8?x6)gECV?Y8@VAhE6Zg08j9RK8 zyubuc2LsgV1TDPDApWee2az9hBIzRuCoAutLX;8N0>D_Exx1D>qsHv%Qq?$U8}e{x z>(P(;N%aQtb-qw4bynojdBt z(6R=WEs!3IqoVXy8poq*wU`K%K%T({YB;CWLIvwXk&-KR{fBm?y;|k4UM(Tb+Nc#O z%F-h9aT)qgXfII(qlpDZC#E&EBJDQ^y)R>ozUKPC$!C_BLPd_10<~#z{p5f!Ps{&( znCIW%Y03hKOkF3<<4;*C^dX%A*WT|gRwp9}!VP4D+<8NSpmHTbr0>c=->xM@6S`6K z9^a2M)T$;zXnclr2{U<+PSz61CTYT_X9_ZIzaR8B?u_UNktkWj|><05PFIqD5**JQhGNBBV zXNdQqidP};q2v`oo4n~SGT=)FY5P{ZQJ<+wTYHvlyiNz}EOH^DKfTmto>ytHc^Qpq zAu#g!HpfnbB#3A9@m@WM7k5xY9sS`c0$=$saSq@qaz{}pbWOBG{_7#SNiLw`x>8ORjIokmv2Zd zpwh$kZYGonIP3Sd@4WLhnI7@7tv@KU?I;FZ>%X5k@M&k<0aZIeg~qOaGm*|E0xMs8 z{lu;4nM&_^^#hk*LGN&A5l&ZW-6Z>+1VuN2710$V$BWFnN7lkrf=I-pH$mo)7S;y>03e?*)UgeoMWBN7ti2m$=r=E*mf8B=pHPkM6%{J{DpHP z%KV{!6v%je^LG+F^CTFm^m>gS8#am|7J0Yiv`rG zSO`Q(LXEnc<6QS?^?2}svxsE;fWPfvf zyU+*gqbIRE&f%FWZQODL1XGaRt*#!}aDvS*COQ0mxJC+QT_>c-m2_;0`V*(!F$sfW zNPJAil_dC7F7_Ok;riWYLf;y8a`9tKeaa5M&AIr@hO3YIwD($owTwBLiZg6rGB*T>=Blxj}chb!&bEDWY zW)*q1-Xwqi#;?n}g8O5U@2mW?f@BHrOB!1Q4&TR=p%jizJRJetw1wMeA>!H8V`17u z`37_aoMUEE3B^7UomwT*FX2`Fb>CiwTC?jW>OezkfE7YdL>{EuCicyse&Szeml<%|AGRz5Dtc5Ldxy%cied zgxRH>_oqMAwzKzV?b=uj?_jxRYF=>^kWT=DyC1NMFc1|aLjnOP$5-q8hct3-fl_>M zd_HJO311Ohqbo`wg+s#zFt0fhU*^m_aicpkSObmfQS80gp6iRhTupZ75b zEWS|wPYw$EWfOzM4e(r$eUq$X`k0S0&&hWef{SJcpS||E|0(efO_(Jn;EtzSY&a!% z{woRCkNc}i=Rbo}7A%#Df2X+0jr2}5IDAr*HkKj>#*ySf*Y^fA3;cZut#5_rAr^pJ zR<%qan7m-D$AFq0_Vu1QPYVp1U1~a?*{+OuZ}IM_$sPrUX6M;>DRmplMt0cnYxP>d zdTMzKrG${LEvg};0LE$FRd+M^rtZR4Uk$Cb=n==t3rD!nQ~oq7yaSjb!KwqUcX%{L z9xxhZ@X)AxY0Z0v%M0(K=k1L8-`L890in|ZL-*^6UCoEs zV65Z+9E>fd0D@@665{-ZkPaj&yE|5}DWT-AKSCe<{;y0XWa?12&b&=Nif%I$1Eig#B^2NOnO+ z;Kp6>Lv(T!oQX&daO%fsqkQo0bhwwQ_B&Y%6CLZuv*{2t?>Ue5sU*ucP8+7+O0o26wa{rh5RY#y@~DPE+*3h1TiG zH{jM{RqidVQU@sJ`W^|0ClGfl;{hE-0qGmx{~zfa>_9mA?}5-9$lv_H4ur>6=5Mdy zaSlBDZca;9AtG$zRA7=gW>J?+bxlzPg+l75@?Ft1A4p_NK)r@=f!MyU$N1>dnpt`@ zTT5@Rl2qLHJ%5YT*q?ABIO!f*FgF0w*MA=)fLB9+`zSO3xDgbBgv($WXfr=?qvllm zwR6aIsKE%$y+ZSi+VHp;1Ll0X?{tK$T6xiZm9Nk9o%VD?Lg8d(SK{ZZVeNDtWTzr? zl9fdfO~YIFXxd&dd{9o0B0?5ls&%e5d9KvEd0gIc&-B-y`W4kV7)D*VUdvq^wB8!o z*46ceO zJKm$6f051}siupKg0ubAfAwZ)L}zb-5!A?!CBAaLMGnBTsvv^Qlr~b$8tw|XXU`7h zyPPAsqQbi&n#&K`3)x}7t;?nD4?FJ>{;&~Z&UB2&k4c^#dI4&zju>F@JjJfYL- z`eS6N$b}dW7ObGH#jm~etA!u@mD<5Xv++Fgd^M4VVB_c0nt?b2+Z@2huJuH3jOsQSsl%}OP!mnPlvu~=&g_uA`90J3hYs*cDWS)l&!_jF?u%$VT zccz8YZbk3|!ObTPQEd)Ac#+skSOt*Ll4ES68%4~zQVm2f1i|p`1zz9At(mk5nd;Nq zkc%N<`LK(pK3atz)1bIU?3xV_{Rzj;=vv>?mOdbOf;c^Dx|N9T{(4~g(1iS*VM+M@ z><#(6Uieywx+ftrRojf${-7#ZTyn$p3hY zXG^`R9-dfz8ote1kuJxoqjtscSb84BI=?zYQ?a>G1)yLwP!6%Nv@98~JG|uVwR#<9 zEP0QTpk3MC#vfz8xda#++VvqcKu@3~allypaGYFdqlC1#o24QG7q zQGXb?#=KY=Dm1_-S}i*Awcbr|8V#=MuR{9o>2;nq$8yrf#vglq?aObB5*fMq4oXzG zvL{r%wt+%ReQwJ;`Re#%w?!6&Hv*4-6MYqYwVg4zY0%y=Ec0pL}AGkx#-YfTk@E zW3KT)sjM>%Ordo@im-v`{~Wt_ znL({uC4jbzA!`ceIM$K1`}DfuE;9nBClXdP;>4;!h_&cPw(uvPO~Y+_X!kHn{qP|i z)dN^lBFV3UT>PI!AAyS=B;#1WL^;>vFFO}&?TAx1{SvNlkAbK+_{ydwg<=T7yJWKa zvacyaF%Ym#t?`f~i&!c_BL@tRY8qh-|X2t6~VsF>lo zuRo>BK3lm3Wc}1%0S;6rUJ9rE&FE?013kc8B#xB?dNqSv3LXR>dD*RE3bF!dO}HsN zo!fhL@)t&7I(Q3upl@efg&%rtJ9dwnrSQ;$PLB79zk1yek^JSmU=%?AxkWdiT`}Lh zA!zZB(*Y|M;dDrFhXy2;JOhef)&Ue1Fj6+gwyq9}J|K)l9E(Ar=SULEqli&(UMDuX zWUuT5ot0jAh4b~c3ui52<4vnjAaPAF^*EdoSG)9`b8{at*a2qC_nH~39*KFoR@U#1nlctCKH#lVfYh*>fYe*Y+L+a%^H5fFe zQl=Vu9!`{>KV-^6#VLc?rU8V|5!5XyOmcoVssEuT@d??rN_3 z3hJy^Fb&9EO2qhxLFk1Pc3*wiRJ%U&xcf*G=5b%L_6zV5uo)cl3nAP{${?I;tS`lb zblj^7$OceluG7B5$3i6S6LeA9hzBv!iD;ONCX^6};Yotg<0qoy3@jBckma-{=G!x1#GFJk z`1$t$b@{J!On0)$_DxdS?is%{5m%Fj_9C>ZY|L#D0ut?ILq!BnCWNo{uHNDi5*J<% zvmF**XqiQ}0jj2`_)ko~ZvoLCa|APZC?tqm5c&1@dYb9b7;Kge{465K3=!+S?A_YO z-9f7UzOnqYGAug6xWS33Z-BlA)=6UdVf_k$nSunVN31l4NVWJiSTNREMpso>66ZPM z@s>OSQoGGoA?O?)1Xb zZBN6^^0ny`r@Qwy2f<4PfV@i3;D6GQjhPX*6mkEu$QBg%GE-d7`+q|gXRT8?G~~BJ zlx48;DvDYI&H+Xc;GSicT^_kQ^x(p)1w)^cjjj&vJerYt1EZy}j!~s}TRBS4SlN_J zgb+(8UFS>KfNPKzp!f@91SB?I?xx4i%pW}~{y34$Tu@n=Qt%J2<}Vg*LXpu~yOF9l zD<|64ZhEJfsYO3pzVH|EZC30Ic^>l9bAMg$L3_XLc*7{|XF;z7qp(=B$5p`{k4^{* zX0IHQs+6@qt744Qo>INGN8+o%Ad4oYUMj5O6za*IS>3y~$ThOM0E9=7N)Qaj!xhMq z|B$T^CbJoz+yCD1DsUU?O+L0)d8ExSj#5NbnX%}FQMgD$OFR5*SC0i^UjPzuP@#>% zm0&06F7r2{Wmr;4)n#FlLCHKo-nnezchJ()>$tsZ&$U5grhc*`AKbyWmFcAc9aYBZ z?)NZkXG8e=|Jd2!1cD*Ozm$KfWCI);Xl?A?SK;2Z_WW2k+nJ%{aNm-Vpoz#g=?No` zB>vQN-)!~-EnyOL7VOc)AuQa_y1jA9QxtBI#?6c>^_{uY!&@Kg|9iT_{>cUMAi;t! zaKZk`vJ2Fg;*(HGIwT9t!!S1)5;(p}8H~4t(m~E-9W2H0R#=PPt08{L^6V|JGK2MB zg^v7+T3MKI4LyZv$&uwJ11aXZsKdB`>jLqHmBl4PvS;5gtCex;fxtCsqxO9uLfY`{aDEXm<^auirsa@@cd z>q)|6t;Y|<8jM9Ae~(3^QtO(xsfHIW}AFfy7lcn};_GO(=_*wf3$*JwW03CfTTV45=%x zX^z6$f{kE^&4)XmdRlr>@oc$MdHLvFfVHC?4-i z2D#>b&(Io=^roQi2F~^^m5QMCk8R~u3nMpEhTK9l8=HAF&bQN-@(pzl7(U-8S1hV} zN6NshkH`}~p zv54bMRKVGElV$^6L;v&uR*2NHn7tye5Wei11W4YsSA(I`Yuq**_!+HbNKy5bC}26wTi1NMChBVznE7y$y_2x8Gp>lU^|Ga{H;&;|rq{m&kpuO%KN>4YVytjnk~apCf6 zzat=xw;q-yrDwEd)@@t2@^pWF#$yoBGn6GRdWbU^7mxe;&oK5~98lnyY57V%U90}v zBP;XT%9gD2{k7jbRPiF&ZPF=2S`giJSIT7*L2r`Q6QH9eVFanGZe4%vo{3)kVU~ z7V9IH?&vxKGfp?!dSmi9`gjg~+2b&*IEoyH(lseyuXc}o2mPNVr36}$< z%uAJ3+9>J1o5dx)v@F_Cl>_MIkk%+2(Q~PT*TiqV{jjOkZ2a4)6d)RE+@aFH^gk6O zhUwzFW!t9L+SazZWg@sQ$~W>^&(rGgp8S0L8F2eiecToK9xxZE zA%CCY`+Bpdy>UW{{)Kam`+q`y5M-A$4rfnkmrbG`0Hx8A-pJ8nfvIaj8c@v_NCFzF zXi4{Q{!RSn(IHzHH|grc7({8~nnNS=nuU>DD>GAax`Qd(pef7T8j#qNxS8$tA(dXG^z|d-d z44tRT-ci$Fd3+PZstu1^J>K|nurq@tqWiT)pPQ~9N7c$R>n_}|R)RMR+$rO|sN8^G zr!f(go$N!@qMjy{EWWd5zfI%<*ZU&K-66-EDpu!w=9A0?+942RTrU4W<_U7mgmRX! zV_yK)!gLya&+~6B)^xE0&}4(ht7jcyzlHt|{dBw}7;_MkyYw%G@zi0gU@jO?KQQ?8 z$ng9c=tG#~G?$>er29nKxrv#CQ!CG?p`Kd$PWu*&=-3t0+yHjB(xK@4? z^eX)gmdxM8SKlG=d%p4okN8lAdTx_NAiPok2T!60(XPV9=YG~6)mTj#q#Wv}3?~yu zie^1Y71?*)?y0Pp`aSdITt|X$2t%6RHuT+@1}EDH1n^MeZfrzi&+x~?H% z@EUN_gVm+>)kLQpOrMQ`%L$>dq|Ysw|k3+U>_X8 z`oLBY8DNQ(X)#3a88x7U_DRScBnvdtyIltNpF;gtBmdp(ti1X*^Ahk z3lIgOH}8!oNGAfB$(}r2?7o5+xm*PCsyr~LKk@HaV}M%t7V7gG4Ag z)FQSE!Vm5R8H!$FZBqjlYY!*>S6It{{84ji%9b|=NF)^_5R$v@z*CY~e?6kbCzP;i1(CcBuWhQegZpdAEm1ZF= z`kw~zUo$EQT%WMMs+>V#;#_HIfajrN-=o#XB`nVU#-8!1;$4f_=+DC47F=eH+!pZ` zor|w)8vs`VC?Hb5lj%p`<-=}i%jI5O&g}jEtSXDcc#D2U2k;+_mwNl)HJ-HfgrWUJ zy>NmGPdodC#c}4M8-)`?m4_u`b)NMeGp%A3NaiisR?fHkT~dG5;eC9MlUfbMoy1hK z#kIs3YOPkmoN=yvoEq?-tr3>yX>7}2SW?mM>VPzfucX?Wr;;-YeDL50jU+Kt&HO9} zbrYm&H@L1m6xSnYgH4eIr(c!cXG_~`@~ynFa|X2HBb|p!pUO~=QyqKT{`91yz&xMU z@Ag0sa)IVNTZNifEPw_4Z!$qq!&^`4*R{D2ig$KyVXdYc^QB8SCiA^ySyC2Pl<<& zd_HDG)Vk=K_sEBdNEUX*?At5-pdRv1f-Y&*Y%E~BdKrOxZ3)Z1G(?R*s{gGlcB|lb zb}BT_8+y<%Qg^!voUM)fFZnmjv1j_U;?f1Jp7d%I!6u2cJEm2_qz+7p-&)K4fzPzK zF>HV1b5OS3#lo;}sqVW8dL|P))!oP2D*bPVJz)0Gl8=n5pA&}O`oKl(b%NnFSFfL15W36iz2iN&Dn`=jeHX z7o9Lc@hYA~ov;u`l~s+>D6p#v>U9#lm6T?(;=1QI#sq~gtso`gM?fpk;O-3|R@%IQs(hGh_dk{Y+rnfp%f)L)*V_o6`!$4d$W+ zIT-r4HI{B;>ut*r)f+n5wSlM4SMBR^F}|gHK;A1ysj5mu#Jdn#DU@IWEqx6Rm+#D{ z_I@Djp>{-kbwVa2KY74WtWOD1CqXeCHFf)MMgDJCSx-B;frBH7qa-V(RZ$Po8ON`~ zM8M^_9()dy9pNi79Vd0Uh2-brFiouVZ|3Z|SrWI#J;lEY^gV5pUfyQ|`%&=crif`L zrG}J0dM7O{KaMh`qDwo-GqfnlX1k@`7AIv2y$zn&#(hd4hEgX6A^szm5D`xJu*R#Dn{bnuCA-dJ z{#n}H*cZuVPdH3xs6up7LtBE)@m|6@LHE8z5h1k7L4Ez$kxw>Fr)&HKYo}f#69@YD z2#QH1NEQ^gSUVe9d7J|5OaUxZAe`_&>=pvn{j}M<5m$GYzXWinDWyOrlUqsILg_G; z23X-iqHhZMcL(*}6-_>-{dIlv#;I|-iCj~>{3ToucL#f9!8k7qY22F)(Dv$GBW$es zKOrLkLoh#p9Hc(16Y`0uDMH8QaSWau<2iegPNH6_v2&urbGjqVp6}};&|v)z zm9VJsZMJlw#gge!$m0}t@K9V3n}#lQ`^Q$r8Q)+bO=fR~y)t#LY4x6XxThliuJXHVK)~C-X1H()V3Q3&$DQh? z#iLt`4AqboUJasS7=GMyW0r26JGauuQ>(VPEs|sz5U%Jo^s`< zwHL7Rz{D=Ji&8{Htv<5qbY}Dzwj&>8ej&`hzfX)UN&Ux&y8P@($TNPKvOW`KTa!}t zMCRZ&Wc1p0C*O@y{T8<^lo@hGfA^mg9>eipI!GaS`eMyy!djr zqMh5lcaA)^;!qy%O7tP=$JTzT?7G+TXFfWQ7*38R$fwY(^xK5I&BUknCBOr)u2jcv5+P4X|3p-WA5D2s-TJM0QDHz@GjOmQUXn-_#`? zkNtMvDCJH}0z&+&`de$`yEkU0bvyWb;hVP^qy;3l9ZVH0pLx0kXe+KZ`8YNx2BgVZ z(USe?ZyRqzjDsdnjXU%kORU|~N*_1btIC|pB)oWC~5uHL! zU7fVik#=e{-%*|SI9ymLh@(^`FwS)B=vQ96tPjf5as*ie4VAEboWI<1vteHW14ImS zT38cNx$1hEGWbvN$E?22zc?ht_8SgK6VM+AWS#uGj9kmC@SHh6KP)nq6GX;nR%V{R z)|UizZ3}v9*uiWCyeV{6w#_`^MP&lH*+EA`{Qho!;{ny;JUixNO5~_qU=XIPSfnkVVq9c~j{^-L@Z{D=J@05LMZta^Z{pp~*7sGe#b0Uqw75WP zKb(io^DedPALAIMWth=0$%6mqf9?>3;6~%DjsNvSv5>;hfB(H}DG~12Om2UEmBiMq zQx0E-tp%K9F1N(?W`EWswZtkWG;a@RL_wN_$yhRXHs<4o>Y}27*-3tD^==imj{J`w z2jE%qaMp$|ja)6}r+%T1VY#Kf!Vb*vMNi=#RJ~oYjRfBr0l)7Y_3Hj%N#Jz+na_Qx z8^1yr9G6|ouw1q2RnzB0xKWT{>Z9T3^j8%lCK)Bv$_I(r)&xeLfJPvO$yO^dZ8gpb zjFJ`(Pr%YgRj*60FqX~VB>pxbW=!r8uS)$UwxR{!eaBR0dRS{q{s^$=3Z}F?9BDXB z9z9ME#)jy7;W@bCatEy!_FrZqe)~lAXg>)x%hqU19566MP%lqCx7I=jX9L+=U0`3K z_A!6o>f??m?;5orDT=@M;SdjU#`fAR9UYK$WvsV#F;G2J<^D+bdsV>y=db^Dku)aj z_djH8DqO+0hJ$m(+Q2g{;R9(B=%l)}Iwp2*t7=*O;%FPOB25Wn`AC1GnqQ^LremEJ zX$&o03Tks^`1P3Ym3>}iq9p?nOxxL|_@5=+OUQ0!Gajeye}r3&dWE>l{MoMVB%({k zr^DHXbRgeXGO1*Os5V^q8a2aX+j{T;Y}J)F(UObfhn);sh)IN$L%D~~4O{vtrnvor zTYV-u8Eu>_*=f(3ZmD>%aszBeefQ^6#4-a55k+feGfABumiyt7X|NZnj30B*# zs!R_2Sta^^;|2F)!>j%N1se$fy;zeo{yoyXPXefxEK-c5}1&3Mx;bhPug1ju(-E4EyEjTx}v*We;+X;*cihpXk zJY@ctj_Td0yWa_u>k#9y`?U}QcM)zM|JJ$DM~98S1&HnTe7DN`o@xhKL~}0=iRH3A z-uv7LipkZ-yC(Tc-(nW;rPClxN9s)zU|chN$epq$M6hcc@GZP z+R3Zj;JXlbW>tsHk?&t}bLlddmT)&C^+Honw{)CWPOytyJG4i(Adl~Aw!|iD!ld&| zN9??z)WvY2S!mwJ+4y?QAlI^aHs1M(#*I!I{ zW+8l~7XJis8_hq{;ZFP<=jpg=Di?u*cV`r1JoJa*DkC7R;#M26D8dc zWrHYq{A3h{%)XM@Yqgo1iK$Ref?w;N9ZW#$5elT1E*t?xii?%E*}HSKXS$x-1Pk|0 zAD=h${;U4@JMaKCOZhF|X53pD>=*O=*$N*Wv8jJBRg=gvADtcibNDR2K4no>>6AIU z?i-Gk49GWj&6|oHzByq$jKX}ng1262w%BAxnLbUgl2EHOtJ1x1IVYxmtT|NHZ4U;=@1pvYmA{^zQ-cCqw+JLRS{u(Kso3qS*JM ze0Enqq&Aw=F}tyAkCwGvVd#^j3G!#Vg^-H4Dg7+v5D1i(f#OwVyBt1?B9{L^lwtWBnu3?Y7M<`$XUZ%Qd5Q$JvO!eIe zE#RO^?~Cc-^(ql7fN~|1DkA9gIWj%UOb;}_;R0LjE*J7_tXrl>aW*dh-RBA99hyGd z`o)3P&aVDkT&D)Ca%efu!fVg(P^ zkXAzJhPCMKMjE7Bx;v%W5AXZ?zWwceu5->`z_qyNGv}Cdj5(%zte$LFzR7sT*fU#5 zj3ISw2hL|aOCq8|{UD&{xT7kZdR87xr8iLeL)w+!*JND_Q0Se!c;P3pr}f79HL^bO zKlCWOZr^KW;{3g}UZ~3%F{YVDaR-74+G`}2SK87WOfYDyeK%4LygD!k&a=mK%s4FD z0CZx3A#yLI$QZGbWs2Mn~2{xHtH=JU(HYtz^tM88t`Dd_Zy?Ft82>#L+qp?pMN zbPMG*BC0jRi~}!43`)?;Lix{RPx97{N;%p5e=_uK0=7IO>1c$dm}7G-?j$Bz6uR_w zPgNY_m%`uM?8qkwa_eUDxtZ&Kh_>SXe?(i&T*ZUhXP(-${*&5b>h9F`V)$h#epqv^ z&N=|l-H>b}J@n>op69X0=hs4K6zL?Tj1@JZ>w~<;j^viy+=Wz^p>4oCUP>|(lTF9A zwpYXqMhj|RN&E6`;i~-$o&U#Yp(Ty%#mYF0g;iVL6$|Gla{PGsem4kWBl>F5m_`n| z`0T+RWEE3y{#2)0W2vh~qHO(=eSBSs2(-PIE82wf}(6eog--o}j3S zTOM9xdR+q;?m=cXQS>>jOz11rv9ybz>$wBbqs`oM4LD9 zb~OCguMeP_>9S-3R2j_?g8j|UCwX77({Hak`~I)(2p&XX-M5yU_EYIu>08%YDuJYj z3xK!T_lj?rExoekUE93rD&;V8K5a|j>OF|K0X4_BO+atckTVt&`rXFNK@D?(C*5<3{;oBh%ftL5|2z)UJ169S+w;_!|Jj~%0{*}MH=~VaXN^MrlwLjq zd(JQ!Zs8&%F^TI?mHfWoM0dV-<$K=PU499t&?++#dlen%NvH3fl?Jm3LngUY!4yt3g z=Hh-TGxRQ_tS(NINKg5dChUMaSGkKy$ay-i0ad2}nbIZdAkHMB8e7 zZP2@$`$WG0D1pW{0sqw#dRc9qkFt#+3&_unP-u>UlNiD$eu1S+1<^GnR;j|Wosl27 z2|C=}7dl6JB185qAQ6%K7$#{cDm)%SrgBg>ZAYQN+Fb25w*oR!8U9)m>gU6F!5CZF zgzxOUxwv^rNhOMRUAME&u_(>Yvd$s*}X@%$iiNq=LK=@#(-g)Vtxkzy(+=RiGb zl_WD8_ZW&&`Eswhx zf9PH>u8zS`&rTrqWAI$`O@^S+t9D&6*I+`Sz$KxO3NYh$YQhrPY`_aArUn%wfd{5Z zr*atV50-=CYc$pu?jLhRMm1vG0ccBr&rhSg;mN(Te62GW@6)RvUoF>l}*bn)K?f!P?9oq!#6cH7=jx_^fYWj`aek{QH zm&srmrcYhw|CFNqDPaw?z!!3jKO@3z=QP@;STn%WJ#)5$uBlNhPi0|dc2++qS^@ZU zr7hu40i&vIb}DVojUgfTF#{4}hRkfC|M37UoIqw$V&x=Pm_IkuR|!0^SSQcFGt!ES zPewOeSEi0}^h1$i(%eAXCUNoOWKeTnrYMUv)=n8`pqc{ZJl(X>U2Q^i1jEr=YJ7Xf z64*KoJf?G9JIt!~*7^x0#_j8%(rZ=`K-&YN(@xg1kb2|EL27&k`ZX4C@A!!w&^(zm+Rt!>@xq@mDOW2klkSZ zsYN>=-TY7=Cs0a?c(t`=KIwgWGGEjX3D-|AE&u&QjJq3c1q+j-)z{fz{SwE=%{P^* zW@_Rp{wk_a(c#9}N?(qdl}?Oh2CR}QSxQHS0ONHfepl{8DTb7hy)&)&t=05}yyzC) z#p@+64Y&ZaPsx?|%4<%9ZRK*t-o+ zbX1_|l1j+^H+3`}i-Tz+hWqCET<%?cQVMkn0OGuja6*h<>Bzs+bC6Xg0GEdfZ|~d=3L#Ay}Ar!%?~;bEqJh`*{2<-R<^nbR4W?I(H(= zDH=~Wcv0JncCup)eQ-dLDoo?}S+7CWJIm>eIEWHi@3@-{o;Cd%Y6U5{1n}7HfEWU% zVA!vq-AuOYwut|Z9sx_~+U=_wWw9BSXWMYCmuQmH%#qI?5I#Z|#qf(@(&zgZ*EHCz z|FYyJo)LV&5k_LmlMS-bW}dgGh%4V)po-W$%1FOnS3mFz%NHE>)v1>

Np=bxq!k zM}^#=+AU;1)dyfL9gwneOa8yl*3F*QlgvauI0MI6Xc3dDaS>fKYfv>Of9=yYXwVrX zK&2_Skh!3o>#%K_vYyv`re}qgD5~2r($Z<{ndIOi#>7Xf75txS*W6q*^^~^0Ty@e* zNqpaz*b72|_xx|H)iy?Be;pU>rf|FBZhIv}M0lIS#70Xq^NEFV4&@ik5E~awG*-A= zc?NnESQeU#BE`ZaY=I&*^W;%M2`bhV7;)_*-yOoA1yc zyR8kpGhUeEV+q6LMNd)x^@dS17r2uUI*(%tEE3`^Aak(;Mm4XPw7Rv=xp))aiPVwP zm3h<7Xw?M2@;i8i?&9w`y}X8Ei>Rm5e~FiKF|OD0#9`{be^P(jt~1T~7Ori@L{qJ7 zt-7htF=A|y<4F;}iq-1o1Oak>rhq|8a8bebEPn?-W(mTI-ee9jp3Q;GCV&z3mt+T* zP{HioZUqy%QX_s(ZTpDBe3#Bs(N9V$9lU;Z{EyAz{^b8*vk3eDuvr8lngLaxDLb!5;fuZ>>sb^M=^&wV2!;6JEIEE%=XeW;Sb_XO?9tVG(W=#!Jc?c@H82rc z@sf-KM{UjYk+|}8HY>fDsu??G)j9gd@#tG$tJyfJ@6|ENuhz$A(KjJ2eBBorub%Es z{8uqD#K6BgUo6gxHdk^qQ@CCjkdGE3^ReV=}=rK70zr5GJ7#Yxf1RnD@K1?=8 z;E?`z9vFpqt8{4$g?t6mCDH+qR8wgIrs=4JMfU9AqN$I!y^F|WmnYj9L?vKb#N=n= zvm*UPB-zX2O~AU~WRPB91ztbD!O&Mt<30LpmoH%l&XF;S1Mc8Vi>qqcQ**H zdpBFIqy)>lm|hMd?);rxu{XGug#J;ZtYeSBDouvIvtEg)pDI=7A7do}sudx?Pxn;j zZteUeMn5v%r*cz=M0jx1OD7|L1PQ@l7c#DkeE9BaOu?0*9$m=o{P)0{f4TQ=v;z-n z3y#8x*(Xoqr~PBxG@_yS>osIIJ8vnQynTdXb4h@%j5FY z=y;ySd7f4geG;6g_XCF3>pxs%pOO4phWa_9xcYI+(%1V#j5|DDq0fD)*A*TRlz$f) z(bQ6#e6*VLUbu9Y$l%`SUReBEP(7UT(Wl(^@OTt^5*R-IcXE`+Z$qxv{ul(aHUm~R zpIfAbM~kBapZ)&f^OkCn;kS??rntp@D;K^GY7AM7wf!%h7yQ}$ zNnGy{-Xuwi$zWh>BRJxt*UNG7#TvnTF+l6)UIgnjU+l;wNcm9-$=nt9U{oZK4&S@Za|emXu4x2POK|M-jMg%eLzN^$}ge>h#VFOe2p>^%d}J~=bQA9F;zZ$HIhS-^u1{; z8a3x%{mxrZt!tUbhi|@{KHiygLgn=-7}yhkTE6N$hY`}NpvoQDhQ@SQgpR97GCKf_ zF3Bs^@-f+A`+)lr4z5e$?(0Y*-j^hQdbM0&J=l?lGZDLgD8jr~^loRbbo+~}lK;j> zzBENAM*OdXoNZxJ8Lt`c!%(^~eb&)s%Mp5vVreJaD$-Y2B#Wm7!5d#9wu#(7{^6GV zk;$)1?GQ?r)Du$}jN6ThHf}6C{p~?DQKCvursa!qC+ZlqR@+CLOxBSxEfC}<_3@yT z`ZqQrCV{NwZxLJizl%_0tnrd598Jw_Z&2=!a)+XzSAx>b^1q=bxp^-y& zjU#j{X7||zOo;e&cO=vJf}1o@E+yE(&X1r$Y$C++LjFY)r@g!Xy-9YJ?|zOk1R~X# zxNw=qjePa8c~1nDfaQvr{Rf$ezdZ#LJTJ5Z%W1R}7m|UIMk^jaCv4`_V@9K`Y6e?e z>w{K#&%|ui&xut#%jM)`Ha)TkY)rI!&9MMSa*9JDYc9m^JGN2F-CFI|C(+J^KX{xHSyq z=ZRl5b>z)0wjN$oA#^*d|KpdDOJV58*6G;WrF7Fw{Gv;|#Frm|H=#0oM{r9*jF^vx zV3#;I@I{s3L?gA%TP#C=2|w54t&}V*;*{(6eU2X&wd?)nz}MtdPZDbNh$tAxhTE}| zhe3~8yi7l zc^lNbr#KQu*oh7$@H^>!^R|!LVV&yxxJsWO#(=bYdG5?F`k;oBKaxvZvDu*fEJCsz zQ@R8xPbz8 zHqMlBP3;Y6B7=4nT6*4Lf1*2Sf)X+!rj*NG)Suu!`XG$&;b2<8VAW`xuf3P11rsPSC{zYv?;#+#+ZnYsXScPS2+6nbYhNl!xoe*D7d#hOs=<}`5sK9LDNLoES4$h49mRfO3gwP zx}V&1y3A)Xmsw8apDJe)ba|?jQH%YSlYHKmJt_=i>wYX7d#{pm2dj(j{pQ3b5@YSK z=-d%MhJRqTD^L6VaB@I{GqsTfqJ!7-56({nQ4%O$lWe*Ec=1D$U(cI=*Pl^3vDxfDiWW`;Y)thS?$H1rQ zdtYx|#;LJo(cvfONtA(z{i)4wal5VAthD8yx{u!fb!cw-@e{wgLGDrK+;_0RLye+7 ze*E7UY6J1dqu*T}e4Q?O>&Qe|yf{ID?zUy@j*X3Vq_U7`jKuz)Y%(~qU@H`y(AnQR zZW!6}e*HOcyHvI8!+AJPSv=H%F(H-W0tY5+HH$Zs9au9DN)kqdB(>2K>f~h_)6EB= zBZ3{vvQlK#yYANL7;vd>)hrP*lMu!%{j}T45n!wzo{vxsj!{LF+g#Z&LLEejY2SuJ zitUN%S0ZGgstge-C860VSzOj|f@4G3Gz1JJ%@qGUn%)%?ek9cSL z=jFTOfb^CF5{0GvhnjhtwG*-BmB>E0=FTBoYo%vsTV5}OsI;Z}Rh?oiT%4%n&`s6l z?tC6A(c_$xA$8OICfwex@9Dqq=l#kM8SX7%hDM6mgX;&l?D^0dYEQ&F>f^&;!}Bzf zxi^a}n#?RszIJ;pR?`ki>`i6WgnaK*-`8@@vJH$!-jM~CAD>S6XfQNq?e^oF%SOcN z60(7L3b5m_p&buNXs?5%y)b>;V@3~bEsR3TTGmfJl}Y;KdTuJQI*;Y2&44MOj%(?9 zC9PTLN8p&NKkfl~4L zL;v$aIe#H~lEiV2aQ%>@f_p*O`6PSoV^x^f-)|PqGLAI6Zuqe6V9QzMIreT;@$B)@ zSdU4Ha2!Z*N}`L3i)eMGys+dK)#;>~+3i;3c z^3eD4&^^vYqP~@EYExgO{A7sGkUM+*zG7F3=i}#yO$4jd*x;Gxl#|JUSe0cA*`00u z^MMK!AU17hRpeTOxV6u8AfSpBSf`K7^@jl zT5W+GepeJZ1TTjx7L+agEjiHn;<)##HqV@=hdk52^`UG9M^|`pq_39_uM}w2#nsxq zQ?|aXX7iBTVFtCjG@dK^@cBH_WUF3uQoHwr8LYaZN*;M_JGu}R@|%8Q?D~W<3$s;W zWdA_yF;u6LY2hUCpOV%FO2S0^hKE;RoGf9_gg3(G_zZcS8%Q)XkkMy}8j(UlQXK=b z;p?4NWjqXAPO`<8PitK?ORZoUjj`~b4fJ;plX<6qSPjo;5c~kpaDM`21<-<4qWm+B&N8Q(3wLH}u3ZABSCpN#-j5$)OP)e4R zmjZone0NW2Abp^nQff`a7#Puz`2q-C@>Qa;RCp?Wwn>MjQZV=15M7@T&a@d?G-6U6 z(mpx1gD)wvsW0kBOx?fYr>43jh%~%A{yaTU^vO5=4-?TLlq`?`ap9n(`ev8B9baY# z$2EClTF_b@9W>U)>{Kth&Wv9ww6e>Ct_k<4WjNdUyBl9z0&Z3J`Bg%r6aRy36c#vO z{$B|C3-xanL`9O0NZHYSr)21%A|u{QDd4dB%(FFr_t>sn;~&QcC|DAK@cxflOuSUk z01vPpRVWGP{PQ4?HBmUuO3X0G3TKy1u6$+mrxQD;Oi_DrVmJt(4He(e<@ zn5L}Rke8KGX^3^MQoiaFDi7cFlhxu3jv+ja57q1@3`*s&#tWbb*HkNSkgPsApmhf;L1K$FW8f&e<7eR3%pdmk~zE=j6^y}Vz?24BN zHUWd_wdi$)TS+as_*>IeyuHX{NXg!llNBu1CN%r!zb3(FYRuA z@{Z_&7acu}*F#J8q@laSVY`sKUfactK70UH`%m69N7IMu#zGP-MCO^t`3Kp=0#hyJD}DoLRBD3j4e~H5;*{W9vXR-t zQ`1++r)i4J&h*W`NbfT_KxVq-{f>_S;u-z;Lf0E&G*^cR(;pU{g_Hk0- zMD3O~+0MDVS2{(BpPE3?x9-Q!bFHg8d`G9tc2MK6qswx1wJzIuKFiJ5vB+Pmg}D^0 z5^zc!qSnw66YUsOM#Pq<%LG6Zs&yHn9oMr00(0M@?J#Yd?g^w0yT*tpj(N36-tOS) zmakb(r7MP9_wWq13X^U8wUR6k0aal>pCT*KdtPcQe7MfeZu`9C1t*Kf3i6&Qd~dxG zXVhx!vi`%4&FFuDC=TRdfq$aU&{@DoTah1Z@nI!oTvR6YTPk^YLkoEbcU74GVQNT= zEdD^eacA!j#<=*XZvo{R)?dE*nQ-=|k1BrOoA_*ozi}-dteDEd(%hGTAGJ%$@G4?N~0(cuXOveU88_XzA=PB*T`!%H&~+^ zn$yUsts=!CS12fJVp^&u)!(0EoQpK>LoQX!{Vja5o>!v?Cp5`~2|9|U4JtK?79T26 zYfivX_?VkC`3Q4auCDwXhoAJh**&hDU(sy(FFXu5Pi)C_kaYMftVaj!}L^J#{*!kyPx74)+HUU1qkmuj~FEADtP}SseI*LNA zz%Nau*$W5u8MbwIZHQ=@15a})6w^s3y%dW%`ooGd-+x-A1>=VcE#| z@_Zx+vyeIt^0U;)sE1l?>o)AL^YPhF+Vp7A7fW0Hn2REPOWc*dtp!?D7g8Yg7=*G=nF{^D?3AK6 zLDrFwdsfyn|L*SV&^)_5c&m(IT{$7knC>OB6{>R8SVje?0`gq6h9*G1+X^%ksOlLL z3D?q_YkToq!GB#oWkgsDb77Lq7EIPAar3C(z3~%k7pZ92xS8YH;9zuPCWWTpo-69G zkk@r1Tji{12`5&?%=J4Pq+OhqO!Bp#h6zjC-L6-isp)N#e>?*u%$~yVi1vdOm`&S! z^~Z<0b%q%ii$Y1becuSNqmG`Y%gEoPUlT&`MP3mU^y6Q%(O_qFuPI)WGM9Yq*W=^-e7j+l32WLxuY>LIQx*-Ktnf zo~AVR9LN!d&KynqmJ)_m=)XZr#_x_{km+>IqUYti^j$FJLw$@E`dnBwo8zHe*)ci| zIEP7m9EC>xou4e72OqRtG^g+@yFl&KtE^6Yqt7p|*xL<__E_^yp#GCHhs9{hpKvx< z>Gm(R5P9@HPWvqSDI4a3%16ItHW|gyX_I3pqxZ@Gl8#&$%wp`gN#aU+lNFGL(uIUw z$uc0wrp#!!J16wjM_@mLaxp}gr!z2MRo9M}AugEgH|JlZ9~KP0>>)#{8?ZR(tdU7s z=<)m&!VU=`GF(%~=)wKSsrHygk`o()#?()o&3p1IwgY$v(_g#gn6}?ylR&vf{zb|9 zp;*sipk>hRheP=+@Vc0tCHP0o()|k_W}l*)W~GL7`2;}B_Fs{?TtsucR?DvcDa(B{ zNByBl7@l2sgqD7o_xSnhx%b7ufyleN*$@#YNm$2icFVASI1WB04njB%1`DPD&)dMS z3;38ax)FZZ3oIe$b~!T{wm;_{D`Vh?6?U;^I0Z9u<$jAj0xQn??^yb%?lwYpkNt6OXrez?fCf9Dh)C^iR*8=`{`|~;sEi0y46cN zd(&3gLaX~sbt$tRiVnJg3^uT`GPmZ-AjR599#_%2^-#=sB)(2M6M_RHv}VmS}j0(z% zLUeeER0z@g+`N1?AEX9+YOKa+M;0?Z#EDnN6iovBltp;j=|cfdF?kfOs?H%QlRqdM za5ftjSa>>q{ep_eUkRVneYK^;&gkPa>L+`*&JvEXu=N8PGH94Q9vtSWp3D%ht+OK?_AzVj6}=Lw0;^8gG7^MW~iLgh%?DFVq%(2KvYG(&J;CbNPUs1 zC-pAaeTQ{zs3|#VOGR{FGhXS+$8c>nhgR^{#tt)Zdv{9Cj=vo zL^t|8(TIo-ISP!ZGZaTy411qGUtYhbwfF`63#dI&6bN^W1e9h)ja(MXqar%Y?NDT< zkStB>0++&o5C&D;9%PT4{6lG z-e>R$j3?uLid%X+g)>Tj&7O?r$U5gA$ERcZH`GwaED&9?85E@s2ZnUh zrGB4Pl51AGVNAZmi_3a7Oq)+HCij4;QD#SQiJr^s@`+lRU-%>`QqdnBjn_5aeG}jR z-c~Ke{WHZfAoN?@pKW<9SuFELR+0%4#qHVTm*_mpVKxC?Nip$S3}mgvPF%>GBWIGN zLb5j7D(;}uT(hwImO3YX)Fk~UTCe0XCU^X+s5+C+(|BJ0_x`Ypn_Lpww5xSF9EeX}s_CN7C{3k#9G`QF?;QQJ1K`!jS#2Zhd5W;;t z>&ZT=z;8UrRyBU2l9CW}>Vs$?=pj9DbsNJOOsTw|J0@72DyV);GEzI_($y3-6@;%e zr_5hgH3YUhjYf1D*L1TN>*p%xI%?GOHfv8ZZ-k4?Q<-~jtK8_Y2NkW}a{^fh7i!~5^mlxEiw8pV=Tb6+ekz@gBi zH!$ZDi3q#`ZFh@7=PVQb`7Gd=b$lF&9v1n2DYbn9Xy&wdVdqzEA$h-5z^?mQjt%bo z!pDC)!yCY$46MJoL4Wx9LASW(<0@jpmIlaDyxeF-_sU|Nv57(PJJD%4&PA!dUU}yB zo?BdeN`#vAai^SBIL`0a#6seKLoa=DXMP*~K;i?FSw1f1kMMsZ;JlA>GN5L;a&t9j zp;c;@9RIQ|I(_vAO&RKYSL2_qBd~`a3d%3UK&i-9D~~IdaKU|io-hzUgcOGW2`1E8 zhJ^X{$tQ4?Ma;0Om19yt!(94yh_g>w4ti84_tExK)}i`! zwEU3z^kojyrWNH!_wV8MeOiO66>4;7aKbV0!gv?6wpeBVOIoM*`|iS~B0Zm|Y-Xap z0{xkEPv0cvCl<;8Aj3AEnd)FiBD~_QpoE!78upu#hh3;=yTN{g-!)V$h*Dakk?ai z_X(KC2~~<2rFRo3GlMEnmgmcGfVR*~1|0pZbP@HHm{|%Rei^aex&7mnyM)Oh2PGnm9Jg1!oW~h|jem65^<&n` zq7nZ#fdmIG7j4##2OfI5p9!L!KEi2&%#nGWC1o{Zn#5tAbLV8FEg$8pslHGZazNd+Jon-h&s*MDVEFY#c~!8$LlMRxWba~uvGeUXKDAqgjLw9TrCe)K z|1$zvi5tg7HJuPTLPSb|>2=cSRBpRHLW@2JxA_2FA`ptxcK(Ld(>#rT5BHsAU)_e; zjescrgADMVaIH{UKzs)%Lk6laty|V%B~=20L%R#0?;75rZ*ku!heHZ9yF=-7Z7H5B zzn}+|n0sp1p%z-)ge-QPD>nG{DZnSAA`dc|ZMnv-@H`t&N*Tv-*?S z7y>B8N+++SVAZ<4u>E=O@wVXH=Xhrl=N2By$n6O&rEny){;ylLqF@&=0@iGJ;I}kc z&T~Ob6I6He6;MAZRBFWRt78`G&2VFDKE9EhCSzQPe!@t$5{>($FIu2U;5NOe5XD>5 z@~qIJTPh?UPqxe|A(=uf1*TR3FWv%Ix_g%0FJsa_8QvJVl=!ld(HhZbL1@LPB;hEb z&^LG~U$0MUj`UBAkXp;vE4m?%!gcXmis_^4cFjC9}+IdrX$`oT(u_)Xdn=+i#0;b+Y4^AXU;;2T?;z*mhwj=b2gjgt8a zxSjAP!su&YT1IDve6-B3wWy)spknJJI#nz((eKV3C1mRWjeB4yY=GBz%@PCPqzhf{lx<0L8$)b+@?&sF*IYfsQr%U2Ny97`6^=m@CTU8xA2qQ7HUjE+iFe=EFk+jo zob?0j$)ue{+{c|pfOB<8&_qHPYT|SU7}~Q_!Nw4D>u_D?lU*IbtO>$;Or^)&C?c~S z<<@5V!u&FJ0wt$6_sR?@xn>ps(`JDTAllSeG@DxYBK(VaQ6ka#D(I4qW5-(+&zGqW3n=N)QjEFt-Wz41%cB?ME=JYJJ>W|{HK-@7Ug zOM1LH$Lg-nm{w=hqSk{1{xZ^udgGv5o~h`Yfj7Qta)Y7AA;=nzv#>oPYT+Iy5|Ag( zs#p>65355$ucGOBWqNfxf#Z937thd`t5)v8riou-ipCV*$Z4)(cE#bx!P5K=>5X(( zuy{G9(=!W84g>5*s-ZD4ilJYO+7~ewzsQ4N>^7Fq{^JEO>>6bSsq?$fUfU!wof_f< z%!)%BW_KC}#ZiR+A}igY`hW%$7Xn}V^ZM5)0x})28Fbe3glBDc@{W3i4AHzU$5$Xqpde3_{=k(^@%S81m=u z6rN9A!+hVO{uZ5a_r6PS=B4-6)~6RV$11(Ea!;)rGV?7CF`~@ZHt5y~50Py3?z{e~ zo8Ktl`rQRxz6eU%_7ps{q+s7`)thPR{5v+??@LBn5jp-3*;{rd`KV~+TPz1*Z6Ek< zk@DU}3)-LwJv3b{8q(ZPH`LZ`Jm{V=8|>UQ6Prj-@8upuEEs!YN$i1=R)zXVyz`D} z-fMkuXp%GvT+E`KQ&l_q<$769?uCbthR;A4oTXtdijo*0#Wh&Vb{mW|pa+`u_aIAPK)Je>6)aJ$d-J znstF7VXiJ+5KJ}8Y*>^`I{deM>081TsGE3vXJ*ox%6T}1#xN{2_c#CDt4&1;+!d|g zl)b9fF%q06*L?(A=6$tl)@}f~%e?DZbX55`rB4>WD#>E605^Pl(jX3O;4vt5DH|$K zCu0-v+kly43pMZx##_FHgV~$hH13ADkK5o{EAwiJl-;nTVQdaR3ofYb8Kh$fY%F}O}BzJJ4TIVE{^e$B|43apDWn{W&;xC@#>LZCcow1 zoG2Y^GGSdi9Z<_>*l4f;>GicnUrArXoD2+sKqW%)T$ecerp_dL?e; zqF~9A;2r&lcHThXx4LfKmAEDM$R>*E4!X*ru^oww*WlLf+*)NyEf*P2KvJsH2-iIL z(N2vw6RGRm%koz!GL#+eNiore!2Kmsck)?DLN2vE6a@5N+U{d6={@*<#lt2kP2l?e z_;nbTYiXb`?Q*u*)zP6{li>T))*!{ksRpjwF$b@*{LJ@qKOcD((Sngh9ydxo9>F%RD;Xe z68v$nt&GhSK8Pd!`h(FY9NxHD>#6P?={kyY1s-^ijJ8XZssSZF*YO6tmMaJ}6J+X4 zx?JlCOy9W?y=LVp#_6%wx`0VAokcNyY#cmW6OOX^wc3o8joo;L1w$KrZ6#U91Su&= z+}h8w-6)0V{}%2tA)bd(+pB@TaFj1-XtN3#s`o1KE#+wh=jF~&RWXGa2c=A1AKb1d zZfKbL*#tS2J2Livl$~%Y9b&e4HSe9A=F&3r9R0OLCv_JSAzA1d4%|Oxx#|%E>kv#f zW!X58d8JG0U3{NLaDsFHZ(GD0CrF^QH{uBJB*POGpU4qn-*NLmL>4^x?<6#e9`^F= zOk0_t;TS*LTUIA8#E^G(B&spbRN_U&t3fp8si@m<8P(mGg3Y=Z6~CAwst-FU*z^el zo-w@hdx4luzlZS49VnjMoX?aTKYM}D06id6A>}fN)HpdzeEkR zM1=2NW)|r`@(-h(mC0!K5w3N4V!vn}Ud8;DJ02pcem?-PMrp!jO0zG$|GOsw@&0ep z=;0jw@wBp;E3$Jxjx_t*d$|{+?N)JLKIjLxkw^dv8jF3u`2z|{el_g(GMB%4BTn<~w-on0GQbvuzgy2p)t>^@-p^Z2*4dGz77 zuy)v^g{tBaBU6%t8-$DRB>6|Yq93m1jsou@DEeR&vB4ASVVe|g+0CelT-ssJx*hT+ zm7`meK*I`p;Ns8|ZF$(@K5ORN?#ZcCaoY9u6Y)FQyV2e`*3xNJWAkZVK)?||o#E#E8w&a~(EN$+w*yUU2 zO7w?IJ1fN(9Tvuv)l=IK;xkGhMd|;QBhtiVNKg_+q-o=heX*Hkz!Q6<@BTtbavPVt zz$}Pm3-{WfYHhZNgO7V-cJt-exB}JNm-Hy)Ut=Xyhr4*yrv@xgjs8d!Q(G2q5cxA2 zv}N+s=F9}4kcTgl9j4;-DXg0pj-)Mj)%wJfX{D{R65@V>Gt-CdRGnI>b<6*?U*l^Q z(Y@Inq~voJnLzN9jqmprVS-M8s$3+B3z;5%blK`k{1m}0sa z#U^)kiA_7p2yoAQ2>%9ny`tAJbQKm+E@(3n!I8;CG}u8qY>$Hz!k5%j#XgKuvc^Xq z%w=6_2?V1bMj>A1iF@-Lkj zvcMPVQ%KSgYWOghKx-uh4C5(Xau3AU!Af1isf`L4W==D1?He;}9DfsC9zXPbstKHf z>Tm-u^)2`x`Bq0JhX17g^!}g5%4zDz{rnDd^k}+&#@11Ih|*;O!Xq8>j=`XeO5Ri}U+NK% zkb;r1aqBRNtC=itlt^OXIab{&NQ2;r$G93W42m#{?9cHLO)ly3jdfEoM`TvU8mgZx z^d(=SZCy$7B5*wZg5;HUY8hoTcp5ZA;Wt{V+6X^d!yEvTnpw?`XW!uB`t94VTQzCI zA9_^P^GXNpf*BjGucXVoCO3`}Zv;I#Nas6 ziwD~~$kiFg(hD4TY+~XR#PB1bsv={O^wyB)GHn6(4H&ZwA+C%*5fbGCQbb{4l_$9! znRU7^Rt22A$K(6$%*q|5A9*is1+wjxE^3mYuTrb_!NDV3WK@=LR%z&=YVE0p*yScvRhw@%VQhu3f4x1%0q zkpJ@hsgYT2qF}3UNEuRQf6{?1-We%w3MaPQ-uIcB)^i89b&`G4ELAgNZ&k-Ku2kGf zrqR@Ex|!nDWaj0OD#6sNYgm(-JsG!S*WJy;3furQc!YzTLuIZNOv3Bp~hDU>=P&3kJ7i zPGg{I#lq~nIWleI&7?rj>zyk2WeKJ_+o|LOI3(>nWs4AqavTIL?al*`$)p+5oC1tw zd%xHU2(B%-$V2g{B_qBSF_!7V16|LhEkQucRiHSxGs*-wB&Z@=2&r@~Hn`xj! z&y^%4VWm4o%-(wui<(;O=~|t!;$;)iz$d%AIJGL|+q#AJ zbRCn9Bue&Jv+_N9%-@TvERB$Z{OKf?O{rC2nmkZT^-?SBxk8JDKnaI`W|0l}*I4oE z97Xl`-pnsCNR_{X!kAc7!^hGgf@xEA+;Fa7k*nh+8tonapWB$&Np#$^U|gY|OxXb8 z5kgXXqL8eX?4#?rlm(GM;U--2fp0BH_rF-~CUaC%m3$Q8T0AeAR9=3**%#(c=+%G` z(7wZo4kt=)<{tD$`sp2m)ue<4Mx4Z$V%o)k;hf-t>@mb|&SQte4UNSnb9TSP0ESw* zyFvgGrkkIq(H4Wsu2>FlVIA0}VUHhk_Rk#XGk;|t*MGdi#KUtjm#a_{G1#p*bdQVg zH^0B%xtsXw4fY`zxD~#7hW6p5`?E4qfvT{}-`t&anq^m)ph>bbn)0HbT_cvvxf7lO z(gzH|fj5^i0lIAd;CKpWT~R{jUD`R}da^R&jm(6%Q?((y{&zw)SrpFe&9}pPb7G&& z27^`j0=}bE&dN!B390u|8?fCF7D7}k)I0zKn&^;Ley>c|w@&{m&RpJ}9XI}mIcl8W z%fQ#auyqwYHN0&(JFAvK6YPuv${aw^69C<|jj<6>aJfOmZ)2FZ%E^1a@<$#yv$Fuk zIgbPU)Z=d0E|GGtINqPua^G)8VYOc7UZxv5wz-9egXu9vrWVPx9442`9dI)bzPv^K z@V@RNIlb^DdM47hP$NT$&&6Xe8lVW6+}Jh;jf||;zov^nKPe7`kvJAlSW*`Si^=PS_Sg3s zC_HhQhA?z)O48pA)9T(CfEKwu{~O;juhxRMnckx6hR{ThEDTbZ66BSfGqJoyqw#tn zlF%aiOPPTf>NlDJl6$HP*bA?4YE0+>?n-L|KUmd4R;q&OW%wt4LW4hfk_@w%SN{aqWD zYHr%Q!47vbp&#_UwFK!mV2>cgP4em2#Scg|H|!!_oXOBKpQ`?duM)Lby1Au@9KY@9 zh2;O8RoWZYh=v@bACwgG;$O&d5^9_|sx?aws#|8yX%@JLTQPCPXTO^;7tS~c$Q(0! znV2pSi+7eA@MhQi2b+571$rWj_K2%~cE88q=Z;P~yO4C7dCC8@CVcG13Kpn-150yy z&9K*0F0V9soE1q^;ffp0IxP`QF&MM|GEyPZZ7gje!NW@2$hc8$y`)`*%PfU zCz_??DJ+)VUa36t!g6kenIgZz%_;Zd^&Jm1Z^T%PCj)3LC14&%;**_`-#Pq#_-^n# z7v5b%W>TNSaD2V|x|-v&`~qxz_^x1#xo9a>*`JSEcQM%h;-*LQ^s)gf$957lE8krp zo^0uUcUe93^vk&epDD(<9^`&P*Bx*%KxY}0YMq<*BM>gIr}DrI^0yI7ZdvEnE=I+}+ zngR0_Do?8GBnY2?i%k-Z){x4xyIs63+(^Bs@=Fv?>LVeS1gi!WFrvI(u`qLIK5_%Y z5mu0IwXy|mg+)ZRH?s95#C7<S72z6?i!FBB&1t9rJ12qI+QNyW%(X9Xy^}~Q^$N;u3h97;6n3Ne|KsKqqXakWm%3$`$_lv_ zatzD9FiAb%+?Irp;Jexr0EQ&bO+PCA zVtBkeI_b99apD|17r{9f!RyPg2&sxGe}-k;v?bsuQ{^~7OM=O`Tu2TvV#5Ab_0k%O zO`g%bU7Yicj>^)MxM^|bUD@WJG2V_a6o@=-5_ej)8JCuasuB5zf+Y{V>g+ZJT|VohL*^D|_XbdK40YyfI=`v-9{gvn2d)s-7ogx=*(? zI~k2eO<|-=obI&+6$gkN%6yT|)mr@#q8omXsMgHNF@6N>2OjQ;Qe!EBj^(OSuzz?Q$Ep z*BJP`rh`-*m*9(}5S!cYaLp7&=;K)l*5RBji6VWVZ2)l{PH(@h)#CdUrKG^=;)2KV zhz3GI3`I$6qvwok^Kt z|BRI3qcnxi0^p9snj&^tnW{^BEl%8CnvoclOR`1~V-rYJh2AJVl+|-kGg9V zKN|vD&O-YdBkKewq9#)+#coX+QApI7IjJlZ8VRwt9zURO7BGqE6-OO(Hn3SeDGid) zJ8`3bQqV7rP=197R#Ahc2mtA+O8=uJh$c#1Q0?rdpEN#TK4YHKwiC zH%N{j$^(_2DM0DAzy0&D9VxBrh>yG1`#S4Kj{lzJq1BxKuEB+dyik-okZ;rQWj53j z&MWH26wSoU85Zrvp z7zuw%Aj^xoPgay6=9IlJ#x1D(jUsWJ4_Fz=;+L|`wi9q2q?k-C1+SI^v!%5O%1Mz+ zVBfHJ#ANlzmrKWhnT!;;!weLxx$H4>tp|iO+n9JQ{$N8K1Uy|7AvRMXUkM~HEZXo2 zjmjjuNroLi^SLk@ny!n*6gVmgM<(sI3!^I7S9X39-zPfXQ+z!cp6fWMi51G)rCpWg zlSt}X!Pthuy8R>s{}_Kp6f`kyP23&lvv06PIYRgB^rY% zi=T+wwdLx&_wE#n_rkj@O(1Fja6QMf?n%_B*g4#)Q-($hv(}iEFqsAI`FL|V=wtq*e8`EKLXrOPtykRzW0J`He~y z!Z$R6wG7?s^NO$4sU(_`Eyg9{mG3XdtMQ+7bh!9C*SuKxL^?^LW*xIK^dEGts#d$d zaEJ9q#XP&bIpsCsj0I1(lY~0{Ja1J7j4AcWtd!;|CtFJJ6U0I}K2v=&n)jwFa`+h_ z)9bi*{-PK8ML1FLj%6`f!cZfup0wDv$%=_?D76 z?{~y6MnBjAT)AH!cU+%*cqzZJS(i2YS)H)b)@V}?3-Z9NbcnC{vL;-I!{3qSkb!6l zH<^r$aegmswm~kCCU#_u(|=m=tQbTzp&%6PDfNPklc*A@RsyOd^}6)}{VFB()@?@F zXil`;W$_L@oXcUcNIujC{Z@JgUZFkp^`TBlPm!U5;@#UDsd?0HY^b zoAA|*nmngwGY&RgOq}8lC=v_rv)vy=Qf0pWz&YjH8Ho7R3sb6Hv5+?fTBuA$F%Mc1 zL_j^D&!L?6tD3czB!%4tpcVQnq%dOC3Kp)3^OYw?F&qJpj!ty;wUHjk7gi2|-P@GC z;4@sZ(2M$3`lnA{I!>N(y1k3grf|t}KP%FzB2X2*QUdT|W(k3!$;iPShZW zVi@}Iz37DoRexm!^Hq0e2RjeWolIkKhL%RqN!k9^An&e+7ZO*(T1jvu*$zci?^DUt zhvk?hjyv$&QYYZ>M7yqWWAD*{1c)XkWklx3rMQm`4hyUcoC!LAx)V*z(TiP$7@!b* z{4b@((DufAYgpmGVVi@A|7elSjQ?m68A{4lN8u3XJmeAc2N&fs>FJ=84#%4Ppv7sj zr}=kyfULQ5d4q+^Np53-u@<@QZCh!W*gH1Nnx!75*yLXbCbe^A!2%H~;n~wARmx{6 zpYEN!b~(jl(1a$#G7NaEB0gfH|LhAleS=129#}4oT--QOE{hJ4KAE6mEI{r18P)R~ zOTdP7Y12LHn3-o^)lW+yQEy=~i+$$No5mfchghn*)nZh@c*FFD3{O2Tz zjYLA$bFV1YsxlRFUAz81z!t1jgE}tpvT!0gAIxE*vZ~66PNq@IWb9|dsdo;tL1xfLEVg2UnFkqq8ovxNx(*Fmb_5bm0elBkbWh z(;O614*>7L9Yp5Y%v?uZjRBcDB$>7%*~zajS02G5c$pvAtVDTep%LA&_DOR2Tby7K zGB@t$9Y$!+`{{Z&izkcC$nI(&vcA%$p4;f@HGJ#b5*t4U&6RRHk1|qaY!mJ#6 z78=0}L=+mI0_>&nb4T{K?e^DV8&n#yGo!jB4=*8Wr_7v9+ksj1$*`q$vSHP58dCZW zCPUFXOym!Tzud~aGfcVW_ILX|Aeyz&LRw@sos+pG2iYG+g!2p9BqC7ogUso zU42hCz<|hiw8#!q=t@C))8DqHg|mi!oMi)1^B&yKkf~#x8lBTDETS(IUmYLUlt|>O zhd`qZd516uTHEw1v7h<(DNjt9j7-BeGEcD~3YgVwemV8Pqr3y|JGj zP1k>xXDz6P_L9S@-f_IDva|4vfT13@WU@`vO{qrtjxheOsaC0T7?C({sKoQ+rSyQC z9R~%dHl9_=)Nj8*8Z2qs%_GV7MHqohy`R>*5;?OWGuDVUK@XEo)newzN9USrNk2Xv z_U}5asylGI)!SMEj*)1*0qk?q`cx#tA0j#vYCVvBw`G{oe1j>&Uy`e0T-iT~!4;O; zXyihAD&LVTl#8X0?Whmyh0Rtqvmo(S+MPBh_jX+(PVl~Ef zHw}lWd%ZTstaNs>l4=R=7#sAHjKmGh9<{R(7Xw^yWneJwY^H4L$)V#+ z`Ik$BupDF63&GOm(p43C$flaKh$Q`5*>MmYq2GjK1!L21^(6(l%GV#r)*HS3-bXnM z$D#0jS5324=n{CYI92*1AJkNYxbm2LR~>``x6hdUCrl%at{U!9Yxa)!!{`!W1{SJ9 zR!pV+HM^Y3379dZY4ZKNi6ewb_tkX)N>k-qUHej;&u_$)b)&*yXP@8h@1r{Al5IXz z3nj1F8OVIWU@4z z7=q7v8LxU^HXMdte2T4n)3B);A`rwShWvL9lU}}TOl%_KZWf(hWdb7;gjsGwgekGc zLXI}6ikf|sTxk%8BGIFmN%6p+;gp*b&z!~zCL3B!#BH5Z8$#9!S1^!H6_Sl4cW4dM z+a1;;M97!+lKfGDFd2bU*bM9FMr`Zww_wWpOeT`Jo8|@XuzXh*FND~lswKKO&I@Ic zUNdB<41e*U;Blu^fS+X-XxgO%#4wsH|(dtH=xMmViqoW6d2|WQ-w82U|1= zn71LT8SW}h+lCMhnz`>|wK&rotBdDn$JA+_mLf$cA_p||(v^jnJXXksFpfB-VfF*@ zSYf%{Dz9IkjnV=(Rkz`1?9Ma)^w)E}+w|a)!;a)=A-zHK$`G8zD%Vve!-|^YdyK{k zI|d3+omVmp+Sw~0)Z;kIZ9C5akM(RBWD^wRwfmbbSzi=1qF&C)osIkN5Bm`U=krUc z+x@#HA?<^TVudW$%@BCt4iK#1h->Lb*eQ?_Ms~p2&uF9WadQzoX*vUpJm-oG*HR8{DBm^flY{jY_R zfc1YHC<3z8nOVPd-#)2e5U%W{ZPx+sG-Gm;XTk!6W_tEQL zN2qVKs~yPrYJrBa_9f3)a41EnPocxI)|^yUx{-MOSmF#`y(lAse7$Cwqv7_UDj;D( zzGCu*j*cSe9EZ}vyGcS3T2!5?rN3hV3UCM$=bwPFE4!mcubINuIcz32l@hbdCeDe~ z!K7&>fc@-b_<#&O`^)?_Kji4APIU)_m9En~>iXSYHgAM;ZMRQ3R6$fHQ-8O=ZCxtf zm?p7SD9QoB#s^XPLX3j=2uGfLOM>-aC+?5(^-G0M&!_6<8|2*#b)It|K<@Gi*BXJq z^S%v3VByNwRl}h&3SX%8TZnTxclAC_UuzVL<-g5$(ia_;(^h}0w1~tT+6XhqTj5yP zV01k(Q_9@Ab_eJVpSs-Y=ruS3(>g0V7OkrpO_T(i zU?CYwhAdd!eN$N>$jonfG)IjUz1@daP2QbZ+c4FR<&E+^vojDxKErR~4@_pq|w(yl>%G z+9R@atY@z@Du((AFF}yZ%2)t^0jZjCDhN6 z#5^PLxCM|eLm$xpH`7J&^7g9i_&-ABiY@SK)%(YdXhgfs)S;KzlFL8K>;Dp6wHu3m zNNvn33#8{-3`2fLzM&4)o=4V3xLh5w2GWfpA24mKO0S03(rcFtqsj%osMOM>UzkG` zHrFl_4Oj0f73kh#qSDszCNNn;^K=WoBr9#xL-^)%I zbZ$v*kACeLb+))*$c5RDd8c;gr?9HGY)HE8XMH=W`Z*lP&~LG$_W>t6nLksMnMkVA zs8l)+#+zh2ag=ZYBQN)@zBw}%5$!s}q_bvQCTHf*@_jF}u=#1}w15+R6J7DTH|A{$ z0F%%y!6Ys|dSVzTR-9Z=IBe|uK>!u}LbIVjvH1BJIg~c}PV@8Yx6cHocuIk2^JeX|{Kl(>qD zG5`6tU%b#fNA1jdQ+2ItH_S2O_xJ$K*Iry3x=ZSeXCqdzak_zu)E`5t4w_|Gq2*x; zXD{A@XFiq~4G1rZ8`b)|;c*#DR$#q}GhqoBJN3~CW)V-nZxkU$h+lxfj`lM@ZE5l( zzCGOBEIox@` zCo$3Qwl>!&JN~9v*1-37kA|Im4oDC% za1VZxX*oNlsuik%XQ0Vs>yjfs!g{4w4UO|HjK6vtR&je%AN>Stnm)CmR~b5D!Eu{K z)0qs$^RXs%^S>VMFVE~n$C5RhWImsQZ{?HAw1o$KKiK&d;?%h9Csci3+VRh03fZuP zZ?DGFCEIyvG29t^8D+dtK1WdDL2i1a0&#pXBNb)83#oQlQ}ff+3zYhJRAH1=uk;DLzgZ`g$~f)7hov1_k<2gs6Z3nU7lQJg z6cs#URJ7N1_34~J(2H(#o=_wfYvpEqBiP{1@jm#Q&}IcV@)*+$V2%B!l2QZ+5w*t! zV5V}sV2qL;&9Z3fhOw0EM<)1&P63g{L(KzmA#SH`+2~FlDkpp($nh7NIj5M4h&MLB zxNO;o_&6IFzq6w*V6LwQSYEa1brrsNuF?z&q#h7V)h45N^;;C}-9tBjn>CC0S!^~p z5`H=9J7|n6Bk`tJ>i9$Q+0ta{4(K3Wo>6HUlB!k%`Wh|QGlk+2Um)}5*;n~4T0`~D zw|+52nA;rRv|k_P`r)#j0SoFapA&JuAMC4SV{^x4JEvSkeojZ6C^H#1?YSym^D7=u zSaX)^-8$3SFd2`_HHZ1ZUT?O!OR>#bCsv(7odpLarTkNO)7ulPPUvEty|P?tg7G#T zJAo#v%n!q7b2PX2&esmT5uCe*AAbIAou$`s(RcH^cfz9e0hFp#>)7`=M1+YQW#pBf z397_o2%uQ8t#B$8(njG#_H`&5MO|*dsBXA_iDN~B!f7_?pg08Ok2nOBg!HaMSI>^* zaF65*PeKA{>_jXnFVAEJl99i}I$U&dV{O~Zj`ea!ixeeWC*LqZR}{2l%`DFM-W}a> z{?-2?|G2M8{9sNf>qo9lH|Fq&dRs!l4i4@<*!0`6wDvPHT&?%~f>>2fAZieaBN{|5 zBZIhi+r*KAdY3jWR%thD3xEWunJ^vmzRO5klt&ztK&{E*tYJR(3D=T&s&ARwZfR|> z!#_7wlca!hhtKBC>}^^YE_$U7_vVky7)j@=R!q5+Awd?X;l%Rx21HXhA6C4$%$F;J zN1#J7kPW`idKdU?Df&rqM3tAUlT29Su;mfEKRKLSm%xdk&D;w8Er-!Hei2m8qG_AuNT-_W{GZ z)GwRfl<0BzSJ#eC|B?$yTXrY$Y5H<8&RX9j!4}iQ)}0*?)6+SN(0gE&1so~V8g*Y! z0g^cD3nOp-OVqtrKJwZ?xneW42O$327}GO>Mfugy&e{1BBQFu4`XMFmqF~?2@X13U zLXH3OcsKZ|L71nT@Nc9tgnCc4Oz$Pu0?`^4%|M2kVJ{wyWtc(_RJ%4q3}|3uNTY+E zHB*}xJ%tAq>O@Ae7VwSv9e(^O2na`=xQ|%&kDjt>+xJqeU5XP4TPMd0-6PykFbsOP z-6ceeI$gOh@$|0MZyFw=?O+} zg`O}Ga$z%$i0BT1{ivgfJR*6hA{;_wX4$|P?w1xQmqog#a~8~GQ=qTh2iW@ThB9e5 zZwqmIm-<~ZWtvCS>R~^0dtbE5aK7iKScKy*!V#`25or6;-&CUrtFzMve@!Rpia%0mfL3UpCntIIy zX1o?r9DSKU&q^Oqo#u-hH||%Q(;7JXOPui|dW8_mtWlSy)j3NcWT=!L0ofki5?t1a5;54g&LtG588CvbUhKD@5ll1Ghr2CxzV>(?tJAo>dW&iT>lj6Kk?$ zKf0sXm%>%P0!gw)toO3gJA_1T3;`p?KU^K5)+JVP$Hi?$)mAL1VKrP1Ngo0w&`~2Y zKfk)JM88~7s1GaIK+v}R7N|1CAhj%)4(D)jx!hNK&28w(ehI z8CWrAhyKIaGy%=@W0R4K+k*=Cw`!*KontZ#=BG+xvz4g_NG&QjUKBGZh(ZTOMgF+@lTtkJX z?8bOtE=(1+>o%s3=TktAw85G3BFV14rvix$dB*R0$d^Le{{Kojy{a^GWPbZU;uS9x z8;Fz$*dg|Yl&H%<-UFT~{QD`PdzTbO*PqCe)`lGd?(T)ZYGWkSV4hMmZ+u&k55A$K zAl_bXP7*~xD=6GKR8p9glu`yzPu?{vkoi%}p<5X5ZK_kDMs8JL8nIaXX~TlS`meVp zf%~YD*tA6WIJ1{Q?o>}9D_%pu6|1q!^;V}6BV;rzhW1yYkN+WfFIQ@aEG9zxH||&p z@ZcnOJNfH@3YS&{dg@b-B~b53Oa^xq-(@~%tT0IyN2lT@^kM)#u`4BTmImUH{Y~sM zyq~0+HnM*cz8~`2#Rus;nh_XI>Y+fi5CwH6`I2$Sn9zTi>ed`O1IkzQARfB zl_WyI(Y>=j66YU*>kBNY8+M6R-{z{;vg5Q{RKwcZU`Mtsu3;%mCcng*?0zl_6=6)I znTky(Gx7!^H%@Lf{Vi)+ci4_hoLNDhP8QViF{N7Jd%XVWJIs}{h=3JtQQSpiQR3TY z2Mbg3MKoR*Mspp?j$U#NBecwKd$*-A*YF4yG%T^@>SYhzNa;^>mzErTj@cwE4wt7$ z@pQ-2?+wxWcU8>wm^pRdeHMT#rK~r~q8}V(B4t}tew&SW@*KM!kSENM^tH)IoZ;+n4hd7@@|zV95RLXMDxP%I`AiO1A;zKIFqi{it`b>1C7e|8!Thcz3+h-2oXzBgD;8>0Uox z^_Jx7p|E8mjkr7w5w+2Yr>PA)sx6&V5y9HKc0&L>>QeJB^#Pdr`P(N-Ay-Zs1R;1m z`gde&fDk#S!TlIOb{(pHN=*=FiKeN6RiVoGW`{F5kYX2vrH_}US?3qwl*nOQByl+4 zaI>`$@ZH>iVt^@TK6%5|hr?iv_05I{dL2Fz;a1uhd~eqML7Gdhi*d2rl6O1-K`xbJGsSL|>z(MAw&c5~iW_FK!m9yUJ@s*#L9| zB;hmZzXXh|AIEq9H;c>Y2s{!*>>(AdX#5(oX3 z4qLwLkM2fZc2>-i4tB9LO%8gLNVw5h1ejp{*j1re=>=#|PZf>7H!48&N>@*=1d&$o zpw6**j$xA+=)IhMnTwDwk)bd}BNgZg{nm{7=|Fl;keZ$mN=BB{`}bo?r9d@Ew^I7$ zZ1QlNoZu{38)AH_z-bpv@av-yPZn;}tv~PJ2FL~yiT4JZijstnmvN=pvu3T^Uyca9 zGjq;Eo^#fe(MVL&*SCx4cS~xPCb$!wmPGSJH;pZqgNk~p5nGQS&5i_s2#0N)D9y*} zd2O{$);Uw$)`l!R6PSM`Wx@fbNY0Hv(Nj?D&d!Wk&5cRu6%6}<2BOEqzqqzrSWkzu zp_Na_)$6SPT??G8^qw}Vm=jFxJ4ow$z4q3J6)CY7$VPBxMgAACXuq@OjZ3u98C;g+ z)O8uoP<@o<)-unyf0fX=vZ}G_weAztWy^@_qcG(CgslcWH3$SH)otk?RAQWm9zY!vRnf%^W5+?H^bw=BS zFbgno)d^{V9!Y7&+rCIK7-33-Ptp5Qjy!SV5!k6LZxJqU%cd8-ku%f`Jk%^} zMBc`vR&Mv-Pk9ehs-JIteS1D(6Nrp{pw%Ulpc&I;V|L#JXt!(CP#0>EIbm^Jq7pr7 zHHDd=-xB-O0pLIz{~S<%wT1AhrnV|?_T9XhrMkd=>QV&PN!^aCZ5H!CDk0>guhqAk zh>1>5HL*Hd7ErG_0-;`nXR25NjL$e_zK`@;=APz#y-lp)`w`v&Yt9e0a&;-P66_&D z-&8F!iKc_MCOd;8iLRCYeDnOVoCkQsY?O9VUC+U@t@-bm}%qD)~bwf*XaV8p#BQX0BWm0(((!kYx zo}*^@MsE|v92rEZVV3}|Qw>HdGPYhk8d%nQA_L>~a`zTK{z3`4w&l`{OQge4jT`nI z`^VvmH0@~9Cg}U~SG6*aKH4G<#UA!D@UStJkIV^n`}TaVmFf!}Gy0eq2W%d>9fGX$_1@o86jtgq9Lxv><#^XE^>16EArhCm zR*i|PJTa<;$ z;u96xV<3ccN&1)ad@lJ_O##-cHP3XHe}xGV*UfJ+ff_R znypD1+*Y3!ZRg67MOtpB{Oc1O4|RW#I=*Upagp{M5{j# zQdcWhXNthn3XtIW)7!Ebc;&q}&bV>N^V*N*5@eW3t>NgYCg$z-D+}Lzxc)UKmhw-C zJc8!m$+6~cTIan|nt`TA$au(IykeW<146$}RjUpZDB7W$;qulQ$i9GAAk53~g=z~O zPgbuz+@70&J~$1N-$06#^}At4PlbeB`_QoBv-^$q@!%>@JWl=G{NK|^*z4i&*JsA& zzGtr;{}L4s2|I0~yzouCaSgZ6b;yVg|N16Fw%EBYJA$k5pGm&Xr@0_68!=AFHPDg-I@3{)~7y0Tc(}mt0KgGY@Q&=~-pby)a8q`h9Vh4G{CDV^BeAYv0=9CZJ0=sB2$Fcl;#J$uYS)2- zw8Sv7O-I-v*)LT_y;RUH5mzwLG;E^gv08Oq?^nh21Xp_;{uMItfgD}e`5o(T#JoKA z`cr(Z7Tb^f#>!W@ggfN33|j2Ur#2*hf@MdoSq=%;mU})bP3aGd5ZzX~C8Wp9wGgg_ z%WeP&V|6)%PBho67!%|xjCK84H6vF_`yDRX%pftIl2m9ygaga79e%0ym5WlV;vH|oa+)B1LQeUp8vdGZf3+n_KE zZbH}x`TA*Bw5oEiKf9;p@d2*VN$~xJIH-btFzztOgX2XdB?v`0ZVAJM62Ef8XlS4~ z-re4kQhE~Fuq=aE)imz#j5h^(T2PK*czpeqkqN_K4ygEDzr*+WyY%1vAFvJ1^A?~{ z-Ct@b1Tm}4e(1tzR5xnMsTdkhQq_oXp45rbh6}nWEl=$C?qI^GzaH!dtExABH0vTI z>NvMgKH!<-YW(nU|9f6=C|0m@89rIdv`t?YtW{ke_GKJeEd(xcr5FVyl!&jC)UgY8(X9)Ye)Sw#rie) zUN|_Jh}UXPCCGoU{m;RW!`-*wt3aRge>e9ZiK{UB{`~$VrPkebV|9|gmKtiVwEc}& zZ2uERd(`&|EUFML=Bum3?nd0t3%v$&dgb3T{3kZADmv1Di#q7MMF5a zmEpWE(4B!ONC^bW8g+t7ULM}HBSRT!2&H(lFi3VV0&+3GE`Zxgc#3$@CJ-%zI-5?k$} zTNPo7_W<0IUf2)1pGRO@vQ3=ulcC$n0Bi4t{u$ug(l=JF`I-tVg!(;e|RxV$$+}cjZYpK{h&lhU?-S zd^a|MGp_VZfGnFtc(DnxOJN+lkW-AD$d^&2Q9gC!{>PEcJ88YO&+Naql_NgF`drI7 z1W8%G`xG(#N=PD2*lMDS-tDh6&gdaIFTXPV?hu8eR}dxVne}A;sU%g}OCjhEF*(YE zsMri|DQKCH+aJo)8&4_%inY|8<_mn4*t+!Ur&|wpbV{pOa6>q@ForV8j>4lrD^dTeBSRmH`&8V`*55JPf z`m4JBSMfhah-S~{NHG0m{ohpmgh|aM6!O$u&G5f_hhl@6%rJK(0cp8uMeC%Mb`=zL z(foQ4*|f4jl2yoW1>Qp}Bs7!-GywY@3mhW@do$IFEpJ0tIrbfk=9|Wlx*rKZ@gPm< zB_18`Y{ptmzaiN&@Kfk%TceAtp;yOvmgpdrc-Rdu-G>4dK# zrJ;pQ^+_py)bv}SfpSoH07mHmH!C~X~yVnYNurZ9_z=JcXM#fv)m?c$_8mm}?@M?MUTxBtN zOGs(ayAw&PFjW4)DtW)~rPm`NMv=4uP^v|OHe}QV%ZM63i%3-qowCCCm2({Yl{w8s zn^OFJUigk+oJ79+Y%01_2#^DBSb5+R&PQxsf@ia(}L zPW)4{$h8XUGMD>1UK+6y23*-~Cp}#qc6~W5u}(~{rHW4zfoXD|*wLcxSU%!Iy;tsQ zTIN?P@#{kaZ|GCH@q9*DUVhcs=cGrIBi8!ztNGR6mU(jHU1J+zQOl$ zG;2ipvjy3bomjB^BE-k6)2AtSPz9nVA8!%j4V%7UecN_8q#tV^$<4*}E0I1h0$myo zd1Pefk&myh(PGBnz;&TbRCL{%Y;MaZ<8 z27Iol_0@ zenjl*4PH&9 zk<4;p=~HSnFumC@;U&44u_TL3%=hk%<1MV`|N0O|!HYnNe1@xpKz^P%1ZfsP zlryP{>7ffbRo+J4n#`)5o4*MbwK|(}+Z5FXV!*V6-_skh+c`wHpA#>tVt|=1fI2Fx z&_{N?({xOP9U8uHWckg=H3@0k*}JT*%J(ju1(JXI2^v|t+W~~pSzK{ z$@l9ZUN~OgWQEDAEptBoHKKt+O_AbMP?=txkYY#S(cIPVxe52KygP$Wh_nEyxw|yj z_LW>Uj>CB;;mIerKcb%|Zij_=xX-!&#SFGfEQt)DCm~)6Y>=Ylot0$bgcdCuWW)st zeq1i7>Im#zWIGG@A#^ch5WYt5VPF{QaGehs~OJL)q?c)!Q$SVa%*)(h85V`_^mMn`o!i zi%Hgka-}BJMF;+uzdV>ZL9|UpoXKRK>h?O0x^mrcPQwaIP2H80O8KI~6GzM)?0%4< z*uI#aBGN!3x}i~IM>?!gG{npj^6NO%07kT#;=GTul~^27DP!o8ZO`_MelPBYOJuxP z@%z-wFON_{_R&i(w=fw6M!uP#xFixs%i2b zD_^4FB5l#n4uJlhZZUed>OW`3h?*C06W2%KpU&^|KLZZCbbf;F z-&G6`u3jw-?VS^9jo3coJ6iuqV@8WjY5WiGVnKs^_nPq8k)87*)mSk*Ry*+RJ5LRt zR#HB#oi5hYO5}#3Z-(XCQVAguWjl|StTUYQoKqyVUy=5oTD~rt7m8-$5b1&l&`3u6 z1P9`GlA2VxI(cS+#yK=E^2?)iE}GkyeVJpk>c;75PN+eM>X7;G8g$g%v3UPpylNwu z#mwU7XE(*8BtFxC3N334_-i;4bcH8B4Cqvcp$*Ws+sVup|BaEaTS|6a*IH)Y#SSVt zichP6ErieV5mpTGhV`N^LJf)+6Gi)Z;7A{)+POR)b+$?XXuiXV*Azje*PkNsO;g}| z7r>jeX~CCWroy;MyXD{`!nn!E>{PV?%RLEEn5*B)WL2o&a<=?za~cI(pN>!ko1fio zhE{l$F=7N2WApQ4xrt=yJLJfsIf8s8$`dSF2Jp%871RNIe_rC$RlP3Tv80*tb<0}1 zh8*fQ{%HVhz3u&=4G>k_&V2Avci$eDc5;eU7?OlBNJ+k zxl#e66UPPDdTy;|yvag8$0iC7kmeq5cxW`>pFhrPWCQdXvcZMH*tcgjcV6ZiDVD>T zy!jGpu-et>{J`qNB}+5jAHGm@;&Sf|e3{tM`cvBEHR9#~KIn^JEo zjL541sQ;CW5Ic^s{wB&G2&t!jFu4EL&$ttp1>W0^AGL0~H6sJ3O;{(yogb3T!v6C9 zdw;R?_qL{cuScT(hJ1hl&VM2M8diLsv+#)}Aw3 zHmvxuVeqt1`NL27H_3r8=m2v2806QpuYF9`pfjRHXlTeL_PmKZ>5@16+RVn~1YA%!@8-`>>)w-Ay98^|uc!&~8J4zm`$LN}YhHg^#zc&Gc z*!uj0)k}hroFB_`c3Bk}(JOf84bA%uR(=sGo;-0jVI4S}?7c9>jgetJ{Z^|v7WdNi zwymZ3JBA7q$u6RRZTM)Xt|}&nOq-GErJBZbP|1i&$?k}D$?i3aRB7(j9bKigAB{D!hW29(db{>&=+9uhEkp(($~me~Up zqV0{h_<%X#E+Ln?=9Sc#0iYf;YKy|WXyUAHk-bnn6 zUE9%4uEe6c`G7Mch+PQlN(9ptUo{<29~3YlMUlBOR+qx#zlQ50nPl4QKs#^nYg=zy zZwksk+|M=@wqyv^64gQr&5C)OCzH1{>{KM=E>sFOZn?Rg%_0~!2N%Win&P_uGz3z( zQTTM>iqkfSW9$+Jl5&ikE!OXru*8wyVAL8|)B!1;wc=XT@r+F%+7Sm*>y;C^nQNbO zb3a%6T|BtAl)>xk>p|UfIhTls!1Mb)e>0i-!e`?7f?6W_P083VZDHlQQpPB#lEa`v zr!#rZt+a3Dy1J51ls`NT1WS!M#wunjr;9aOqHgnEmQ-PwnsC~j$)zU9HHw8m=jJX} z$>b4;obRDH1*=k!@h+v+OJVwvZ`zIiMD5-ofl6#grF`$7jt=?unhGk2imrKSB)WoXY2di$}Q3$Y{QVK8OI8seM`UEa#}@nj?f_;nZ#}Tl-}vlzF#x zmF^i+$!cm2yIt+sAN-n4MaA|&4LLF+ytUd!Xq8rS7M~b_E508*QgA(at=dL!_V~RX%yPj zb~q9AQFLloZY~5ebA;t+NhDS=i6U6MBa1L2JI7cesN*Yun@}ylR6X=rjBOVmXPV2^ z@0G-|jPSgHa$|67)ZA>9&TJLmY!$gELj<3MBJ5=Z3F#(0mpvIi`i=6B0%(-rSqt^- zp&SmME1sV!zw?YU>YzT0(^f3?aRh0$Z0c{Zk8^+i%3Z1o#eKl(1+(^&9eTSj0l6p?71Pe17JiikqX1FfZ^tr-_ZtJ6rUviXTo`+mngwy$5xRY z^(3KWk#GGb{~Mi!jzYo@IDIe>M;F8jES&yyBd5?BDl|_?zJ&@^3_iui19d7KATA_)%nx~`EDkf{ z>`JF5zcMAd#5osNEw(LT4|4^A+^Q0i=aLi3ax25&;9XVD*l!4uUq};qByzHOe zH3IJMc+F;&#AFwku6LR-K)MzjZrWH7*VKyy7i!rE+23y60`YmmEt=N+_PD#c%#w7j z_(O)5OaGHE_7;AczpP#lR=+!DI@ww4^lW#Rvv2_jq$s)%*mS+Iw*aHj8u7`&K3A3N zJG8;=i634drr*U8pwOWlf09K5G5nQgu_FFEex+yACUwls!wkcuyv$Kb|L~1W;>JmM z@(Q_A4<~doTSTJhv|n4r1NRpXoNKbPL;C!wbRcg@{f$#`fY2Hu-pIgh=LA9DftR zIR~cLgz8qT1n}#BLCQhFHEa%%WojRN`)~a+6LrtD<1W$0dQHDbjz6CO*E?;vV)=Zi zhmOE_K~dzMiPdYs<7#Y-0u`5R-B8a=Spr@f9J;Q+T;=iP(K|jb8rt_vwR~ zyP1@ZnY<63Q#&-CzhwfQ4%+!^oIX0fm(-3GbuzQ%pin9aRW1?A!NdO`X8x~fT_hlR zeY7Uc{m8W%B~7uK`dKj%yH2S;Y}T?Fjfto{^@5gUG>zqxSgu;GMsjonz# zPipO)+|rd5n*hs={D<=o{P2c)yTxC5jX;p(Dc(Om!b!Y)d2&|Qg_xu2JNX^SfBsH` z$`E64!vTxo@Ha^CtXlnYbQ7 zq2}}NxopP)xowGN^uceQG_DVi4Gf??Nj~-tSl}b%-x5};%)<)za(N3Fv8Xi0rel|~ ztl6>o#?I@<1F9FRorxJZi0Wq-g1(Hh8imG~8lM7QcQVmQSpxpDANP1yE>|Lh zw;Fnq7&faW(lawOrgQ@2ahapVuzR2aKz#PS`AQgzc}!w|_&ud(Us=S4;K-28M@LBq z=IK&Fbt9zQHCsNMEOnM1GYwofA`uQyvichBiMI(;kLEmJa>~nfrIW>dso6}sE4-WG z3V^}^_@o-Ti=Y+>CBsD)2sdvJdI&kR3U&YB0hcvmr5odFU5Ck-0xNHk&G<)7)Yq*k z3K&@7960@@*i8kHZQz^MZa|@=_v6CD`-87l9b3f??Rmq!|7=#9d6x6CZAq!5df!xQ ziWx?C+u?TL$)_N@3BFFhJ!R`pPm;GUy(DN_>`PC(uX2K4%){^98Q>X@UlWEv#vLOm z6>eAs2*Ty9x{#fb7-8s$bjqQinbryPGnE{hFh0!94qiC}jK> zgw-%a(Eo3(`~CF+@Bkdw_2;;)P|}~n@#E}<)I|wa1S{T`hPvyfvAVeTZjYXwGLor{ zFze!Lk4w?oN-GV7Ty665*r9YFAsfKiVb)CMFcsz4eoMp@*Y*lUhemJkZDa(yXvb?y zCb?&2`rl?@FGP|misCTn$SL5kj69cGdxB_is!d*&>qm(i)WUj#X`T~gVtXZdJi57F zIN?R0!OJYQ7feCBH_D(tettVtBDNB8cbOz2o!!{w>Ajm@ zgSKhUCwWPES8pm({={2iwp`RN5!Gt!szKNw4R>5*(!~5`-$q;d$dx@xHq~b6TXjA+ zKBT^-_M_{Q&koP=f`6Mw<=kboWpQgpgu;4qi#?en*8~g4tItBq_R0`K#$|=SM(x|* zD^jI;hvQVA26V=%FSTSsPKUibuO#8Iu(#7zX`H!j^LiGq^oxR#MWe=lAX)12K< zV0^kjWS@ZjoN7WNct_ou_sJ$FjA@)cgR=hFf4%>)FY8?>3y=9MA+8^gBj$@ff=$+3 zo!WuG&c?J6CH(dbC+zu{V?^CXl#0C2)ElcmRAAzt2+tuP&Z-5!eG?B>X*OIW#nDx$ z(#Tv;*9_hnlr*Iq@r|-J!1Lt-GaE85w=m%uVNtFQpewL(Kb^R$v+Nk_v@&R#to+58 zN>rWiTr=lrcI0QEm(v0&{ zGGqe;D5FcB-E&O@4?%;yI zy7a$a_27n(XjkWdvxp#KyO^zrXB`V|oe}E6ZIm>_h~ojPd1wr1wiw9o*k=#$Z+ybd z$53chaaEcBg8LorF|XQM;NWGELmY;i&tyZPUj}h~*kyFvb*{n}DGSub+anMy3y5MJ z&AEDVW;eDAj%qKQpU2h%$hMSeDrtn>mk0dZPiz`-Ta@eowfxGrJo_L3(~OYNh@?pC zzf5Ep(lxu|2CeK8KJl2enH)2}j$~QwZ2uDv6PXYH5EZDXF5Y?oe4TJH`DS!)*O$_a z+kb~gD$1u*IW}om`y(ydBp?Yk6n2)#hD$VrkCg=1v{7<6R*@JKDQN8rgVCN1W%hSb z+srCWy;UEgoOhnwY0{AYI5kN*uNQ#T#p>;Ir(s~7@m8m1;*&mQ)P28ni+IBj3@ZLCR6K^QJNTh9uLmNocyJ zoV5xQIi{;_!WyX=5$~rnKEGa8$*-l5)!fG6Hf%`1s0C6KFGTVDl*y;?_l%ud`r+Rk zo6}0UuQFI+pHAth4P6F$m4D@DeBS96m!#}j>irU`P?eb(ZL(ICcIiHRcO3x12C!HR zgY4_dg`Vl2=a9qq!i6KLL_7sp!21k2iy(xt!kc%UqlAJ&I1NV?>fZD6EeF(r{mhPq zzvxH4z9)8DyrK9064b=$DUFPd3&G|x4=;X*>aIz+03&$tl7hBjwG%O~d|E47@80kH=+d(<8Z zQx6>Qr*f}h5PmjKYDnXL(hFjE!Tlg0ypYBVimhw8AvE)>LKe5W{F@%V?ALoXPEvk` ze)=(Guyzw^IqtfrVG!HnXjeF$5~1@OyY-m&3N!($zhT(yLp6N&n$qp2Hj#Y%ECK7O zrSUvgx=*-0(2>h<^7ryREbTHK{knlGd7csUx@6kARRc^$6u_#L7x8f;MQ+94EMFB? z@sd#H%e88M3NLC*_GrkQf&LCkpF~HSI_2Q=wtRg>2=Ff6k5_xQOm_qzMetX5lx$JN z^9sR11Y%_}$FubBCB`|+aBL0B_cms{m@sWSGMuz%xK55-{URV5QBcpcuXz+Bj@K!w z>*_Q1^5p@O=j$n>@LO7fcJ>6R4s;$0$`~HD%0joFhJI%&n~~oK#XTnsCbTW~ zgU;U?jo*Gb09uL3Pjs)jd*lL-ejmP;Gv(RtGc{ygmS>@7uJSAMgb-w*8sV|WI$Oc~7cdng4P0UOc{}2< z_;bCjCQ`=wOiK50hR_*h4T|qnq2c_c?8#0@k=zJ*U zxh}3?bSniquYI{peJn{y|9Ry*;1CFE8!BCyZd*JR>b}%k`7SeldXX<4SYYY;nhr6r zu(+IzRGgg zQZbyg+QUW3VX4^cdMe%H=7pp6(j_wVcq!Vcy)*02sLAqY#*Qlz#nPYd4ySzHM5oOY z*ZiH=!}C`xt(3J7Q9?>2Z}+8_e*MVucHxkc;V%`_TGx%R#lU#!{@#do7t*rKoV)HK z_WiYM+MsV!V3;OIvHe3TT;NYzC;Kf`zM&hR1%l=GSWIu}IbzDi&}I83D$+22{-p={ zT9DZdGwPxaLYwt}Q7iF`sdxXwtk9@3*xQ#TkyJ#+mI40>SY(k9^GfPJ4fMztcna(r zq4^%4cuX1GtVddSPAP~Yl0x$ALc~Yk22w~Neh>IuA&j+YpQOQX4xMojOXpFH3(JDd zy%T%=^2#QdwOhl?I+(Tk``_gjs~%wGFXQKMMS*fOzu%omOc^IV;4remTLhq&F)(`u zD&l&YKzswATLehbcC&8^%}kTxo&Crd0{N7@dWY=G_LRkOdv+<97pI|kxP`1juXJIK z5mX57*R}_J=n@Dgo9krRg!P}fhPqIYSGkDX`jxT58{~UND5^Z#q#doo=onW6|U zpl*u_1q+Je95sPcz8m``e3izQFdqZV3c)A_G)33WA488ej5ymjc; z%+pL;G?TCV%~(t(%2G_=4qQUMV*+))N`#9eci4reQ*N$)fS)I7+Q@}OTWV=NB@JL5 zujN^?jNWNmDl2c1j+s?fn#S(#t5@CoAXfGLY8GD}M=Vre+0_ex$jZi~O_z^tB<9oP z{j;y%)%tiKUz6bBJ&(#(FfdO4=Va2CVFA0Zeh%waqttKL6xiWh>|DI{Q?Cgbi%Zdb z(pzj)0{JcEjdahPO@(Urb4J)kq?s4o>*iZC8^nZ6IS%k-H4HeHq(C!{WwEp4?c=IU z9?YF{tG|CM4==7{VM>OQ5H-ebok&Y_ah<#vhRwSr)En%xB035Ezt$&7e-KTE9!zbc zA>=VWsH6s7>K}Z$M|Fd8DozLc-n~WVvHglm6Rms{SI+ebOM>gj#nUl!TGLZr0!mB4 zkY1)w49`^NbYzMRsNxuShHuZK9O3=n~rh9$UTSOrlNfxj?KfxK!HFxX8mz54QTVIkMw!?Roswbmx=Gtmh%8I4suel4FnTOW;hlxy}~+Bm2hbb7Mmhq)MgZ(_4My@bnhpf1LN*&2GE->!+Zy zJgLej1O9IkR`uOKdF4r$qE`fRts%jqBsvqP;7F2~WKT-b8BsIdnn_CZ0&(Hk4;rGR zdR)sWT(t6_uI%$axA`Z=X_>FR3hOE(bS2e)0V&-+%o8t}UGM&8pftW-BJuEm9A#b*)nLu| zvl{ZMR%Q$udh53t7U?+P%$Tecc&5IZG*7v{R+DZOZAB>t6p*WENV%mzC5uO?3}yi~ z^!p!kLB<;gexRQla(7m96fL=879PHkuM3Q0#_U)DGHwB}MfG$$aJK=k>QX<-sUA|@ zDBY^y0^A(A5lY=Njj}`sn^W+R;y@KQFFSoUb34Pyl=);%OFKp87iP{aKaSA-Vb|u8 zt0SlE>UN8E56t5pk6icyguN6o)$J#XU4DDRS5!aNx9M)l9;s>z`X-!_DL%WW-@Hs@ zzHDR2PChPpH4OqluG-7v=VY)aei-0)krTaNaWwJe> z@;82r1$MnSF-{Zo^Wv6A(k_mK(Cc1o{ZaShuT9vYBFfm=Mjazj-P8LTU$@dbvttiO zTUgtz%}^C@emUvP%GYMT)^wGm;TL=Xv5~7Z27e~9)YPIw9fpa} zgyP`w3n5wrj+?b7W!I{Nny1S?k6_e76#c19-X8YJ@+^0?ZdlU}y~F&uxy!&3e{x89y8RAj|OOIR)ew^|uS4WWqEZ?u9{bnmYPJ49rKO>es1Qu^ zhapzgU{{0Ml8tAHcmXBG)-+q)ob@9+j2zBRR-KqQHh!28P+B#goaqx&ck|ZRquud> z!HQ#xVpipa*8CO@7>EtNM6R*t+Rxq)&8~~;;Ux?ct;_(0u;E4D^c48c&{}9~=9?>9 z`HvTvoQm}89DQVg;2$c3DG7tCDHZhhJhv$ZpPshVY=|?MxP8k?vyCt|VdJKP=+&Y) zV7`w}{U5G$h#0sBYv!2T+75=nBdkR#AjKUXPU9V`k0bN1cjEqCU^5#R)SIO8j(j{2 zQO#D|O^iWK*Ci47HLl)!U2r7z5e(uQ@Ma5YA-?^C4#$kT*;?Dh+75z?YGB+h&%itQ z%f(uzQ+r9KxB**lm)VqIB(6e&>p8u#14)fU6u)F=s;k80G4DCSBjEu$9+u+(4Q2d% zZLURy-b6Z|JuOv)}#46?-0ygwdNZ?+ACc!)Ia zYhShcLFuDHt2Y?JJUV&zd#prvCsV3~!?Gj(4hsD?+&?fA$up)@W&zy=z~vgcW< z*UPrBGmx3(*QPGeX&D&C&Two)o_~0pHTYE5Rsz4c{OpKgrT3S5Fc@Kc`C>FB=WeN~ zcQ2J@Ug3)ws#?FjZkH%vmA$w&>I8m_n+&abFAsS7q8j9&DGF#U@l*mGWW6AJ%f1Qx zW}RGKfrA<}gOZaRT_6mr$>=8lHWf%%Jv$G@;K3vhq%9GW?1--zDoBLosms93+e6I4 zsO|L0@0bjdlK9Oo2lzNxrY5b3RciESB@P=5IDi0hdnSWvq6xFvmrM{6Sz^1)dtFE^ zh14&W(y!YpM1p3fC^eNVn+Lxgt6bQRsz_ftezekT!M!Tyz^b4mZ4(zV6)kvIt;|^X z1ccBjn_MgHD*6;G&|<|2GJ0K-?#gwhO@mvU{d)9hU!jj6gM84H*3@}|6s<(|A84|a zP8VY&$oaIS@C~#n|h2$4RmEf67>I|NE!4Om&PeZ zP>BLiP{%m7Be;KLm-mUJ>)=jRf5+6~)#L*5k-wXm8WXilNNUAVfUHWFQ^dWiY^TrC z<7&y{BKhL&+y^@saNU_`NL&=HG1+6DI+x6Vr|8irmPta|iuwLB82{cLVz2+zrPxp0 zc%sxv{2L)MSX4yzh@xtN-N@C6rOX<5kIaP%XV66k<8U($CNJoe8|6Fg;OJpM;h#dS zf&*#8Pk#LjNVmWZPA9L9RRQA)h)0sDWVMq0{aM3`oQ4Z92#vK5&P^hu>6wc>825ZV}kvN9(Y3Q|RPWx0%r; z34()%T299&fSphGZEs|{bGoM4fjjHFnjT=jo%Ti-!?uGo$I<>Ht4r9#pDoJl7Z+pF z++Ex8a#E6ey+C6l1McVNVeI^Qe|;XgM<~NlAL8IzGbfHM^-%R@al07Pw_M8{`Axs} zQ%dQ?X7zRHii|z+3x2a40M!qG-a3m7R#@fB?N-qiQsJ}eqlq|++BsAD3Um;>(Hz}tYOx7dXlC5eh8;p`b=bL+MPeMguh6vESmGP63P`s zD#;aOajrh+~ylJ92izTM1c!ieIZb@nm>p=gO}x3S3C_@ciP^^?Lgei zl>j*TnSS{lvTN;l)v$WKXK_A&U6KEjl^ZMHaWH+*RzotZ>CxrJ$d4{g-0|ICukLWbp`euY#ZKe`RsJ zpVbYhecD+umNUgbRgBNotNB28hZbtVxQ{RfQsO=g0j+Y#wD0$84^F@Wn8iu(zyB4( zn9lxl`_E1-hyDk1Qlnb2yaMF)uJcmr%bIO+bZE_c%PiDeD0bu*J7uc z9?pqV+8|4eH0cCMHxKv)%A}F*k1}y-&81W>F$IKT1H*``4#cy6EBI=s%Hj&A0gL9ekr`;Jmzvvtn^yw!=p2Sq&Ang8TqPLN$9ME^0Nm&} zA&)F<5%C{d?e;T|FUfN`sH8A`@w}iZtg1VxwX^qtx}!f7o#to`^GJt z`^TH*q0wJit5?^eO4>vSx5l>_GgYVKjkmNDEm(dd5Vw)Z*$&>vjW-id^2rTPnCo*1 zr%Dz*{rJ_XfB1>MS+Yf-Y|xgo_(M8W#8|}%I(WjcfqsO0U-BQbI)NkM6*tu{-H6Zm zG3zTq_4I*sWFJZ7S%ZNXk`L_FtI@q=4a^=i9I@t6>cwBmF`KK;`%{v`|hRdS6x~M^^$V2RIN0#{e4a2ag`2z z_1Yfmf#&`fm*Id2g7A%F=vD3)+}vpPvj0Hbu)FbXa!_MNV_q5jGcC%%#rJl%zY1BchLP9Gr*cbvsY z6#vd_%r|4ry`cnc5Esfz4`KhkP^;X4D4T;F&(B263*T)^P1!I`Cg<)o`HS2kDM6aK zNdK|%Tes)yVUg4fP7ZF8+!s={Uv#?~i?4=zg6k`<-MH1_^e{+fQ%1NmqKvA%j{s_~ zKf2KH^#^Gu%C1e$wtfD?e~VO{#Q!F3aB;3Wc_o+AG3*`L@ujiDhQtSXT+W&=O2uc5!M|Q_k5HT0bX!FzG7FFjasr}fgj{@au*K3A%1gG?a8evaLg&izX zHSulH(-N(sZ}wI}g!*fIRRP&j&4V3eIfT=b6%j5JI+!-Z7%$$wA0nUJmfeK$MM>&B z6o>C%b1U_)(I67Jk&E!*K0WMfPtcna9^c0MTezFxtuWu!F4|aUg%W5EXc*>EEEpz1 z)hXCRP%HmMr3ml=s{t+_&aIu~Q~cPb)NU`N+DvV)xwA9mCVq={t1<$vW7vUnj8(c? z)5Ejr4GsKu(?HLF=)9GFRwzpPo_Zf>?UD*rZ?$Q#C-}Q6-xPT%x-zYg;OMhh*tIDQ z%`|$4*hLTvswkL)zLL?Ktr&0sA_5=Ms1}xPyJ>G7th#bDMALscNnLkHY>eNRl`G(^ zb5&S;J?R9PsBd@%+!ikkt{z`KT|kt8zD~RO-?`i9s3=KY#MYq4Z6#XBW}aYG#L1nN z1cxiW54}%RJ{Undj4K)${tDafM@`X-j{Nf(`xV(?+*k_cnuB3&19+d^YsykpsQw)7 zmLsiH#)7DB#Yu(r^(4vjYWG$0$6wsioEmOxI=;V7PI1l;dH*=;o1uD+v#e0#L)@w63Bvs1ikPfR@$y@)f*G#K*W@p}sAuzwS_ygk&u2${0 zNPbY&=uhTdd#X|9j(T`6bgaoeuAOLwuZ(M$kr8rp!d3d31kPkI{EJ<>Q5{aaVrLXX z<{YLv-k{d01g5GQ?%`-&w%6-eym6Tj2NFPpo%1 z^8p>E-bg78N<=~YgWS?o1<>8N{MAR(%3z(qha8zEj=wW>1w_LXb<0Q=f8~Fqj&~^CclNv&G9yhQBE&Wl-SiK@_PjW5R(RUfhS8q0z<1EHtXC%)I#v z&?4D9HmoH>-c8{j$sdJfTn80CT!{C^-ZgZlxmPUWEchECbdJttwZBjw^6h*F*1{Of z=Y7ncL}gDN*Tj+&dbjQlU)x{fm-Ez54FOSN9$LN=Azz(W&!}RB*{}1yQb}I`31nro zH4(8LnRw`EY?T?+8!_1QDIAHI=6`ozDYb|94pTbMILpy0wOIw9S3hgNPX71WD$SY=T( z8b;xKT&8ALr`>6>=(i$=3k{O-slZE=IbTp_Bc!>*MDtgcKJFpDwc+6$U z{fQ}3$V>e0!LNgk)-av32g-qCDoCd7vV$HJB4I_l}(B?gt$Tgxmz>-GN@vI^|Q}Qs}7{_RV@g;U&PZ!fC#@_c~xc zC#{7KNspw7f#STvO`(A8%5b&9C}I7%^*yoCID02C?%!Ov(?RFS!BM}w$NU=}t@gOu z*oANTVC@{|^**_;ht8ANg!Yx_bwlQR2B~oNvc0&v`k{Qal{vB@h^ds`&isG;NREAc zj`v%1{#=?)Kd5@(QB1;R$H&?sU=*Wo!1$jug(}⩔AKZtHIqj5~3Z^JR8-seM_XE z95ffB$Sw|FZQ?RYnBjv+ ztK>7cp3%SK^>ma)d52PLA7M>0&!*2S3mu37a#Y+219YpH%o`Le&3$MV$?xQBWy*2! zWm{ohx;2@&#J#Pu1e9|4C0Ziz%_$S}5Q8E?jdpYW!D)f5`Pb?d68V_vd%H>Wzr%UU zjbilJXP}47CJikUm_+E@CNYe2CG>c&t#4<@s+ZDI;L5$^LDh^ns)Lc5;cnL4zVj%R zE`cfvz>3#4#c1z+L!SXX!aG)R45{PKK7ttvKE4ZXJRm3IK8RP}`NsWAY}j#cd0BZe z-K`jQ@gFlnLQn!>#ib`i+E!1QW5ey&Wm>R#4 zuJ9pzg}EaOref>(7XINWTa!`6SKfH;4}4O6-+B);9AZhVqoOJiY^zuK0yd~@$Xv@r zJI6K4b`LD1>6MA_6AThzuvrf0P0Ta)L@C*umQ|XapQE_~oodQ^+#Fh!>tWE1>Q+Ps zPkQ-(4h^V9M*O361;IeXvis6 z5s*nNB-5AV} z&&&j;wK{Rh(868ohNl$yz_w7x3fudu)E>)}s`aWj2BSI|g2atYwEtAwwx$}i%6cTO zqeeaamppt|ld6iLWljEc{_$j0Y z&(I+I3^poPzdGqekZwS{iml)HugLO)EE6#ewbzABs*a>MYSTXHyI(JGxyuIHC3k}! zXLcH$!G#AC3$!&v0^+ZXea|NLZ+Iaauy6n}>b)bjOa7hNumqj)tqL1=yOem8UhT;| zf(5nO-tzVLojF!nRpRHlgJOgutg#ZO2;axgQhfpD@eRe~jRE{-{smNawOiF_0Mr&>;`Nxap}@Sfr@sZmcHCS1Z2wVwWYA)Pp6PnG6FK4Pz!&&Uy3bzMQXQ zG8_)Fnt8jD3i(lGo%nT6%*xO`n*-0I2gk36!c$a!#Ps8{OG{=puZ1u4f{`;6B^F|A zjU2JV`~=&8mk=h?SHi^^BBk%68D#Q-S(!|E#YfunV&7hwJ$RFq*mf4C%sRlm@TzLv z$ino29XA8t6U4}_-@vd}AjQiOwadK6yr@XI;zir~a@vj!*rX6_$-cjS_H+1-tnrd? zO#PDc)o|>(5j5ysm;XbFLJ2vE!zY~AaV%ysZm+{hFH&W>X2Vb`p17U2gO4kBoeA_sQhi8b-cKqq7JE=-| z)G9K}wIrWxo%j*3UfSGjm_5>V`|)@MIe+g$&c1OzMBfunuEI+aB||?sitYUnEJ@2lyAEeqyNEHWo^GdukO44 zSNVA@gmy-+5xZ*GBEBB{dA!3diGwv3RW!IG7Tm8HMyQcQukT5(f0LZnOJM&)vbijm zfoIl~$p{vyfP8*VoM$%`ZGMdSTR>;Q4dmsv#RrOUx(X(uKAEg)!+#6X^91oLoCo*QTVz5!_V+&Ia~=5{T;icc33z^g$9{x5O?d9Pj+nh3)>kq) zzyL8}PUXlkhuF{pO@aBXEKryMyD}3>MX7N^xZk{`Vh7AqY9g(~mA>`emFEOE=cARh z#+h&Cq_T7xNHN6UB@=6S?7&|vmu~2=!)x)ELp|#?DQYd^^3`69g!=!<0zhhyC%Fb1 z4Nj6kfI#^ocwD-nw*BQuFwr{rNv(rBJV;$0@J8sF3UFf*-<8cTAs@L{E39gWIh{5L>yQhM-e{@LHI|Tpg zKU#wtwV272H&olSK?O=?e{1Py6 zivzEnw|GDPW?WSruXEsI`rRUeKOkIKk!S^v?Cz8cVeB6Bdbu(L+R6X%(P#dE7-3EI zoinx%j@bvm{8Z;EORy#ikz!TU$(oMMw#DF7MjH*FX-DH?G#U`EHw+Xn;UW`>0~hhptGW5^g!7!c%6l%j&j9iFRf@RhIU}CQ)@;{Bxe8V? z8Sm>!vzV7l3&n2_jXtvcTZs{^3lLrs_s-wXvBLIvq&TJ%-Ti`%Mf3w4 z9lV4W^5GPKE*q=|Dr}PLu&;EW(n#xc)$dRtl)L=q@#+HMkaY9u~aU% zBDjQMiUPF|_86D2Rn}8W#~!srKcIUpd@K<|vbD_EQ`M$!A>y>~zP1M)R~EU*6itdm zGhsH6Wwsen^^TpRJg3Xg>6F1G#_j?M^M&pySX=IjB|p>s=PNvFqq{%W7Hq`&MM9}H zYMuMTh(%Cur7}o@PM+C=xHMvxPW_dPd^6d)B_8rP!MDZ2#rFM)3E&1KEhC{&1GKh8D>BS!5S7Wkp<&Pju{k&39-_9@=;S&Kku%e3qbZzTF(vMWijm#V< z<;)j>)}LcgYE{ut082%+A~9Rp!&e^fi8XP(%1ut+lTWn@6jCH(8{8T^f-l28t}Yi6 z>;qlDJy`L5TX%K{l$_m0@sD-nTwC?zFJJty*y1-?cn!g4eWFYCWQ(vCmQe13-RCSu z=`T3f6dD7qwg_0e+jZU{Is66C=XJrZLA3kM%=TOooiD^;&?CXSbf1Jahl0!ot#mwP zWb}n>)4yt#!l?f!N917tk8&iTpCYndrF@+SNg*OfW$7P}zZ7loPZGI7Vmo2%?M@}o z9O1p@VF%yes&N$y+IAB2y+)HhT66liKj1yldit>D+(w7!>REMvtcA2ocSLd4Z?3=v zS`WIO3M4N!ES&D++hP^|Ry9%m!g+O*Y1kYvU91|%4_o}Pbde*yZTi7YG$y8V{`f9u z^BX%>hj+!PzdZgkIa3#OY4@$)_!fV{7zIs-Wv~YPhe5DTN6woNHR^J6Kr~n|W-Ulj zisLcEV5k2bp9I`f>XhY7^1JSj7Q(z=vgkFlN-P-W zDmnvAt)fmq>dDV7woT9N!T6?F4FHX*gEB~r4|`RTw*dt?0)}sT_}Wd+x0dqqx40Kz z2ux2s1m068bxJ&pa=}0uO)2*{$qP~drYh)%2)o&Hv!N662e)5g*nl-@6G_ve_54=m z%H5ySZ(k%wsos=lU)ks2jnVql-+S2Z2)#uF@^{YTygc;YpazKdrc%(GvwYyjV6O~t zxOH}k_l)H`HT{sZWIWj&izr0Tq%mR^-}2z3b+|-tJq)@Us8^L0>qermwri#YbqADG;(cHQ=Gk`xG z3Q|I{=GNP^RP;0VevZXC<$(6YL(D?r) zBwV(HQvXqqT&Psa{)cyzz(#-5fby^HGO}BfcqqA%%xwR>>9HFUxYLrYJ7@IK)U#Xd zyDy^cVVbComGQ5ZnCH4&_T=5OD7Uq7P{Co~p@WH6-TED&^N$BC3hkA%6FNkz_t!f) z?^DKymQeLqs<)$)f&CDvJ2kPVg=?c)aiVXq&0*^%sdE|c?XWiogdMwD?+9F@^!}3! zk!Oicy_n1WlRS%tdU6;v5hXF46Y+)dJD){J)2ElG{SsM~M>W;}UtXiT@9lR8eh7|J zfUW%3kR9WTyDw)!Z_ThyqfMfz_0pnLf_)5bQEYvmeiH7hge^zE@%yODQ4Ao-U=p}h z{&RFF0jF+GTcIGqIJ4Huzp{HLbU0EGS=-!JKXGrybEVEZq7Y6&-V%F$WT@O}-c2_N zqR^SOm)Cq z)H%#6S-YIxr9vB9tL$-r8XGvwGbGds<_) zOGvRyn+;$=q{wm038_=|U_A(OQa+FkppqNR6Ev7AakI(m({F!|*6H0!AR&!Zu;*^e znQp#(PfD&oWu$a~bhS(&U{%n1drd3kN3XcoPzF|jmw_9!5N_It)+~qCU&7oXTsf~B z0zOMVy-YWmb-H)p0kxe~eOeG5ehk{{97?GeIk|2`YkAu|)4~LKNT*s)+xhiEn2^7IVytCW8}KzGjB=2?^&e(2jXUJYHrOr0~BDz$98^cF8 z$DHLk+V>2??HN(MsNIE26tRVlB1PN|IT&Vy@oId{=s&nUZ4B0Xg@?DFJ4-4HU0-Nr zm&>C!8&Wnn<7bv*B?@<9dfXAaZ=MeBEURtvD8|HY!5dh2Jc zE&h+?2IAa>_Fv0QQ?OIbx4F=u4&ScA#kK1g8yi0Pjg{X^&ymBn#PZK;$`az$jKq|@ ztckLq*1ubeKOCNZE&kZq&AqfNII$(}|JC^f@xI(~Zp%V$%DdnLs_*$&&mq!2Q47D; zA3NKE9(@*x#|=iF^LM=l6FS28&Isz zlnZd4I?%AAWZt(*R#VS!1En{Iuz(~v)wHCqA(d<a5TAC9_D@^0c@f-2n_-1 zcApE#(i#np7CWTY0gxIn6+NVbuAaR?9wDBG3p6f&J2Yd>fTb7Zurulvqc6T>o2en8 zxeaUo_*x^lSZN3APXh_hv>$U$SBB(B>b%iN5cOp!5({-njLq3>A22YuU_`RZJwxyL zQ^+eg0z%AqM&<8(Bt?6OY}V;~8opCpaNShx$MvL`({~faRHI75W|pc+HArStE__iH zs|PJnO@z~G#d$^p6VYg$xLt8JQMBy!4@!+aS6W6DvHj&Tj!RoQQ(A|1=2naOPHP%w zh?hnue$TvJ9;Ap;#w)ZJo0M(DZMx19;CsZW^q#9SUijM=y2s9!pr&Y(uM*=J>P4}w z^H(`3k6c?*9*(4nGy}^E9%>i?sb z#1xM|(DR@f&$&KG$y2WAFBiij5jWB5lt4r0YD)BZnipkg?N?5nvZ-UoQJ=Q$Cgv>I zU1Q#QWzhAuKOG=gL+4FmN(GZ9P{DN3p9|)02jMA*Thj!ONQ8Lm2K0he8pDTFcxIXIWTvj`u)u!& z^HWbLB=Eo`zwZiR9P~)^<9-T|+#(iPqWw~=5?}uS;su~ynu`Q9$EMhdSuEq`FNyN{I86tB&7AlQy@G=5R zr>koKb)D9dl2sbwgS-idCVniQXA}I`dVhmN?V%!W))*KiF*QRP9c_2E_w9^(zlaGO z2enuTk(+NEyH9(e`q5YlNerFX|Z0P4wN-{kmA46(Noo1&wgpkJslPHb; zg90jLICroU#i2y&&>uG5K_!X7zkgkPcC6g7(t;Cpz)Ti)MbBHPH9EB)xJQr!gMpQC z<SVCu;oB&WY5l)nhZ}hqMaVgYGoZms71Y5fX~W7{JJsQi%tuMT_{}$Ms?1 zUi5>pznxhA3~L(yl4F@;nfij529%bmrQ;X~b)c)Sg*j3I3J3*d>%__s(`iUc03y zjrTpFrjp}wyo7558L4EG)~hPHjDp~6GBo3k<{R{frY5S}6uvPV#_4yG%E5^dO1g~G z$Yy}*e-7h2#4-rpvU5{7d)8to+{Degq`m2}7Qb~}wB8n^u~PM05eCeBrW>=WPdAgA zx@*rB+E{E|yv~R;NZ&o@1J>do{l-|V&JVA1UrD=6La+;8_qG;KzuWji7Nv2kIFOj- z`_HmsR0tGv{9l$8y8mTaAz%c7{_b4zRKDBy@P6jn(Svqkqm5K#VD^o~{!5NKWw*2L`xLJ%VsV;z!U`j(PFN-avg5`Ze zQW5bFk6HU>9$lKkRvje; z($^eIA=4wXbA?+dHu89lcJ}!8ps^h% zkvy+XT2bZxLkwp={fap}!hvw~V5vE)eYeBReSBx0s_PvcVm~nG2isWH*N3#h-$@!l zD-1!u(Gn{zR-FGOx8a9wGXF+Skd5EKS(Nsz7vU#Eu5bC@Yau!~@)bmT<~_njOXCq2 zU4)0}2ujPZ5?_ex+F8=xjr!;4@Myza|}`k#^goXBqo(nL;?8hZc~ z|B1$}+ow$elThJt`&IhYNFi5&xmAoDUQfjV!mUxs#6NQ8#1TTbsHf$Da*^!Airomv zX@9Y_$|H-Qr`~eO=paTm84pZ!YG`?~LCYc7NcVTb-+q=aw%xtgz#7nMM{wnJp`fcN z-&^eB9}VC|9*f%t5-vHMmM5AI&H|`N^%gx`VO71FXGz_c%xd%PS958>pYZ#Un*93i zvL0>ip{ei~NzJ)pkix47OvgBReJW=7m*PrshnuMrs3ZNyd8q#GNrB%d@i0zbADMlh@VfJt z?(0bL>^~k7+XgxrfCXQM!Q=8a&MqFj+X}_97W*lJ}t$`SChpqxhwMAz%~Ws6SVBPLzL!d5g*ssp@WjtIrDQusoPpy)Gp z7y3+ga(;yDmoP7YD7SH%@4>5oBFBLZ^Hx^6Wy{Dldyb1kZw!KGf*RNl87CdY&m7EQj<=2UCgLoW$;$|%8NaczSeLPw1V>t|W(Lfw_!ZJQm4I)%as%a+ z4q<7$W5^Kqr*|JyfgS)GuwP=jo-@HxIdv9TWgIcxKsQX#?k~*?j_?eDgi-{kLN$z> zOcb1mbUOk|VLsZ?zO#ZL#B`?75UQgnQu&E|bXt>^Hiq2wLB139*vpLlC6>u4cvmg= z_ryY*!?9;|)qHlTj^kY(PEmiHSQ|cn<+873z`E|1q8!#DYVJfc^S@)`^bmcJvQdzl+!Q=!}L8Ow#0ceTGjZ;MvVJ^@1CxNFjnjkI9A zHSPi^5-2Zb3Ft25gI<5GF27!v3;c{iOrTD}+HSmVzPuAwnmLFWGQ7|%$pz%10Lm3` ziMyDk;sY&b;TOmA;v3~kfXc)&qG8S@4*hof0erozB4~=+Fi6h%JGMZptM@eh)qYPx z!mcq==AI=-P{{6WL6`cvEbJGG}+leup0&Oy%76I7djbHxKJBom{;;n?r`r1~rc zQh3S1Z~&jz;t#wJafqAkRr|UAmP&JCV8k#uF#Wu2)4c6Ri$8n)&Gff>#3htFGBy(4 zTKCI4$JpIKK}D??OW)eg_-@`~e|w{6ozOJkqy=W4}{ zf7C|x+i6VUbXAw5H)jKJWNJp-$0NDXZ(DMHkWR({H&8rlu7MsxbH}9cPJsl3h4==! zr0NL{SK2gwSHH9Mq&73Gy95T|bYx&w4@H8}xj?&Jux3Tl=9ng1R=btvRg#oX%nmUF2|@o0 z2}?hsJEL3kFSmfdbaTG|ZypGqJQ8(|K&h1_6|B*bsX{Nvj2||2@v4C)k=AbD!5y4f z9lrbZu01qW^aYqjMi3}=k- zA;E;Wy+UA!Et}#9O7@{r9WqW))3U|T_>T;-$}=VY(DpuW_M}Z?mSrGOIfMDO>D4VG9C0;XjE^F;_BZfc3`7<-%xahwqIpl^Q{%TxP9V8KPaZgOaXMMEi(o zV9Y*PNY+Xz)0Dz{bVxd4UARraKWa!X!7200=DlUvKe^hV+x5&> zuiMIM!`stH6Q)Oj5_8u+Xltku=%HC*+080><`ml~y%+G4-fxke-}TQpNu%#^*6wOI z(Gz`G}dx1!26VU}0Ny0sh!X)SV*@`N_8{cmTl&@W~YTN=5{V~9<#{&(&3(_41rY8gB&>Cg5OFz!(KBE-hoKA zD6%4158)3gkuueY=-&7c5DJNU$oGi?xV_t=`2H0nl<)wK53J?Wu8v6`Z*`WH5-R#` zq(_Sot&gba2l@fmZ&67lvgSNI7MPd(XvAkBUnT@N5C<82`sZM1H%g73s|tdI1pvM6 zJgC;q5IsdxNrg?w91}+o_;*ex+WXxYZxtBvu>`iF2nANOf*Yoo>PGesLq2T(QBM^`S-u+ zY=Jp{Qz(_fZ1uOEnWlc?-%3ExFv*yt?sOY;Z|W`JPzi0R&E#+fQhH8pMl!dj+gQnD zSq*Hy2`+^lVfJ|5ma65_`K{78$Q(w`maDwLo3oq-%sg*TC|O~%t-I09?n2$|g~J6V zJQm|fKtt>N>&8*7rYh7_Kw?wzYz~=>^?RtPqpGpDFDbxu@p^jk77; z_kYL*u_G4n8S7o3C}As(;Hcy2c>0bxlhhNK{{8kq&YOP8~)1*TAM5XO* z#O^1b6igC*r+aXTIzogE`p25nuHsHfONc}z{s8}ssMq$Y|Mf%RZB8K}dS|7`_PMQH zHV7H7Zei=C9HzD>h=Hscd|4`y9}V;?^eEfL4db(kn8dj`K8Ctxv&iJjLzg&oAxMq7 zFRltK{d0C?mcd6X{uTl`t)H7viBrlLFQhj95=s>#322Z1O(=B$kb+hVdD}47FKJTz z6Y075S?1$3u(&U^q*|&)(aUYg0q(ZjHg-FF&`b)CT{zpSX#F*K{pp52Wa!&xTc=%I zk$7{j36h&~I|=D4g}unA>_Z%E(9Idj4vVq~ zvw`6G5b_~E+86VYA=OJ1tLxZ-d3EA-8Vyay!%{A`d+$M(9JEE`{(4F!Q2J=zx)$Hm zERiLO)vt!`nLm-an~gp>0`&te1jj!uv!sCCkzMSq$W@8cIx!s^ooO(ZsT({EE$cM9 z$%j5@#)l7+P?y@B%p_M=e1N(OOO+GuL}O_fQpQYb)3;W2^1zr>O@~N9q;7!ZHXy_c z>;dSslSV~X*AP=9S9xtVb|pR~@gY-?WSVD&@-P7==q$B0{*s+%+EHOna=Dd2*jCjo z2vGi(GGUKY?et>r#IAFinFOr2^MV)H1$eHZt4|CsV@!DD=(RKB zzME=%)5diCd3{aF`D5#Ym5Ad;dcf%mefQ!e#+2IG@7ttmAWq!O$x7 zu1mf6rKEtTjqerqI7b1$8kyOCN8Xil#$!;k(#^)a(7^L~o8KqPm((XC;x}zGEN%PN zvyLRo&A-kr!{fibW-x1lCa=h^%Qa2kGi%qIusREupe9(J z_X*fzN;$j&dPGv9B-FJYAJVzqbCR=>c+6cwFX`HG<8Bes2(w57q> zG^`Sc4R31Qhniv6-QxSvCiQK5q3n*tZm85-@Oc%5ft~Q&vStQv0ACs+h&uvt5yVN7 zP=EIT^@!7wVEZ1~ zwm^cnO1F^&zf&BZ{6vcMMZ3Ai3fW{;la|ty=+{y(S1wdz$23Nd@x?Q&xjS(64}MDV zg=m~5t0=L=V$2t1k~!t_ITGV2x7ZPzaG`;~>&b>iep|{-@VTC}pabXj{I4^O(Z|Vd zoyLK?Iq7~w_Bco*83#1!nHb!X8IM+IZ8L6*+Py8j8F4WE1 zgr@2%XRLqtDR!d%uqR+>3RL7pI>!qWMVoelBe1--aw3zq2||V&gf;WxeT^tpk-!}v zt)aPrN6r<1@UG6;aIjVhY=FKy15Qp4K)PKw-t z7R|>zCh}aUOid!~-IVJ*Mp!Zg_{t)GZbVrX|B`($jQ>EQ_7xEPd;OQ z%HwfGOZ6POW<}2o2QIa*iZaz^!U-sKg4dzAA0*Oh8BNCkW@c){f0?@TANWKR&F{jf7GY(4=v0|pjh zj#t=I-}$q$pw&AfS!^(1ldSml=Uq`b=Roc;gXH|CRAnCgLo%6Vey$@G&^;CYs=rV%r zpa6Hfruy+3Pu0u<%-7>%y^zX+OI)(=LlJnP+^G?G7Sw`*urH?6f6|H0d{`jW9deVJZP49xe9_(?iWhqemK-CYi8fGO6*MHV z5}46xiRN7bZX%(D4pB?x`5#=y+Tx&5 zHB5#Ym3NBe34uVDQ->r8b%Q~=MOafhGfCi$__e(hmD7yt2S1OM?re+hry*I_?Q(Gz zeEWu#A%73pb|B93cY+1nLnPng*lwG>-yz}Q-n&+3kXGq*R=tZwg)i{Gu$)np#DUW(FDX5NLA+LJ|0IqwD(2S;dYGO6U$q;n5mKqSevr>+uhieD)3@LUADgul0~iZ_72?ZyODDx!DDL2J_N0g)X&Rz-|*);%7t+&}zV1wN9+1 zqRej%Umo%WWr)1+61nb5s1stjLtFa2|Mtn`ng8Ba^=G(JS(UzJweic>knMp4Hop2i zja~0JVPNKx5;pHTImbt3LuEz)ROMPm_s*)5rLe`kjjxot#7HQ8(hWeHtF=lC_AbXA zIV0j~<^9swc3G!<(Fg3^xR*g-ZG}P(6-@?_$|Qb6Vd4Tzaw+U_;Q~x#C}B4H3lkZe zc0Ewi7;q%Bo=G}6eSB7cS*%{I2ncjs42rzShZzgTonuu9%?<|_H-xB;TqG3zCVL7( z{(e^Xd|L#oV~&8Z>{Y|`nV;t85EK0H3yQiJONW3OQVwa|a<2Ej;>!WOZSjUH-})@R z1RoGh!6r?syDN(bnhf9usTK;f(L#if?Ow!kEwG2{wbbM0M@%V8D9i%Il~HhBn4)LG zQTHphX79en4coQdJQ>WIwfDgFQ=rh;!1lD}ES>(;CxU%GU$;qtKd0>CwblodJXC?V z5NRjf&v6GB9Rrv-PJ)b(G3f9ofLA7yU}T3~v3mgmpY9B9stEZkA0Ks{c81fcyg#7@ zv!s+yXWrG+E?=^ zZtdv{7~9>YsNrC@bMquTn@LljB$?udjv6MK^-x@j+VfZLU9a*NbLIViX zA7eIC27n$IXK8D-vyq@GPCdxluwZ&&k~JJWP%DK^nf)fGJ`Xfu>=08%LIU4WuNrZF zEP@^GwE|6Ms`SB0YkOO-O8WG|e=UNdBe0<5d3;}P$a1eD67%>l=|{^0kk_$DQ*>Bf|Oao9=TZoX64xv9SG;6jyW z?9E5TfEoJ0y^C3(y!0BD<~J;-eL}iTvATB=wC`*XePO|(ppu+4Cd}iXs)6YUnW3kA z;=)3zr+a&6g@FaHtb$9U*6*>9?oQb)a~kTYM?S+Ezc*jPof~*2mNn`SqdvVL9NX@h zi2V7TCBnXl16I0g@?&1Ad_>hMP^N^PwS-+>R+0t^ETF(X1CK}9cDWigMk`RzYeE(k zpg<$o_VHGnqA(MMiRKDW)rC8L}H@oEAuekrc~!kf$X zha6iI5u{X@jfML=E*RIl#B2{Ik@emYUtoQ?p{W0I3cME|>kZnUn&h1fhySFk1173d zAF| zzrOn~c5~&vo#8UnsO8V1M|_#mrl=C`rySBMlj6!N%vG&?Q`{$%nK$SSrKfZF!2oCJ zQTxMV;`dk2>aUEIiQ4Uyr>JYzHZw7NfhB*~2R7(G7I0F}1k1S!M{_lT%@l_X#ee5D zo?KG3Eq~k%aG6LzhU!$~Y(x%6^bKr%zPgv@V$o3QH@DP<{WY$tDz+2 zQ=Y(;`9(cyZD&sL#%cDWD$*zFX3%=^V=^lTU4gc4w0Q(AmLDCQEB1uAjGq6g_zMm% z71@Pc8ch3%nIt|Qi!-AYRxs$UkNran;k$6r&|tbCbb$ezJO$1Ex`)40#(r;N@i~+Q z_Btm}Q!SwrYZ=V!SA2QMiX{AjS)?b6dncOkOVvUghFj`SBy)N)a*=--@qh-zgjsm+ zN3>L=9(PeWS<#Ln`-@*pQrNG?Jfen!>v_gUkmMz>mz~qVwT>>`8%duIVLUbNHt+$V zscjY+hkm!>0GaJU!9~b7pDbD%V5>TRRPc6|CHfDvmJ>qT%WT@%r?s=u6+)k}R67$B zaVa`0*nC>RqMoh({Cj_dqnhLL^ydJ!Ft9V+G}1O_I55|s`{Oa}lw}MtiG2OU4Y5q?Pn7Q z7LL;Q8#g;HHl=}bu;^Q2G3yG#pZkU=y>5b*2MV=Ovc@q(G0yQL^WlI zgpv`7^7{pdB<=T}`pbw93cc+mslVw)Nb24N%Cx+vOF7a-)^n0<4 zm-Wde`N^821G9dKJA0Zeng^v6SDqmi_-tsL7{2pLVj->mB74EVy4HVvMHSC8S$<2b z;CN#UW!BH?Z$?(~frhXY{6-!UH$7>rfhQ~)!`Lj`i4$)s3@}ONSLd(S?iFiz?hgtI zKL8lZ$jyb%9r1+dzhl)uB-;TVW3Z(r4VcY|Z&xMF3AJ^^k2EZ!708{Bx|mXB0g4u! zeNRJ((_2IOs%b0lrUugDBaFR=sHlYOnAuPSp_7zVle_;vj1 zv%@ip0~@mwMy?ZW?w&=~g+rMywD~+QG`u4BgUv^K&QrLo(+TV}G5t=}u65H3o*s`2 z1gc3&@q6&59xFm;%u973jJmVe>k)AlwM-Ocl}t|o{XM>=C!rY+h*MCYE{Xj08C``= z%^f5f|66={>QG}s1xw7<^qhTZWK6lJLzcTk#jHZ}+>VqKz-?aJ?K z4A&S2MSlmB(5yq;Fc#~+Nn`g5N8U|^RcdCpbA%|5Q5GP`ACj`&&VS>xFz3mA{sDlK z$6UXr3oxCVKc&qzq($I4sOIi@*3CWkp0QMOf5r|w-)y$*JVLwqkxo7YaT0+VM7>4_ zt49gA(E8n+jbjfDhX%Rs*F^J$9o5E@tdXoh^^d1R3cfS;gjWNm;C~}Q4K6URwL#y3 zR+kC!L+`!W<_rSpyt|_^h)% z65tbo4qC}}3RGQ4F12iIKxE%~h^wEt zFc)~OTkH}~tKf_2z$vnr>HO5Loz9yA<>a^gra2y`K9OJdHY4A1`#%77Hvx~$0;J!T zCe4|mJH%9H;H2BlXrMM(KkdeCty8tlctr6H-2^=RGJ(9Ib<6y5Wo44@Smh?+(VrwF zVZ69&F>BJ+7}9_8oq=J8@TJR1Vs{E5P!E#eb6}zQ3S<(n5y0LEh}h{#KYcjkOQ+fDdVu zB~V!DTD(>&jKeSv$^79|Q0v5ZOC|AXbKP?Io}5_11*~jbn_Bqw^hsG%#^^*+IJvpf z9%@Vn5S|Q+U7^Kd372@4A7#Im=T&jpb~9`!d@}Q~4>IQ4HS2nL=VUVP+TLk#U9|Z= z8(H$|eB<&H@e@$-l+v4zC1R^l^5kB#;_FOBDdeUq2$P-;!Hb?CG~L_)y*B+`NK-;S3Zsz_K2`3P8Ux()cXKK+uOOgOB@${igaw$K zE=Ynqk!FyV`-f>PnyH<*uM}d2uuL&Y#uq>EJtVjE=g&fs)3|uPd|{$mE|e_ldT-6F zhv}3e{qVLnEnlgD*MD4jWdC4$IFbi95m|vWY%!=9 zGU4FiV{Se&zWK4WT6S|0#5WOm`tHuRqH%ouk+&pRBiBoL&wGM4_Wp1+Mq_hUaCP7v zTzvF!)OXw9URu*f`|UMMY_}X{9E20rXYYN5i{d*AJa7BsHE>yRC2haE@7A3t;5hXC=~zmH3b^(L-8 zKLMHa!HByty2s(?fW^c6LF4gw_S4uQOd#QVtsiuKI5}N$^(C_I-Q~nVl70Z+FU>cC z-0NhT58DsHUP16WU`m#B-Vg8QSTm>h)|k?bSF{x9Y_xH6>!KNEr0(U7u*Cf4E4Baq zYdWDniQNtn-n2#^9-Jr7P~`Bk&sA^ayOle>d0(uyg|>1!sM8KcK!C|KoJH^g`5SGw zN22UQUr%7cAdrm6$03A(zrIi%dsr#eWHuu^1|CAda9uG*DD1jlZsUtBJ~yx~F$xoU z!i6R0FEF@fN*?K({_QL3m(i#WlwCz6!o-deokg^k0>Q(Fuc@kL_c4p)=K8p}NYD%? z(%pPgqj>_Bk{}>}zX1C8vWx`|=TBel6ZrjC3&8iC%DXotK^>P-H-)W&7s5Ou&U4oT zYg#RP%K_})quPGgI-)=S{_VBEb(G^cRPT;OxaY2R`D>c!s65~pes$vWNCaA;id>_c zm`XF~uG{uEM#dppNlCG;!rFg#dwt&;x(25pjU2NKQn5E|Zmt+%qGz0yxFRFX7{9ju zO!vUPr;4Ay)tsHE>#0~s6Il?Fv5j)2 z@(KQvnJ=?8ILnA1@2PNc1&r$P=vt;|Txi`O$S&wrEEz`P?nbP$DFjvB8Zk}v?`b;Z z7}K8{GQrq-r6?#&A#4xzNqSv)Lp?a|z#iV&jct=}#cCAAD~-O$rTQgR0>vF$swCw~ zk*NM%)z0hUN*4DkXLiGsr`0iZ6C%yOx)ea}*FIW|JXw(h{jZmHc+;LbVKlX>d=Wts zY%*42;Kv#8gZ(~g6QpxCCOkXd$PsVk;8C#mN-J)&##b!TfDEOFt8x_9p3f+u-_F62 zHPLo`cN=p}C2}W?C>1GD&fKwf&3;oy-MAB??IJB55_WVOwoqbc@f!3mY{CD5L0P^xJ8 zP$Yd}S(;{2Q2WnVHE$(~f$xv!&{kh>G)34L9VN1SmNEH8F%{cSLJ#L1M@Sl|+Fyr7 z*rrb#Vg{0jXd~U|shEl4A?1Vx29_X{4b=n5Mne09()Xq%u+&#+g}5Bt%CUYwT5Am` zo|2t1^+65~?xQfRk>kO1-IRUpN1+R4g+CLW823{@#ki}=dT@P0vqRyTvl7JrPLaeA zQ$iTMJJZ^&0NEEq96qL+3T$Ost1o&wQ<_ZLke|qshntDtRAVH5GL`oCJk!Gqc@0{d z)x1lsPD@l^CIZZ@vn#m>V)TLO+!leJqdm7Rylb5aif%1FlRp=a#K3PqiiT(QgV~7^wyUj`jm>97UyxftB=|!n5_G zbd}JP9Q5PBz~1|QyAf_<>kGnY+xLYNsDRHV<(I!s^k{fw3qD#s?q=Td~U9L5*g)r zzfAkTZv00WE4B}n1Uiz`tvqfRI3YV%!1dbQ}9pwF5#` zGs7Cc<{(cdTd#U`9xWphrO13gqN&JGbv>`FLnY=})SIpo&rEg&4tzcQ$cv4{oR1lT zdg&s3M{M45!loj^xVkqxW!;bP+uk~N4~UEP91UC#1y07^?WkM2gIm9WLOz;{x5cR<99VCGx$H+sof@oN;K)ZmmBE%zSL#@`{)Fb zW*G&zeYD1u#X}S(xn`;%><;8!uXYWAhbSqMu~Z#zD*6l9fI{eAtX;m&ik?zCV{C~X z#7zpA9jd{-8%OmdqML)H8ZvjfP_q=lmVoSJ6Zptya)$`;w<;g%oUAP508G&A&E;n1=TVQk+p4Z|6?MS|8V`R=NeuD+e8uxnHK=lvBU-kTaIh__!!wb~tIc^bZEGkNlXq%`30B**|~>J>6YPDddcfKIW=I|x?JNgU0A=uo6g3n44g z)Zy*vQ`gnk(y2!mGm^SFFAlP2n~oynx9ZJChU9XEU%EbuxmFtz$&meIr}fW1o=jNy z4CNI@sUX&)UWo22xx2NyZmnU3;pfZA1dHHC(c2V+M}v7jU#qW|2)8?V_}HkvY2Vy^ zsl^Vdkl%6oh8AvVU7_8?y8T@&_zzLxQ$Jo(0vpoX z+zp_jCs?zLZr|y+MMj;11dJ!R8VNTV6wOFdfX{0~();?Z8)zhOoHF4!81p-pEv`;8 zb+-KiOL*SmxL%{(Cr#6;k!{ zlA1`ek$IuNu$qHXsTC_T`U7~ErB%ux+$#}#JJ|wP7AX})3DHdE7JRvb#1B#`y)P}h38#iNDN?|(Y6TBsF&dU!K&V^W=TtVDiM*&`!`VdK=H>hd(tK`{Vu;&4wKYu^7@PX|@~)Fcd9`oHWru}Ps)6hdSRRt7h&v6U zKuM%kLxUML5|w4`+v&J5tJI`Yw;C$(Jxs{g0YN7M@7VERgET3s@nEA2hQqGk`))4? z?c1x8r~)v6p$BcT_k4^y_Nw&d99&N^Gjt{$3XD?kDr`NJ`^9NZ1@z$eXImUCjI}4& z6fxuc{cLZdFoK4P4GCY?v)~{@mvjHzJW3AV%zM&ob~ikX5E?u6;B80T%U6t$s3H04 zP9QrcX{UNAJ(J%_%Llen-$#kT<$f6@p~ACI7mNDo_dPZLOqgM4*j(CA&Dk*Yk{qmeDtfyj)n8*H-q z2pJ;-J{&vggLfIcat|+Lm+B-d7cn~B`tav3Y;1-yGCFYqMDgwhCiqjytW zU;^mWJr>{ZQ8qp2_n-Qz`ZnFtd-C?DCk+Cw-YV;WYsd5hDL<-RYwJ&JG6VkdFPr`I z6GALjm}TNmDG7?TfvH+T|j(H?>` z258E|?ZO5SZmG36H5Q!fnvIOwxU9Mh=)07t7G_6G(Mhbde3LM9%%6yml@fppyR(ew z>od>>H$3Zufb1m_kgKt}8T_vFZmCJZM`9wP-sR*_IGty)LV_qkZ4>Li2NaE#xF4Z= z(WVlx{LD|JH1xsuLMM);XQZX_O;RpaCp-K*IM{U9rFI-u1bI35x2Btz{V$B83%t9LlazKy!djb&L@Sj)ld1mxUl>%w!};)Y!3VKwe?e6(MZ z-OksjDH5(X_Q5dB0Jj>1vZkBFex*^aSpHrQ^yG~l1m_x*g3O0n?qK-LBq6X+Rfg|H zO{NggFh*D^F$z-}{|k6x8aqDNfKTrav%Ep~0-r#xn06$18FA)aYDX;Z25NqN?qNBw zV^{h|U~06T}UsGM2 zCzbya=cBbMKTM0X^i zHy(GvR5s3gRJ`JRA>=24)fNF8^e_X^X(4ADp1zeT{|yXXv&N z?~pV(`)U$~0WFJu8}Y#lWzy^bL4eVbUWgBz_AU~y zq8R&P5hf2&;NQ90lrwql))m<3YZCw=w<}{9eRadUKhqcWT=!3)?`j9vI|$H|dh0)x zL}Y@eE&)df6l1AedH-gv_e<9&$hxiCn6nmEIuwjJJK%3Re+W9yi+@iA8oAX0azE5sL*g6hYY_pJS-p;NhZ1u=}EV%5wRagmODCb+n zLY-~?NC+!L*H5C}AquJy3`k@bNEvXzEpYAIIq=y=XjMh3aKeUxik-M>oMgiJJ@NmF z+cn`pnLbia;PAny`=OO7*<_?it^YyGc)RU;PYO96e#C0C#wI0Payi;$jKDjG3ac3x zv&VJd)bJ$h;^B?G)s3~7^`(D7TmdWvRUXp&=WE3ci~G`Ve8-x^YqpMQ;={gX=gcCK zz@H=bQ(>iV)Q^O0-Tc0e>}e};T&FS|H=oEYl~$^jP!ak>5R&RBfKd$I*XSuW zG|`rcA0qx1Pqo@}St;_FF0PZt1+^@TR0p=~_RC@RLW+JI9(-G$pUHkkH2SDqx_nw* z%1r_s5F=5H!cwU2@7F{)cffl7dT~HBMo{rzK(K^^WPPa5pp?4j1cQtr2>S@(dNN5M z@0}U~)~RSksNkz_5=jWNdI~w6Puxf|*^n^=`5s%JU2hbR5n^U#@JE5ZS;?ri)4Rad zl}q27njc;SzY2fbOy}hKT=vyuY5V;Lpt2KKSdvZJoa&OIXlC747Q&zUjz3|e=ZDIMhtoj>?TOuR`dLc4QDsCV&TD37cJ_wzNyqn~i z@LuYHF%zsH7g8FcZz5Y3y$lAXgKMK@kda`h<^A#!GswLz=DNOW-l9rneIVoatCxfE zzi9wS;C?S8Ik~Y!+hqwLYQ}>f83|c0PJtVv1LjlyEEVhq>Y9^iYf9&9LE+o`XGE5X zC`~^%JNbXlG%XqI3Uf55Yb3spXn5-FP8RW_d}`@kk3>FmZ7d~xhNu1E(~mhOCW|5z zsBjxHosYh$Q70&1=j;SRK(N=_{=4zp0z31vfq>!bUyQ-{V#0!jKji&YQOZPs(XyFt9)7(T}dyx9zxb!j?XbU*197QDxZYW1q< zs9sfP0!ylxg2sdI+?t9@4F=s3Kgh{Xb(Y_nuq~|r8}u|1ASTX1ls|=UBa{RSPW8EH zZXLIKq3h3FGIi8VPX_M*(0RQ!p(hJh7g#N0!~F zlelw6QzX}G@$naf0qj6{mDvQlb-3R z1VMsDEeSt73Q(1qFy=RPbzDN5C7X9QPpx)|pO?G(&(9oow^@93YNNGN&$&viKK~K1 zdX9F^sO?6ooa7k?moD>pE|(m*ojsC6XWBi}8A z0hl!6`>gm4=o877s(AXUAMzdGU?1WztgeA>f32Or6bWzwOCludQ=H_>g`TeLOuokp z<;W6LO7fuFQtgAd5TQ6=iQ1NZP)2QC;Om7lnuS?Xec_~vcYb;Y=UYFI-{~WaBk=ZG z+*>cM85{W#JN;SNy+vW_p-=C$T?keaC zM%a9hJ$1?k*f_VT4BGW>!ZcehZK53gVOFI>l8-_5VqR{G%9MMLl!rtn+wx1+{%>Gu zuOpD}u+z8kkRrJI1dCNyC191weIoC&dP@AT^1y^@axhDP6~oZicJOjtAaKCnZQ*ki zmOuH9$=PyFv@e7+;6&qWpW#9`x!5;#v_Ey~UhE`U7sJ)p3VmfX8?2EPz+At}#g8me zYEcHMJLXzjQu)QCf6)RZJa2w~R2YJ&J|B%x$seCV$fmf4-Y`K}6Q}Hqa!wR0J z4NxC+DF}l}%n(xZJFtQ~Tm}6ISO-}`KV*Ss8$*RGl}-n%iL4J&^xIAQArc60F-CO3 z2DQG_L@CIU4-!ZkFl>woraPijoENh@K@EI#g66&+Ay3y7#Vx4+n29c)qjmT~(U1B29L}b^ZP3_S3z_ zDhn5qnPl7Nn}?KhyuT#%(-;jG5VFF!F@?>%H~elW`ypf0A(XpART@BKdFyqxrj~J) zcXcNbsn@Os2?OAKaK00NjBD(!MnlGb(Zlc%fk(0dUtP~tx-JKd?F)hxE~hV{L~q|g z2^dBUb+brgFHyo7%)5ejuMD#Qx`L}+1=iP>7&>A6pk8wv?6VXmtzN<0h!9+SMQ+K% z+-PS9X24~HUCksO{wVEI*KEg^TSf$4Pb90(D^z#VN)xlwk2atqVAzSM5e!YoziCk@ zRstyBr3QBdJEEUWs|-W>ps}M3E|j%W)ggFfsH_hA3LPvxv(zY29GjZEtfgR2CD$*n zsbgSlI%Pr!MhSpq0DG(V#z*1V8u#<>9|LDRpu$(N7`Kf6?9Kl_qpv6@ytOu_IHV02 z82+8CJ!>@0A;n**Md&7CX9{;QXFI0t#QB%Y!aL!hW7kh7gXUk4KGl-x1m4J1;`@)v z6xDuo?bKIZZSM-2xw|JE-0G`hoNg(LG&q`jc{t}BFlyF&g4X8-!hSR)?~&>57qw~Y zA9lhBp4%`n85`US$s+r4OZxy$tv(PQ8skKfgp8j{3^FY~!F&C{r!FoDxGMcWY`u3p z)zKe6&UNig7uj1~6v--k%MACrMzXWV6;Who?{1L@E$+25vt?&huDvqLs3^Pmo~u5e z-{<>%{Qk=Qhuiy{_jx^E&+$4??p{Rdeo{y}F7Li=i6eipZ@W+eRW}#v58rjxd~tV@ zo_A)YllrAp?mdX>l&_?{_0NvH~v;si48rI32gjFY%jxe1ZyS?RHLQy9d1<=FA%(&MB-a;314n#F;kD3dZBs5 zjrm;ZjYX}oavF1QwnRRNbru-ozo_AX^)+ug{$d>}FdT=%wlJD}|Jw}aO;K_nH| zwp$pFOP(8({$}0n9nX<`Cf+=|>7qS?&>ICvL6R3EQhzcVD^6ym!!AKqx?6r)@lu75 zTtlv2LR2o)F1fGXC(>);ne-kWP->x=tE3P8yh`Kw>u&x?Ixdz=(xW=n8Z!<7wfYZO zi4yz5L%>fk^eiM@1p~R3Kjr(L?$nQ}d2_u8{ZU`n2iD6-3=TA>EjIyYpayG>Z-w|o zT9fV~9uj^F>AH-VUFJFS4=fCG>T2mYUN5IOK>1*D;>CHzi6iYU!V}A)z-N_G%bb4A zI5bS$4XDJ8X4Pc+J&2WGfO!#S{S?&a_VO7+;KIgE_Z9rj^A`S8t~1)F9itz#vht2W zS-X0N=W&8jdNhtbN`U>IyNw(qYwsuM(_f#To z&woCI;(#h3{Cz6d76Wedi+kNHWBxsIYg4=H()F^#1wB0;e1rFPf4-BF!y)3=1Zk__DOKy82LoKQX1`t_dGvxQ8t)9U})j`|!LKPvaNkYQY$;!9G) z%=QauLqf$>xx>H)MZ?l?2BUbm`Bq?obu=x|iK|J{aj9+z)9`vKCW`ANXbj1Hvl8*V z#dfN4H|jbvG=>_9a4Jg0DHNKgSz~~mIrg*#;uKq4SBPpQm$6|6C515wfU3PajVL7bS_>nr%szpDgv8fM}y519wJbR2_w_~KyT zK*b9mN#+ec#v0lP-J`VZF5jY&&X?2`LA2QA*b*q$8#V~BTGa`$twQ-E#*g`b7Pc=O0<`WDR9CoQ&L*_ymf|1f+(fpp+35u+}#sBF=Bv`o83{ z-2dSGfw=noU)Xa)JIpa2h$A7M)H--L#3IFd`6EmDArCV;HOJnB?opWBb-mIEd)VZg zYd{ryVUKsVQemwLs|F6tGbmA&bR3$pzK^)R4=l=q3WH@JGR%_JyB!+yO~zYm?qFX_ zCR@6TXN4w5U}QudU}sY+5|X?g96pALu#@Ub%k z`REGEkA~8PIEFHQ^>&z%g$b;t$eEORbzHrirNK$BDjw1iU|d#t4Jjm&94uB&ZEf8V zEfJN)y}K}_1&8h}$xzn+`6awOywo`QUjP=y(8C@cNjrk8d>g6=sGM(K*>0VZIO}zW zpVkl_+ECnGm$j!1<4zQErS7}o*8Y1#Cz_}I=PI=)U+T*nwkYdp#d%TZep(UT`wFDws^f~NsZi9J(GRduj{vWao%D;!Gm+iosJsM z1lUhr##Zl|2{AI^gjO-*fBem{Y8VzVBWNm-R%Wk1h z^BXo`3B_DTdl5VzS$;S#gE35pt8)gjcMl1LL!~BreWa=3;9kC|!$`;}I{R~4>kU7r zb$B)MI8E=J3%X3BA>iW{xtzA&v+$H zlFDAse$v_vYy3RbauB?>=c!Tt_IJ#j?w{Q2oK53}*Y)3XxtqPl|5RP7(I{VQHbUYA z%n0cHl4h7jPKdGQC0?tuj$BH2e-=~5yNl0#f?NC<^q%l;yYpw`nUuqwS$AvYG4$2! zi*?6?*(<<3uK9B1C@SiK{EuNCS@p#u8QHN9pj4Khqv0f}{C!sZIk)y&I&6b2O%N8_FG@S2=dT(KOC zUhq*^Pk9~kYhEQ?R_2=}_&Ezc2lH15UR|V}W<5}!8*h-qa1zsf1QaI-2qgrp5tuDR zm$03!3EP8cptZStNI-*z##@{Aw2t zmr(Co?p3swq{DNO=RZ%sRhx>fN5?&!sX$TEoXngQA0q9B<|UTn9~Kb`vt>z&OoeP? z;et0SXgieR%){e;q8_WfVpnTQ_g+wRAJ<+4rQboGqVi}mlP4P!8rG=CBxB#8+JI^o44?#TOVj>={%7sMTLU*0>;0<*iqdlr|sWzAjCDS#q( zy?b<1@fNVcFxrS>_jc;yw6cZy)qZ7=aiW6@E7lkBVfGGXqED_v@{WX7~R zVI#AeE zmE+rmehi%#A_9xs>ac~#eBJK22+kfe7#fDa?R%1Uh^FE=J+$}jD3Pkhd={z|*hhqY zRNyCC4=)rlSCzD*3z;^or+VE=tv+NH)yhHICgK>8HK zysi7RhtFxkJbshljrCsfr}gD7`d&z!)_K*=ehta^>N;Z=pwu-)w@d8DA?P>-X8I#T zg@uMKwjRW1<7n%h`>@T&>v_p+`c7dCf!~%ovcw)pDSm2F&|q1-{!d{5OG*9nRyHmZ zUDD|{!BD*B@3(9s11i$qQaz?R|4-7|?S)1PWR8l(9#Ea*iFA>RcX(u_jE+4S?tSat z8{KWz(lYjbyWbRg>;C@9@9)mb3k^EtRgdZG4kI7;C%ykJl|OtNxI!kZ zR|U2RA@l1c@w=hOYN{o;M0qh9BXs?GQM`sI1C9xeS+|=yCJ(^)sskzF1N$;IUVQ$V zoJlpQ{zCxc?=DvISP?)5kWs^=c|D2a-hGwANY`5Y7^`o(Xo`8if&=|m*Cy;%?>ie0|>%U$Z>RNauL0{QWdqUptKi=AF)sqIza zDAlGuxCQ2EVDx6|1z;iMFi$!~6)t#%D^=XL(l!RshgJ4vR7{8eKor;@w*jo656%Bn zTeX+n_+6v>dxCsjee&EH*$%yj1Gu+lSIiuxt zd#n{z+!Yi0qaf(#L8voQsd4xcgi&m#w0ZYvrK|nrs}_9v!-rE+ZeAx}pSd60cfC=* z{rM=M|M9UcU0I7rJHDjau)jNnQ<{GMY~tLXv%f1@>#@>MgcSvi3}l~6GP{$)i7}HD z`&WnNaDCIWoDB1dN`79<44YrJGI9ZXRo4b+a8b9P*G|< zZaKNSvQu`#{N}As#Y-8Mp`mQ2@eNJYqOa?X`F4KuZO<;%r&YWv(|xnBhDlhVP2Abu zYnhU%k4L3&H@{Q(RhujMympQjCaEu7c>20s(o$K%CK4n67IAJ>UOsdGtBkd#XxL}{ zSj4oAQBSyjrziU5298e99Kq~$p@gXIX~7R-xCI*ag3gvU*7Gu$UP1M4oKn0Vt5y>a zVIv{y3?sLrr%=Nr@Ni>rZgcRx(cSO0&7RV)!hE7uox`+f$Qg^IZ4@4 z8|Q!*MeR_H`$Mtye?C0(#@?UA@F#ATcprN^6FYdC*Y~~S-Q|4yPT@EHq;o^wr_M8> ztE&4I!J+m=94e8r?=q>#)CTv|`0H57+hgEmESP!}R{vCf>_UA!6}fe$iyo7&ex1+~ zyfmnhUxRc95^EuOKqiEZhw;`_nDd#1qckz_bnNh}YO>qYg#~7$7`Q}*N>TfLsFg|~ zYPqRgUs`m8>|GzNS7v23^o6u7XqFKV&wZuh zRGZ0W{}z5|{j!=tVkPd)7|*-aTcsH&V_G9YoFSM`drG0pF!ZeLc{jkHb11rpipKq9 z>#l$`&@Y%RFozv{inoLfSqb4<;pW$ysAh*pStL^Yb6LV?Ejd|>Y?r$TwuGT~HVjMH zy@?+|$CiWAJF*B3H#dkGC`FunP_`+MUP~acF`0*ECu+isYy%_^HOU$L} zzjstHPt`P_x7uq~1JLb&`uQJZ!EyeK9ovj$+jKrb9VHkZTv$=#cZSo8wd^d+rB>&x zN${3-t^?Mgs|J3geTD-M@hKOwQ-=PuK73RrYdSOP>lY+N#*b=Jh8W0>YH4DTH$qow zQ+tq8b#K#+^4-SY-8_0Wko#4=oOGmYy(Vl#Q_Yr@H|%WJ#0?33+UtM8go#?18{*cS z*b^@tGL0a^4nGp-lsT*4*;Hp7e&*NYd7hF`Weqs1!v zqMSA&!(r_-3|Iu?S2Mm5x()UQV_bnIF9zGr3x9V7JicKlFEZSrlLvuD(A9(qnlE$l zyuYRcjKwU3Pfdx(!b|W-LGGA#jag8zJinQ@_)Yjtn=?X6_!-z=%P3E;Eqmfwd$Ho< zx6}S|iymsN>8lxmbgR}3h3T0?rQ$_k(sgW(&Z~k;MM4(g*56ZIqFqv`kj7;TXcAAF zcicXd>yp5v%EzYpe41>^xl}iJ_0Z~f{-t3Gg;P0#$j=BC?kcP>iWC1W&dL~`T87LhUjWWbeP?w*g!+YdfnFY!mw;zc|PJ4mwE(c~#YDjEWeXSK!8QYKY=u&FB(z~Xi(!n$u9t|BIl8%JL%8+;u$z_^-LCH z$&Ow%D4=XAP|j}BtyR;NwwB$htj4{lt>4z~pTAI?pR6X$-YUG~d*)$3jGl(7y5AsFu7cEe3ohW{)g6}bev9M zyLl$|E+QIWZg`Jnw4f1Ulx)hlbU`wvTfO)l(KSj{rGR`3O<5Hb>7)<^M51`QlG+}_ zx{vEj)es$tEwrxYs$WZbe_tP%d&Tn};w=9N7T&)K^h;93%jYesNpbSh|eoM{q&0k66c1vC^#LUR?3g1tcah|$6lWZqpnS}BO!Kw$HY)j&Wn&w zL*Uwj+$q0=>qq121mh9bS9wkAFFeE*)AAM*;eZRCD+=@+|Hy7$yH?7`luG(EyN8^XuSd5q8N{See2O%&?8kK8g z+izc=p7T-^j|2ci|;1N9@b+@ zcc!R(o+g+`H4jn&nxj|?(hGR34i-+DPmYJPiiuvxHK#%YZ%KqT0_b_cLNt9|X7Hfv zVU9P{PKgOFqWV5w*V@~q>2(=s#GT_C8@mkw8IE}B-4_ji4%~l(tpxgm_p(o(UG*f5 zBQc$=N3OlLc)ay2X-nskcEuKNtm!V>u3|c_+$Sl{R*d)z&WC0V@x!YKw70Ygtq}z8 z0vuJpaJI|+yWe7?WH*S=$#&;l3%WAV`gUi`#r?wQN~x;t-Ib-%vp6z1VVxKt z9RIQ;qo)WW6ISmA6r-dz&&j@c>t*&kUY|<5Cjb1`h{~o_Yok%_#swl^;P9b^R4sC} z9M#G1-a1Bq5){3Eo(n?h=C<-vTHLd#{#VsjuX zPsdpr45pzvk358tX~vjBdqLOQBEQjEidNe0d!qp6n1r#zjkU5 za(Rih)b+HDA|2?3F>FqXR^ z^2`yVW@P}awr zR@L?4`r`wVWh)HkKAi+)DCenMM(2-Lx48QYEdHYsC?=~Q*APi!5Z0llCzO|V#2{bg zFq-mPBV#LT)eV(0RW1LlCPEg_8tU9qVE(tgMUl#I+4no8-%#k*d*_p_m_+5vRh!w0 zf}G3UT?4;JlwF1M$43gIUP?O}yM2OB-L#9dVm5U?iPKJ0R;Uc=qL8Ov&x$y zBY7#mC8=5ym2X}(rL}6dKsm+bgz+$&wqND99Lhd^85Wlu|N4o;m)&g+&E5UqOc?lM zsg(aBDZK_CTolm=smF+^oypGk7|%4^_^X>`b~&fvjmrsuVanLcxqRoO9jHzD zwxz9;7p)t@y~{bo=c&tkH*+q>gl;QK65RS39A~HG>IabbJ=*2&4`!lWv)QF%(hw6; zA>;Z$T6_{YGtr;4QSFd8{9Ui*5YufkGGFEa_V6@_+DF4;aU7EUxl9^m7zOiH>-tXpmP32Uia!8yd zkZ;YzZMO%6CscaWF45Ah%o|7&KM-Ueyh7K5*fkMpcc*cIoR#yzSqGljL0)4WVGBS* zT9BWFu>Zi>9%=VthaalX!IAZai)t0vST|mzJonnY>up>4&edmvj((qOS?a(Um%OzR zGRG)!#+RUBV#KSq#p(n_W&bwfE-7((ao&8`w~p6WXI|5nA)E1BlVz-lSCX2(fh=|F zJpZtsuPyGJ6+dE!Uk_WBrM_&Zhi&H(n<6Z0( z`CEF0fSH;I*UV#P&9{ZwsjA$1obWi&BI0v@c*zxOrxtJjYu$$&#KDQ&Grt(8w!#2v zL3eu=5_T4$Fd9Ki8y_`m+6wHz`7zPp-w6D6=blNZ&R<<+F6k9K{O0o~1JAlS9`U23 z=^wiRaGMH)nBAP*%Zf>qz3kETP0qPA=P2CbMxBe!rEMw!=cO8fQ7xa7xM!CEb1`Ek zE1az8qI7rhNZ9&D(GixRs#H}^ue1Hhv_@xlGFP?LiDcq z{Zx0}Z?%&Ox@cCKw;O1YM9Cozn(1Ef>arKx>N5j7wgZ)rJHhR&o7MS9z|k@~(TpKz1k z&bLHx|15~BWDO}_=`=MxI9IpPOHAjnG9V{ZcfMU1hO>LeGqd-P(^$wo{yK|h2;#YwiOq}?&2>K#t z=62r1NS%c0qYGFRdfIE|hV-l?TCJdPown*c>JH#dbT&^nn8Dz!^ zJly-*AZ~D12eW4UBIr63Mb zeBgBfBr#0GgTF_*M6*gckR-Hz{%vUuzYB5@i|Z?I>+KYzXwdj0_i{=F@BTH$bIF6@ z{!$_9I@qnr5|P^)6eGL@!-uocKdGSHiUE~rbh9aboC}z4-ZD6AmOhaSu%9{tKeX#5 z`%TTB5doRp@ey|WnyU9Y`4sQI?Dhr7f7!MCggD(KaELrZOrfw7&EwtoJ#AA1o8`bZFLCyS4*({a0M;%lZi_cNS-f7-)8-AppzL57(XEfl14LzIM#3vDrH zjzc_i-##WVRDYc)ghS_$I^0GO$*7u+b53T-b!8aZ>_9lU~Gdc^TY7a^^=gayWm34iEY-O7de zp_}Kj)sI5NntoQAjq$`^IXxK1J1J5SG8#f^)9Hq7QWu|{-*WzO!I77eiK<{Eunoz3>=yW`) zQzldA`G{GiCxR-ygFQv+!vmogCa}>sH_T$zxmbQV=wx3x>E%&&hV^pJkHQR(!jymw z*F0PcgMvidi1q}XBJ1F#)Hj!$7+z18(_r9Oq^6-oxFy6&Ds$J1F6gelIR@v49^M0E1+jKOL}d zt#s~%ApRojad`-j4XuHi(s+>j*BYO|D|`A(Knl%a{nzR)FLVLRw6KWZhla7~eTP&WU2&pD zcS?Wv#plnnJ&B)nRTxlS`O)EF(Opo918Fr`hA*V)u+TBx0j#VN{aQ45>p@%(*g_4H zn`LCCgKz2;Yb&q`&@JgbTCzSECnMoL8s&eh)kL*p%lO4~;PPrKMH?~TV7z&Ce&v~I zPPMw%-ADJtnO{8n^V>s7a;&U=agOXigMm!SE3R0e+$`CKzxZ&9lcQ8V0Vl6`NJ6XD z8e+qm1*Cr|kndUN+}R&Hi)s;4zi*p44EX5utDyuI1{;f&1h3LvmhcK7NY@o^I71;c zxnFDgCbc8PRUIfY;Ii0I$8O#|gOB*O2o5l2G(^O|8tA?ANGP~s1IEmC$}ipK2)7YN zR)aS0%RrY$b?=*j4mcc!5@m}RjIz+Iu+_uE!7vmjb(@j`WOP@F8*GG@e=Mc7`;Jd~ zK$xr*M9cU^fg8r0>;qLz7_Xsf36=&AFIMzw_-u*l$eEEq`9A-h$-|Pf>;hdwEK3$5 z#re*X>$R@)nj7%IHD1!k*iy>zWWOsXeM)i^#QrB&a_g}~)awcl)l$op8@`sB6u$%s zm5T9UlrdZym-{=m&=j2Tzt|uk*4d@iSo>DmtSj;9pEr_ro+2)^29*d*p3x;3+zSOY zdD(}}i3mh=`2xTEpr!Q>No5M^wzz4Nr*|6bpv&;Vv&ku{(YKvupI(Uxc$IX(v>?av z4nJ!;MG{REu{9CSo=OI0lMq_^8=AbLU{NCbB5d(raz$A$UdE>P9>~S3#(gibUmTfT zOD9Y*Ib}A%vu#{kii?S3Zh1Rk=_D4UpE-RuI+-F|Rf|_CXZMRn`8AH~f1{u-JjB9% zgE!BWWzbA|pU)O`Ju5}XPdy*aB3f)0oNz|NU^w9I$Q7vXLob&U>2*RF;!PPbSX;_S zX;zVlOhEYYUeDKMDy(~@O!E*6>5hYVv{4LqKG8FqNVcVm6{3jbrRO49yyryUz);@4 zFd-!IGWEU99QC$wMc+G?a!xUAAr#~h0WjPHeggmjUG4@&T6acG6M|J>42~pYef+t? zUsAxy&xadGL61@CA_cRs|MqGK^pUrtl+b9Rn(LojX{GmazJB8AKNt`B8gWnk^6lLy z5h$w&r4)xExMe4ItZ6ed3S)}1i4OdbUMgX(laA0hYAs++0oOuu$v(GM1#$1A#;qU+ zqzL1C-+KuQ9=`f9QLWB|^r)B8ix9BS322J+FSnvOPDfB@{&6dw&n%Hk@Ugo8pn|`X zI=0-Jj%2QgfMuij`Y%{=o0Rx!c!Vp9Vea&$sR(XBc_`Z}J;bHcu>?mx{xluPNWPtV zWS9(a?%%Z}pw0U#VI6QLs!9|7@$+P6B~NmDnugDPekAx_##F$=geGTPzgR8g)tJY} z#DK8I3sHvv#4FGhkZ-#QQ~j7^U$f*;DHQ-A2{fXbszbC8`I+sbM;*KS1S-% zk+E2Lw+kg(E7NeK2_k5mI?_kH?_iwnhV^Qs%d*zXt!0-pFW@LLtxjLrS#mT&V!6A` z=^R%g@*(KTDXJp_iT}J@AdacXvPKy6pT8g^2M>u>DLZG;d#SukNoK7W>|J2)cW5@G z6^dg$D}r?B=qnL*dzFV5VA7bsH6%a-ZGda3+~OSBQ+XSaSX51ssQO}ih6pyz_V8$Y zAkBKYxfFk!*~9f>QDE_llHi`7n-;i-m>G5yg!$k@gdGyLkYtnoT&}xlF1bm zf6gaQ8Iuq&7$BNLDICGQZ9V)1cW71^Vu;@d7)15I1uYbn4dC4aojhGy?tBu>-f@!+ z{g!Z1yqCW`sLt%)6of06-J8G3{F_J|oYP{mmTt#^x z{zs*Xp+4SKb33&p4_|>a85S?Z7FzlfnEM^AnCZttOhS@YCU>MNsE!IieYaGD&>nS;7So^5xK| z0*sJlxST&uXx>e-514JN!%Sd3X7{+5iXRoctmIhLv$-svl`tDEG~X?7zmW=!&w0KI zyajrIbp{klhHqR>|5?9`ju$-*gN8x`CDNR}k_yr1x{-4zHvVIW= zCRwkpSYN@xiT3f-okC=^E!}7S6^e=^Hipqq21(kP@t+JAC$AV6+s9H~Ld-r~Fque* z0d4`cQHf;C?5LRsQ|6 zfmCzOii4eUy0o(u@6nCtC{0S7+ku_`0F+JE=viqoE~c6Z5)wlcQ9IGRUUfz6?~7%{ z-x|eMLR9aV#yPe9N}T&eMBC@D(^6oD$o`aL#!Zj6?Clc`spmSQ-Jl3f2+mNm`C-yt zO$CXDaAJ{^J9%s(`zT7x0$X4V3pZY_W2lPIfeC9t(u-R2sS3?Vog(@cYC&&VhRKFY(Di0K*OVc}-(8R7jN)lfoFeKvZz;tnP{KNJa z-`TD-;~aakzr-ONMdpl8{yPig@u6v3|9nCc`_2r- z!~NTO4E0xqQN8>UGq(yx|DIoY22ihbVr>rnx7Rax&kG{6S_fOk)t_oqa9Fd7Ruimu zpD#`cI<}1YdHl))nlJiMy)-Z0fL?zQEzdNtJsmZ@|X!dfax zym8{fsKcRBUR?g&VXFGb7}w{Gf~GmM9Z1;Ll)bW^@oO>QKN@-YyclH6zV=HeL)oSKz)WzbF|gl1KlXdq;^=PB ztVFZaCs@NRjxi@`pXw=p=#PV}&ATEa#D9ww8{4U312qQl@&6eL0#8bgx5bdf|9u(D z6Bv56 zZKsp=RO{iH+YWlA{1&0;SxX520n0_7JYI9(YT9}?E{@UbjF89(z`?R5fFX!mCt>sl z99EIyRyObEUz|b47Vg!NGSwM&HC#T9r#69$QJdXob8PhwDy2dQhAmd)t+m_J zzSK-D`=v+KtoQK+)k-Rr;t27oYMR@H=+1i8#zM*m_Z1>`;1mdlAdw3Z*8cS#{Zx}= zultO%Zqv2Rh}Fol8t`~&X&hPc0S*F7|QN5mQybL)N1dw)+d=DO_5Ne17sto2v*(Z&-yjuf}Uq}~pA5rp%Kx9%h_BN#OZgWL!L zOk`;L>jp*R^y2*^jG$-9zsS#dmS`{ehZP6?g3f+t94FG8rt~Z6p1Vpa$tT8AzY|yY5XX zt?P5Y?+WL5I5O@JQV-oi&w(Dx+~(Lzcu(O}6O#VDL2~rH!GE`67B-M`1TsfgYbZ2= zaX^DrQRXd|c-H*bT{+PzWcmihxGWeWAllm_-@CB2DOK#rUv`C9UEYU>`9g(Nd|P8Y zb)aaHcV63+5O=qgYwu0~82X$$Q`8$XbJs*0m=XAmvlyg zSZ(4$j$v~3PpAuJ9gth&Dg@=b0)ejy&mnCEb3h{YcfkyA!?VOpcctB zS;y*>f}W6AH&m*A%{BRmMhn5LkCgRJR>tkS2tmmh{|zsD{%3V~b4#e@mTSXb{rbxt zX!@LMA`df@>+5c_p86s%L!mPzkt<_At>U?OA_t?g+TcjnW192wJTB^59f=CJdXa*N zphN*z`dE3Db5lQ`$f?{oH{+Kpaly2W>OK1-gvcy)Izn!#gO$*N%j9`gh9Es^X$}7a zMNY}WnM9)fBZ9TxF|0Mg<>!aXQ#mJHV*?w?`>$PPifW=%tSElG$<(yFbRnyLenbw2 zD^HO>WB9^M2Wh9jVCGbjajukgbxVdFt1dYm7ej4a^|q7SXSP2dvT5BMrTI?;J3Ue? z_**;JVC0!+)*n~qxFOWA3vx21tZWTif$41Tl-4wS+=>`Vu|bzmb!Ix-$Y`RmR2L@% zxzAI|OI59z-ZOLzk3t!j3i-##KU1w1A)`;!=%Khw579!C>z!1GuTZUTD%TKZMfN$o zG!Fu2qXiN<4$Z)=tuXgx#qx5_!n1PkKTssMF){cq?LG>7oo6W>^J@V@XeD~&(mm1* z;%cLAIaTm-6DT%b;E(J%Ve{I<6vxVkuA5{Mpasd5m0P?!3pbUO^29vqqX9fN@)`Njm=9TCVr1pyP&oC$e_m z>vD95H&*Df4g4ZgtSx5E;kttw(M{;qz(s0KM`nU^_zePDncl=Xv)EKU%PKtkWAtnZ zQIU1$5|<-wY9Mu5syfvGbU%Zt&71q(3(CsC%rM^t>Hf*Y`I0Jz4B^kl7cLyxaMB!X zxpA-dmV)6=Bp@TPTVzDhYv@Fz#Ks;0UR%X*q6~X83ix+j0kth zh;+)8mc$|C2A9~5jq%`8`6_Q!+O8mmvtd7ZMyFd9+diqe(bQa|(=4t_+Bo~kU4D*I zPjXTD#YY9s=wJjJsi0$<00YQLMe`%%tu@_ff0Yi?tJFa9`=mO{6`=Nr!FSR&W)b4^ z+;`+H`6{Teh4nl!2=>CSQiPfp$#!x#Lxug44L z7v!K4n2IXkfxKYjVkVUF)Ax*#(?=hwBml1eDGpt6lMDDO#O-s(bSzTv$)7!#^!phB5x*aImx}&KA9BLHaE&K1N6rw zIJ-Kj`sY{y6!v96B{T-54u~ljWPf<@)r=iu@kcE)Le`5gp%cG^dcLZ1we|U(rJFiy zc;Rlg$DS9gn1S~SQFGy=ElTRXx)O~byu-VZ%!K3-k`&S;WX5+<3lrhr2^W6v!x&x8 z&+7W7$AKluRR#qufOUl1(RG~fxXbs8%7^Qt>-g&kyzxc^3ZZpFR_u&1c?UMz3>R-) z!afURTOhHpo%?;t6#xX*mlFBjSVBTwQk>YRjj8;JOF~O@TQJb~IFxhA?m-u^;+)Q# z%{9@jUYe&CK#AS@c#OYyp~aSkAUvY7U6`G4CjO#={9%wCW%|DUdGy*wtnJX?0iR42 zR7|h`)*^mi^14Eakhx@T6(9eC8ZdIHej&JbqCz^7ZCYKV`W&vdD&&*fW3ojT#camJ z1adnR79=Q;$Cp-UF)i-*#7~u==t=B9JB?bN5`^3+ zB>#SN;-Det)wJvHr)1ZGjppdhM9$gs;~QJa-eKcrgefi~jImsU?_S&Jcv%4LC_rK- z8QT`FVf*d-$v}Ts=MsmAdFHh@-na?fGpU6VcOo40M>-`h)3g>;>RuW0M zTBEG}$pnh|%zc>?7Ou_b@k8Ah^Pa$t{T(l{n%sMatkJE!a^(C0OI?G`Sv&QB0;7qq zJ(CoA=Dn`~-e5gRF^&U(@E}hnWy@U!*B^M)r8u6=-#p0^C*>;&yf=$MJkx%I!c-ZW zD)`m(@XQF@%e0!x5k9GH%7xJoy&A3~Z7PN!E)HB+ukG2+b&m~S9E904I`%hg zEU_&A`kK9Pwezl(?MPV+T=2)Qd&-JGZ4Z;Mntu`Smmp|m-zT2HIwdfscL&>hR3n0N zdTHZ-N-5nNp1bcT#=L_CTGOxVfd9K^NQE5We|c9ePn#k0=;)*=>koO0+nLtA9+C;6 znY_D}$n?h!0^i=EAioHY%B94a5qiGiI}~OJVw3N+vuj3NCIn2` zN+d8$k0qIEYnh;v&`qqk54_Gt`cb}~x*+_)sosdkC0y*K(3x#Ha{Q{!`Tk*P=o|cUhFR&H7K+A*5g&9QqFM(TKo&+)l>IB0kZ7__P59M&rEe|4`!&j_W3Od1@jePW zARxLlEl!EtSquf7u zHz3_bHdIzF@W=Me5;DEiS@`41?9&M5zZQF(F1#51a%$$Hd~@J8yyF{(b&MYCm4LEI zjuPPR-t)>TDK7aiE75B&;+UEGuHBK7(tKC^^2K=T#(Xkgh*;+Jwztrr@0`LWbxUav7dc$5AtpgV2Z)J0hS}b% zM2(InIXoPJMHd<_L*)xiRRiHYs!des2-R59B`liz$SyWS2U%ZVo){=GpTfk_@W2gG z1SQ3BJ_7UCfX$V)Pw&GICFPEKe>}FRDA-Y)ShW<)$GgmKpN=aGuym(%F^uFwlL?fZ zPqe7G_o?E^Cqfu5j}XVZsdSI)#PP1)K>SQb0Y>Q+0Wrt5A|)M^yOKK2#j$4iGf9o6 zZ)en{t#6KC*_6>kH%sH_!!EPw;IFb10*5j9EIjiqz0!UcuLGseeq=J*oB25vk_b%{dp8# z-EuVw+*XulkjH#-lymv|_esn4_gl)iux4)Ll`(FJ;1>S=;H3M@ErfQ;QYf>Z(9^co zr#hNVRLsJe8sQ7F!kWST1uDrz%$G7H+C)7HDdK#JC4-pSgj3&=iSBnh=h;->ag^Uy z+Fe{`%MgCHySR98u-_LUDPuQUCyM2R-eo5?qD{+usnpV5J0J)RX8) zT2ku=5Pq0&OcL(Q4hmf_Q0U~|3b)Gs^s=6;%2gEksm55tXNlCE8U6w;ma3!{6=JB^FjA>D46 zF$`Z&bhUS*#9&#z!8w$?%|q?&;zWONe}z__Jxry^Hjo;R<={nf&8bz#_mSRgpU<>@ z(}9&5FMQ;d?#cChsg5vuU_?959A7-{e%=xv8gNAv^uRwU1byvWx)Xb2yS3ns!QZ@- z&t_cF-RP-Hdnps3VO+;3m4-x|U*B4MyKr82UFJb?KZ$*r+Lv1-nI{gvfoo|@aNVi^ z=J1ZwL0gP>BZk{vL+6QQ0fGU;=%9ulc~?<7N6@r3>&5UPx|N>G@PCDa3}Uflz%fCBe(HU)^0SD(ma%PZo9DjB&OQF0 zNA}ObFna%D!f2#LC&PcYAE=Ns&Ofe7Zrn)!&9r(Y$ld<#iX(Xp){2%L^020Hdr~xO zg4hdKgJbp3n2?O5-nZj!{NpdUR!s9yoyFWeX2GK38kdrm;7lcT$qt6OfsX>Ep`@a$ zNQnrDmJ&n+B_#wlT0lxV1SJFs2^l2Vtr zKhOQVc;4^Dw%_;nz0P&cxz06g*+{KLM-$yDv>5{#V5C__OL7gGF>eGSW0d299Amer zsP$z+Ndz85$veh_Y~KI^8GHb#7O@*#e>_d!!k88-06cKgQp}y*z($}GJmPb5FTY?x z-e+xmrYsfSkAgu|Tf(spB=2P3)9^-AdXYOj56JwyXH*$c*}i2x>gkKe4if;pu544k z!(%HK8m07a;G!8!arbk}qbR-&f971QJ5+q~5{2Trfwd(evVDBAk2brPqYjlEp78A`R?s8dEB_FrVQ69qaEd;)Q;T zR*&puwV%ab2Dn4PA$u(n&fi*sv<+EO_Iv%mgQ>aGIe`q*v7_7Y(7lvQVW`DG=q<`^ zqMB!0IS~C@bPfj&x7Kh(vW7*tQQ2CY?f~2x)XSNkDS+N|UUd;ti37cEb)%qjHz}WW zZsG$G6B8Jd6+U_Aq4T%p!7Y%x_rKtv*ls^92h-;+xliGSp}yk!(Z6Qatz*}j>Fk7V zb0fYmL1baJU6LZiWF1Y9-G1pD3qP-9p5+d58@A-Gu(X6iGm^Y#-VMb=4p-*6$KPp% z;UoSn`6M+>vA{1F3iIiG=@+E&A{cnKdJXFWejb5c!w@_TlTq^bAkw_8)*pY+D?8Yt z-?V)sP?YX&5$H=NFwX$|{f_~txqY}eOj@-zdh>POQ~KCjtMTrCFby#+TKy^0`!L0` zyjFY^TYn1vQ7Wvq@Wa0`2?UJIXiiv(y{~|)M~_r>S=#XbuxtB3$uQi5fnb4bVcZBU z5V+@QiP3O^edUhVnN*1cy)$Z z3q&lYPwYCuw0rA$B)SwcijPZEcVjUz4CFqUm%pVG$gpkd80CL$QAXCTz(2@7fAMW@ z6n0OWfY!>xKi^uL-+cYEXnhi_RQxMOuO_Jj`lra$^;w3VYt~)DU%F;0d*9Yso+26_ z3TMA!?_(u>2Y3*LwKh4u|C4d)XS@lynu2X|c8M8&EkV8geleVHyu!E}WQ(_j=>JV2 zr*pNaL{P4Fd8Ag(()*#N z5ELga4SHDtyT$E3c#nDvve1osJid1Yda4PZ{k)s`{j9-FB=ZX(1@!dA$47Gs9F;AT zfo?)*Ac5@es;nMr|`t8{;lg^G3w^^k&V<{c*FgcB=5E#^S+}KD-L`AS-Sn77Z4(5*L5dm7+gi1pZUb@woDy?P^}Nu?QMq(BZrW>(1mZt%s~6dvoTg=6#}(j zWinU2@S7E=N#pQXP1Moh4k#H0u5m2mcdny~hg7cNO`sxyxm;V!LuQ+Hh&s z3!m=s#-9Hy7gEjG6f(<{GR`r%PkI1<*>fhlI$4-_&_lQZZZK%EK$}IA-+irCL#|^@ zoq|JZ2QV;#r;13KEKkd^E2R6`ujzou3X{v7 zMAz$jlNnGj3qq`EP5qDtwSm*$wR_PB~9!t;W-htKycEEl8(vPmr_gjmGPFVb671SS-<7d z6U!hH-z}xEt>^>)}%I70iIXHZ(daN0lhV|)HYgNx2AI6TviS7%cJT|<_Hm5jEUV@l42<2WJ}SOD#h_Jaj#HSuV;gG9Nh4Xga-Zr zf-v*oSC~~fNlPbTyMep-c!Ld@fFlf#tYCGoYi?I{vN0g; zS*P7=qu;-?kpdgc=8dC!@Z;zHV1#iwtg)8{_j;_F3`x#T*OB2%6d4=KsVot~BSQ9T zfIF)LviQY7(8&Gb#TLO<1Z<071<4>tIso^pmK=a8BKtjiDHH#4;_r^`73lsK?==&e&;Ss6n$H|WMQ82% zJ>35n)cw&(pJV1s@h^5J)*+kcqSEMTMS`YQ3ba0VltkJkBTM*I+3g)v{i-Unm0*Z? z?kq&n51e;D@j71pc7SBm6CfRE^>17xKGz?jh9ojUC04i`@Fgrf)OZ62QlmESPh-|R z8>>jIViiBF+;2Lw$x)aPFe&4g3lOy)qVtrvUw0HW^xM6!<~=LAqygGL;C2t?);$ce znl8+4`nR!ug|`STeWkFoU+WQWg(JrXBa$i_??2$IAWI((g@^}WHRc&B=Cu>GvkBgU zV%I;mT5(ShPim(L^okG<;sW#%L|i=7O)0LVdL1YdxmSsF%@eVvOmCD2*C}42r(C79 z5&CyM;S^>Jgsw7k=GF`|X3w1U_h_MF4(zg10O2A+FlJkiggJUgYV?!wdLG#Pl(o^; zv9kj?KDhwS@l;Z4K`h0I5Cr_(v27w-V2?06yj#D}GrrS+!TP3bVdl-` z(sV3>?y{$@k`hg*rgpZXA6RK_{izsMsCk}eHs6k}uK83s&enQFxvN)hVPb zNV5ql4EBQ~30+v#P$o)qW-%r>0ketl!dFwMT>09%(?T89NE=Uvaof)n<;r znzZ@4AW2>O##j)f$W)r@B7+s{|8}RvD&Nk-5mX2rToFXAu5DkmAhR)6X(L+9hv#|mMMY}aYj%NGn45kOr^k;>%|f_M^*TdV|c0l2G*>D1YJ z8qN^B2I777+u1UYP}`5|KCrJ_LF?>?p51xmQ06_|w7VC3QT#ybru%`!lvjMWB`+#( zvFa!Wx(R>7uwdv@_Hk|oqBsv0GFgXXy0<;-4?=5=L)VK92!96Y*<13v|7wFDCt}nl zi^^c3ydnkA!sxJ^FF{{4?vHS^nmE1ypHEklT}UbNvKt=eh`+_<2UjJv@+GzfsSLD1 z&-jog!WFbsPuZinW6#!~Bk{!s(G>ODZT|E2oP3>14m9iNa@B1n%ApH=G`ZKeakO}A zU%SOBL*zcXU6^x_g1E)%;Jn(91{i1eV{Y$fFP%N-MV|bd|K2QLba6As2klk|H4^t& z@_10@tC+H(^Kx+Bixx6vzb6xdS4+#vPyF27ClarY#XifceqYX*)_v8DVP*1`0}Up1 z&)$%VBX06rty38+bAQe80MOHHt^o{Hv~>kQdzAQ#QdzZbC$a(^a_3)~mT~+qLE^;v zuba%C9?06s+~Js+6yTt2<^?U2{l(xH@=Ujk5oO-ng%2@Gw(rwD1I&n9rICXLT9m{K zU)(F_mgE3?pnP6y!!Jw29kaL~BFPvyw19hI#9!vY;Q_f8 zqP4v+P;azNDL6KWp@ww}pb!WeYoJ~*i)ed6RYy9D(SA!9TUX!*qLs88!mpnTA_xw|naIl{qa zDl*X-T4m-x$9I3x!5vlptd+=5wRexA{v=~f zL26E8HB9FLv2-*7C5!6Q$B|w_Xk%CEyz==R#o~kDxzFD9d>|Wd`Em&$7y6BoXOrXBQs-zB?5oIwBftarmZ`NqG)i|BeM;gJhxj8ua~BBgrRB zEF4~p;oL;m>;Sl((nYc(ma9jo(;0X~`n&m<2c2aV<3VEJx9HT$H2}$%S9jTKtjj0J z#(>N*GEb+X-U5#}utLpundy^h=tW}T$4YLLxrt3>!Z8k}@C{!)I)T=BMK726Dk9{d zMgDC^;@<-EvZn}Cl7gcoR-#{R$N7Pt+`}ennUSdo*?(yqZN~Qo_>)jHLB=D3oDjda z{-yO}l8BTQmdvzj7@`OQy)!$V23>HQK3f9mt&VaXBsd)U4%jqOv(LLO{K zs;jEsR+zT#*P0#SMXEZD#1~O0C0|YaWC_};ID*OpPfjZ2jKL(hK*1^+{!VB}-3y0Ga|ai{baFZ0dV{fH;# zLDa9AymOrytPYU)vI7p-kpb;;5d?-m854(Flldf3xM^>yvqcAvuRi*0UvguXz1SrQslBNrS#Uc8`0d*`z zxeRVJRxr|tVe3~chrRazmqnIV zgxLoLh9WX7v{2Aq6HGD0(2GIb{>&-^PJa`^7%pDfmqjmqkv}f~&O+}huORi$j82Y( zeT}rd-95!y5iL#{k+8k0Zik#y^{_@TgXqE4ZX1~I1y z`#k%Y+$Ut;nbq=^<9D`1X9$HmyqfOWswJHW*C@4HjH|@8Y_HB}2pE^+Qi5+yAARv| zF$t=DUYJ-J*+;!S@^!g%sp!8I2+jRJ*jDlLha?4sr}XGP5F%G$neBEB^sy?tyr#q%Z0XQ4DE}9=k?k7Z-cwyj7o`pWT)C% ztfwc5tGaT{#O!LO-{CPqyH$&o)W<=<0_$!>R#%kJz~Jw>c5c*XNX`$UHx0NXuF;J{ z;~Rg1O6QXpuWWB{3Mm0)N`Ai@3!(F7_Ks>xuy}-l7TnsDSqKA*c460SC=mwGl#)T9 zbTn}0sf;8pD#aWwXvs}f3QbQ|a-kW(#0r>Lt|b|F@Npq;fFi1=>29?1KJA1O(VzSy z@n8WVWEC^LffuT3H6lJ(Y%al+bF9Qj$>M1pEGiKDT}zK@tH2I|XyR!>tgxEoSG&?9 zdd6h^HPyzyiU)IrdVCFzg3OBCOmPiIlZ;qN`@XBjKLYkApfqf577x}*WH)n}_5FW3 zpT~#1)K-v%(|%5=EH)z`0SU(H z3!i0n)Wwsm(8XG==dY))M^rd#acZVhi<7;vd@;3U*>AC)k4ry#L-sQQ_Acn)*{}4D zdZPms1gUm0(Jr_XY!*R0JN5WOuCeVs6X7vx=vevjhrP!mBoYdY#T{m)M5RGlNmx!@ z)-9a+X7C98AIQTCHA_X@=k>ZpfTaa}g&?@tHMhZvlzhaovi zpzS7HsxDW}4S&CRxR6+F-^(PEQpYdEJJS0Yry1tAcz^&W+r%jK!-elqtb#m&&Y@QP z`mwr8KkqZ}%HgO$2+x)*9pBRQr>nmPbza!dRc$Qj(e=?wN!;%)CN+Jr{N9^G(YL;o zFb=($l^eSFp<6oJ?d+}QpRtj%7<44jGw#JCEYnU)onkYU9|hy3-~dd)uAMx{%xKI>V^G$pzRoKu8hEo`VrLS}$y=+i|edKr>=Z}Wuj zX4a#b4>lC}hABo>@3QcYj{^-2B z;WnNY`-h8|le(*S)*jZb>i-|S-1+ryfgpcPuM+O@a%S;bd)K4$#j&v>nu{H~9N&nJ zF5U|+S0MI!&iWa*q&#Tfb+r0dNy7um`>8Wp2xr1?0!q)prbr^e_%5x_ zRAV@6jUGBbU(qrxRIP>&q!@CjqItHV1!x^?f z14swT6$#MC?moS#z&7(Q{z<}iKcxU+0Mclwej-1C8hAh|?>-khiJ~}PhNc0B@V)5A z3aJJh&X(5GLW{(*LT`I9IxQJ8a<~MDWj=BFVH})V%{ptMo~#!$Q@PV~5}0m}#)swn zt=s}^si~=7vEHXgR6Ff3MciogRNG47nH*0w2isW0rKq8g9HnZ;1L{l-wXVNrp|!{Q z`8zG62D^$2iQjE=l9$^6?=c?|F^^AqLV0r7lIZNv($6JUHe%rwVE6TxTL8jRG8Yb7 z!3j;hI-!W@+gWb`>nhstPyzE;gt5y;|6c-R<+>_LS6S*6x+stj1Ygf}lqY zB>_(#FvNF<(nOX0-7!(NwRcW8uIj6|?IU`@oVIbRg9cB`(Ha5Y1FI%V^{5DIVLl}k z@nmL;7ID)r001?BzLvUW0sfp}hJH8GS@Fw(tN)DJwq^SF8nagtNIbM4{4*>oo`&p4#JtBg?AH=iw9*nU34ZPH6#`?jT z#u6a#)+TYxN*pg^&0C(-;-sWY>S!S&rr*i?-adl%!9nSQ#q%Ql?uCgMp;x^~moQ|5 zL7l{6I7}#5?Oipp<0OGYK)Yy!}G6q0vHNx`?w)iVt!>oW@5fPcW{)T)yuC@9H0C_3(?+G z!0YcEO3V)k;=7+fC7%+#xoFAbeNthUSI+uohXR*eV?1j2xQF+OV1B@e638foy%u6& zFR9x`A0Fhga}!8x(nCoS=SHsR7%gwX(G!x8K?NEU%DbC9!hkJ}0Vv!0uf#@mNF2%v zL}(V<`V#?<5rIdy@^O$8I-)qzzeZBT-ueO;RvgUGI|B}(MLjh=h~6l(lej^{+&eeH zm3D4ZOP)w2KSESh@d5zPV{bHo9H)C-@r7{^Rn_*3okXb@(;K1|2j_iD$oF#aYF%*T zI|X`68^Rx&=<(y)e*!R28Uj~G64^wA zkaa7u@7O(xj;OhEq_7s41rBp~L)J56;*~fwj(DP|LtPgi zK0KHeyMt?yYC8L$6D1A~Kbk+a08s_i^2lfE(I^@^G(9@}P#-C3opZTTr|*b-Otq8k zET+%+s@tn7ao#1qn;Y6Kfx0}=VG=1NqY!Q&i*DcPsV=&{TpGAZ{7}o@>-tTY5?#-O zdaQDOsH)+Q${UL(bD!5R6Q;e25-{TY$)#0>HPRSnu6bd?Sve1 z1pE=Q+GM#m!bnyrOIbub^izZN>wLQFmrwYf9fiRN>RnL?%&Iwv`o5`-lNE>9POr!p z^}?2Y@dt_}7VfKrz;}-izf(q!W7nd{u<0kS#!BJ682Dm`{V4}I(R<&2G>9=RaO#D8 z-DPXmtzR35yJyxx?_s>I7rG1t8;QAp^KLWLS}de(%W<8Am{|^ZE55h9U`xV$Y=B3N zS4*m-a|Yf{;|+@8=t${#%ZmZ_{3=cUk;EQy!YnS4MwgaYy=S?RjQL0|Q#0Y1Jj9b9 z1h=td=o!ZJv(Y8CqTCbxA5L_qi|$cGTvM%NJ{=_hr%Xt zV|XHc`VJbK>0C$8bqXTlFJeA=mFNlHNeC(s4)_RQZ-B3+M<{u&HpXYWWa5n9L#Rdm zUnSK6oRZRAikCs!qx8X5@%ceJKgJU%U+JlW_gbGCDx>vBd%S=}^DpAN0iaVNV<*6C zm}xXIBq99C#_pc-je^1c_u-q-sI}@(5XA=YSFaWCO~y5*h>h^yS?r&bk+wYpck611 zx1?SM;~oGU8JOJC@5fUij)jc&_3Ug-*7!!#1K&~wFjxTH0$7Kt`-fG&Yt3?obzrmJp`mbSV zgh;A`Uy#<)lP<6V+i50>a-LQ`aJKOKVkr)e?UGBX7@>ZF?l=m;^ezJe**cI#Db8Wh zE?}SG1F)wjnVKT=I>b^cCk_GHgv>>!adc4FeWjVB-Gs#sr;9_x5fo3}?3NR_pi+iL z%TQP^HN|DBL*h`Ie}EoxS`CA=sf)vVC3+5iDrNZN+r2RJ;PAirx#xv%EV53KPsrbmKKuJ{9anOfK4{-z=bz444~1grUOlzn znu*>Y*ijE%K|l&3M8TVWsx+N{-relkNnT35nYP9?>kFg;V&Sy~_Ed%K;ms#x?AW>% z-ZWC{M8|Hb@k(yJgiBd-cgbhH7zz!MZ}F%)jmK&pm6Oa1AV0YN#9il7QXgJ-2SQJR zE#xtGbm%j!<3`GWX=Co1`TLy(K{RE8at;Sp)pK*j!=xi=FK(|U%PTeYhsPc|mR z*JJbMVyy(%W11J}L3m8AE-mkst1in-K>YC~)6Q}vE?meH;3|9lCcbVMudFfeJBg-v z+A)K6c0XAU*$chwRg5kA=`--_NnUwH33bR@N5&!36X(J$B#sq8C7?neP6VYYO$N}u zTQ~fX(4D*HmIpzmqU%Nd+m+^pN;R~oG(-h47+ysk^h@wN2)naVx6b_MW}KpA%*m^7 zLKi+H==xiURm~ZLt^}UQ=Jdfbbkv;A3)Qy7lUvzjk~V6l6o18fF%~l5mmx7u9yN>z zZpj~s`vAW7X}rG11V5GvO@x zHH1*wep%Azt6?Dxlk%H)iE83M2?c4hh~!O>?rboxJhe_bIpyJs;}eXXOafNtRbIMq z?L4A~H@WU8I9r<9Tc}b%;sUS zD^whQI_nOxS)!q8)hCE7h*lL3B)`rXi_#s_d2~S)3!i(gDWFFrlm_Qa=+FpILH!gQ z`>QIvmZriwWnW0obUJT;_4_Es{h`PI5UzFP=-Hdq@#vk%Yxnl~z4t3SH{tF$`kP~t zaT3*w>qEk!U*(4nubic-KIQb{+=9pYW zy|uu@$gr)O565CcW}QzFcpfH069B|mjfv?jw}w}qOtnDZD?A4!qj9->)amj%=3^ja zxOyl=y~7q<0uvup(~H~SZFv>+O~Ax1L=iE)TH3r6yAOL?!f@`jWgI3$K0Y&fOVNsw z)~b4?A3IRE>9iJ|sY@l-4%Ku}28XY0+}_D~Dk1E5kO&WYADEc(cZrC$1-WDwzQPbD zS}~pc>^Fxm$Wqo-z2V{_PgV`je@N3sn>Gxnvd{{xaejOXYBAk-cLU!-3N;sNG9*GM zF#Q8$y5@!AhmPcfvFCNc;&Px*v~?nZb*wS2kb5f^Ag|L}6gGOi0`xKbJ*87P=vIMaifS9X`w_` z=&c3PsMog_ECA~9eDl`nCutVY*?Un;BBL; z&7c9+`QcmNbaoSi5@Uo5fAhH~FO78Hq*T9UlMQ$;|F6RUa~KEh5n=z8!o0oO|=DwjulZ;a2#fKdCM_X~YzOb>7Vdg>Z;15s6v z&wjo7IWBJR(?QW=`TfTTgi&K3(NX6C#QLj{GQ}dJ)Z19%$%GrDMHm)XAqdCc0{L-? zUgYEwa&3sF$wqjLH^aPb=xbm3ISroIwlhBMjZs&5SmxNNuTjk&(s8P&2L`;{xN=(A z`f0uNR992gl`P|3lJ|3NKvC_%W|F5-#QlbPaw=b0pt-(U{$ihX{m;h;2UuLhA|=ob zS8s#RsN^xgqgV%PeEm#klbofGqVVI#c?5HSapF-EEYdPDnU4q>V)4|&Kwmv9c1I0- zGO(thnnfP%yZ}xcg(@Qab!^zu2%9)c{c8O6%su{ewpIwB>jlhv8Xw~1AuY1v!yCQ6 zfJ(q8^?GzKW^%}ty%v;+cb3I>n}UpsC+?~r+`=T)hTr$fv&>NXX19^3)f2kv62aKb zvaF0piPvTCzT{qk;Ha2Yr`HBFsbZ72r$)$jyl@`BP*2vgA4OA*^(igDqa-d+SsD;h z?*NaX1A4fbGFRf1atqsl1*d%}8$7EbN&Lt$UG2d}fKJt~m426G$lm+@{G=cmqM~s& zZ#}LL(;65L@ShSl3U!7&;{ao%mI{d!oURyiBqK_!;gY3hJ?)|7$S?&szciZye8;zb zUymKaY<6wT3I1flg7litykiz;o!HGY6m9G-BdSs^(j>0Wvxn}Kjk^C=tTKe0{%krZ z2X`n&cWJkz}aKGGp z^h6uMyuVEHtY%rM>F`FmAusN||N9cF$eD8i^C0Eq+D|52sS9;jMN)8g^>$&l@zBlY zro@oy%9TJUw{b$@icp2)8={vh1oZ8})z8c4|J0?H|ENo!q6!49Kt4Njd=i(j?o_|M zZi#xAw2!;@B3@5Ma~30@1y7xl^HF%m2z-= zfT5N=uFDE95z(baNi;WDMkid9;7HzMYWk^z>zAYN!G&xVBtfEz@)U0lj9qx1!7#P)MB;JM|UzcrTj66$OvNL()#-qi&A1OYb-7K-!Gxy zI^vvUJ9rP*R_M79yNNyUz7?kyYj+5bg0*$ic6{*no@yIVq*c?y|G}02qpF`I;Q3>T zy}j-WWiMJF)-k*Bf9*LYjCr{q`{`bPWv>zdRT<&cq-X%4@abE@v5k%OA(~D-OZgd7g|DIepF1lnEI_oElRWevthX*{=BhLVUP( zF7*8AeGQ-4!&YOKZ`H*!OTumkwnx^#>wh0{PgKMqyTEQg^3s#2$F3BzcP1It#5NW& z&?mh-Mr>(e z$=gk)22*_TMX-?vDhvJNkMr`WR(-4%?wYl_RTK45i5A}hg%BL!b{0r=Bu0{uf^llO+txV%o3I!jSYpwCFpR={A z69~7tZv|8)eRy($N4zy)1@jgMv_x+;eMh=?&{0-zW=aF)YAG&t{n>}Do3E93wc)US z!AMIp20ce{s&dN4=2v_WAKI;(AN2(Zi_^F6856z*;JvzqK!`B$z)CQou@z~{nsjKX z#EAc3Q#j$9#)NLJfod;uKR7ixmHNK?4B%<+Z)k!ue4l4W8>WVs-CwmmhdsQjeIO#Y zuuH1m-P+b3$u#un)3Y1wGS_CXeD;5(TPgMbnulioi6M1f|1l4Beq}nt_elurI$G* z&y{p8@`mRxTv9Z1^;|jld2>4}y%Mpu?x{Rvs;~GXCt6YEm_RN(u(6-eY%wd3Z|LK@ z{Xm$->iwN}ZDFNN>_ngwR=zFbo;*o|y&jNFQva84+wNGa^>SB7yH@+5Zzm2AhxA{I zfxq!g@7acVQ3_hpoHu|y3X&x1*@c$oyqh2#C;(bz79e!LyTzPN0x|08RGYjB(YBx!J-|8WYT19&Gy8u8+0+$Uxc>My>ng?Sc_1TRv&U((FqM6et-geJ*k5B<9Fd9Z#3 zRqG68EQXC8;Ld(h25Kmu-Q6^!g-&SF-q}~S>~|&m4UdKyT?{#gf=|Ud1T5|i_ z3mzAeRm6%!YesYbFzw+rlASs=Lp~ewZ{=U>V`Ymw$>TEU35C##eE8`+@)LlZVuCjX z1ak~kRA5q%@9ZY`2_rGmV65Az2SKP)@&g-O>UQ93UuVknt&(~jz0MY}dc^BT!hnOE zZWcwj@WVwjvk0#D7@S7>G7w6S2;kWFo(mL9ABcrhk%TGyuQ7@OJ$vu2_>psBDL_{evKM zYEi3@@Gn`Rz%IU;d*r?F@JHbXk0BA1_Chpy;OHSpRm+uD^Sf^pX5J*$t3S7WFyxdgec^KJ~Qfb>f&B zHE%Rli8Ib?CHKUTCeycR@%Zz8#D*_W!Owz|3-o%|yw%gx5Pq5Ga1c+frv9Dmj{u{Y zTdyq@oPo*fls8py0TLUtO&yKtkRnvTHokN<0`0nm#lc5SydMQgjm_TpBEY2Qx0YMwJC5 zNbnncc#_IpjK@AnGRL@tmW8hLfECT&b2#dQ{q7a_-aq68|DNjPgd6MPu(Jq=V!8leKC z@(=+S`%@m{i4?*BBX}WUB{b^A{Lbm$Frph!wegqV zMn{N`0VB%g;TjuRZLVG8!dEAM>W7|mk+E$gZgl>+4(R&p*%c9hQL?+(A?LuO$%nfeN<3Tu5|!06|TLPjr(VD_cmuU{d%?q;C`qAxW9UYZlji|_!8}h zUp}ha22I&0lv&Gg)iOR&i3q9rWAIJ!Q=DJ{`~Rr}q`J6Ci0MR%!V7hdS)U5Zs z^arP`KXV)%oz+2yKpksr=P?VABoWX_agtbdT7Bv5y$eL(T;e2hc^V$GqmyUJ6|k?W z`Fz$$i&_!4UoRwTc}y2iz?A6(Q17C@z?sLKa-rdbT$}l#&v=xqA~Za&M&bKJ+(=naMmO^*t2tmy(=rQP zf(~<)1AY8j4bzDdHG7YjpP)T-tdvHCEnvs#eCEgJLgjYTW1cOUJcNcGEG&X2mh(ysRV z1NM9WH2vC8`7|_t{Y5(pnPK&iu^3|jfgf;ds7RwrL)SyYgOoGUOwAt?!Cewb4fTiR zZZ&_VyK0P-jz4q|Wt_yhB(?4L3Jvu<2d0a|eKe(`t2hLnE7`X4yzk{$foX49-X2Vn zYUXKjYV$ze))(mL+Oq0%j-? zE~MwK_E~J|PDFTxIagJw9YgY;A^0wG&CXwn3ZI+#`xa=QuvywqRQ&&^8)W}+U)w_< zg9nw2ME+0!}>UbwIN(lxp&o`}CCZv*w#HY#SCaRFLEwy(7y&uGtBmo>mtO7OY) zXHJyhBDAVacL*j*B4w?xjM-qq*-jUGGcekz^iFw;-KwuvER z@EmpkH!ita(ml~3v&wUIwq`DE$54J;mU}{??rZIn*Skx*8>dicF$^>uIzXNGY}tX8 ziMa72%yh%ep(sVuJl!D)tJoaxycA$;svV*65W|@bBYoNPB#NL~1ue}^L>O&eMB%GU zGSwYRpr`C+S`M?f)aC+?-80@?EeE)eZ6e4KHsHjJXJ3qDWA;D9(T@ugx@;rhppVS# zx2Gg1aR>!P#;A9`5W{ejR-*4eNCs!-(wsbz>{KUy_}f{Ri@*RM9#UxbO6KKX>c88+ zehCs0Y$UA*Gi9$S#0jqc7#>!7P=u9EqLyEQ&*L% zki)}zC}2pkM6(nV>cmyO9@arz`khb#WL5IwZmSbHQi|AWU74qzC&op!g^L}lR19NS zx%daslc)`d`MEn2F*)auziMNN{LTRt+caxyw*4FQuNakt0e93RuqdUC@$|Rpoa=~z z0L@e01c?9tN8&Q1N9&h?*&lq#gjiGF-kcdNq-K-Oh*19U*pG+K1Z5sn_Zr$F-^5+V zh1a+^hgR6CMJ_ceg$ciDykP*c^f07^ZPh>QyVYWIPi?;YFLz^c$+T#v@p8-rCf8^$_j*tw|UG&EUuu z`0$huJ>s|G5GDaOI#sEer>LIWBxmf{>zj0s`UFv=GLhMOAYfTIDuA#>wRbE+3lyP}wc zH1}c2D^79bqjyfC;RDIl4);4Xe1Fd>r>#1KxK z8wxf7F~#N5Pwp~oT?-XqVs2}w6p{pJG|vafmKiWiT}-E+KV#3Ryct2#ytH}L8?pbP z;`>-&X(H9OBe4Ni*{6u3vKWqp8{g=1M|>c)f+>q`ksmx6Yq2a%tD#+++Gy-!d!Mdy z?wD|I+}57^Vg6qJ{7F z*Z&z5z%lbqD1K0+Nx(R!e+~E^!)WFkdDMelz8ak(B(l8yqya|RUZ{TcPaz{YUiP=I z?|jrXHFto2kdmaJ{p$M7hnBGj#V^UTG3mW@Z8p^o7n9CTI$z01uQs2|T!46VA)z;C>qA|_TJ@1XzX$I0S_Q=L$jQF!v-g=&>6 za31FN!Ph06-^OS`JZgK4V>X1CplzW-yjU#1C*bsO;3$%`z@c4|X;VQJ9L=~*jR)GG zV!TUe1Vq4yUsakpz{rkm#vxs7`Z^`5R&DkPJ@BZj7msTd!7U7~cN8MoW*3($nL0Pw zYw#K<4nF~eH#)Hq5Py<2)scG9Mvb!<6QnQC!FXgx5ZBM?!m~i1P=;Udb8TM_I)KB8 zppl0gg~b3=H4Im`DrSIjo%U#ivcsvb3CDwsoxBeAR`Tvj>D_%p`;1TCIJ0kGP~-t)mM@Yhd5 z#SN$Fv!v-;n@Mw#zcXN0{ihUmyXm4T1M(%)I>BDMZEIeAF<;&W~(fa|b^!w;^Sh0o65aVsqiA z<^gAGxx=LEVkVg(D_xUe=7|?Jl8@l}2c_4Ay{$nrmkiFkwSB@Z*Rw?L&W2MH_g#s( z3Y9p%_a^%pz67%5Hgz&|w4I9yt4A5GM*2j83&A2V&y0LaJ`00qYZ2J5rnvV$cO=nO zEhr$ThUu8WccscFm9yi1nz7zlQq~7&kUU5#-3cY`W*mDcq`LX@Nn1oiNf3W;_sAvE z8P}Pqdi-nr;p&5;jU83yEf}XsD;C@b2FshWX8PlgoGxPR&F;6G32Hiz2H!g=7LpFN zfcu1WK}!)z-KZC-8bte$x)kU_vOIB=eE-uG^Yoa)QK-dJcb=Z`B*DYiMF+D_7vdSd z`9{K`MjM~`3siIt;zy+-ZNtINlx21zcal@Ox+l%%dU2DX7j^Jg2(Ff= z$&X$9>+m5~nUUraH&9BK0@~Gu+&d8x|qGx{5-OMi7M0z93YStK}EpLr} z|7rO~&gk4?MaG3xk6Cgx?dV;aWleKy5ZC|{Ojs5z4+W2VwJd624V_uvnYPtz^Tdi2 z)84vBmnvH=PhYVp*{D3+*bu!7fzq3C6!!ItcF;=ilRgBZKgFLuZLxW!<+vf$!c9l_ z=?%NNh+Ew#%}JJZ_yokM`MSUIo?7r#4kz?r$AV6J15EL9=864W+{ zmiAh%yFvmq4WViM*1OekM}-6(W4{a0#6j_t1o@Xw%*sPYpiG&bvn|myyOjsQ{yR}w zK?lGz&fRy?CamqQA@it&>9hs^Sns~OnXNx=0&rJ?C)9Q1Bi5JNI-%|!Mh63~&VR0) zXv(O~x4ch2i&}T17!hjHOI+MAJ-yG%l_MB6V_s5IG8R_S;*AXr8jrx4n&ix$~M`5n+qBis$4nb(>lh{y;uT6(vGD zrk9Q(-dlm%;SYd$vDVNEa~nt|bJ+FXMRqa+P!DaSfiDDEenZdbozwFzz9sxBhq0z0 zk{*HV*4p6@5T|B;@-U3YaZ|7k9D+t`EDavfG$~S$)=gSm?GYDf;Z!H9s5mJ4C@Rp&WeS0u$snHsbS@*!g^KpAnbx z6F}&+;IV}Ulw649%CCHxcY2u(^%l3h_xGk|k9 zB7D)0-kp7tEj(vO2vcl!GrhW+mIHCob)VKeM=;LfTz20g(J`~x456!HWd|7T*pgGh zOH~{sz(!;xLEel8^zp)iT%i*On%?`pJ=2>Vx0%OTD8uHF&0m{ux_+i{@w6mD6qn8H zz2I-KF@?XOcxh?T^d+z?C5*jpH9-W3xPb6V!pFS*b|n;>LAYD-}K6 zox2(j-IF~6F~SW@P)3kZn1?nPR@imfB~Q++5U6q>g7Y-Gmg`U>5#6~OhUa_qGg(j4 z9jc<3s& zVc;St{urth1(UyB(rj`-1fQ@Eb@`C65=OKXlL%IsG3F89p5{4~Td2QRGV?MBGrUrr zl^8-DeB4WdXcLbZCBGIe?)Oa<(YbBkCOSqtN>A`CK!69mc-`xAj3pvJELa($Q(9;m zh=4e#C&q;f*3o%kVP5=q!c!fLzmAYOz7Y)S)s$Q;@;61;kdp~^t9-9v6ZN6u}()Z88AVgYkZhZ5oPuWhWH2h05%8K`M63-(~ zxNM4VXxozRmxZ*7SIn%#y($UHP|q8d{jvN7

0ybUymw)5FscP zZr3HVbTS=WK?jI;jf{|`($9H*@>UG_nBWv7eA<4vD)jBd*gK2K2}l}C@K_NPNrXEX z0$sBgE>VjTsK&T;9aBr99k^APPvWinZsUXNdH06S{WkEcDq zgMiPt`H^}tZlFcO1k*AS zv8(yiL*oNYa*dl!W=rBjc0;N)S$Hzrx2?lmxjWZbC2fn=#*-?UcGY*BMkF=r)*VhxKT%)OlWsW zUTB~Nmg^x%(o<0|>CRylS{%inD>P>(wDAjE-a_IL)(w42UDLNo6ZdfZS^3q!g&oR0L%VEJwAF4TwA|>( z1w1~b5KtF=UsmC%9kI<}Gxo49PFNKN$vIGi_0S2%*@T3X%UqV>NB~PP#OF&l3D>0f zi-);&Ye<0GxA~Y1!LaANb@R<4vB@Fsm$RvsL=Wv(WnlV#mcIkTLLeirLPxz(&W!~V zik)y3^r3|+d?vaurYI-y4JQJaec9lP+=Rp6J~!>oMH{8V3y_^;_NzOzd^nSZ*oLQQ zcIJHreNbkI$JWN?r^ajjNuvwD<$a;HK^HoJ1Gz=EXY_=g3aT>(^;6Vx-s!3Hua|=MLV02d5&Urv1f+DAR$fjAK zd4z3)1(~un8}Fv1W9a8+p2`qVkt`Hr4ZX%+J|>dqu}0e9&0cDjEgVl@V^AA9aSG}o z&h>ng=;(8^x1u%o7mQ;L4Fq;GKN#JcSTw`-SmU;}+YC);<^9v050w&tQQtTFmt3w=hcU=5Gz9!Dl=qX6K_>6Ibm zxw1NWQC~A{3L#K<6GMN58JalvZ2ltzaYFQ~eo)P`-GxrUG>E%Vtkr&6ythlDgY9WJ zD&qD`EkDy_6A?*}ja=3rrwH(89s(_@E`~4?bH6FRcck)N15;B0YbA=O`-4 zrzNBhfwJClHMhNov_m&99WDJ(FAMaM>>aRoKNF|%e8)tYwJ3c7THM@7Z#ce@rV+g!miDacZZ4-ui@ayr9&AgOfKh49ynZ2W61&0MURQ(}?uTcxm7d#guEiN#r| z8WV->kD7CFgS(-WiEC1NFR}OoHL7>!!=1Lx&{xxX7azUz2 zEv6WJ)-#}y+U(E!xonPReweV9Rqp9UG8wkj9?Y(vQ2)PTs-vR?D*x!aa1>$`@LulX(WvFTx7$Q}}_U>TWne|87)nnUtXQt~(#|hF`@@E~7 zOY2}nVOgU0GPGq%IW6x`vsy$%9rqA0h<2j}(!5Le zzsJ@O^D>{e7+u<4X^WOKITYhumrv&t#d5vGWT{ON(_7%P0ZinSM^CIBw#4*Su}}Pa z1e0T#yAIg{6bN=2VXIch2r6&C8`{~gjTc1?k_b1M0}kC>)eKPdVN`~A=Q_SEgxoQj zuX%ADjD<;(#lNBPoD0V~gyLT##x5&0J9RZEnM4Gsye7G!_fjJ98~L`3v;Bj&pJ|eV zs^~!Zv`U&QY)2LZc2aI3G<%xoQp&O^lI(ppIgsdxk8Hmu3jngEDA`0Lh|U+)6fiYb zw^N<XP`prNsRWmDTE{#KcWP(t>i`6o~SBCVg<{;LYNmLSzuH z+imKkoeCRZLK!6}^RdTI)?muX=`U?Hcyjsw?p=Zlh>QOY$8K|BY`;=pY)uT0-oTeQ zZH#ILDRhJkpp-@s7?BE(7r*{SIMiOnTR`q zQEyln%dxTRL$(7t`A>5k4dT?jgr@{{6q4_7LI&qRlk!h9mwDOKp^yxv?-|lfkz6J+ z859x^Gt%%GEo4&^fIiYaLZC1G)efFM&{uciah@w@9}?E#(hfIU&?6@q8)BtON`b>5 z#AN}aTLz@70Nx$Jqb%oC2f6MLkObH|EIK{Zu*(_(00Aq}b-%{7jkT^rpk1 zL6YF9k;YU0rpGh~?z95qGJlEa-%Af^+?A?11M#edrIz5tr?^uP zLTLH%PYGYj&S%0kt)s^VkR^1>*0167H2L)WSMONcD^9QL`Lp*{b4>xPV#yU(#Fi)25HBn1BcIkwE(jlri4MHAjIwBQS>(M z7OM!dLPyfpJs?Gc&u0&zbExII+zw-()}9{_Ab8+hCM8f|qiEsAm`ay|Pm1Qs^KcH}lms7b=U2}w{J=TS!7^7qH^JQtBv}aPp`q;07i{xB(?Qp}_WTSyZt^$l z-eDuGQ+w*xxQvJ|poXsw_sL*@tM)#0G2cR%nM@^au;ZTer&0wMlwh& zlOvYbiaDJu3=e$Sio2vclpJE3p8f8{nOF~#Mk}r&8=^GURCx#2`PdclbmIVsk9#Hx zRU{J~IxEH;(^#a|-mQ>;DoJ)AX@(HOZu@)yHdktPF76VCE{YXe}KC#hp` z|Dd7^Fbt%Wn8F&v_|9KQcC}Z*%Tyw?oySl*w2#_Ah?ci^gsPs$)b#k)2GEKks%JdZ zTEI*3>Hdd=pefv^am|pprl>>Pv=F|&h$7kEswd6Cbb6UJfmZ`FJ-E7PO=P7I!a}q^w;+=nUgAwc;h(UKHFiLwD5y4UIS@0D43%hCOu4T#TT{G zA7qi9m&o`AT$7DNyK!=~eDqM6jgroTzPZJbzifm&rtTe{*MZlh_9BH)^ix8*LqlKL zq#xkBz?~8NgTdkG_;d%LI(-iJE_r{07y_L#1W)G z@#F-3w`w0M=BqEZm9P9Ad{m$cuv`h_v$+%q?&|lh9nqkz5q^K9xo35CwZ-EV-KJCZ zdfBTJw6|yZn+~?~71K5y?gu0Yakl?lW&fZ0a#2$q`Ig$8yDe0Ivc#`r&(ye4-K1qD zmPq^%+^bS{Y^tfz-GrY6h(5URP;{H7+YC)hW_hJ{5we_Ql5Zmh>f<{Uv#jbXrWc&Umo6Md%81T-fW_X zr9P6Dj0Xs>4{?osjN9T*@K&Dr=GZTXqVmG%t*k}l`gW#o;xj5?DqKNaIWM@FUl&-s z%A!$|-Ye(N&QU_D4SY~i#qyNum;RcH4NcqiZxLVLp(SoVQK*tI?x+f$6ZGjfSE`a# z#OmkH7_&XUM)qQ>L738yd$Z#vSRdElA`x3XkT?;M3GO1sDZuH znDh2{ebdR|bZAJO_bd>>i9`S=8^YkhU8dVIJO?O#AJtuFQ0>aBue0IV6SS;0#)QaF zKAb>sThg6Dq)(rHTBu{m*zF@eyZ)qxodt5`R{R}0)TYX&s_g5553mk>=Y*<8WymfO zR~)1!|8S}nZcV+}fCs_FIh@UC<=Q}&3&OvX_une8o~ci@k;+zqlVeU_m54VzJ5?JR zd?kS3H}oz*YtRG{&T~R?Pj?Je-#$(`%zPAKrv1Gs;LB@tOauyO;jZORDV=4@KhJjy zaQ(La`2rXK3m(0|_T3>~|KtuXJmNnn;9{~yvyx$-2-HAa%MH5~>CmR zQ^2Y(Bqm2y&XyF*3!WrStpMth5|OsI$zD9NHd7qI?|nX{l1;e|OCn8v#I?uS*HYop zGF(Jp=j75=@bg@{?z*dc!K1%VAzq3A%h~xWoq53#%e(CSA-trn22wp7O*Yk(9GQ@P z{tw68QojkqfMtY3XhhVWX$!dO9x?Cf9j2@zH-I(c6CHsku2U|RtbDJJvC z_EvyuHG_;g&#za?{So>VuIAlPtr6d^b}T>fS4aY<5`wq08jt!z&Y}@C#}@YXNt{7@ zoos>BK6AM6cOA$=u6t&7~s&oy!H>Fy?6q_ zR$4!HfuogHIwicANx9HDu}UVY*w9xoxdLg*O`KdSyKgdno=StGVDH`7x@fsfcI1wIJIV`76`GBJ{s~Q zpMVNg;(M=XoiOZPLptZL8#_DZnTJX(^r-Ql%psbY`LFw2#dcKo-7j;PVrgWEr?uEa z=HoqN66L{GN38+W`m@RNFE1Y?+182XdSx5n4|BzjhNc&0eyY`(4NxNQ<}GU9Zy?qt zE&X4{<3Z8oJ zQg|K{4pzx#`kRVuE!nLa%lwBOx>eHF><-Nan%7+r4eIv)m7u2n)v&#Il&$k%>94>2 z*VbH(33VG^?>V2j?r;wre@}#JScxHBh@*5A6W%3Y&|YWk2NS~ySWouSc- z$aDIM=l6@R3nylOKku?N2`$=%77sgk-r_&tfR<+^Z!{XwoWDB6eE#A?XgzU$Lbz4B z<-CDlP8={00_oM2nKD&w6IhJB2`jU)Br+HS3V+uPxv(XlqZouU>QazVkJz)r%Hj)p zD3It2mYeAAoH+DKeaZbyH+y4Ib(ibudm_avB=#&(oG;!U<#LKGFvr86 zCx^rhX`UoTpawCP?c1XRVX%D^FXQ~^H+SAeo&K`cfSW%98{Lp=r_wpd)lUQvnGX7~ z6g3zyiMkURN*@0B?oX1ITWZF^0g5bybSO8hc1=o7<}eR`LAaqq)mm^y>W>85Pc{&` z$;a!kh=J3sLpHx#H?;IVSZ3CcXD&a7W|o;XxXdH%+?+%HCQezh(7Hn|9T)B@?59d- zFWjpf069CNEiJ#eG#@`+`A~hfJ09YdxU^8)DsZxJU~#sqPU3Bm;KVs3xF-L<;cgfg zmznRv@J-6qB{yu*9-%?N9&75{8My`P>p6u_7ODvL4U&?Y5VA7e+q4#NSqpt*qWYqk zcw>Ra6%?|}>r*g!xer%kntBlbmo^y}Hh)YX-^q6_jY)%GV=-0JE|xai>AUp_=j?A_ZnfIqH% zHX@9#L&5??Y?5S#sNacvGO)p0i~|l^@+mk=yY{(#yI~YX>fNV zd*@J4>9Wj14bO6(C;|Iyb>dG9oun3$||UD*x%n1<|75Yw%X%4*y{>m6J$t)Z*4iE_Bq z(}vfeQ~rh4OJAcY8b6r?A}0KVqBA)_0$<9V;golLmpQ}q;Jdnw>~*#?dfe{oR(JOx zWgnR*-eo5}i_3gjygt(0tfxM=-VK&fsZ{lR?`ex11h!hwi(IsDSC^2z)?W*$m^NYzxWPP`te_%{1-g+xI6N z0~BRb(1{vjEp(sOa~o$Y2BER3j)aA0ZnTD{TcgmN$3MkX4~naGzH~{{v!nL~3g*n5 z0#!^tc5r@g!Q#T$WqmfazB!SybAq(2yaN@4>J5Ge8gwB4&tX&9f}MZ{T(*wG|Hn{L z6XDM`J~nw|QU8tVb@5A4lL!N6qZFZUl1kWPa*N%uB~O01G?)ojUS6^B#p)C1WK~$r49>|VbW-A zzIf1TP|TIExU5i{R!Q}DG#1c06EV{UDCd?hFQqb$`O9|#Zlj0ln6;uWz+QCy#^IB> zFWr-%B_WcA6Dx#y!u(9mCUZ$j^bx+)^v=FCr);1I{4ib+ne*DIDcBCukg zpa*$Uv69YT9^5%<5vNMT=&~1#hUP#eyulBZ^oHfWnLA0LI3rZ#?#wQMV8L$1Sus7n ztjl-DTXzsORG=uK;XBX@BrK$%jLpxXuE|&LJdjS{ao|zFdeL5K164F+*NJ8L4RZc~ zR$%3*VbAPOt@s6=fzU4Vk%0-jTyb0X7At$~5k}2+6rDy{ZH|zZ1F09Abg$oPxaHE2 z7sv@2+ThqFGksqpbBeUx2Criy@F8^ru+h}Z^5Dk1nQTRUD&5efO)k76`y*~Vk2@Z{ z=7EFrX+77YY+oMcdVHcTyCSuFc>k3qWNAm{KG=7)lf}O*g1pwec6Ba*IeQ|}_MnHx zUkf2H3?6)+agvxLFdM*d=gsc60)LtfPBea5Cb`0r=w1&U30Xlh@05xi^+v$edjboYXbMB1md~~pv&D2_@m=$l3v3!Pi=5-%4xdzzWOmJVfB}+7 zAKmP!D{9`B5OSnD_Oz|VW%Li`pLxjpn+quX-?#w2iV__y=5v?O0JAjm6PSMl54^(q z&L$|=Z_o!cN1y&Gi!(1tGxpDp`#19XexK_z~-0vl_IuiOakL`Xs=je7u z8Dj+2>iwFk(F-G5sHkP=B1wJVEXs1EnW?W07CGUFr}gVrys`7oIvaQ02pc;nBaU`s z&MgM*L-8IdN?I(xmBb`%s~C%{pU8Y{FaAvOmE{!DM|WuWr>}X2udj0ydC~P8)2+Pq zn=c(H^{60Tu>)t@#$E&gH@Ia2YXeWLZ%iR?UZu# zPV8*tsK##VC=2<)tnai5*O2&L@O^#A>WKkDc$quQY*51{CF;+-@pWe4f)#2@DK~51H}lAzk`&4y!+*_ z>z0SE5#JYgk`(oSQpQHRP06`_JZ{B>9E5x{s!M*l(R*Xe^@3g8$hnm`aOB~~W&b|u zZ>a9yi#ikjY)1xso31QL6c2ERcDfNT_P1e>Z<5B?V~wpJv8AI6B&9Xe?)7ujkqF5e z;_{vfO~0s{+TUA)z3Hq@Z-(F7{y^;tpA(7~thYQO5cX!7k){f`%OPs~)sE<3Inf(J z8n^|dHRs4DC@6T~ei>O4tmI9bpOhb{lHNI^ic*lnt zA4B6W-t(2qgi~TzbR-ODI>>yRm5mGnybOz)9@c$|#)af@mFd4Ax+605q>$xMoj3zd zLnn{`ryg&!MAWStEk+X&%b0CtCclL!!d+zVcwy&(`636WHzF3YC}y(8iTGhtg|7{d z@;tO%y1DD+-`_jz+u?GxUh4ZGR&Jnf6t5p|{Ns6}UhF(E<8}wP*;mD2TCoLSUT2rj zVs!D!PuLY~)irfkir~IXzMvGkjRvEMFsNFO<|R%SMP+%Y_9mes_nnL8Vw^YJMMT_H z_fw)^&W0_wR^T>h->;?Ad3F-u1ePB*JrZ_6onajRC8Gb)MgXWo^Ojon&f@0bRJeKV zKP2P#_5#?I1n3FLyPo`g0@&(3kpo-H2>9wiD0n6KD-tJ!9T%S+ew_Ygb42=OZtE8E zOPag6Sy_2mn8-e+;4R(?>jF6hWV|}E8{yU5dY-`$xjNX)w9-f$i=zy8|CA#NDn6jU zk0QD^P8#{;I=K-8Rq{`T>Py0HFAK>{ZutG!KjBKNs&QlPbaGT#`~Cr&U6JTCl8R5o z5VW_FDVTe@A{D=8$DOQL#kM=}oI$xZe?>5IV2-3^G|xux+pq)D8=b$cdgByDZuH&)}edXNpl?B;&5;tNR?A)+#XaJ!m>hE*aNMwwt-wOD>Q>? zVQ*3-1DT$`IJT2H?0HRzf|A}3Opo;i*dNyAzY};uLbS^rb|6-p9KJddO^B{3+#aZ4 zE86f`l*1gKTu<{1)b!8y=!G;E@(5e#n#5plW+B#_pQ0WKo9iWvR5baGIsO@p+bWtbwHuAr=!6}!Fyx%dphh$2apZ;anWGQ`sOO0Vkj1$b!(ZyIq_5>ycIS8T`9A@ti^ziX6scDu z^W_a-34IX>1uWTA(Hg zrL_p}6{lbk&NgnbgM7LFX>}~lJQJT?qKInc10#OmZX7L7WCSF*%p&HiRm5uk zW={xL^HL6^VNQB6+inYWPOFrF7fX#GdbO)&{904y$48S+yKa#ni=Pw~(v8Ar_H3HN zDtM)xA8F>}&WxKB$i)^8zw?N^4=MhJ+iHs!&>*o{sdj+a`%v%;w;N=fwz~HKvmF!c znAGpva%b9n$m+@ra^TLk38GcM-;_0$fRow`cqpa}sYxMTIb4sLuPm6!#C6bvfe|%- znCrg=pPvcewd7zS+{*W&1M0uo*#rpzht91VN^pgzOyxEsjM z(-yX~e(5wR@^!$6{DbjGwn_d+TIZj^wU)1`E9nL8d(?=`7k$=oEj8#f>qn4)?arro zJ+7h5NV_R|8F$Ml{o*2Nr1#6q&y$^AbW|)O&4b0`a6D-?xeTd_?SgCV-{OLvHB%}`FY2aNymkNzXtR8R@Y|7-5 zmrLhm9ZoaZxW6Vv%ZHEZp7dd+#1HWB*lwK}W7*VaW8tq84G-p{R)suOs;K6o$C|Qv z3Ul5F=h^QR9^Vd5LFF8tBx-x{ExybVWX$3X4iD1a`K*w|>(P`IiVT7)Kdzc?{`kg*j0Io2&0~`| zY?Y2)oLmsdWYtEH;=P#@-HmzbVUtO1U5LuaV9*Bt*su6e^A%~Awe?@mD&TuGPCIX{ zu%i$<*zk?T=9GAAyg2XWw`PJ6R#IDAxOm)JMkuo{B+}5i^pY;-^an{Gr)c&Zv!c?s zmD|{<6{ySyPSY|NsC%@LNdnw?u_wjWVeA`0g;OJC@=O1)VW0$G0(AS5=~f!t4URQI zl^x3uFBJ$$H=bB2S@wf?yR8=iBO)4Xrc2gfXR5efg!6%+wdXXGwflw0EZdEX zLlH0TxiILII1gMSPbgk^)(Fw$FB*I4XN}q{{3ZU#7WrU;<*Z2b7CT0T!8bP7&nwpo z$w8!e|AX*rRuu2{sD9)=JAzRkQzBet&*ndOEbBBmkihHFt|u{k*-T3&YfJTi9Fp=| z(NZsiGfr__$XZnPYJb_c0HxKCjgtcqD$_u>DAXV=!L%U2Z*P z4klQK4N?V}fcQG}w0S%p4(4Y@49h5|iv^%rOO*wh`e^))))>?=F}9&a`=AzP8q@uO zvgJe%ajo&75y3}n3kp~Xsm*PmA;I=1RXis5o+>sIjmI*6zWioMGH%pL9Csed>puG6(+w542iF#lw=Xd!x)nJd{wDt6<>;=R{Gf-8{X zk^O7x_aEnK3d&6_rVBj=TIPp>UoQqr^CU=R-eLI9UHog<5<;tQu1c9s{$XDK6>+QT(KK~_+Wq$i7k_Er*U(irtxskvdi76lAFPbzAU9@JP3#~cK*)SVZ%#c zBg}B$9Y;E?p+a7U7v5pdCrE%FMN6e%c94grqBIZpzFFh68wzjybV^*ZG zd&8LrnIrw94WCdjabVqpHf{~r66WS0==thm=BfFl>$u9Cy+5EYrUO0C)@##Vmdm(F`O!5DVU2>u#-%p163cq7zkeV@>AQDHqL z!WnK$T~xxpI{FYKgN!M3Kdg~ef3G6Zx@Z;kj(buIYW}da-pVk!i-Th=%`B?=Kb8V_; zPUz-=N3TVrN%wUMDdEc(;?G6u8o773%_N-qy_r%5E8e$MjQ3NaY4@R3pAbnMAdL1@ zjfZ8{lt%lHTeCk`!96V)UD|0sl{Rpv1`LI-nBp&XAb0<7Q2wtg$|DUMIEWCeyI%jN zVCEkW1@xF51C`6Iw-cDJd6W#ic^*NNJb>T74Is~X+myFWxJBGp>pzhGxF} zqfI-rlfxzW(2~1NKC{G=$zx`9P6~Vi5b5u4#wQRHPCPX~iYgn(rMvj4->>*R3X>?^ zcu6D;^s61d0RQdM8rcktmrjT}}?3q!8uK(n7mt%kW2>@MqRWQM^^eT1yF~y8REHYj#P2N*o*vd#->N#_Yf|ACNhp$;GOUEH%k>= z8#@#~E_)@6il<+>7srhIM7ED{C6Tas-2qQ{=vDc5aTlxHR3P|1oi@FcV=nmbvcse< zG25Q4n?&&ZM5T-JHE;VAnrR|)>ncn6x(&?lRlj&p80-)vAY?E1jenx&O@K>6QuaW0 zR-t%t>=y?++aeT5^NUk=Ijs7sAmbxNgJ|bTTLL(xl)^&W=p^UfYBHGYkKD$IbSx*U z1Fe^u{J{C|ukHgYDPi{1H1qG$u6Xt#>BIKAY0vu_aoLbw^@7zjDaiiBCC(1L$1!yC z^6gLGSaJU&5H6+cXr-;-e%m{?|GhF*f_b?Xk-Md_BREOFH3TMSBkkSjfW$qr_qhrs z_s0yfB)v^vxR-2+@>SkD?K#z?f7?^#lw@Vsl;F|nUO&=a7`VIzbN`Z1r1eMm zf$7aZSI&Nj0d@W#Qav>rYOhu`NWL^_B!AoHVj2?q#= zTuCR1^=SW^`<;2y|N3F2nHNf179NGQ0sm2S5UCdE>IOr7k5^!Z)*2J#RYZAt|Ml)F zx72jBC|`qP#Ps7F*jg&ab-SGMna^HVW$$?KpHEx)64)BMA6U)#g^FxlD4^3^tCu|| zm@{uaLCR~?{a`#T-bwrK-ub;>KM-Iae;RWc`_Go*QnGrlw0EkqfB*gbx-86fHGva= ze(TBfJile^$}3K$3*t+t-AGDzQy{9IcYdS#pD%2tC%ZA_p1Dl^8uG6S;}evLehqYe zp7AqeC4Ka&)?VjLocJ`6Ki=52gE{eoqnmaO!B)TRIWkeF!srpxXeVtxA`0{}c| zKdCV74aL4>kuf*xJ};Av{lGelze_(Sh_xQMOi+TQcgC&H4vpLAKlx)L{v+l8DCvLh h`2Sz;*_ str: + random_string = "".join( + random.choices(string.ascii_letters + string.digits, k=token_length * 100) + ) + tokenized_output = tokenizer.encode(random_string, add_special_tokens=False)[ + :token_length + ] + + if len(tokenized_output) < token_length: + tokenized_output = tokenized_output + [tokenizer.pad_token_id] * ( + token_length - len(tokenized_output) + ) + + decoded_string = tokenizer.decode(tokenized_output, skip_special_tokens=False) + return decoded_string + + +def generate_unique_prefix(base_text, index): + return str(index) + base_text[len(str(index)) :] + + +@sgl.function +def text_qa(s, question, gen_len): + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0, max_tokens=gen_len) + + +def prepare_prompts(num_prefix, num_samples_per_prefix, prefix_length, suffix_length): + base_prefix = generate_random_string(prefix_length) + + tot_input_len = 0 + all_prompts = [] + for i in tqdm(range(num_prefix), desc="prepare prompts"): + unique_prefix = generate_unique_prefix(base_prefix, i) + prompt_list = [] + for j in range(num_samples_per_prefix): + suffix = generate_random_string(suffix_length) + prompt = unique_prefix + suffix + prompt_list.append(prompt) + tot_input_len += len(tokenizer.encode(prompt)) + all_prompts.append(prompt_list) + return all_prompts, tot_input_len + + +def test_batch_by_batch(all_prompts, gen_len): + backend.flush_cache() + + tot_time = 0 + for i in range(len(all_prompts)): + tic = time.time() + text_qa.run_batch( + list(zip(all_prompts[i], [gen_len] * len(all_prompts[i]))), + ) + tot_time += time.time() - tic + + return tot_time + + +def test_batch_by_batch_with_hint(all_prompts, gen_len): + backend.flush_cache() + + tot_time = 0 + for i in range(len(all_prompts)): + tic = time.time() + # Send a hint to cache the prefix + text_qa.run_batch(list(zip(all_prompts[i][:1], [gen_len]))) + # Send the batch + text_qa.run_batch(list(zip(all_prompts[i], [gen_len] * len(all_prompts[i])))) + + tot_time += time.time() - tic + + return tot_time + + +def test_send_all(all_prompts, gen_len): + backend.flush_cache() + + all_prompts = [x for prompt_list in all_prompts for x in prompt_list] + + tic = time.time() + text_qa.run_batch( + list(zip(all_prompts, [gen_len] * len(all_prompts))), + ) + tot_time = time.time() - tic + + return tot_time + + +if __name__ == "__main__": + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + backend = RuntimeEndpoint("http://127.0.0.1:30000") + set_default_backend(backend) + + random.seed(0) + num_prefix = 10 + num_samples_per_prefix = 32 + prefix_length = 1024 + suffix_length = 128 + gen_len = 1 + all_prompts, tot_input_len = prepare_prompts( + num_prefix, num_samples_per_prefix, prefix_length, suffix_length + ) + + print(f"Total input token length: {tot_input_len}\n") + + cost = test_batch_by_batch(all_prompts, gen_len) + print(f"Latency of test_batch_by_batch : {cost:.4f} s\n") + + cost = test_batch_by_batch_with_hint(all_prompts, gen_len) + print(f"Latency of test_batch_by_batch_with_hint: {cost:.4f} s\n") + + cost = test_send_all(all_prompts, gen_len) + print(f"Latency of test_send_all : {cost:.4f} s\n") diff --git a/benchmark/benchmark_vllm_060/README.md b/benchmark/benchmark_vllm_060/README.md new file mode 100644 index 0000000..b480dab --- /dev/null +++ b/benchmark/benchmark_vllm_060/README.md @@ -0,0 +1,89 @@ +## How to reproduce the benchmark results for SGLang v0.3.0 compared to vLLM v0.6.0 + +In short, with multi step enabled, in online scenarios that we benchmarked, the Median TTFT of vLLM is **3 times** that of SGLang, and the Median ITL is **10 times** that of SGLang. Lower Median TTFT and ITL are better. vLLM's multi-step optimization did not improve throughput while ensuring lower Median TTFT and ITL. Also, under maximum throughput benchmark, if vLLM does not set gpu util to 0.95 separately and uses the default configuration instead, its maximum throughput is **lower** than that of SGLang. + +## Online benchmark results + +### Llama 3.1 8B Instruct 1 x A100 80G + +| RPS | Num prompts | Engine | Median E2E Latency | Median TTFT | Median TPOT | Median ITL | +|------|-------------|--------|--------------------|-------------|-------------|------------| +| 4 | 1200 | SGLang | 1564.17 | **31.98** | 13.17 | **11.93** | +| 4 | 1200 | vLLM | 1691.97 | **100.48** | 14.14 | **129.32** | +| 8 | 2400 | SGLang | 2175.02 | **35.68** | 17.85 | **14.41** | +| 8 | 2400 | vLLM | 2137.16 | **120.39** | 17.09 | **158.63** | + +### Llama 3.1 70B Insruct 4 x H100 80G + +| RPS | Num Prompts | Engine | Median E2E Latency | Median TTFT | Median TPOT | Median ITL | +|------|-------------|--------|--------------------|-------------|-------------|------------| +| 4 | 1200 | SGLang | 3005.24 | **53.94** | 25.03 | **21.67** | +| 4 | 1200 | vLLM | 2915.60 | **179.15** | 23.58 | **231.23** | +| 8 | 2400 | SGLang | 4064.98 | **58.11** | 33.07 | **24.45** | +| 8 | 2400 | vLLM | 3752.38 | **207.12** | 29.15 | **275.32** | + +## Offline benchmark results + +### Llama 3.1 8B Instruct 1 x A100 80G + +| RPS | Num Prompts | Engine | Request throughput | Output token throughput | +|------|-------------|--------|--------------------|-------------------------| +| inf | 5000 | SGLang | 22.03 | **4281.51** | +| inf | 5000 | vLLM | 21.27 | **4132.37** | + +### Llama 3.1 70B Insruct 4 x H100 80G + +| RPS | Num Prompts | Engine | Request throughput | Output token throughput | +|------|-------------|--------|--------------------|-------------------------| +| inf | 5000 | SGLang | 19.84 | **3856.01** | +| inf | 5000 | vLLM | 19.04 | **3700.64** | + +## Installation + +```bash +# install sglang v0.3.0 +pip install --upgrade pip +pip install "sglang[all]"==0.3.0 +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.4/ + +# install vllm v0.6.0 +pip install vllm==0.6.0 +``` + +## Notes + +We referred to the reproduction method in https://github.com/vllm-project/vllm/issues/8176, and added the `--num-scheduler-steps 10` parameter when starting the vLLM server. The `gpu_memory_utilization` of vLLM is by default 0.9 at both TP 1 and TP 4, while SGLang's `mem_frac` is 0.88 at TP 1 and 0.85 at TP 4, so we manually set it to 0.88 at TP 4. + +## Online benchmarks + +```bash +# Llama 3.1 8B Instruct on 1 x A100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests --num-scheduler-steps 10 --max_model_len 4096 + +# Llama 3.1 70B Instruct on 4 x H100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 4 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-70B-Instruct --disable-log-requests --num-scheduler-steps 10 --tensor 4 --max_model_len 4096 + +# bench serving +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 1200 --request-rate 4 +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 2400 --request-rate 8 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 1200 --request-rate 4 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 2400 --request-rate 8 +``` + +## Offline benchmarks + +```bash +# Llama 3.1 8B Instruct on 1 x A100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests --num-scheduler-steps 10 --max_model_len 4096 + +# Llama 3.1 70B Instruct on 4 x H100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 4 --mem-frac 0.88 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-70B-Instruct --disable-log-requests --num-scheduler-steps 10 --tensor 4 --max_model_len 4096 + +# bench serving +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 5000 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 5000 +``` diff --git a/benchmark/blog_v0_2/405b_sglang.sh b/benchmark/blog_v0_2/405b_sglang.sh new file mode 100644 index 0000000..4918537 --- /dev/null +++ b/benchmark/blog_v0_2/405b_sglang.sh @@ -0,0 +1,24 @@ +# Create dummy weights: +# 1. Create a folder `~/llama-3.1-405b-fp8-dummy` and create `config.json` and tokenizer under this folder. +# 2. Get `config.json`` from ./config.md +# 3. Download the tokenizer +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer.json +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json + +# Launch sglang +# python -m sglang.launch_server --model-path ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --tp 8 --quantization fp8 --disable-radix --mem-frac 0.87 + +# offline +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > sglang_log11 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 > sglang_log12 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 > sglang_log13 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 > sglang_log14 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 > sglang_log15 +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompt 2000 > sglang_log21 + +# online +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 > sglang_log31 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 > sglang_log32 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 > sglang_log33 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 > sglang_log34 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 > sglang_log35 diff --git a/benchmark/blog_v0_2/405b_trt.sh b/benchmark/blog_v0_2/405b_trt.sh new file mode 100644 index 0000000..1950bc9 --- /dev/null +++ b/benchmark/blog_v0_2/405b_trt.sh @@ -0,0 +1,17 @@ +# Launch trtllm +# https://github.com/sgl-project/tensorrt-demo + +# offline +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log11 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log12 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log13 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log14 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log15 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name sharegpt --num-prompt 2000 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log21 + +# online +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log31 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log32 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log33 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log34 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log35 diff --git a/benchmark/blog_v0_2/405b_vllm.sh b/benchmark/blog_v0_2/405b_vllm.sh new file mode 100644 index 0000000..d1ef048 --- /dev/null +++ b/benchmark/blog_v0_2/405b_vllm.sh @@ -0,0 +1,24 @@ +# Create dummy weights: +# 1. Create a folder `~/llama-3.1-405b-fp8-dummy` and create `config.json` and tokenizer under this folder. +# 2. Get `config.json`` from ./config.md +# 3. Download the tokenizer +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer.json +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json + +# Launch vllm +# python3 -m vllm.entrypoints.openai.api_server --model ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --disable-log-requests --tensor-parallel-size 8 --max-model-len 10000 + +# offline +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > vllm_log11 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 > vllm_log12 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 > vllm_log13 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 > vllm_log14 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 > vllm_log15 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name sharegpt --num-prompt 2000 > vllm_log21 + +# online +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 > vllm_log31 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 > vllm_log32 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 > vllm_log33 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 > vllm_log34 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 > vllm_log35 diff --git a/benchmark/blog_v0_2/README.md b/benchmark/blog_v0_2/README.md new file mode 100644 index 0000000..7448554 --- /dev/null +++ b/benchmark/blog_v0_2/README.md @@ -0,0 +1,164 @@ +# How to reproduce the benchmark results of SGLang + +## Prerequisite + +### Install the latest SGLang + +```bash +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout v0.2.7 + +pip install --upgrade pip +pip install -e "python[all]" + +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ +``` + +### Set up ulimit and HF_TOKEN + +```bash +ulimit -n 65535 +# Change the token to a real and usable one, with access permissions for the Llama 3 models. +export HF_TOKEN=hf_token +``` + +### Launch the server + +```bash +# Meta-Llama-3.1-8B-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache + +# Meta-Llama-3.1-70B-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 8 + +# Meta-Llama-3-70B-Instruct-FP8 +python -m sglang.launch_server --model-path neuralmagic/Meta-Llama-3-70B-Instruct-FP8 --disable-radix-cache --tp 8 +``` + +## Benchmark + +### Hardware Requirements + +- 8B models: Single NVIDIA A100 80GB GPU +- 70B models: 8 x NVIDIA A100 80GB GPUs with Tensor Parallelism (TP) 8 +- 70B FP8 models: 8 x NVIDIA H100 GPUs with Tensor Parallelism (TP) 8 + +Please ensure you have the appropriate hardware before running the benchmarks. + +#### Offline benchmark + +```bash +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 3000 --output-file offline.jsonl +cat offline.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +#### Online benchmark + +```bash +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online.jsonl +cat online.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +## Other + +We tried using vLLM 0.5.3.post1, but it often crashes under high loads, and it seems to have similar or worse performance compared to vLLM 0.5.2 from our partial benchmarking, so we are using the older version, vLLM 0.5.2. + +Preparation for TensorRT LLM can refer to https://github.com/sgl-project/tensorrt-demo. Specifically, we used a batch size of 512, a max input length of 8192, and a max number of tokens of 8192. The instance count for preprocessing and postprocessing in Triton Server is 16. + +```bash +# vLLM +pip install vllm==0.5.2 +pip install jsonschema==4.21.1 + +# Meta-Llama-3-8B-Instruct +python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-8B-Instruct --disable-log-requests + +# meta-llama/Meta-Llama-3-70B-Instruct +python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B-Instruct --disable-log-requests --tensor 8 + +# neuralmagic/Meta-Llama-3-70B-Instruct-FP8 +python -m vllm.entrypoints.openai.api_server --model neuralmagic/Meta-Llama-3-70B-Instruct-FP8 --disable-log-requests --tensor 8 +``` + +```bash +wget https://raw.githubusercontent.com/sgl-project/sglang/main/python/sglang/bench_serving.py +``` + +```bash +# vLLM Offline + +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name sharegpt --num-prompts 3000 --output-file offline_vllm.jsonl +cat offline_vllm.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# vLLM Online + +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_vllm.jsonl +cat online_vllm.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Offline 8B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_trt_8b.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name sharegpt --num-prompts 3000 --output-file offline_trt_8b.jsonl +cat offline_trt_8b.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Online 8B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_trt_8b.jsonl +cat online_trt_8b.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Offline 70B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_trt_70b.jsonl --model meta-llama/Meta-Llama-3-70B-Instruct +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name sharegpt --num-prompts 3000 --output-file offline_trt_70b.jsonl +cat offline_trt_70b.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Online 70B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_trt_70b.jsonl +cat online_trt_70b.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` diff --git a/benchmark/blog_v0_2/config.md b/benchmark/blog_v0_2/config.md new file mode 100644 index 0000000..3faf600 --- /dev/null +++ b/benchmark/blog_v0_2/config.md @@ -0,0 +1,100 @@ +### used for TensorRT LLM + +``` +{ + "architecture": "LlamaForCausalLM", + "dtype": "float16", + "logits_dtype": "float32", + "vocab_size": 128256, + "max_position_embeddings": 8192, + "hidden_size": 16384, + "num_hidden_layers": 126, + "num_attention_heads": 128, + "num_key_value_heads": 16, + "head_size": 128, + "qk_layernorm": false, + "hidden_act": "silu", + "intermediate_size": 53248, + "norm_epsilon": 1e-05, + "position_embedding_type": "rope_gpt_neox", + "use_parallel_embedding": false, + "embedding_sharding_dim": 0, + "share_embedding_table": false, + "mapping": { + "world_size": 8, + "tp_size": 8, + "pp_size": 1, + "gpus_per_node": 8 + }, + "quantization": { + "quant_algo": "FP8", + "kv_cache_quant_algo": null, + "group_size": 128, + "smoothquant_val": null, + "has_zero_point": false, + "pre_quant_scale": false, + "exclude_modules": [ + "lm_head" + ] + }, + "kv_dtype": "float16", + "rotary_scaling": null, + "residual_mlp": false, + "moe_normalization_mode": null, + "rotary_base": 500000.0, + "moe_num_experts": 0, + "moe_top_k": 0, + "moe_tp_mode": 2, + "attn_bias": false, + "disable_weight_only_quant_plugin": false, + "mlp_bias": false +} +``` + +### used for vLLM and SGLang + +``` +{ + "_name_or_path": "dummy_fp8", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128009, + "hidden_act": "silu", + "hidden_size": 16384, + "initializer_range": 0.02, + "intermediate_size": 53248, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 128, + "num_hidden_layers": 126, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "quantization_config": { + "activation_scheme": "static", + "ignored_layers": [ + "lm_head" + ], + "quant_method": "fp8" + }, + "rope_scaling": { + "factor": 8.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "max_position_embeddings": 131072, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.41.1", + "use_cache": true, + "vocab_size": 128256 +} +``` diff --git a/benchmark/deepseek_v3/README.md b/benchmark/deepseek_v3/README.md new file mode 100644 index 0000000..8863142 --- /dev/null +++ b/benchmark/deepseek_v3/README.md @@ -0,0 +1,285 @@ +# DeepSeek V3 Support + +The SGLang and DeepSeek teams collaborated to get DeepSeek V3 FP8 running on NVIDIA and AMD GPUs **from day one**. SGLang also supports [MLA optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [DP attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models), making SGLang one of the best open-source LLM engines for running DeepSeek models. SGLang is the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended). + +Special thanks to Meituan's Search & Recommend Platform Team and Baseten's Model Performance Team for implementing the model, and DataCrunch for providing GPU resources. + +For optimizations made on the DeepSeek series models regarding SGLang, please refer to [DeepSeek Model Optimizations in SGLang](https://docs.sglang.ai/references/deepseek.html). + +## Installation & Launch + +If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. + +### Using Docker (Recommended) + +```bash +# Pull latest image +# https://hub.docker.com/r/lmsysorg/sglang/tags +docker pull lmsysorg/sglang:latest + +# Launch +docker run --gpus all --shm-size 32g -p 30000:30000 -v ~/.cache/huggingface:/root/.cache/huggingface --ipc=host --network=host --privileged lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code --port 30000 +``` + +If you are using RDMA, please note that: + +1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. +2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +Add [performance optimization options](#performance-optimization-options) as needed. + +### Using pip + +```bash +# Installation +pip install "sglang[all]>=0.4.3" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python + +# Launch +python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code +``` + +Add [performance optimization options](#performance-optimization-options) as needed. + + + +### Performance Optimization Options + +[MLA optimizations](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) are enabled by default. Here are some optional optimizations can be enabled as needed. + +- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--enable-dp-attention` argument to boost throughput. +- [Torch.compile Optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#torchcompile-latency-optimizations): Add `--enable-torch-compile` argument to enable it. This will take some time while server starts. The maximum batch size for torch.compile optimization can be controlled with `--torch-compile-max-bs`. It's recommended to set it between `1` and `8`. (e.g., `--torch-compile-max-bs 8`) + +### Example: Sending requests with OpenAI API + +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0, + max_tokens=64, +) +print(response) +``` + +### Example: Serving with two H20\*8 nodes + +For example, there are two H20 nodes, each with 8 GPUs. The first node's IP is `10.0.0.1`, and the second node's IP is `10.0.0.2`. Please **use the first node's IP** for both commands. + +If the command fails, try setting the `GLOO_SOCKET_IFNAME` parameter. For more information, see [Common Environment Variables](https://pytorch.org/docs/stable/distributed.html#common-environment-variables). + +If the multi nodes support NVIDIA InfiniBand and encounter hanging issues during startup, consider adding the parameter `export NCCL_IB_GID_INDEX=3`. For more information, see [this](https://github.com/sgl-project/sglang/issues/3516#issuecomment-2668493307). + +```bash +# node 1 +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 10.0.0.1:5000 --nnodes 2 --node-rank 0 --trust-remote-code + +# node 2 +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 10.0.0.1:5000 --nnodes 2 --node-rank 1 --trust-remote-code +``` + +If you have two H100 nodes, the usage is similar to the aforementioned H20. + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +### Example: Serving with two H200\*8 nodes and docker + +There are two H200 nodes, each with 8 GPUs. The first node's IP is `192.168.114.10`, and the second node's IP is `192.168.114.11`. Configure the endpoint to expose it to another Docker container using `--host 0.0.0.0` and `--port 40000`, and set up communications with `--dist-init-addr 192.168.114.10:20000`. +A single H200 with 8 devices can run DeepSeek V3, the dual H200 setup is just to demonstrate multi-node usage. + +```bash +# node 1 +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode1 \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 192.168.114.10:20000 --nnodes 2 --node-rank 0 --trust-remote-code --host 0.0.0.0 --port 40000 +``` + +```bash +# node 2 +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode2 \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 192.168.114.10:20000 --nnodes 2 --node-rank 1 --trust-remote-code --host 0.0.0.0 --port 40000 +``` + +To ensure functionality, we include a test from a client Docker container. + +```bash +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode_client \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1 --random-output 512 --random-range-ratio 1 --num-prompts 1 --host 0.0.0.0 --port 40000 --output-file "deepseekv3_multinode.jsonl" +``` + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +### Example: Serving with four A100\*8 nodes + +To serve DeepSeek-V3 with A100 GPUs, we need to convert the [FP8 model checkpoints](https://huggingface.co/deepseek-ai/DeepSeek-V3) to BF16 with [script](https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py) mentioned [here](https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py) first. + +Since the BF16 model is over 1.3 TB, we need to prepare four A100 nodes, each with 8 80GB GPUs. Assume the first node's IP is `10.0.0.1`, and the converted model path is `/path/to/DeepSeek-V3-BF16`, we can have following commands to launch the server. + +```bash +# node 1 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 0 --trust-remote-code --host 0.0.0.0 --port 30000 + +# node 2 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 1 --trust-remote-code + +# node 3 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 2 --trust-remote-code + +# node 4 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 3 --trust-remote-code +``` + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +Then we can benchmark the accuracy and latency by accessing the first node's exposed port with the following example commands. + +```bash +# bench accuracy +python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host http://10.0.0.1 --port 30000 + +# bench latency +python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128 +``` + + +### Example: Serving with 8 A100/A800 with AWQ Quantization + +AWQ does not support BF16, so add the `--dtype half` flag if AWQ is used for quantization. One example is as follows: + +```bash +python3 -m sglang.launch_server --model cognitivecomputations/DeepSeek-R1-AWQ --tp 8 --trust-remote-code --dtype half +``` + + +### Example: Serving with 16 A100/A800 with int8 Quantization + +There are block-wise and per-channel quantization methods, and the quantization parameters have already been uploaded to Huggingface. One example is as follows: + +- [meituan/DeepSeek-R1-Block-INT8](https://huggingface.co/meituan/DeepSeek-R1-Block-INT8) +- [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) + +Assuming that master node IP is `MASTER_IP`, checkpoint path is `/path/to/DeepSeek-R1-INT8` and port=5000, we can have following commands to launch the server: +```bash +#master +python3 -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Block-INT8 --tp 16 --dist-init-addr \ + MASTER_IP:5000 --nnodes 2 --node-rank 0 --trust-remote-code --enable-torch-compile --torch-compile-max-bs 8 +#cluster +python3 -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Block-INT8 --tp 16 --dist-init-addr \ + MASTER_IP:5000 --nnodes 2 --node-rank 1 --trust-remote-code --enable-torch-compile --torch-compile-max-bs 8 +``` + +> **Note that the launch command here enables `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +Then on the **master node**, supposing the ShareGPT data is located at `/path/to/ShareGPT_V3_unfiltered_cleaned_split.json`, you can run the following commands to benchmark the launched server: + +```bash +# bench accuracy +python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 + +# bench serving +python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0 +``` + +> **Note: using `--parallel 200` can accelerate accuracy benchmarking**. + +### Example: Serving with 32 L40S with int8 Quantization + +Running with per-channel quantization model: + +- [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) + +Assuming that master node IP is `MASTER_IP`, checkpoint path is `/path/to/DeepSeek-R1-Channel-INT8` and port=5000, we can have following commands to launch the server: + +```bash +#master +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 0 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +#cluster +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 1 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 2 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 3 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +``` + +The benchmarking method is the same as describted in the previous [16 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) example. + +### Example: Serving on any cloud or Kubernetes with SkyPilot + +SkyPilot helps find cheapest available GPUs across any cloud or existing Kubernetes clusters and launch distributed serving with a single command. See details [here](https://github.com/skypilot-org/skypilot/tree/master/llm/deepseek-r1). + +To serve on multiple nodes: + +```bash +git clone https://github.com/skypilot-org/skypilot.git +# Serve on 2 H100/H200x8 nodes +sky launch -c r1 llm/deepseek-r1/deepseek-r1-671B.yaml --retry-until-up +# Serve on 4 A100x8 nodes +sky launch -c r1 llm/deepseek-r1/deepseek-r1-671B-A100.yaml --retry-until-up +``` + +#### Troubleshooting + +If you encounter the following error with fp16/bf16 checkpoint: + +```bash +ValueError: Weight output_partition_size = 576 is not divisible by weight quantization block_n = 128. +``` + +edit your `config.json` and remove the `quantization_config` block. For example: + +```json +"quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [128, 128] +}, +``` + +Removing this block typically resolves the error. For more details, see the discussion in [sgl-project/sglang#3491](https://github.com/sgl-project/sglang/issues/3491#issuecomment-2650779851). + +## DeepSeek V3 Optimization Plan + +https://github.com/sgl-project/sglang/issues/2591 diff --git a/benchmark/dspy/README.md b/benchmark/dspy/README.md new file mode 100644 index 0000000..a2b213a --- /dev/null +++ b/benchmark/dspy/README.md @@ -0,0 +1,51 @@ +## Install + +``` +pip3 install dspy-ai +``` + +Turn off cache at https://github.com/stanfordnlp/dspy/blob/34d8420383ec752037aa271825c1d3bf391e1277/dsp/modules/cache_utils.py#L10. +``` +cache_turn_on = False +``` + +or set the environment variable + +``` +export DSP_CACHEBOOL=false +``` + +## Benchmark SGLang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_dspy_intro.py --backend sglang +``` + + +## Benchmark TGI +``` +docker run --name tgi --rm -ti --gpus all --network host \ + -v /home/ubuntu/model_weights/Llama-2-7b-chat-hf:/Llama-2-7b-chat-hf \ + ghcr.io/huggingface/text-generation-inference:1.3.0 \ + --model-id /Llama-2-7b-chat-hf --num-shard 1 --trust-remote-code \ + --max-input-length 2048 --max-total-tokens 4096 \ + --port 24000 +``` + +``` +python3 bench_dspy_intro.py --backend tgi +``` + + + +## Benchmark vLLM +``` +python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_dspy_intro.py --backend vllm +``` diff --git a/benchmark/dspy/bench_dspy_intro.py b/benchmark/dspy/bench_dspy_intro.py new file mode 100644 index 0000000..2b0936e --- /dev/null +++ b/benchmark/dspy/bench_dspy_intro.py @@ -0,0 +1,192 @@ +""" +Adapted from +https://github.com/stanfordnlp/dspy/blob/34d8420383ec752037aa271825c1d3bf391e1277/intro.ipynb#L9 +""" + +import argparse + +import dspy +from dspy.datasets import HotPotQA + + +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class GenerateAnswer(dspy.Signature): + """Answer questions with short factoid answers.""" + + context = dspy.InputField(desc="may contain relevant facts") + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class RAG(dspy.Module): + def __init__(self, num_passages=3): + super().__init__() + + self.retrieve = dspy.Retrieve(k=num_passages) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + + def forward(self, question): + context = self.retrieve(question).passages + prediction = self.generate_answer(context=context, question=question) + return dspy.Prediction(context=context, answer=prediction.answer) + + +def main(args): + # lm = dspy.OpenAI(model='gpt-3.5-turbo') + if args.backend == "tgi": + lm = dspy.HFClientTGI( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + elif args.backend == "sglang": + lm = dspy.HFClientSGLang( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + elif args.backend == "vllm": + lm = dspy.HFClientVLLM( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + else: + raise ValueError(f"Invalid backend: {args.backend}") + + colbertv2_wiki17_abstracts = dspy.ColBERTv2( + url="http://20.102.90.50:2017/wiki17_abstracts" + ) + dspy.settings.configure(lm=lm, rm=colbertv2_wiki17_abstracts) + + # Load the dataset. + dataset = HotPotQA( + train_seed=1, train_size=20, eval_seed=2023, dev_size=args.dev_size, test_size=0 + ) + + # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata. + trainset = [x.with_inputs("question") for x in dataset.train] + devset = [x.with_inputs("question") for x in dataset.dev] + + print(len(trainset), len(devset)) + + train_example = trainset[0] + print(f"Question: {train_example.question}") + print(f"Answer: {train_example.answer}") + + dev_example = devset[18] + print(f"Question: {dev_example.question}") + print(f"Answer: {dev_example.answer}") + print(f"Relevant Wikipedia Titles: {dev_example.gold_titles}") + + print( + f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys {train_example.labels().keys()}" + ) + print( + f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys {dev_example.labels().keys()}" + ) + + # Define the predictor. + generate_answer = dspy.Predict(BasicQA) + + # Call the predictor on a particular input. + pred = generate_answer(question=dev_example.question) + + # Print the input and the prediction. + print(f"Question: {dev_example.question}") + print(f"Predicted Answer: {pred.answer}") + + lm.inspect_history(n=1) + + # Define the predictor. Notice we're just changing the class. The signature BasicQA is unchanged. + generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA) + + # Call the predictor on the same input. + pred = generate_answer_with_chain_of_thought(question=dev_example.question) + + # Print the input, the chain of thought, and the prediction. + print(f"Question: {dev_example.question}") + print(f"Thought: {pred.rationale.split('.', 1)[1].strip()}") + print(f"Predicted Answer: {pred.answer}") + + retrieve = dspy.Retrieve(k=3) + topK_passages = retrieve(dev_example.question).passages + + print( + f"Top {retrieve.k} passages for question: {dev_example.question} \n", + "-" * 30, + "\n", + ) + + for idx, passage in enumerate(topK_passages): + print(f"{idx+1}]", passage, "\n") + + retrieve("When was the first FIFA World Cup held?").passages[0] + + from dspy.teleprompt import BootstrapFewShot + + # Validation logic: check that the predicted answer is correct. + # Also check that the retrieved context does actually contain that answer. + def validate_context_and_answer(example, pred, trace=None): + answer_EM = dspy.evaluate.answer_exact_match(example, pred) + answer_PM = dspy.evaluate.answer_passage_match(example, pred) + return answer_EM and answer_PM + + # Set up a basic teleprompter, which will compile our RAG program. + teleprompter = BootstrapFewShot(metric=validate_context_and_answer) + + # Compile! + compiled_rag = teleprompter.compile(RAG(), trainset=trainset) + + # Ask any question you like to this simple RAG program. + my_question = "What castle did David Gregory inherit?" + + # Get the prediction. This contains `pred.context` and `pred.answer`. + pred = compiled_rag(my_question) + + # Print the contexts and the answer. + print(f"Question: {my_question}") + print(f"Predicted Answer: {pred.answer}") + print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}") + + from dspy.evaluate.evaluate import Evaluate + + # Set up the `evaluate_on_hotpotqa` function. We'll use this many times below. + evaluate_on_hotpotqa = Evaluate( + devset=devset, + num_threads=args.num_threads, + display_progress=True, + display_table=5, + ) + + # Evaluate the `compiled_rag` program with the `answer_exact_match` metric. + metric = dspy.evaluate.answer_exact_match + evaluate_on_hotpotqa(compiled_rag, metric=metric) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int) + parser.add_argument("--num-threads", type=int, default=32) + parser.add_argument("--dev-size", type=int, default=150) + parser.add_argument( + "--backend", type=str, choices=["sglang", "tgi", "vllm"], default="sglang" + ) + args = parser.parse_args() + + if args.port is None: + default_port = { + "vllm": 21000, + "lightllm": 22000, + "tgi": 24000, + "sglang": 30000, + } + args.port = default_port.get(args.backend, None) + + main(args) diff --git a/benchmark/generative_agents/README.md b/benchmark/generative_agents/README.md new file mode 100644 index 0000000..393a9ce --- /dev/null +++ b/benchmark/generative_agents/README.md @@ -0,0 +1,38 @@ +## Download the dataset + +``` +wget -O agent_calls.jsonl https://drive.google.com/uc?export=download&id=19qLpD45e9JGTKF2cUjJJegwzSUEZEKht +``` + +## Run benchmark + +Ensure that this benchmark is run in a serial manner (using --parallel 1) to preserve any potential dependencies between requests. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-events 1000 --parallel 1 +``` + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-events 1000 --backend vllm --parallel 1 +``` + +### Benchmark guidance +``` +python3 bench_other.py --num-events 1000 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-events 1000 --backend lmql --parallel 1 +``` diff --git a/benchmark/generative_agents/agent_functions.py b/benchmark/generative_agents/agent_functions.py new file mode 100644 index 0000000..9785fc1 --- /dev/null +++ b/benchmark/generative_agents/agent_functions.py @@ -0,0 +1,300 @@ +import sglang as sgl + +# here are the top five agent functions contributing ~70% LLM calls +# reference: https://github.com/joonspk-research/generative_agents/ + + +@sgl.function +def poignancy_event(s, persona_name, persona_iss, event): + s += "Here is a brief description of " + persona_name + ".\n" + s += persona_iss + "\n" + s += "On the scale of 1 to 10, where 1 is purely mundane (e.g., brushing teeth, making bed) and 10 is extremely poignant (e.g., a break up, college acceptance), rate the likely poignancy of the following event for" + s += persona_name + ".\n\n" + s += "Event: " + event + s += "Rate (return a number between 1 to 10):" + s += sgl.gen(name="Rate", max_tokens=2) + + +def poignancy_event_prompt(persona_name, persona_iss, event): + # return prompt and max_tokens + s = "" + s += "Here is a brief description of " + persona_name + ".\n" + s += persona_iss + "\n" + s += "On the scale of 1 to 10, where 1 is purely mundane (e.g., brushing teeth, making bed) and 10 is extremely poignant (e.g., a break up, college acceptance), rate the likely poignancy of the following event for" + s += persona_name + ".\n\n" + s += "Event: " + event + s += "Rate (return a number between 1 to 10):" + return {"prompt": s, "max_tokens": 2, "stop": None} + + +@sgl.function +def generate_event_triple(s, persona_name, action): + s += """Task: Turn the input into (subject, predicate, object). +Input: Sam Johnson is eating breakfast. +Output: (Dolores Murphy, eat, breakfast) +--- +Input: Joon Park is brewing coffee. +Output: (Joon Park, brew, coffee) +--- +Input: Jane Cook is sleeping. +Output: (Jane Cook, is, sleep) +--- +Input: Michael Bernstein is writing email on a computer. +Output: (Michael Bernstein, write, email) +--- +Input: Percy Liang is teaching students in a classroom. +Output: (Percy Liang, teach, students) +--- +Input: Merrie Morris is running on a treadmill. +Output: (Merrie Morris, run, treadmill) +---""" + s += persona_name + "is" + action + ".\n" + s += "(" + persona_name + "," + s += sgl.gen(name="Triple", max_tokens=20, stop=")") + + +def generate_event_triple_prompt(persona_name, action): + s = "" + s += """Task: Turn the input into (subject, predicate, object). +Input: Sam Johnson is eating breakfast. +Output: (Dolores Murphy, eat, breakfast) +--- +Input: Joon Park is brewing coffee. +Output: (Joon Park, brew, coffee) +--- +Input: Jane Cook is sleeping. +Output: (Jane Cook, is, sleep) +--- +Input: Michael Bernstein is writing email on a computer. +Output: (Michael Bernstein, write, email) +--- +Input: Percy Liang is teaching students in a classroom. +Output: (Percy Liang, teach, students) +--- +Input: Merrie Morris is running on a treadmill. +Output: (Merrie Morris, run, treadmill) +---""" + s += persona_name + "is" + action + ".\n" + s += "(" + persona_name + "," + return {"prompt": s, "max_tokens": 20, "stop": ")"} + + +@sgl.function +def generate_pronunciatio(s, action): + s += "Convert an action description to an emoji (important: use two or less emojis).\n" + s += "Action description: " + action + ".\n" + s += "Emoji:" + sgl.gen(name="Emoji", max_tokens=6) + + +def generate_pronunciatio_prompt(action): + s = "" + s += "Convert an action description to an emoji (important: use two or less emojis).\n" + s += "Action description: " + action + ".\n" + s += "Emoji:" + return {"prompt": s, "max_tokens": 6, "stop": None} + + +@sgl.function +def action_location_sector( + s, + persona_name, + living_sector, + living_sector_areas, + current_sector, + current_sector_areas, + daily_plan, + sector_options, + current_action, + next_action, +): + s += """Task -- choose an appropriate area from the area options for a task at hand. +Sam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Sam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Area options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For taking a walk, Sam Kim should go to the following area: {Johnson Park} +--- +Jane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room. +Jane Anderson is currently in {Oak Hill College} that has a classroom, library +Area options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe} +---""" + s += ( + persona_name + + " lives in " + + living_sector + + " that has " + + living_sector_areas + + ".\n" + ) + s += ( + persona_name + + " is currently in " + + current_sector + + " that has " + + current_sector_areas + + ".\n" + ) + s += daily_plan + ".\n" + s += "Area options: " + sector_options + ".\n" + s += """* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim.\n""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + " should go to the following area: {" + ) + s += sgl.gen(name="Location", max_tokens=10, stop="}") + + +def action_location_sector_prompt( + persona_name, + living_sector, + living_sector_areas, + current_sector, + current_sector_areas, + daily_plan, + sector_options, + current_action, + next_action, +): + s = "" + s += """Task -- choose an appropriate area from the area options for a task at hand. +Sam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Sam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Area options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For taking a walk, Sam Kim should go to the following area: {Johnson Park} +--- +Jane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room. +Jane Anderson is currently in {Oak Hill College} that has a classroom, library +Area options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe} +---""" + s += ( + persona_name + + " lives in " + + living_sector + + " that has " + + living_sector_areas + + ".\n" + ) + s += ( + persona_name + + " is currently in " + + current_sector + + " that has " + + current_sector_areas + + ".\n" + ) + s += daily_plan + ".\n" + s += "Area options: " + sector_options + ".\n" + s += """* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim.\n""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + " should go to the following area: {" + ) + return {"prompt": s, "max_tokens": 10, "stop": "}"} + + +@sgl.function +def action_location_object( + s, persona_name, target_sector, target_sector_areas, current_action, next_action +): + s += """ +Jane Anderson is in kitchen in Jane Anderson's house. +Jane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For cooking, Jane Anderson should go to the following area in Jane Anderson's house: +Answer: {kitchen} +--- +Tom Watson is in common room in Tom Watson's apartment. +Tom Watson is going to Hobbs Cafe that has the following areas: {cafe} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For getting coffee, Tom Watson should go to the following area in Hobbs Cafe: +Answer: {cafe} +---""" + s += ( + persona_name + + " is going to " + + target_sector + + " that has the following areas: {" + + target_sector_areas + + "}\n" + ) + s += """* Stay in the current area if the activity can be done there. +* NEVER go into other people's rooms unless necessary.""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + "should go to the following area in " + + target_sector + ) + s += " (MUST pick one of {" + target_sector_areas + "}):\n" + s += "Answer: {" + sgl.gen(name="Area", max_tokens=5, stop="}") + + +def action_location_object_prompt( + persona_name, target_sector, target_sector_areas, current_action, next_action +): + s = "" + s += """ +Jane Anderson is in kitchen in Jane Anderson's house. +Jane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For cooking, Jane Anderson should go to the following area in Jane Anderson's house: +Answer: {kitchen} +--- +Tom Watson is in common room in Tom Watson's apartment. +Tom Watson is going to Hobbs Cafe that has the following areas: {cafe} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For getting coffee, Tom Watson should go to the following area in Hobbs Cafe: +Answer: {cafe} +---""" + s += ( + persona_name + + " is going to " + + target_sector + + " that has the following areas: {" + + target_sector_areas + + "}\n" + ) + s += """* Stay in the current area if the activity can be done there. +* NEVER go into other people's rooms unless necessary.""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + "should go to the following area in " + + target_sector + ) + s += " (MUST pick one of {" + target_sector_areas + "}):\n" + s += "Answer: {" + return {"prompt": s, "max_tokens": 5, "stop": "}"} diff --git a/benchmark/generative_agents/bench_other.py b/benchmark/generative_agents/bench_other.py new file mode 100644 index 0000000..48f6ebc --- /dev/null +++ b/benchmark/generative_agents/bench_other.py @@ -0,0 +1,80 @@ +import argparse +import json +import time + +from agent_functions import ( + action_location_object_prompt, + action_location_sector_prompt, + generate_event_triple_prompt, + generate_pronunciatio_prompt, + poignancy_event_prompt, +) +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_events] + mapping = { + "poignancy_event": poignancy_event_prompt, + "generate_event_triple": generate_event_triple_prompt, + "generate_pronunciatio": generate_pronunciatio_prompt, + "action_location_sector": action_location_sector_prompt, + "action_location_object": action_location_object_prompt, + } + + arguments = [mapping[k](**v) for l in lines for k, v in l.items()] + states = [] + + # Select backend + call_generate = get_call_generate(args) + + def get_one_answer(arg): + answer = call_generate(**arg, temperature=0) + states.append(answer) + + async def get_one_answer_async(arg): + answer = await call_generate(**arg, temperature=0) + states.append(answer) + + tic = time.time() + # we always sequentially execute agent calls to maintain its dependency + if args.backend != "lmql": + for arg in tqdm(arguments): + get_one_answer(arg) + else: + import asyncio + + loop = asyncio.get_event_loop() + for arg in tqdm(arguments): + loop.run_until_complete(get_one_answer_async(arg)) + latency = time.time() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "Generative Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + # to pack weighted functions as a single agent + "num_requests": len(arguments) / len(mapping), + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="agent_calls.jsonl") + parser.add_argument("--num-events", type=int, default=10) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/generative_agents/bench_sglang.py b/benchmark/generative_agents/bench_sglang.py new file mode 100644 index 0000000..b42a32b --- /dev/null +++ b/benchmark/generative_agents/bench_sglang.py @@ -0,0 +1,74 @@ +import argparse +import json +import time + +from agent_functions import ( + action_location_object, + action_location_sector, + generate_event_triple, + generate_pronunciatio, + poignancy_event, +) + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_events] + mapping = { + "poignancy_event": poignancy_event, + "generate_event_triple": generate_event_triple, + "generate_pronunciatio": generate_pronunciatio, + "action_location_sector": action_location_sector, + "action_location_object": action_location_object, + } + arguments = [{mapping[k]: v for k, v in l.items()} for l in lines] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + states = [] + # Run requests + tic = time.time() + for a in arguments: + # only a single key in the dict + for func, arg in a.items(): + result = func.run(**arg) + result.sync() + states.append(result) + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "Generative Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + # to pack weighted functions as a single agent + "num_requests": len(arguments) / len(mapping), + "other": { + "num_events": args.num_events, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="agent_calls.jsonl") + parser.add_argument("--num-events", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/gsm8k/README.md b/benchmark/gsm8k/README.md new file mode 100644 index 0000000..c110f53 --- /dev/null +++ b/benchmark/gsm8k/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 200 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 200 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +CUDA_VISIBLE_DEVICES=0,1 lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --num-questions 100 --backend lmql --parallel 2 +``` diff --git a/benchmark/gsm8k/bench_other.py b/benchmark/gsm8k/bench_other.py new file mode 100644 index 0000000..a8bbcfb --- /dev/null +++ b/benchmark/gsm8k/bench_other.py @@ -0,0 +1,151 @@ +import argparse +import ast +import asyncio +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def main(args): + # Select backend + call_generate = get_call_generate(args) + + # Read data + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + + states = [None] * len(labels) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + answer = call_generate( + prompt=few_shot_examples + questions[i], + temperature=0, + max_tokens=256, + stop=["Question", "Assistant:", "<|separator|>"], + ) + states[i] = answer + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(questions), batch_size): + tasks = [] + for q in questions[i : i + batch_size]: + tasks.append( + call_generate( + few_shot_examples + q, + temperature=0, + max_tokens=256, + stop="Question", + ) + ) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + states[i + j] = rets[j] + + tic = time.time() + asyncio.run(batched_call(batch_size=args.parallel)) + latency = time.time() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Latency: {latency:.3f} s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/gsm8k/bench_sglang.py b/benchmark/gsm8k/bench_sglang.py new file mode 100644 index 0000000..b6bdbef --- /dev/null +++ b/benchmark/gsm8k/bench_sglang.py @@ -0,0 +1,141 @@ +import argparse +import ast +import json +import os +import re +import time + +import numpy as np + +from sglang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + data_path = args.data_path + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.time() + states = few_shot_gsm8k.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i]["answer"])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/hellaswag/README.md b/benchmark/hellaswag/README.md new file mode 100644 index 0000000..cb7e653 --- /dev/null +++ b/benchmark/hellaswag/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 200 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lightllm +``` + + +### Benchmark guidance +``` +CUDA_VISIBLE_DEVICES=0,1 python3 bench_other.py --num-questions 200 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lmql --port 23000 --parallel 1 +``` diff --git a/benchmark/hellaswag/bench_other.py b/benchmark/hellaswag/bench_other.py new file mode 100644 index 0000000..04be456 --- /dev/null +++ b/benchmark/hellaswag/bench_other.py @@ -0,0 +1,118 @@ +import argparse +import asyncio +import json +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_select +from sglang.utils import download_and_cache_file, read_jsonl + + +def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def main(args): + # Select backend + call_select = get_call_select(args) + + # Read data + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + + preds = [None] * len(labels) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + preds[i] = call_select( + context=few_shot_examples + questions[i], choices=choices[i] + ) + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(questions), batch_size): + tasks = [] + for q, c in zip( + questions[i : i + batch_size], choices[i : i + batch_size] + ): + tasks.append(call_select(context=few_shot_examples + q, choices=c)) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + preds[i + j] = rets[j] + + tic = time.time() + asyncio.run(batched_call(batch_size=args.parallel)) + + latency = time.time() - tic + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + print(f"Latency: {latency:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "hellaswag", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=20) + parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/hellaswag/bench_sglang.py b/benchmark/hellaswag/bench_sglang.py new file mode 100644 index 0000000..798521f --- /dev/null +++ b/benchmark/hellaswag/bench_sglang.py @@ -0,0 +1,109 @@ +import argparse +import json +import os +import time + +import numpy as np + +from sglang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import download_and_cache_file, read_jsonl + + +def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + data_path = args.data_path + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_hellaswag(s, question, choices): + s += few_shot_examples + question + s += sgl.select("answer", choices=choices) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.time() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + preds = [choices[i].index(rets[i]["answer"]) for i in range(len(rets))] + latency = time.time() - tic + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + print(f"Latency: {latency:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "hellaswag", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=20) + parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/hicache/README.md b/benchmark/hicache/README.md new file mode 100644 index 0000000..f2aa6ff --- /dev/null +++ b/benchmark/hicache/README.md @@ -0,0 +1,91 @@ +## Run synthetic multi-turn benchmark + +``` +# SGLang server with radix cache disabled +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --disable-radix-cache + +# SGLang server with radix cache on and first-come-first-serve policy +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --schedule-policy fcfs + +# The default SGLang server with radix cache on and long-prefix-match policy +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 + +# SGLang server with hierarchical radix cache enabled +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --enable-hierarchical-cache + +``` + +``` +python bench_multiturn.py --model-path Qwen/Qwen2.5-14B-Instruct +``` + +Note: The performance gain of hierarchical caching depends on the ratio of reusable tokens to GPU memory capacity. The more tokens to be reused, the larger the model, and the more constrained the GPU memory size, the greater the benefit one can expect from hierarchical caching. + + +# Benchmark with more datasets +## Download Dataset +```bash +./download.sh {sharegpt|ultragpt|loogle|nextqa|all} +``` +This script will automatically download the required dataset to the current working directory + +## Multiturn Benchmark +### Supported Datasets +- sharegpt +- ultrachat +- loogle +### Example Usage: +```bash +python3 bench_serving.py --model mistralai/Mistral-7B-Instruct-v0.3 --backend sglang \ +--dataset-path longdep_qa.json --dataset-name loogle --request-rate 10 --num-prompts 10 \ +--port 8001 --enable-multiturn --disable-shuffle +``` +This uses `mistralai/Mistral-7B-Instruct-v0.3` model with `sglang` as backend. The dataset +is `longdep_qa.json`. We send `10 conversations` with `10 req/s` to port 8001. We enable +multiturn chat without shuffling the order of conversations (i.e. following the original +order in the dataset file). + +### Note: +The requests of multiple conversations are sent in a round robin fashion. +For example, if we have 3 conversations A, B, C whose rounds are `[2, 3, 4]` correspondingly, +multiturn chat will send the requests to the backend in the following order: `[A1, B1, C1, A2, B2, C2, B3, C3, C4]` +This has implications on the cache reuse patterns: the cache reuse distance is the largest +under this request pattern (which means a prefix-aware local scheduler in the backend can +yield the most benefit compared to a FIFO scheduler) + +## Shared Prefix Benchmark +### Supported Datasets +- loogle +### Example Usage: +```bash +python3 bench_serving.py --model mistralai/Mistral-7B-Instruct-v0.3 --backend sglang \ +--dataset-path longdep_qa.json --dataset-name loogle --request-rate 10 --num-prompts 10 \ +--port 8001 --enable-shared-prefix --disable-shuffle +``` +### Note: +Shared Prefix benchmark sends the questions for the same prompt together. For example, +if we have 3 shared prefix A, B, C, which have [2, 3, 4] questions correspondingly, +the shared prefix benchmark will send the requests to the +backend in the following order: `[A+Q1, A+Q2, B+Q1, B+Q2, B+Q3, C+Q1, C+Q2, C+Q3]`. + + +## Multi Modality Benchmark (WIP) +### Supported Datasets: +- nextqa +### Example Usage: +```bash +Server: +python3 -m sglang.launch_server --model-path lmms-lab/LLaVA-NeXT-Video-7B --tp 2 --dp 1 --port 8001 \ +--host 0.0.0.0 --mem-fraction-static 0.9 --tokenizer-path llava-hf/llava-1.5-7b-hf \ +--json-model-override-args "{\"architectures\": [\"LlavaVidForCausalLM\"], \"model_type\":\"llava\", \"mm_spatial_pool_stride\":2}" + +Client: +python3 bench_serving.py --model lmms-lab/LLaVA-NeXT-Video-7B --backend sglang --dataset-path \ +NExTVideo --dataset-name nextqa --request-rate 10 --num-prompts 1 --disable-shuffle --port 8001 \ --enable-multiturn --max-frames 16 --tokenizer llava-hf/llava-1.5-7b-hf --fixed-output-len 2048 +``` +Note: for the server args, `tokenizer-path`, overriding architecture are necessary. + +## Supported Backend +- sglang (oai) +- vllm (oai) +- lmdeploy (oai) diff --git a/benchmark/hicache/bench_multiturn.py b/benchmark/hicache/bench_multiturn.py new file mode 100644 index 0000000..6bd0bd9 --- /dev/null +++ b/benchmark/hicache/bench_multiturn.py @@ -0,0 +1,395 @@ +import argparse +import asyncio +import json +import queue +import random +import threading +import time +from datetime import datetime +from typing import Optional + +import aiohttp +import requests +from tqdm.asyncio import tqdm + +from sglang.bench_serving import ( + RequestFuncOutput, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Script to benchmark concurrent requests to a server." + ) + parser.add_argument( + "--num-clients", + type=int, + default=256, + help="Number of concurrent clients", + ) + parser.add_argument( + "--max-parallel", + type=int, + default=128, + help="Maximum number of parallel requests", + ) + parser.add_argument( + "--request-length", + type=int, + default=512, + help="Length of each new request", + ) + parser.add_argument( + "--output-length", + type=int, + default=64, + help="Length of each output", + ) + parser.add_argument( + "--num-rounds", + type=int, + default=5, + help="Number of rounds per client", + ) + parser.add_argument( + "--distribution", + type=str, + default="poisson", + choices=["poisson", "uniform"], + help="Distribution type for request intervals (poisson or uniform)", + ) + parser.add_argument( + "--request-rate", + type=float, + default=1.0, + help="Average number of requests per second", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Server hostname or IP (default: localhost)", + ) + parser.add_argument( + "--port", + type=int, + default=30000, + help="Server port (default: 30000)", + ) + parser.add_argument( + "--model-path", + type=str, + default="meta-llama/Llama-3.1-8B-Instruct", + help="model path compatible with Hugging Face Transformers", + ) + parser.add_argument( + "--dataset-path", + type=str, + default="", + help="local dataset to sample tokens from", + ) + parser.add_argument( + "--log-file", + type=str, + default="performance_metrics.jsonl", + help="File to log performance metrics", + ) + return parser.parse_args() + + +async def async_request_sglang_generate( + payload, + url, + pbar: Optional[tqdm] = None, +): + """ + Sends a streaming request to the server. Gathers text token-by-token. + """ + async with aiohttp.ClientSession() as session: + headers = {} + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + output = RequestFuncOutput() + + try: + async with session.post(url=url, json=payload, headers=headers) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + if data["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text = data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + else: + output.error = response.reason or "" + output.success = False + except Exception as e: + output.success = False + output.error = str(e) + print(f"Request failed: {e}") + + if pbar: + pbar.update(1) + return output + + +def gen_payload(prompt, output_len): + payload = { + "text": prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "stream": True, + "lora_path": "", + "return_logprob": False, + "logprob_start_len": -1, + } + return payload + + +def log_to_jsonl_file(data, file_path="performance_metrics.jsonl"): + """Append the data with a timestamp to the specified JSONL file.""" + timestamped_data = {"timestamp": datetime.now().isoformat(), **data} + try: + with open(file_path, "a") as file: + file.write( + json.dumps(timestamped_data) + "\n" + ) # Write as a single line in JSONL format + except IOError as e: + print(f"Error writing to JSONL file: {e}") + + +class ReadyQueue: + """ + Thread-safe queue that can pop requests in different orders based on given policy. + """ + + def __init__(self, init_requests=None, policy="random"): + self.lock = threading.Lock() + self.requests = init_requests or [] + self.policy = policy + + def append(self, item): + with self.lock: + self.requests.append(item) + + def pop(self): + with self.lock: + if not self.requests: + return None + if self.policy == "random": + index = random.randrange(len(self.requests)) + return self.requests.pop(index) + elif self.policy == "fifo": + return self.requests.pop(0) + else: + # todo, varying thinking time of clients + raise ValueError(f"{self.policy} not implemented") + + +class WorkloadGenerator: + def __init__(self, args): + # Construct the base URL for requests + self.url = f"http://{args.host}:{args.port}/generate" + + self.tokenizer = get_tokenizer(args.model_path) + self.distribution = args.distribution + self.request_rate = args.request_rate + self.start_time = None + self.finished_time = None + + self.sent_requests = 0 + self.completed_requests = 0 + + self.candidate_inputs = sample_random_requests( + input_len=args.request_length, + output_len=args.output_length, + num_prompts=args.num_clients * args.num_rounds, + range_ratio=1.0, + tokenizer=self.tokenizer, + dataset_path=args.dataset_path, + ) + self.candidate_inputs = [i[0] for i in self.candidate_inputs] + + init_requests = [ + (i, gen_payload(self.candidate_inputs[i], args.output_length)) + for i in range(args.num_clients) + ] + self.client_records = { + i: {"round": 0, "history": init_requests[i][1]["text"]} + for i in range(args.num_clients) + } + self.ready_queue = ReadyQueue(init_requests=init_requests) + self.candidate_inputs = self.candidate_inputs[args.num_clients :] + + self.response_queue = queue.Queue() + self.pbar = tqdm(total=args.num_clients * args.num_rounds) + self.performance_metrics = {"ttft": [], "latency": []} + + async def handle_request(self, item): + try: + client_id, payload = item + response = await async_request_sglang_generate(payload, self.url, self.pbar) + if self.pbar.n == self.pbar.total: + self.finished_time = time.time() + self.response_queue.put((client_id, response)) + except Exception as e: + print(f"Request failed: {e}") + + def request_sender(self): + async def request_loop(): + while True: + if self.sent_requests - self.completed_requests < args.max_parallel: + new_request = self.ready_queue.pop() + if new_request: + asyncio.create_task(self.handle_request(new_request)) + self.sent_requests += 1 + else: + await asyncio.sleep(0.05) + continue + + if self.pbar.n == self.pbar.total: + break + + # Calculate Poisson-distributed wait time + if self.distribution == "poisson": + sleep_time = random.expovariate(self.request_rate) + elif self.distribution == "uniform": + avg_interval = ( + 1.0 / self.request_rate if self.request_rate > 0 else 1.0 + ) + sleep_time = random.uniform(0, 2 * avg_interval) + else: + raise ValueError("Invalid distribution type") + await asyncio.sleep(sleep_time) # Wait before sending the next request + + # Create and run the event loop for asynchronous requests + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(request_loop()) + loop.close() + + def response_handler(self): + while True: + try: + client_id, response = self.response_queue.get( + timeout=10 + ) # Block until response is available + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + self.client_records[client_id]["history"] += response.generated_text + self.client_records[client_id]["round"] += 1 + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["latency"].append(response.latency) + self.completed_requests += 1 + + if self.client_records[client_id]["round"] < args.num_rounds: + self.client_records[client_id][ + "history" + ] += self.candidate_inputs.pop() + self.ready_queue.append( + ( + client_id, + gen_payload( + self.client_records[client_id]["history"], + args.output_length, + ), + ) + ) + except queue.Empty: + if self.pbar.n == self.pbar.total: + break + + def run(self): + request_thread = threading.Thread(target=self.request_sender, daemon=True) + response_thread = threading.Thread(target=self.response_handler, daemon=True) + + self.start_time = time.time() + request_thread.start() + response_thread.start() + + request_thread.join() + response_thread.join() + self.pbar.close() + + performance_data = { + "summary": { + "total_requests": len(self.performance_metrics["ttft"]), + "request_rate": self.request_rate, + "average_ttft": sum(self.performance_metrics["ttft"]) + / len(self.performance_metrics["ttft"]), + "p90_ttft": sorted(self.performance_metrics["ttft"])[ + int(0.9 * len(self.performance_metrics["ttft"])) + ], + "median_ttft": sorted(self.performance_metrics["ttft"])[ + len(self.performance_metrics["ttft"]) // 2 + ], + "average_latency": sum(self.performance_metrics["latency"]) + / len(self.performance_metrics["latency"]), + "p90_latency": sorted(self.performance_metrics["latency"])[ + int(0.9 * len(self.performance_metrics["latency"])) + ], + "median_latency": sorted(self.performance_metrics["latency"])[ + len(self.performance_metrics["latency"]) // 2 + ], + "throughput": self.pbar.total / (self.finished_time - self.start_time), + }, + } + print("All requests completed") + print("Performance metrics summary:") + print( + f" Total requests: {performance_data['summary']['total_requests']} at {performance_data['summary']['request_rate']} requests per second" + ) + print(f" Average TTFT: {performance_data['summary']['average_ttft']:.2f}") + print(f" P90 TTFT: {performance_data['summary']['p90_ttft']:.2f}") + print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}") + print( + f" Average latency: {performance_data['summary']['average_latency']:.2f}" + ) + print(f" P90 latency: {performance_data['summary']['p90_latency']:.2f}") + print(f" Median latency: {performance_data['summary']['median_latency']:.2f}") + print( + f" Throughput: {performance_data['summary']['throughput']:.2f} requests per second" + ) + log_to_jsonl_file(performance_data, args.log_file) + + +if __name__ == "__main__": + args = parse_args() + flush_cache_url = f"http://{args.host}:{args.port}/flush_cache" + + for request_rate in [16, 14, 12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]: + args.request_rate = request_rate + requests.post(flush_cache_url) + time.sleep(1) + WorkloadGenerator(args).run() diff --git a/benchmark/hicache/bench_serving.py b/benchmark/hicache/bench_serving.py new file mode 100644 index 0000000..90f4c6c --- /dev/null +++ b/benchmark/hicache/bench_serving.py @@ -0,0 +1,1029 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --request-rate-range 1,2,4,8,16,32 --random-input 4096 --random-output 1024 --random-range-ratio 0.125 --multi +""" + +import argparse +import asyncio +import json +import os +import random +import sys +import time +import traceback +import warnings +from argparse import ArgumentParser +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +import requests +from data_processing import MsgContent, SampleOutput, get_dataset +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +from sglang.bench_serving import get_tokenizer, remove_prefix, set_ulimit + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + +global args + + +@dataclass +class RequestFuncInput: + prompts: List[Tuple[MsgContent, int, int]] + api_url: str + model: str + lora_name: str + extra_request_body: Dict[str, Any] + + # For multiturn chat, store the context + prev_messages: List = field(default_factory=list) + finished_prompts: int = 0 + + +@dataclass +class RequestFuncOutput: + generated_text: List[str] = field(default_factory=list) + prompt_len: List[int] = field(default_factory=list) + output_len: List[int] = field(default_factory=list) + latency: List[float] = field(default_factory=list) + ttft: List[float] = field(default_factory=list) + itl: List[float] = field(default_factory=list) # List of inter-token latencies + + success: bool = False + error: str = "" + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + queue: asyncio.Queue, + tokenizer: PreTrainedTokenizerBase, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "model": request_func_input.model, + "temperature": 0.0, + "best_of": 1, + "stream": not args.disable_stream, + "stream_options": {"include_usage": True}, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", + } + + output = RequestFuncOutput() + + prompt_idx = request_func_input.finished_prompts + messages = request_func_input.prev_messages + prompt, input_len, max_tokens = request_func_input.prompts[prompt_idx] + prompt_len = sum( + prompt[1] + prompt[2] # input_len + output_len + for prompt in request_func_input.prompts[:prompt_idx] + ) + prompt_len += input_len + + # Messages + messages.append( + { + "role": "user", + "content": prompt, + } + ) + payload["messages"] = messages + payload["max_tokens"] = max_tokens + + # output.prompt_len = request_func_input.prompt_len + # print(payload) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + actual_prompt_len = prompt_len - 1 + actual_output_len = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + timestamp = time.perf_counter() + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["usage"] is not None and len(data["usage"]) > 0: + actual_prompt_len = data["usage"]["prompt_tokens"] + actual_output_len = data["usage"]["completion_tokens"] + continue + delta = data["choices"][0]["delta"] + + if delta.get("content", None): + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft.append(ttft) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + generated_text += delta["content"] + most_recent_timestamp = timestamp + + output.prompt_len.append(actual_prompt_len) # truncate + output.output_len.append(actual_output_len) + output.generated_text.append(generated_text) + output.success = True + output.latency.append(latency) + + # Prepare for the new request + request_func_input.prompts[prompt_idx] = ( + prompt, + input_len, + actual_output_len, # changes from max_tokens to output_len + ) + prompt_idx += 1 + messages.append( + { + "role": "assistant", + "content": generated_text, + } + ) + + # Move the new request to the end of the queue + if prompt_idx < len(request_func_input.prompts): + request_func_input.finished_prompts = prompt_idx + request_func_input.prev_messages = messages + await queue.put(request_func_input) + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_openai_completions, + "vllm": async_request_openai_completions, + "lmdeploy": async_request_openai_completions, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p90_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p90_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p90_itl_ms: float + p99_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + + +async def get_requests( + input_requests_queue: asyncio.Queue, + request_rate: float, + num_actual_requests: int, +) -> AsyncGenerator[RequestFuncInput, None]: + for _ in range(num_actual_requests): + try: + request = await asyncio.wait_for( + input_requests_queue.get(), timeout=300 + ) # Wait for 5 minites then abort + except Exception as e: + print(f"exception: {e}") + break + + yield request + + if request_rate == float("inf"): + continue + + interval = np.random.exponential(1.0 / request_rate) + await asyncio.sleep(interval) + + +def calculate_metrics( + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + output_success = 0 + for i in range(len(outputs)): + if outputs[i].success: + output_success += 1 + assert len(outputs[i].generated_text) == len(outputs[i].latency) + assert len(outputs[i].generated_text) == len(outputs[i].ttft) + for j in range(len(outputs[i].generated_text)): + output_len = outputs[i].output_len[j] + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode( + outputs[i].generated_text[j], add_special_tokens=False + ) + ) + retokenized_output_lens.append(retokenized_output_len) + total_input += outputs[i].prompt_len[j] + if output_len > 1: + tpots.append( + (outputs[i].latency[j] - outputs[i].ttft[j]) / (output_len - 1) + ) + + completed += 1 + itls += outputs[i].itl + ttfts += outputs[i].ttft + e2e_latencies += outputs[i].latency + + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p90_ttft_ms=np.percentile(ttfts or 0, 90) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p90_tpot_ms=np.percentile(tpots or 0, 90) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p90_itl_ms=np.percentile(itls or 0, 90) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + ) + + return metrics, output_lens + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: SampleOutput, + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_name: str, + extra_request_body: Dict[str, Any], + profile: bool, + enable_shared_prefix: bool, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, queue, tokenizer, pbar): + if semaphore is None: + return await request_func( + request_func_input=request_func_input, + queue=queue, + tokenizer=tokenizer, + pbar=pbar, + ) + async with semaphore: + return await request_func( + request_func_input=request_func_input, + queue=queue, + tokenizer=tokenizer, + pbar=pbar, + ) + + num_actual_requests = sum(len(r) for r in input_requests) + print(f"Num of shared prefixes or conversations: {len(input_requests)}") + print(f"Num of total requests: {num_actual_requests}") + + # flatten the requests for shared prefix + if enable_shared_prefix: + input_requests = [[r] for requests in input_requests for r in requests] + inputs_requests_queue = asyncio.Queue(maxsize=len(input_requests)) + print("Starting initial single prompt test run...") + # NOTE: Just use the first request of the first conversation for warmup + test_input = RequestFuncInput( + model=model_id, + prompts=input_requests[0][:1], + api_url=api_url, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + test_output = await request_func( + request_func_input=test_input, queue=inputs_requests_queue, tokenizer=tokenizer + ) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + # Check the states + assert inputs_requests_queue.empty() + + # Flush cache + if "sglang" in backend: + requests.post(base_url + "/flush_cache") + + time.sleep(1.0) + + # Start profiler + if profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + for request in input_requests: + request_func_input = RequestFuncInput( + model=model_id, + prompts=request, + api_url=api_url, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + inputs_requests_queue.put_nowait(request_func_input) + if ( + not args.enable_multiturn + and not args.enable_shared_prefix + and not args.dataset_name == "generated-shared-prefix" + ): + assert len(input_requests) == num_actual_requests + + pbar = None if disable_tqdm else tqdm(total=num_actual_requests) + + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_requests( + inputs_requests_queue, request_rate, num_actual_requests + ): + tasks.append( + asyncio.create_task( + limited_request_func( + request_func_input=request, + queue=inputs_requests_queue, + tokenizer=tokenizer, + pbar=pbar, + ) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + # Stop profiler + if profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=base_url + "/stop_profile") + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print( + "{:<40} {:<10}".format( + "Max reqeuest concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P90 TTFT (ms):", metrics.p90_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P90 TPOT (ms):", metrics.p90_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P90 ITL (ms):", metrics.p90_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": request_rate, + "max_concurrency": max_concurrency, + "fixed_output_len": args.fixed_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "fixed_output_len": args.fixed_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + "duration": benchmark_duration, + "completed": metrics.completed, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "random": + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p90_ttft_ms": metrics.p90_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p90_tpot_ms": metrics.p90_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p90_itl_ms": metrics.p90_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + } + return result + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "lmdeploy": 23333, + "vllm": 8000, + }.get(args.backend, 30000) + + model_url = ( + f"{args.base_url}/v1/models" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models" + ) + + if args.backend in ["sglang", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/chat/completions" + ) + base_url = ( + f"http://{args.host}:{args.port}" if args.base_url is None else args.base_url + ) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + # Dataset compatibility check + if args.enable_multiturn: + # TODO: Support multiturn for random + if args.dataset_name not in ["sharegpt", "ultrachat", "loogle", "nextqa"]: + print( + "Multiturn conversation is only supported for sharegpt, ultrachat, loogle, and nextqa datasets." + ) + sys.exit(1) + + if args.enable_shared_prefix: + if args.dataset_name not in ["loogle", "nextqa"]: + print("Shared prefix is only supported for loogle and nextqa datasets.") + sys.exit(1) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + + tokenizer = get_tokenizer(tokenizer_id) + + input_requests = get_dataset(args, tokenizer) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_name=args.lora_name, + extra_request_body=extra_request_body, + profile=args.profile, + enable_shared_prefix=args.enable_shared_prefix, + ) + ) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "random", + "generated-shared-prefix", + "ultrachat", + "loogle", + "nextqa", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--chat-template", + type=str, + help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--fixed-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument( + "--multi", + action="store_true", + help="Use request rate range rather than single value.", + ) + parser.add_argument( + "--request-rate-range", + type=str, + default="2,34,2", + help="Range of request rates in the format start,stop,step. Default is 2,34,2. It also supports a list of request rates, requiring the parameters to not equal three.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--enable-multiturn", + action="store_true", + help="Enable multiturn chat for online serving benchmarking. " + "This option is effective on the following datasets: " + "sharegpt, ultrachat, loogle, nextqa", + ) + parser.add_argument( + "--enable-shared-prefix", + action="store_true", + help="Enable shared prefix for online serving benchmarking. " + "This option is effective on the following datasets: " + "loogle, nextqa", + ) + + parser.add_argument( + "--disable-shuffle", + action="store_true", + help="Disable shuffling datasets. This is useful to generate stable output " + "in benchmarking", + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--lora-name", + type=str, + default=None, + help="The name of LoRA adapter", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + # videos specific + parser.add_argument( + "--max-frames", + type=int, + default=sys.maxsize, + help="The maximum number of frames to extract from each video. " + "This option is specific to the nextqa dataset (video benchmark). ", + ) + args = parser.parse_args() + + if args.enable_multiturn and args.enable_shared_prefix: + parser.error( + "--enable-multiturn and --enable-shared-prefix cannot be set at the same time." + ) + + run_benchmark(args) diff --git a/benchmark/hicache/data_processing.py b/benchmark/hicache/data_processing.py new file mode 100644 index 0000000..e2951a3 --- /dev/null +++ b/benchmark/hicache/data_processing.py @@ -0,0 +1,589 @@ +import json +import os +import pickle +import random +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import numpy as np +from nextqa import NExTQALoader + +# from nextqa.video import , VideoPrompt +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + +from sglang.bench_serving import ( + download_and_cache_file, + gen_prompt, + get_gen_prefix_cache_path, +) +from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path +from sglang.srt.openai_api.protocol import ChatCompletionMessageContentPart +from sglang.utils import encode_video_base64 + +# type of content fields, can be only prompts or with images/videos +MsgContent = Union[str, List[ChatCompletionMessageContentPart]] + +# A list of all the conversations. Each conversation is a list of +# tuples. If multiturn is not enabled, the length of list is 1, +# containing only the first Q&A pair. +# For the shared prefix workload (synthetic, loogle, nextqa), it +# is a list of conversations sharing the same prefix (synthetic, +# doc, video) +SampleOutput = List[List[Tuple[MsgContent, int, int]]] + + +def common_filter_chat( + num_requests: int, + new_dataset: List, + tokenizer: PreTrainedTokenizerBase, + min_prompt_len: Optional[int], + min_output_len: Optional[int], + max_prompt_len: Optional[int], + max_output_len: Optional[int], + fixed_output_len: Optional[int], +) -> SampleOutput: + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = [] + l = 0 + input_tokens = 0 + output_tokens = 0 + while l < num_requests: + for i in range(len(new_dataset)): + if l == num_requests: + break + processed = [] + for j in new_dataset[i]: + # Tokenize the prompts and completions. + prompt = j[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + completion = j[1] + completion_token_ids = tokenizer.encode(completion) + output_len = ( + len(completion_token_ids) + if fixed_output_len is None + else fixed_output_len + ) + if ( + min_prompt_len is not None + and prompt_len < min_prompt_len + or min_output_len is not None + and output_len < min_output_len + or max_prompt_len is not None + and prompt_len > max_prompt_len + or max_output_len is not None + and output_len > max_output_len + ): + # Prune too short sequences. + continue + input_tokens += prompt_len + output_tokens += output_len + processed.append((prompt, prompt_len, output_len)) + filtered_dataset.append(processed) + l += 1 + + print(f"#Input tokens: {input_tokens}") + print(f"#Output tokens: {output_tokens}") + return filtered_dataset + + +def sample_sharegpt_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["conversations"]) >= 2] + + # Keep one conversation in one list + new_dataset = [] + for data in dataset: + if len(data["conversations"]) % 2 != 0: + continue + if data["conversations"][0]["from"] != "human": + continue + chat = [] + total_len = 2 + if enable_multiturn: + total_len = len(data["conversations"]) + for i in range(0, total_len, 2): + # One user One Assistant + chat.append( + ( + data["conversations"][i]["value"], + data["conversations"][i + 1]["value"], + ) + ) + new_dataset.append(chat) + + if not disable_shuffle: + # Shuffle the dataset. + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, 4, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_ultrachat_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Load the dataset + dataset = [] + with open(dataset_path) as f: + while True: + line = f.readline() + if not line: + break + dataset.append(json.loads(line)) + + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["data"]) >= 2] + + # Keep one conversation in one list + new_dataset = [] + for data in dataset: + if len(data["data"]) % 2 != 0: + continue + chat = [] + total_len = 2 + if enable_multiturn: + total_len = len(data["data"]) + for i in range(0, total_len, 2): + # One user One Assistant + chat.append((data["data"][i], data["data"][i + 1])) + new_dataset.append(chat) + + # Shuffle the dataset. + if not disable_shuffle: + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, 4, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_loogle_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + enable_shared_prefix: bool = False, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Load the dataset + dataset = [] + with open(dataset_path) as f: + while True: + line = f.readline() + if not line: + break + dataset.append(json.loads(line)) + + # Keep one conversation in one list + new_dataset = [] + # TODO: Add shared prefix support for loogle + # NOTE: Now we preprocess it only for chat + for data in dataset: + chat = [] + if ( + "qa_pairs" not in data + or data["qa_pairs"] == "none" + or len(data["qa_pairs"]) == 0 + ): + # If Q is none (for summarization), + # We add a question for summarization + # And keep the summary up to 1024 words + chat.append( + ( + "Input: " + + data["input"] + + " Question: " + + "Please summarize the input", + data["input"][:1024], + ) + ) + new_dataset.append(chat) + else: + qa_pairs = eval(data["qa_pairs"]) + for i, qa in enumerate(qa_pairs): + if i == 0 or enable_shared_prefix: + # Combine input with the first Q + chat.append( + ("Input: " + data["input"] + " Question: " + qa["Q"], qa["A"]) + ) + elif enable_multiturn: + chat.append((qa["Q"], qa["A"])) + + new_dataset.append(chat) + + # Shuffle the dataset. + if not disable_shuffle: + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, None, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_nextqa_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + max_frames: int, # Specific for video + model_path: str, + disable_shuffle: bool = False, + enable_multiturn: bool = True, # No multiturn support for now + backend: str = "sglang-oai", + chat_template_name: Optional[str] = None, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + """ + Example of messages: + message = { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": base64_data}}, + {"type": "text", "text": video.prompt}, + ], + } + """ + + if fixed_output_len is None: + fixed_output_len = 4096 + + # TODO: Check for multiturn + dataset = NExTQALoader(video_dir=dataset_path, max_frames=max_frames) + new_dataset = [] + for v in dataset: + new_dataset.append(v) + + if not disable_shuffle: + random.shuffle(new_dataset) + + # TODO: prompt len can get from server side + filtered_dataset = [] + l = 0 + while l < num_requests: + for i in range(len(new_dataset)): + if l == num_requests: + break + + video = new_dataset[i] + + # text prompt + prompt = video.prompt + + # NOTE: Chat Template is a must for video benchmark because we have to + # add special image token for later expansion + if backend == "sglang" or backend == "sglang-native": + if "chat_template" in tokenizer.init_kwargs: + chat_template = get_chat_template(tokenizer.get_chat_template()) + elif chat_template_name is not None: + chat_template = get_chat_template(chat_template_name) + else: + chat_template = get_chat_template_by_model_path(model_path) + prompt = chat_template.image_token + prompt + + prompt_token_ids = tokenizer(prompt).input_ids + prompt_len = len(prompt_token_ids) + output_len = fixed_output_len # max output len, not real output len + + # video input + base64_data = encode_video_base64(video.path, video.num_frames) + + # NOTE: This will be replaced by the expanded length from the server + prompt_len += video.num_frames + + # add to content + content = [ + {"type": "image_url", "image_url": {"url": base64_data}}, + {"type": "text", "text": prompt}, + ] + + filtered_dataset.append([(content, prompt_len, output_len)]) + l += 1 + return filtered_dataset + + +def sample_random_requests( + input_len: int, + output_len: int, + num_prompts: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + dataset_path: str, + disable_shuffle: bool = False, +) -> SampleOutput: + + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * range_ratio), + output_len + 1, + size=num_prompts, + ) + + if True: + # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["conversations"]) >= 2] + # Only keep the first two turns of each conversation. + dataset = [ + (data["conversations"][0]["value"], data["conversations"][1]["value"]) + for data in dataset + ] + + if not disable_shuffle: + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + input_requests: SampleOutput = [] + for data in dataset: + i = len(input_requests) + if i == num_prompts: + break + + # Tokenize the prompts and completions. + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + # Skip empty prompt + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + prompt = tokenizer.decode(input_ids) + input_requests.append([(prompt, int(input_lens[i]), int(output_lens[i]))]) + else: + # Sample token ids from random integers. This can cause some NaN issues. + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts) + input_requests = [] + for i in range(num_prompts): + prompt = tokenizer.decode( + [ + (offsets[i] + i + j) % tokenizer.vocab_size + for j in range(input_lens[i]) + ] + ) + input_requests.append([(prompt, int(input_lens[i]), int(output_lens[i]))]) + + print(f"#Input tokens: {np.sum(input_lens)}") + print(f"#Output tokens: {np.sum(output_lens)}") + return input_requests + + +def gen_prompt(tokenizer, token_num): + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return tokenizer.decode(selected_tokens) + + +def get_gen_prefix_cache_path(args, tokenizer): + """Create cache directory under ~/.cache/sglang/benchmark""" + cache_dir = Path.home() / ".cache" / "sglang" / "benchmark" + + # Create a unique cache filename based on the generation parameters + cache_key = ( + f"gen_prefix_{args.gen_num_groups}_{args.gen_prompts_per_group}_" + f"{args.gen_system_prompt_len}_{args.gen_question_len}_{args.gen_output_len}_" + f"{tokenizer.__class__.__name__}.pkl" + ) + return cache_dir / cache_key + + +def sample_generated_shared_prefix_requests( + num_groups: int, + prompts_per_group: int, + system_prompt_len: int, + question_len: int, + output_len: int, + tokenizer: PreTrainedTokenizerBase, + args, + disable_shuffle: bool = False, +) -> SampleOutput: + """Generate benchmark requests with shared system prompts using random tokens and caching.""" + cache_path = get_gen_prefix_cache_path(args, tokenizer) + + # Try to load from cache first + if cache_path.exists(): + print(f"\nLoading cached generated input data from {cache_path}") + with open(cache_path, "rb") as f: + return pickle.load(f) + + print("\nGenerating new input data...") + + # Generate system prompts for each group + system_prompts = [] + for _ in range(num_groups): + system_prompt = gen_prompt(tokenizer, system_prompt_len) + system_prompts.append(system_prompt) + + # Generate questions + questions = [] + for _ in range(num_groups * prompts_per_group): + question = gen_prompt(tokenizer, question_len) + questions.append(question) + + # Combine system prompts with questions + input_requests = [] + total_input_tokens = 0 + total_output_tokens = 0 + + for group_idx in tqdm(range(num_groups), desc="Generating system prompt"): + system_prompt = system_prompts[group_idx] + input_requests.append([]) + for prompt_idx in tqdm( + range(prompts_per_group), desc="Generating questions", leave=False + ): + question = questions[group_idx * prompts_per_group + prompt_idx] + full_prompt = f"{system_prompt}\n\n{question}" + prompt_len = len(tokenizer.encode(full_prompt)) + input_requests[-1].append((full_prompt, prompt_len, output_len)) + total_input_tokens += prompt_len + total_output_tokens += output_len + + if not disable_shuffle: + # Shuffle questions + random.shuffle(input_requests) + + # Print statistics + print(f"\nGenerated shared prefix dataset statistics:") + print(f"Number of groups: {num_groups}") + print(f"Prompts per group: {prompts_per_group}") + print(f"Total prompts: {len(input_requests) * prompts_per_group}") + print(f"Total input tokens: {total_input_tokens}") + print(f"Total output tokens: {total_output_tokens}") + print( + f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + ) + print( + f"Average question length: {sum(len(tokenizer.encode(q)) for q in questions) / len(questions):.1f} tokens\n" + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Caching generated input data to {cache_path}") + with open(cache_path, "wb") as f: + pickle.dump(input_requests, f) + + return input_requests + + +def get_dataset(args, tokenizer): + if args.dataset_name == "sharegpt": + input_requests = sample_sharegpt_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "ultrachat": + input_requests = sample_ultrachat_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "loogle": + input_requests = sample_loogle_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + enable_shared_prefix=args.enable_shared_prefix, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "nextqa": + input_requests = sample_nextqa_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + max_frames=args.max_frames, + model_path=args.model, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + backend=args.backend, + chat_template_name=args.chat_template, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "random": + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path=args.dataset_path, + ) + elif args.dataset_name == "generated-shared-prefix": + input_requests = sample_generated_shared_prefix_requests( + num_groups=args.gen_num_groups, + prompts_per_group=args.gen_prompts_per_group, + system_prompt_len=args.gen_system_prompt_len, + question_len=args.gen_question_len, + output_len=args.gen_output_len, + args=args, + tokenizer=tokenizer, + ) + else: + raise ValueError(f"Unknown dataset: {args.dataset_name}") + return input_requests diff --git a/benchmark/hicache/download.sh b/benchmark/hicache/download.sh new file mode 100644 index 0000000..340bcc3 --- /dev/null +++ b/benchmark/hicache/download.sh @@ -0,0 +1,66 @@ +#!/usr/bin/bash + +# The usage function +usage() { + echo "Usage: $0 {sharegpt|ultragpt|loogle|nextqa|all}" + exit 1 +} + +# The download function +download() { + case "$1" in + sharegpt) + echo $1 + wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json + ;; + ultragpt) + echo $1 + # Questions about the world + wget https://cloud.tsinghua.edu.cn/seafhttp/files/be1d7b87-22ca-449e-a6a7-c61d1ea7e010/ultrachat_release_230407.json + # Writing and Creation + wget https://cloud.tsinghua.edu.cn/seafhttp/files/61742d2a-25e2-4d08-b2b9-15f47ae50ace/ultrachat_material_release_230417.json + wget https://cloud.tsinghua.edu.cn/seafhttp/files/f71f6aa6-d346-4b16-85b7-8502efa3d608/ultrachat_material_release_230412.json + # External materials + wget https://cloud.tsinghua.edu.cn/seafhttp/files/42d22e28-e899-4975-a70f-5eda163e265d/ultrachat_existent_material_release_230420.json.gz + gunzip ultrachat_existent_material_release_230420.json.gz + ;; + loogle) + echo $1 + git lfs install + git clone git@hf.co:datasets/bigainlco/LooGLE + unzip LooGLE/data.zip + ;; + nextqa) + echo $1 + git lfs install + git clone https://huggingface.co/datasets/lmms-lab/NExTQA + unzip NExTQA/videos.zip + ;; + *) + usage + exit 1 + ;; + esac +} + +# Arg check +if [ "$#" -ne 1 ]; then + usage +fi + +# Invoke + +case "$1" in + sharegpt|ultragpt|loogle|nextqa) + download "$1" + ;; + all) + download sharegpt + download ultragpt + download loogle + download nextqa + ;; + *) + usage + ;; +esac diff --git a/benchmark/hicache/nextqa.py b/benchmark/hicache/nextqa.py new file mode 100644 index 0000000..4db6caa --- /dev/null +++ b/benchmark/hicache/nextqa.py @@ -0,0 +1,159 @@ +import os +import sys +from typing import List + +import av +from datasets import load_dataset + + +def find_video_files(video_dir) -> List[str]: + if os.path.isfile(video_dir): + return [video_dir] + + video_files = [] + for root, dirs, files in os.walk(video_dir): + for file in files: + if file.endswith((".mp4", ".avi", ".mov")): + video_files.append(os.path.join(root, file)) + # if file is dir + elif os.path.isdir(file): + video_files.extend(find_video_files(file)) + return video_files + + +def video_frames(video_path, max_frames) -> int: + container = av.open(video_path) + total_frames = container.streams.video[0].frames + return min(total_frames, max_frames) + + +class Video: + def __init__(self, video_path, num_frames): + self.path = video_path + self.num_frames = num_frames + + def __str__(self): + return f"Video({self.path}, {self.num_frames})" + + def __iter__(self): + return iter((self.path, self.num_frames)) + + +class VideoPrompt(Video): + def __init__(self, video_path, num_frames, prompt): + super().__init__(video_path, num_frames) + self.prompt = prompt + + def __str__(self): + return f"VideoPrompt({self.path}, {self.num_frames}, {self.prompt})" + + def __iter__(self): + return iter((self.path, self.num_frames, self.prompt)) + + +class VideoLoader: + pass + + +class VideoFileLoader(VideoLoader): + """ + Load all the videos in a directory + """ + + def __init__(self, video_dir, batch_size=1, max_frames=sys.maxsize): + super().__init__() + self.video_dir = video_dir + self.video_files = find_video_files(video_dir) + self.batch_size = batch_size + self.max_frames = max_frames + print(f"batch_size: {batch_size}, max_frames: {max_frames}") + + def __iter__(self): # (file, number of frames) + if self.batch_size == 1: + for video_file in self.video_files: + yield Video(video_file, video_frames(video_file, self.max_frames)) + else: + batch = [] + for video_file in self.video_files: + video = Video(video_file, video_frames(video_file, self.max_frames)) + batch.append(video) + if len(batch) == self.batch_size: + yield batch + batch = [] + + +class NExTQALoader(VideoLoader): + """ + Load vdideos and prompts from NExT dataset + set: train, test or validation + """ + + def __init__( + self, video_dir, batch_size=1, max_frames=sys.maxsize, dset="test", task="OE" + ): + """ + task: 'MV' or 'OE' + """ + super().__init__() + self.task = task + print(f"Loading the {dset} data of {task} from lmms-lab/NExTQA") + self.ds = load_dataset("lmms-lab/NExTQA", task) + self.ds = self.ds[dset] + + # self.n = ds.num_rows + self.video_dir = video_dir + self.video_files = find_video_files(video_dir) + self.video_to_path = dict() + for video_file in self.video_files: + video_id = video_file.split("/")[-1].split(".")[0] + self.video_to_path[video_id] = video_file + + self.batch_size = batch_size + self.max_frames = max_frames + + def get_video_prompt(self, entry, max_frames) -> VideoPrompt: + # Get video + video_id = entry["video"] + video_path = self.video_to_path[video_id] + assert os.path.exists(video_path), f"Video not found: {video_path}" + num_frames = min(entry["frame_count"], max_frames) + video = Video(video_path, num_frames) + prompt = entry["question"] + "?" + if self.task == "MC": # add choices + prompt += f' a0: {entry["a0"]}, a1: {entry["a1"]}, a2: {entry["a2"]}, a3: {entry["a3"]}' + return VideoPrompt(video_path, num_frames, prompt) + + def __iter__(self): + if self.batch_size == 1: + for entry in self.ds: + yield self.get_video_prompt(entry, self.max_frames) + else: + batch = [] + for entry in self.ds: + video = self.get_video_prompt(entry, self.max_frames) + batch.append(video) + if len(batch) == self.batch_size: + yield batch + batch = [] + + +# main +if __name__ == "__main__": + video_dir = "./videos" + # video_loader = VideoFileLoader(video_dir, batch_size=16) + # for batch in video_loader: + # print(f"Number of videos in batch: {len(batch)}") + # for video_file, num_frames in batch: + # print(f"Video: {video_file} number of frames: {num_frames}") + + video_loader = NExTQALoader(video_dir, batch_size=16, dset="test", task="OE") + for batch in video_loader: + print(f"Number of videos in batch: {len(batch)}") + for video_file, num_frames, prompt in batch: + print( + f"Video: {video_file} number of frames: {num_frames}, prompt: {prompt}" + ) + # break + # for video_file, prompt in batch: + # print(f"Video: {video_file} prompt: {prompt}") + # break diff --git a/benchmark/json_decode_regex/README.md b/benchmark/json_decode_regex/README.md new file mode 100644 index 0000000..26acaba --- /dev/null +++ b/benchmark/json_decode_regex/README.md @@ -0,0 +1,60 @@ +## Run benchmark + +### Build dataset +``` +pip install wikipedia +python3 build_dataset.py +``` + +### Dependencies + +``` +llama_cpp_python 0.2.19 +guidance 0.1.10 +vllm 0.2.5 +outlines 0.0.22 +``` + +### Benchmark sglang + +Run Llama-7B + +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Run Mixtral-8x7B + +``` +python3 -m sglang.launch_server --model-path mistralai/Mixtral-8x7B-Instruct-v0.1 --port 30000 --tp-size 8 +``` + +Benchmark + +``` +python3 bench_sglang.py --num-questions 10 +``` + + +### Benchmark Outlines + vLLM + +Run Llama-7B + +``` +python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Benchmark + +``` +python3 bench_other.py --backend outlines --num-questions 10 +``` + + +### Benchmark guidance + +Run Llama-7B and benchmark + +``` +python3 bench_other.py --backend guidance --num-questions 10 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` diff --git a/benchmark/json_decode_regex/bench_other.py b/benchmark/json_decode_regex/bench_other.py new file mode 100644 index 0000000..d80ea1d --- /dev/null +++ b/benchmark/json_decode_regex/bench_other.py @@ -0,0 +1,98 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +REGEX_LIST = r"\[(" + REGEX_STR + ", )*" + REGEX_STR + r"\]" + + +# fmt: off +def json_decode(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": ' + s += generate(s, max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + s += generate(s, max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + s += generate(s, max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + s += generate(s, max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + s += generate(s, max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" + + return s +# fmt: on + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = json_decode(generate=call_generate, **arguments[i]) + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + for _ in rets: + pass + + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "json_decode_regex", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/json_decode_regex/bench_sglang.py b/benchmark/json_decode_regex/bench_sglang.py new file mode 100644 index 0000000..4139ebf --- /dev/null +++ b/benchmark/json_decode_regex/bench_sglang.py @@ -0,0 +1,101 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +REGEX_LIST = r"\[(" + REGEX_STR + ", )*" + REGEX_STR + r"\]" + +# fmt: off +@sgl.function +def json_warm_up(s): + s += "The information about Hogwarts is in the following JSON format.\n" + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen("name", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + sgl.gen("country", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + sgl.gen("latitude", max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + sgl.gen("population", max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + sgl.gen( "landmarks", max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" + print(f'The warmp up json result is:\n{s["json_output"]}') +# fmt: on + +# fmt: off +@sgl.function +def json_decode(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen("name", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + sgl.gen("country", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + sgl.gen("latitude", max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + sgl.gen("population", max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + sgl.gen( "landmarks", max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" +# fmt: on + + +def main(args): + lines = read_jsonl(args.data_path) + lines = list(lines) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Warm up + json_warm_up.run().sync() + + # Run requests + tic = time.time() + states = json_decode.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(f"tmp_{args.backend}_json_results.txt", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_decode_regex", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/json_decode_regex/build_dataset.py b/benchmark/json_decode_regex/build_dataset.py new file mode 100644 index 0000000..1396e5e --- /dev/null +++ b/benchmark/json_decode_regex/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/benchmark/json_jump_forward/README.md b/benchmark/json_jump_forward/README.md new file mode 100644 index 0000000..38fb67e --- /dev/null +++ b/benchmark/json_jump_forward/README.md @@ -0,0 +1,88 @@ +## Run benchmark + +### Dependencies + +``` +llama_cpp_python 0.2.38 +guidance 0.1.10 +vllm 0.2.7 +outlines 0.0.25 +``` + +### Build dataset + +When benchmarking long document information retrieval, run the following command to build the dataset: + +```bash +pip install wikipedia +python3 build_dataset.py +``` + +### Benchmark sglang + +Run Llama-7B + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Benchmark Character Generation + +```bash +python3 bench_sglang.py --mode character +``` + +Benchmark City Information Retrieval + +```bash +python3 bench_sglang.py --mode city +``` + + +### Benchmark Outlines + vLLM + +Run Llama-7B + +```bash +python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Benchmark Character Generation + +```bash +python3 bench_other.py --mode character --backend outlines +``` + +Benchmark City Information Retrieval + +```bash +python3 bench_other.py --mode city --backend outlines +``` + +### Benchmark guidance + +Run Llama-7B and benchmark character generation + +```bash +python3 bench_other.py --mode character --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +Run Llama-7B and benchmark city information retrieval + +```bash +python3 bench_other.py --mode city --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +Run Llama-7B and benchmark character generation + +``` +python3 bench_other.py --mode character --backend lmql --parallel 1 +``` + +Run Llama-7B and benchmark city information retrieval + +``` +python3 bench_other.py --mode city --backend lmql --parallel 1 +``` diff --git a/benchmark/json_jump_forward/bench_other.py b/benchmark/json_jump_forward/bench_other.py new file mode 100644 index 0000000..9eb5c58 --- /dev/null +++ b/benchmark/json_jump_forward/bench_other.py @@ -0,0 +1,288 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +import guidance +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +# there are some FSM bugs with json regex converted from pydantic model +# here use a string regex instead +# regex_string = build_regex_from_object(HarryPoterRole) +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + +# fmt: off +def character_gen(name, generate): + s = name + " is a character in Harry Potter. Please fill in the following information about this character.\n" + s += generate(s, max_tokens=256, regex=character_regex) + return s +# fmt: on + +# fmt: off +def city_gen(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += generate(s, max_tokens=256, regex=city_regex) + return s +# fmt: on + + +@guidance +def character_maker(lm, name): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + {name} is a character in Harry Potter. Please fill in the following information about this character. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "house": "{guidance.select(options=['Gryffindor', 'Slytherin', 'Ravenclaw', 'Hufflepuff'], name='house')}", + "blood status": "{guidance.select(options=['Pure-blood', 'Half-blood', 'Muggle-born'], name='blood status')}", + "occupation": "{guidance.select(options=['student', 'teacher', 'auror', 'ministry of magic', 'death eater', 'order of the phoenix'], name='occupation')}", + "wand": {{ + "wood": "{guidance.gen("wood", max_tokens=16, regex=regex_str_no_quote)}", + "core": "{guidance.gen('core', max_tokens=16, regex=regex_str_no_quote)}", + "length": {guidance.gen('length', max_tokens=10, regex=regex_float)} + }}, + "alive": "{guidance.select(options=['Alive', 'Deceased'], name='alive')}", + "patronus": "{guidance.gen('patronus', max_tokens=16, regex=regex_str_no_quote)}", + "bogart": "{guidance.gen('bogart', max_tokens=16, regex=regex_str_no_quote)}" + }} + """ + + return lm + + +async def call_generate_lmql( + prompt, temperature, max_tokens, regex, max_len=4096, model=None, **kwargs +): + assert model is not None + import lmql + + @lmql.query(model=model) + async def program(question, max_tokens, regex): + '''lmql + """{question}[ANSWER]""" where len(TOKENS(ANSWER)) < max_tokens and REGEX(ANSWER, regex) + return ANSWER + ''' + + return await program( + question=prompt, + temperature=temperature, + max_tokens=max_tokens, + max_len=max_len, + regex=regex, + **kwargs, + ) + + +@guidance +def city_maker(lm, document): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + Please extract the information of a city from the following wikipedia page. + Page begin. + {document} + Page end. + Here is the name, country, and symbol of the city in JSON format. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "country": "{guidance.gen("country", max_tokens=16, regex=regex_str_no_quote)}", + "latitude": {guidance.gen("latitude", max_tokens=10, regex=regex_float)}, + "population": {guidance.gen("population", max_tokens=10, regex=r"[0-9]+")}, + "top 3 landmarks": [ + "{guidance.gen("landmark1", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark2", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark3", max_tokens=16, regex=regex_str_no_quote)}" + ] + }} + """ + + return lm + + +def bench_character(args): + arguments = [] + with open(args.data_path, "r") as f: + for line in f: + arguments.append({"name": line.strip()}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "outlines": + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = character_gen(**arguments[i], generate=call_generate) + + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.model_path, + n_gpu_layers=-1, + n_ctx=args.n_ctx, + ) + + def get_one_answer(i): + lm = model + character_maker(**arguments[i]) + states[i] = lm + + elif args.backend == "lmql": + import asyncio + + import lmql + + model = lmql.model(args.model_path, endpoint=f"{args.host}:{args.port}") + call_generate = partial( + call_generate_lmql, + model=model, + max_tokens=256, + regex=character_regex, + ) + + async def get_one_answer_async(i): + states[i] = await call_generate(prompt=arguments[i]["name"], temperature=0) + + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.time() + + if args.backend != "lmql": + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + for _ in rets: + pass + else: + batches = [] + for i in range(0, len(arguments), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(arguments))))) + loop = asyncio.get_event_loop() + + for bt in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in bt]) + ) + + latency = time.time() - tic + + return states, latency + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "outlines": + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = city_gen(**arguments[i], generate=call_generate) + + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.model_path, + n_gpu_layers=-1, + n_ctx=args.n_ctx, + ) + + def get_one_answer(i): + lm = model + city_maker(**arguments[i]) + states[i] = lm + + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = executor.map(get_one_answer, list(range(len(arguments)))) + for _ in rets: + pass + + latency = time.time() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "json_jump_forward", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "mode": args.mode, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str) + parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/json_jump_forward/bench_sglang.py b/benchmark/json_jump_forward/bench_sglang.py new file mode 100644 index 0000000..10cf269 --- /dev/null +++ b/benchmark/json_jump_forward/bench_sglang.py @@ -0,0 +1,143 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +# there are some FSM bugs with json regex converted from pydantic model +# here use a string regex instead +# regex_string = build_regex_from_object(HarryPoterRole) +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + +# fmt: off +@sgl.function +def character_gen(s, name): + s += name + " is a character in Harry Potter. Please fill in the following information about this character.\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) +# fmt: on + +# fmt: off +@sgl.function +def city_gen(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += sgl.gen("json_output",max_tokens=256, regex=city_regex) +# fmt: on + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + states = city_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + return states, latency + + +def bench_character(args): + arguments = [] + with open(args.data_path, "r") as f: + for line in f: + arguments.append({"name": line.strip()}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + states = character_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) + with open(f"{args.backend}_{args.mode}.json", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_jump_forward", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "mode": args.mode, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str) + parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/json_jump_forward/build_dataset.py b/benchmark/json_jump_forward/build_dataset.py new file mode 100644 index 0000000..1396e5e --- /dev/null +++ b/benchmark/json_jump_forward/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/benchmark/json_jump_forward/dataset.txt b/benchmark/json_jump_forward/dataset.txt new file mode 100644 index 0000000..c12421e --- /dev/null +++ b/benchmark/json_jump_forward/dataset.txt @@ -0,0 +1,50 @@ +Harry Potter +Hermione Granger +Ron Weasley +Albus Dumbledore +Severus Snape +Rubeus Hagrid +Draco Malfoy +Ginny Weasley +Fred Weasley +George Weasley +Percy Weasley +Sirius Black +Remus Lupin +Neville Longbottom +Luna Lovegood +Cedric Diggory +Cho Chang +Lord Voldemort +Minerva McGonagall +Filius Flitwick +Dolores Umbridge +Bellatrix Lestrange +Lucius Malfoy +Molly Weasley +Arthur Weasley +Nymphadora Tonks +Dobby +Moaning Myrtle +Peter Pettigrew +Alastor 'Mad-Eye' Moody +Horace Slughorn +Vernon Dursley +Petunia Dursley +Dudley Dursley +Argus Filch +Sybill Trelawney +Gilderoy Lockhart +Fleur Delacour +Viktor Krum +Bill Weasley +Oliver Wood +Cornelius Fudge +Barty Crouch Sr. +Barty Crouch Jr. +Kingsley Shacklebolt +Quirinus Quirrell +Nearly Headless Nick +Aunt Marge +Griphook +Ludo Bagman diff --git a/benchmark/json_schema/README.md b/benchmark/json_schema/README.md new file mode 100644 index 0000000..e6790e1 --- /dev/null +++ b/benchmark/json_schema/README.md @@ -0,0 +1,15 @@ +## Run benchmark + +### Benchmark sglang + +Run Llama-8b + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 30000 +``` + +Benchmark + +```bash +python3 bench_sglang.py +``` diff --git a/benchmark/json_schema/bench_sglang.py b/benchmark/json_schema/bench_sglang.py new file mode 100644 index 0000000..5a5fd66 --- /dev/null +++ b/benchmark/json_schema/bench_sglang.py @@ -0,0 +1,146 @@ +import argparse +import json +import time +from typing import List, Tuple + +import jsonschema +from datasets import load_dataset + +import sglang as sgl +from sglang.global_config import global_config +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def schema_gen(s, message: Tuple[str, str], json_schema: str): + system, user = message + s += sgl.system(system) + s += sgl.user(user) + s += sgl.assistant( + sgl.gen("json_output", temperature=0, max_tokens=256, json_schema=json_schema) + ) + + +def contains_formats(schema, formats: List[str]): + if isinstance(schema, dict): + if schema.get("format", None) in formats: + return True + for value in schema.values(): + if contains_formats(value, formats): + return True + elif isinstance(schema, list): + for item in schema: + if contains_formats(item, formats): + return True + return False + + +def convert_dataset(path: str): + raw_dataset = load_dataset(path) + dataset = [] + for data in raw_dataset["train"]: + messages = data["prompt"] + schema = data["schema"] + obj = json.loads(schema) + + # skip some corrupted examples + if obj.get("type", None) is None: + continue + + # skip schema with format "email" + # which is not supported by outlines for now + if contains_formats(obj, ["email"]): + continue + + system = messages[0] + user = messages[1] + assert system["role"] == "system", "invalid role" + assert user["role"] == "user", "invalid role" + assert len(messages) == 2, "invalid message length" + message = json.dumps(system["content"]), json.dumps(user["content"]) + dataset.append( + { + "message": message, + "json_schema": schema, + } + ) + + return dataset + + +def bench_schema(args): + arguments = convert_dataset(args.data_path) + + if args.num_jsons < 0 or args.num_jsons > len(arguments): + args.num_jsons = len(arguments) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + states = schema_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + # Check if the outputs are valid + indexs = [] + for i, state in enumerate(states): + try: + schema = json.loads(arguments[i]["json_schema"]) + obj = json.loads(state["json_output"]) + assert jsonschema.validate(obj, schema) is None + except Exception as e: + print(e) + indexs.append(i) + + return states, latency + + +def main(args): + states, latency = bench_schema(args) + + # Compute accuracy + tokenizer = get_tokenizer( + global_config.default_backend.get_server_info()["tokenizer_path"] + ) + output_jsons = [state["json_output"] for state in states] + num_output_tokens = sum(len(tokenizer.encode(x)) for x in output_jsons) + print(f"Latency: {latency:.3f}") + print(f"Output throughput: {num_output_tokens / latency:.3f} token/s") + print(f"#output tokens: {num_output_tokens}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + with open(f"{args.backend}.jsonl", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_schema", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="NousResearch/json-mode-eval") + parser.add_argument("--num-jsons", type=int, default=-1) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py b/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py new file mode 100644 index 0000000..b61240b --- /dev/null +++ b/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py @@ -0,0 +1,403 @@ +import itertools +import math + +import cudnn +import torch +import torch.utils.benchmark as benchmark +from flashinfer import BatchDecodeWithPagedKVCacheWrapper + +from sglang.srt.layers.attention.triton_ops.decode_attention import decode_attention_fwd +from sglang.srt.utils import should_use_tensor_core + + +def benchmark_forward( + fn, + *inputs, + repeats=10, + amp=False, + amp_dtype=torch.float16, + **kwinputs, +): + def amp_wrapper(*inputs, **kwinputs): + with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): + fn(*inputs, **kwinputs) + + t = benchmark.Timer( + stmt="fn_amp(*inputs, **kwinputs)", + globals={"fn_amp": amp_wrapper, "inputs": inputs, "kwinputs": kwinputs}, + num_threads=torch.get_num_threads(), + ) + m = t.timeit(repeats) + return t, m + + +def time_fwd(func, *args, **kwargs): + time_f = benchmark_forward(func, *args, **kwargs) + return time_f[1].mean * 1e6 + + +def decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits, + warmup=10, +): + + k_buffer = kv_data[0].view(-1, head_num_kv, head_dim) + v_buffer = kv_data[1].view(-1, head_num_kv, head_dim) + o = torch.empty_like(q) + total_tokens = batch_size * kv_len + req_to_token = torch.arange(0, total_tokens).to(0).int().view(batch_size, kv_len) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32, device="cuda") + max_len_in_batch = kv_len + sm_scale = 1.0 / (head_dim**0.5) + + attn_logits = torch.empty( + (batch_size, head_num_q, num_kv_splits, head_dim + 1), + dtype=torch.float32, + device="cuda", + ) + + for _ in range(warmup): + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_seq_len, + attn_logits, + num_kv_splits, + sm_scale, + ) + + f = time_fwd( + decode_attention_fwd, + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_seq_len, + attn_logits, + num_kv_splits, + sm_scale, + ) + + return f, o + + +def decode_attention_flashinfer(dtype, head_num_q, head_num_kv): + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device="cuda") + use_tensor_cores = should_use_tensor_core( + kv_cache_dtype=dtype, + num_attention_heads=head_num_q, + num_kv_heads=head_num_kv, + ) + flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( + workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores + ) + + class FlashinferAttention(torch.autograd.Function): + @staticmethod + def forward( + ctx, + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + dtype, + warmup=10, + ): + total_tokens = batch_size * kv_len + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full( + (batch_size,), 1, dtype=torch.int32, device="cuda" + ) + + flashinfer_decode_wrapper.end_forward() + flashinfer_decode_wrapper.begin_forward( + kv_indptr, + kv_indices, + kv_last_page_len, + head_num_q, + head_num_kv, + head_dim, + 1, + pos_encoding_mode="NONE", + data_type=dtype, + ) + + for _ in range(warmup): + o = flashinfer_decode_wrapper.forward( + q.contiguous().view(-1, head_num_q, head_dim), kv_data + ) + + f = time_fwd( + flashinfer_decode_wrapper.forward, + q.contiguous().view(-1, head_num_q, head_dim), + kv_data, + ) + + return f, o + + return FlashinferAttention + + +def convert_to_cudnn_type(torch_type): + if torch_type == torch.float16: + return cudnn.data_type.HALF + elif torch_type == torch.bfloat16: + return cudnn.data_type.BFLOAT16 + elif torch_type == torch.float32: + return cudnn.data_type.FLOAT + elif torch_type == torch.int32: + return cudnn.data_type.INT32 + elif torch_type == torch.int64: + return cudnn.data_type.INT64 + else: + raise ValueError("Unsupported tensor data type.") + + +def decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype, warmup=10 +): + # Prepare data: continuous q,k,v + dims_q = (batch_size, head_num_q, 1, head_dim) + strides_q = (head_num_q * head_dim, head_dim, head_num_q * head_dim, 1) + q_gpu = q.as_strided(dims_q, strides_q) + o_gpu = ( + torch.empty(batch_size * head_num_q * head_dim) + .half() + .cuda() + .as_strided(dims_q, strides_q) + ) + + dims_kv = (batch_size, head_num_kv, kv_len, head_dim) + strides_kv = ( + kv_len * head_num_kv * head_dim, + head_dim, + head_num_kv * head_dim, + 1, + ) + k_gpu = kv_data[0].as_strided(dims_kv, strides_kv) + v_gpu = kv_data[1].as_strided(dims_kv, strides_kv) + + seq_len_q_gpu = torch.full((batch_size, 1, 1, 1), 1, device="cuda") + seq_len_kv_gpu = torch.full((batch_size, 1, 1, 1), kv_len, device="cuda") + attn_scale = 1.0 / (head_dim**0.5) + + # Prepare data: paged k,v + block_size = 1 + blocks_per_batch = math.ceil(kv_len / block_size) + # [num_blocks, head_num_kv, block_size, head_dim], num_blocks = batch_size * blocks_per_batch + container_k_gpu = torch.cat(k_gpu.chunk(blocks_per_batch, dim=2), dim=0) + container_v_gpu = torch.cat(v_gpu.chunk(blocks_per_batch, dim=2), dim=0) + page_table_k_gpu = ( + torch.linspace( + 0, + batch_size * blocks_per_batch - 1, + batch_size * blocks_per_batch, + device="cuda", + dtype=torch.int32, + ) + .reshape(blocks_per_batch, 1, batch_size, 1) + .transpose(0, 2) + ) + page_table_v_gpu = page_table_k_gpu.clone() + + graph = cudnn.pygraph( + io_data_type=convert_to_cudnn_type(dtype), + intermediate_data_type=cudnn.data_type.FLOAT, + compute_data_type=cudnn.data_type.FLOAT, + ) + + q = graph.tensor_like(q_gpu) + container_k = graph.tensor_like(container_k_gpu) + container_v = graph.tensor_like(container_v_gpu) + page_table_k = graph.tensor_like(page_table_k_gpu) + page_table_v = graph.tensor_like(page_table_v_gpu) + + seq_len_q = graph.tensor_like(seq_len_q_gpu) + seq_len_kv = graph.tensor_like(seq_len_kv_gpu) + + o, _ = graph.sdpa( + name="sdpa", + q=q, + k=container_k, # Container K: non contiguous container with K blocks + v=container_v, # Container V: non contiguous container with V blocks + is_inference=True, + attn_scale=attn_scale, + use_causal_mask=False, + use_padding_mask=True, + seq_len_q=seq_len_q, + seq_len_kv=seq_len_kv, + paged_attention_k_table=page_table_k, # Page Table K: Tensor containing offsets to the container with K blocks + paged_attention_v_table=page_table_v, # Page Table V: Tensor containing offsets to the container with V blocks + paged_attention_max_seq_len_kv=kv_len, # The maximum sequence length for K caches (this is optional, but recommended) + ) + + o.set_output(True).set_dim(dims_q).set_stride(strides_q) + + graph.validate() + graph.build_operation_graph() + graph.create_execution_plans([cudnn.heur_mode.A]) + graph.check_support() + graph.build_plans() + + workspace = torch.empty( + graph.get_workspace_size(), device="cuda", dtype=torch.uint8 + ) + + variant_pack = { + q: q_gpu, + container_k: container_k_gpu, + container_v: container_v_gpu, + page_table_k: page_table_k_gpu, + page_table_v: page_table_v_gpu, + seq_len_q: seq_len_q_gpu, + seq_len_kv: seq_len_kv_gpu, + o: o_gpu, + } + + for _ in range(warmup): + graph.execute(variant_pack, workspace) + + f = time_fwd( + graph.execute, + variant_pack, + workspace, + ) + + return f, o_gpu.squeeze(dim=2) + + +def calculate_diff(): + + dtype = torch.float16 + batch_size = 64 + kv_len = 4096 + head_num_q = 64 + head_num_kv = 8 + head_dim = 128 + + q = torch.randn(batch_size, head_num_q, head_dim, dtype=dtype, device="cuda") + kv_data = ( + torch.randn( + batch_size * kv_len, head_num_kv, head_dim, dtype=dtype, device="cuda" + ), + torch.randn( + batch_size * kv_len, head_num_kv, head_dim, dtype=dtype, device="cuda" + ), + ) + + _, output_sglang = decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits=8, + ) + + attn_flashinfer = decode_attention_flashinfer(dtype, head_num_q, head_num_kv).apply + _, output_flashinfer = attn_flashinfer( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + + _, output_cudnn = decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + + print(f"SGLang output={output_sglang}") + print(f"FlashInfer output={output_flashinfer}") + print(f"cuDNN output={output_cudnn}") + if torch.allclose(output_sglang, output_flashinfer, atol=1e-2, rtol=1e-2): + print("✅ SGLang[Triton] and FlashInfer match") + else: + print("❌ SGLang[Triton] and FlashInfer differ") + + if torch.allclose(output_sglang, output_cudnn, atol=1e-2, rtol=1e-2): + print("✅ SGLang[Triton] and cuDNN match") + else: + print("❌ SGLang[Triton] and cuDNN differ") + + +if __name__ == "__main__": + calculate_diff() + + head_dim = 128 + dtype = torch.float16 + batch_size_range = [2**i for i in range(0, 8, 2)] + kv_len_range = [2**i for i in range(6, 13, 1)] + configs = list(itertools.product(batch_size_range, kv_len_range)) + + for head_num_q, head_num_kv in [[32, 32], [64, 8], [40, 8]]: + attn_flashinfer = decode_attention_flashinfer( + dtype, head_num_q, head_num_kv + ).apply + for batch_size, kv_len in configs: + q = torch.randn( + batch_size, head_num_q, head_dim, dtype=dtype, device="cuda" + ) + kv_data = ( + torch.randn( + batch_size * kv_len, + head_num_kv, + head_dim, + dtype=dtype, + device="cuda", + ), + torch.randn( + batch_size * kv_len, + head_num_kv, + head_dim, + dtype=dtype, + device="cuda", + ), + ) + us_cudnn, output_cudnn = decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + us_sglang, output_sglang = decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits=8, + ) + us_flashinfer, _ = attn_flashinfer( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + print( + head_num_q, + " ", + head_num_kv, + " ", + batch_size, + " ", + kv_len, + " ", + us_cudnn, + " ", + us_sglang, + " ", + us_flashinfer, + ) diff --git a/benchmark/kernels/deepseek/README.md b/benchmark/kernels/deepseek/README.md new file mode 100644 index 0000000..fba775f --- /dev/null +++ b/benchmark/kernels/deepseek/README.md @@ -0,0 +1,19 @@ +## DeepSeek kernels benchmark + + +### Prerequisites +- You should install [DeepGemm](https://github.com/deepseek-ai/DeepGEMM) from source before run `benchmark_deepgemm_fp8_gemm.py` and `benchmark_deepgemm_fp8_group_gemm.py`. + +### Benchmark +- `benchmark_deepgemm_fp8_gemm.py` + ```bash + python benchmark_deepgemm_fp8_gemm.py --run_correctness --tp_size 1 + ``` + +- `benchmark_deepgemm_fp8_group_gemm.py` + ```bash + python benchmark_deepgemm_fp8_group_gemm.py --run_correctness --tp_size 1 + ``` + + - You can use the `--run_correctness` parameter to verify all kernels results's correctness. + - You can use the `--tp_size` parameter to benchmark all FP8 w8a8 block-wise matrix multiplications involved in DeepSeek V3/R1 under the current tensor parallelism (TP) setting. This benchmark compares DeepSeek's open-source [DeepGemm](https://github.com/deepseek-ai/DeepGEMM) implementation with SGLang's and VLLM Triton implementation. diff --git a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py new file mode 100644 index 0000000..7151f86 --- /dev/null +++ b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py @@ -0,0 +1,398 @@ +from typing import Tuple + +import deep_gemm +import tilelang +import tilelang.language as T +import torch +import triton +from deep_gemm import ceil_div, get_col_major_tma_aligned_tensor +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + w8a8_block_fp8_matmul as vllm_w8a8_block_fp8_matmul, +) + +from sglang.srt.layers.quantization.fp8_kernel import w8a8_block_fp8_matmul + + +# Adapted from https://github.com/tile-ai/tilelang/blob/a8cfdce92795cb861c9033573534653ee040b5ed/examples/deepseek_deepgemm/example_deepgemm_fp8_2xAcc.py#L1 +def tl_gemm( + M, + N, + K, + in_dtype, + out_dtype, + accum_dtype, +): + assert in_dtype in [ + "e4m3_float8", + ], "Currently only e4m3_float8 is supported" + assert out_dtype in [ + "bfloat16", + "float16", + ], "Currently only bfloat16 and float16 are supported" + + TILE_SIZE = (128, 128, 128) + block_M = TILE_SIZE[0] + block_N = TILE_SIZE[1] + block_K = TILE_SIZE[2] + + A_shape = (M, K) + Scales_A_shape = (M, T.ceildiv(K, block_K)) + B_shape = (N, K) + Scales_B_shape = (T.ceildiv(N, block_N), T.ceildiv(K, block_K)) + A_shared_shape = (block_M, block_K) + B_shared_shape = (block_N, block_K) + C_shared_shape = (block_M, block_N) + + @T.prim_func + def main( + A: T.Buffer(A_shape, in_dtype), + scales_a: T.Buffer(Scales_A_shape, "float32"), + B: T.Buffer(B_shape, in_dtype), + scales_b: T.Buffer(Scales_B_shape, "float32"), + C: T.Buffer((M, N), out_dtype), + ): + with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=128) as ( + bx, + by, + ): + + A_shared = T.alloc_shared(A_shared_shape, in_dtype) + B_shared = T.alloc_shared(B_shared_shape, in_dtype) + C_shared = T.alloc_shared(C_shared_shape, out_dtype) + Scale_C_shared = T.alloc_shared((block_M), "float32") + C_local = T.alloc_fragment(C_shared_shape, accum_dtype) + C_local_accum = T.alloc_fragment(C_shared_shape, accum_dtype) + + # Improve L2 Cache + T.use_swizzle(panel_size=10) + + T.clear(C_local) + T.clear(C_local_accum) + K_iters = T.ceildiv(K, block_K) + for k in T.Pipelined(K_iters, num_stages=4): + # Load A into shared memory + T.copy(A[by * block_M, k * block_K], A_shared) + # Load B into shared memory + T.copy(B[bx * block_N, k * block_K], B_shared) + # Load scale into shared memory + Scale_B = scales_b[bx, k] + for i in T.Parallel(block_M): + Scale_C_shared[i] = scales_a[by * block_M + i, k] * Scale_B + + T.gemm(A_shared, B_shared, C_local, transpose_B=True) + # Promote to enable 2xAcc + for i, j in T.Parallel(block_M, block_N): + C_local_accum[i, j] += C_local[i, j] * Scale_C_shared[i] + T.clear(C_local) + # TMA store + T.copy(C_local_accum, C_shared) + T.copy(C_shared, C[by * block_M, bx * block_N]) + + return main + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def fp8_gemm_deepgemm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """DeepGEMM implementation of FP8 GEMM""" + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + + # Run DeepGEMM kernel + deep_gemm.gemm_fp8_fp8_bf16_nt((x_fp8, x_scale), (y_fp8, y_scale), out) + return out + + +def fp8_gemm_sglang( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """SGLang implementation of FP8 GEMM""" + block_size = [128, 128] # Matches the block size in per_block_cast_to_fp8 + + # Run SGLang kernel + out = w8a8_block_fp8_matmul( + x_fp8, y_fp8, x_scale, y_scale, block_size, torch.bfloat16 + ) + return out + + +def fp8_gemm_vllm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """vLLM implementation of FP8 GEMM""" + block_size = [128, 128] # Matches the block size in per_block_cast_to_fp8 + + # Run vLLM kernel + out = vllm_w8a8_block_fp8_matmul( + x_fp8, y_fp8, x_scale, y_scale, block_size, torch.bfloat16 + ) + return out + + +def calculate_diff(m: int, n: int, k: int): + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + x_fp8, x_scale = per_token_cast_to_fp8(x.clone()) + y_fp8, y_scale = per_block_cast_to_fp8(y.clone()) + x_scale_col_major = get_col_major_tma_aligned_tensor(x_scale.clone()) + + out_deepgemm = fp8_gemm_deepgemm( + x_fp8.clone(), + x_scale_col_major.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ) + out_sglang = fp8_gemm_sglang( + x_fp8.clone(), x_scale.clone(), y_fp8.clone(), y_scale.clone(), m, n, k + ) + + tilelang_func = tl_gemm(m, n, k, "e4m3_float8", "bfloat16", "float32") + tilelang_kernel = tilelang.compile(tilelang_func, out_idx=[-1]) + out_tilelang = tilelang_kernel( + x_fp8.clone(), x_scale.clone(), y_fp8.clone(), y_scale.clone() + ) + + diff_sglang_deepgemm = torch.abs(out_deepgemm - out_sglang).mean().item() + diff_tilelang_deepgemm = torch.abs(out_deepgemm - out_tilelang).mean().item() + diff_tilelang_sglang = torch.abs(out_tilelang - out_sglang).mean().item() + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + print(f"SGLang output: {out_sglang[0, 0:5]}") + print(f"TileLang output: {out_tilelang[0, 0:5]}") + print(f"Mean absolute difference (SGLang-DeepGEMM): {diff_sglang_deepgemm}") + print(f"Mean absolute difference (TileLang-DeepGEMM): {diff_tilelang_deepgemm}") + print(f"Mean absolute difference (TileLang-SGLang): {diff_tilelang_sglang}") + + sglang_deepgemm_match = torch.allclose( + out_deepgemm, out_sglang, atol=1e-2, rtol=1e-2 + ) + tilelang_deepgemm_match = torch.allclose( + out_deepgemm, out_tilelang, atol=1e-2, rtol=1e-2 + ) + tilelang_sglang_match = torch.allclose( + out_tilelang, out_sglang, atol=1e-2, rtol=1e-2 + ) + + if sglang_deepgemm_match and tilelang_deepgemm_match and tilelang_sglang_match: + print("✅ All implementations match\n") + else: + print("❌ Some implementations differ:") + print(f" - SGLang vs DeepGEMM: {'✅' if sglang_deepgemm_match else '❌'}") + print(f" - TileLang vs DeepGEMM: {'✅' if tilelang_deepgemm_match else '❌'}") + print(f" - TileLang vs SGLang: {'✅' if tilelang_sglang_match else '❌'}\n") + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096] + + for n, k in weight_shapes: + for m in batch_sizes: + configs.append((m, n, k, tp_size)) + + return configs + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "tp_size"], + x_vals=[list(config) for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "sglang", "tilelang"], + line_names=["DeepGEMM", "SGLang", "TileLang"], + styles=[("blue", "-"), ("red", "-"), ("green", "-")], + ylabel="ms", + plot_name=f"fp8-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, tp_size, provider): + print(f"Shape (m={m}, n={n}, k={k}, tp={tp_size}), Provider: {provider}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + # Preprocess data before benchmarking + x_fp8, x_scale = per_token_cast_to_fp8(x) + y_fp8, y_scale = per_block_cast_to_fp8(y) + x_scale_col_major = get_col_major_tma_aligned_tensor(x_scale.clone()) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_deepgemm( + x_fp8.clone(), + x_scale_col_major.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ), + quantiles=quantiles, + ) + elif provider == "sglang": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_sglang( + x_fp8.clone(), + x_scale.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ), + quantiles=quantiles, + ) + else: # tilelang + tilelang_func = tl_gemm(m, n, k, "e4m3_float8", "bfloat16", "float32") + tilelang_kernel = tilelang.compile(tilelang_func, out_idx=[-1]) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: tilelang_kernel( + x_fp8.clone(), + x_scale.clone(), + y_fp8.clone(), + y_scale.clone(), + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + # Print shape-specific results with TFLOPS + print(f"Time: {ms*1000:.2f} ms, TFLOPS: {tflops:.2f}") + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/fp8_gemm/", + help="Path to save fp8 gemm benchmark results", + ) + parser.add_argument( + "--run_correctness", + action="store_true", + default=True, + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Enable TF32, adapted from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py#L148 + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(64, 512, 7168) # Small test + calculate_diff(64, 7168, 16384) # Medium test + calculate_diff(64, 18432, 7168) # Large test + + # Get the benchmark function with the specified tp_size + benchmark = get_benchmark(args.tp_size) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py new file mode 100644 index 0000000..2c3e8df --- /dev/null +++ b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py @@ -0,0 +1,486 @@ +from typing import Tuple + +import deep_gemm +import torch +import triton +import triton.language as tl +from deep_gemm import calc_diff, get_col_major_tma_aligned_tensor + +# Import shared functionality from the regular GEMM benchmark +from sglang.benchmark.kernels.deepseek.benchmark_deepgemm_fp8_gemm import ( + per_block_cast_to_fp8, + per_token_cast_to_fp8, +) + + +def construct_grouped_and_flat_fp8( + x: torch.Tensor, y: torch.Tensor, num_groups: int, is_masked: bool +) -> Tuple[ + Tuple[torch.Tensor, torch.Tensor], # grouped x_fp8 + Tuple[torch.Tensor, torch.Tensor], # grouped y_fp8 + Tuple[torch.Tensor, torch.Tensor], # flat x_fp8 + Tuple[torch.Tensor, torch.Tensor], # flat y_fp8 + torch.Tensor, # output + torch.Tensor, # reference output +]: + # Verify input shapes + m, k = x.shape + n, k_y = y.shape + assert k == k_y, f"Incompatible shapes: x({m}, {k}), y({n}, {k_y})" + assert m % num_groups == 0, f"m({m}) must be divisible by num_groups({num_groups})" + assert m % 4 == 0, f"TMA alignment error: {m}" + + # Reshape inputs for grouped processing + m_per_group = m // num_groups + x_grouped = x.view(num_groups, m_per_group, k) + y_grouped = y.unsqueeze(0).expand(num_groups, n, k) + + # Initialize output tensors + out = torch.empty((num_groups, m_per_group, n), device="cuda", dtype=torch.bfloat16) + ref_out = torch.einsum("gmk,gnk->gmn", x_grouped, y_grouped) + + # Quantize grouped tensors + x_fp8_grouped = ( + torch.empty_like(x_grouped, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, m_per_group, k // 128), device="cuda", dtype=torch.float + ), + ) + y_fp8_grouped = ( + torch.empty_like(y_grouped, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, (n + 127) // 128, k // 128), device="cuda", dtype=torch.float + ), + ) + for i in range(num_groups): + x_fp8_grouped[0][i], x_fp8_grouped[1][i] = per_token_cast_to_fp8(x_grouped[i]) + y_fp8_grouped[0][i], y_fp8_grouped[1][i] = per_block_cast_to_fp8(y_grouped[i]) + + # Quantize flat tensors + x_fp8_flat = per_token_cast_to_fp8(x) + y_fp8_flat = per_block_cast_to_fp8(y) + + # For non-masked input, merge the group and M dims in output + if not is_masked: + x_fp8_grouped = ( + x_fp8_grouped[0].view(-1, k), + per_token_cast_to_fp8(x_grouped.view(-1, k))[1], + ) + out, ref_out = out.view(-1, n), ref_out.view(-1, n) + + # Transpose earlier for testing + x_fp8_grouped = ( + x_fp8_grouped[0], + get_col_major_tma_aligned_tensor(x_fp8_grouped[1]), + ) + x_fp8_flat = (x_fp8_flat[0], get_col_major_tma_aligned_tensor(x_fp8_flat[1])) + + return x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, ref_out + + +# Since we don't have a group gemm kernel in SGLang/vLLM, we implemented a +# custom kernel based on the Triton tutorial. +# https://triton-lang.org/main/getting-started/tutorials/03-matrix-multiplication.html +@triton.jit +def fp8_gemm_group_triton_kernel( + # Pointers to matrices + a_ptr, + b_ptr, + c_ptr, + # Pointers to scaling factors + a_scale_ptr, + b_scale_ptr, + # Matrix dimensions + M, + N, + K, + # The stride variables represent how much to increase the ptr by when moving by 1 + # element in a particular dimension. + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + # Strides for scaling factors + stride_a_scale_m, + stride_a_scale_k, + stride_b_scale_n, + stride_b_scale_k, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Kernel for computing the matmul C = A x B with FP8 inputs and scaling factors. + A has shape (M, K), B has shape (K, N) and C has shape (M, N) + + Note: Block sizes must be multiples of 32 for optimal TMA performance. + """ + # Map program ids to the block of C it should compute + pid_group = tl.program_id(axis=0) # Group ID + pid_n = tl.program_id(axis=1) # N dimension ID + + # Compute the M block ID within this group + group_size_m = min(M - pid_group * GROUP_SIZE_M, GROUP_SIZE_M) + pid_m_within_group = tl.program_id(axis=2) % group_size_m + pid_m = pid_group * GROUP_SIZE_M + pid_m_within_group + + # Create pointers for the first blocks of A and B + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + # Initialize accumulator + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + # Main loop + for k_block in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + k_offset = k_block * BLOCK_SIZE_K + + # Load the next block of A and B, with masks + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k_offset, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k_offset, other=0.0) + + # Calculate indices for scaling factors for this K block + a_scale_ptrs = a_scale_ptr + ( + offs_am * stride_a_scale_m + k_block * stride_a_scale_k + ) + b_scale_ptrs = b_scale_ptr + ( + pid_n * stride_b_scale_n + k_block * stride_b_scale_k + ) + + # Perform matrix multiplication in FP8 + res = tl.dot(a, b) + + # Load scaling factors for the current block + a_scale = tl.load(a_scale_ptrs)[:, None] # [BLOCK_SIZE_M, 1] + b_scale = tl.load(b_scale_ptrs) + + # Apply scaling factors to the accumulated result + accumulator += res * a_scale * b_scale + + # Advance pointers + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # Convert to bfloat16 for output + c = accumulator.to(tl.bfloat16) + + # Write back the result + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +def fp8_gemm_group_triton(a_tuple, b_tuple, c, num_groups): + """ + Perform matrix multiplication with FP8 inputs and proper scaling. + + Args: + a_tuple: Tuple of (quantized_tensor, scale_factors) for input A + b_tuple: Tuple of (quantized_tensor, scale_factors) for input B + c: Output tensor in BF16 format + num_groups: Number of groups for grouped GEMM + + Returns: + Result tensor in BF16 format + """ + # Unpack the tuples + a, a_scale = a_tuple + b, b_scale = b_tuple + + M, K = a.shape + _, N = b.shape + + # Configure block sizes - must be multiples of 32 for TMA alignment + BLOCK_SIZE_M = 128 + BLOCK_SIZE_N = 128 + BLOCK_SIZE_K = 128 + + # Calculate grid dimensions + num_pid_m = triton.cdiv(M, BLOCK_SIZE_M) + num_pid_n = triton.cdiv(N, BLOCK_SIZE_N) + num_groups_grid = triton.cdiv(num_pid_m, num_groups) + + # 3D grid launch - (group, n_blocks, m_blocks_per_group) + grid = (num_groups_grid, num_pid_n, min(num_groups, num_pid_m)) + + fp8_gemm_group_triton_kernel[grid]( + a, + b, + c, + a_scale, + b_scale, + M, + N, + K, + a.stride(0), + a.stride(1), + b.stride(0), + b.stride(1), + c.stride(0), + c.stride(1), + a_scale.stride(0), + 1, # Stride in the K dimension may be 1 + b_scale.stride(0), + 1 if b_scale.dim() > 1 else 0, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + BLOCK_SIZE_K=BLOCK_SIZE_K, + GROUP_SIZE_M=num_groups, + ) + + return c + + +def fp8_gemm_group_deepgemm(x_fp8_grouped, y_fp8_grouped, out, m_indices): + deep_gemm.m_grouped_gemm_fp8_fp8_bf16_nt_contiguous( + x_fp8_grouped, + y_fp8_grouped, + out, + m_indices, + ) + return out + + +def calculate_diff(m: int, n: int, k: int, num_groups: int): + print(f"Shape (m={m}, n={n}, k={k}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, out_torch = ( + construct_grouped_and_flat_fp8(x, y, num_groups, is_masked=False) + ) + m_per_group = m // num_groups + out_deepgemm = out.clone() + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1).expand(num_groups, m_per_group).contiguous().view(-1) + ) + + fp8_gemm_group_deepgemm( + x_fp8_grouped, + y_fp8_grouped, + out_deepgemm, + m_indices, + ) + torch.cuda.synchronize() + + # Prepare inputs for Triton + a, a_scale = x_fp8_flat + b, b_scale = y_fp8_flat + b = b.T.contiguous() + # Ensure scales are in the right format and contiguous + a_scale, b_scale = a_scale.contiguous(), b_scale.contiguous() + M, _ = a.shape + _, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.bfloat16) + out_triton = fp8_gemm_group_triton((a, a_scale), (b, b_scale), c, num_groups) + torch.cuda.synchronize() + + diff_torch_deepgemm = torch.abs(out_torch - out_deepgemm).mean().item() + diff_torch_triton = torch.abs(out_torch - out_triton).mean().item() + diff_deepgemm_triton = torch.abs(out_deepgemm - out_triton).mean().item() + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"Torch output: {out_torch[0, 0:5]}") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + print(f"Triton output: {out_triton[0, 0:5]}") + print(f"Mean absolute difference (Torch-DeepGEMM): {diff_torch_deepgemm}") + print(f"Mean absolute difference (Torch-Triton): {diff_torch_triton}") + print(f"Mean absolute difference (DeepGEMM-Triton): {diff_deepgemm_triton}") + + deepgemm_torch_diff = calc_diff(out_deepgemm, out_torch) + triton_torch_diff = calc_diff(out_triton, out_torch) + deepgemm_triton_diff = calc_diff(out_deepgemm, out_triton) + + DIFF_THRESHOLD = 0.001 + all_match = ( + deepgemm_torch_diff < DIFF_THRESHOLD + and triton_torch_diff < DIFF_THRESHOLD + and deepgemm_triton_diff < DIFF_THRESHOLD + ) + if all_match: + print("✅ All implementations match\n") + else: + print("❌ Some implementations differ:") + print( + f" - Torch vs DeepGEMM: {'✅' if deepgemm_torch_diff < DIFF_THRESHOLD else '❌'}" + f" - Torch vs Triton: {'✅' if triton_torch_diff < DIFF_THRESHOLD else '❌'}" + f" - DeepGEMM vs Triton: {'✅' if deepgemm_triton_diff < DIFF_THRESHOLD else '❌'}" + ) + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [2048, 4096] + group_sizes = [4, 8] + for n, k in weight_shapes: + for m in batch_sizes: + for num_groups in group_sizes: + configs.append((m, n, k, num_groups, tp_size)) + + return configs + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "num_groups", "tp_size"], + x_vals=[config for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "triton"], + line_names=["DeepGEMM", "Triton"], + styles=[("blue", "-"), ("red", "-")], + ylabel="ms", + plot_name=f"fp8-group-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, num_groups, tp_size, provider): + print( + f"Shape (m={m}, n={n}, k={k}, tp={tp_size}, num_groups={num_groups}, Provider: {provider}" + ) + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, out_torch = ( + construct_grouped_and_flat_fp8(x, y, num_groups, is_masked=False) + ) + m_per_group = m // num_groups + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1) + .expand(num_groups, m_per_group) + .contiguous() + .view(-1) + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_group_deepgemm( + x_fp8_grouped, + y_fp8_grouped, + out, + m_indices, + ), + quantiles=quantiles, + ) + elif provider == "triton": + # Prepare inputs for Triton + # We did it outside of the lambda function to make it fair comparison like deepgemm + a, a_scale = x_fp8_flat + b, b_scale = y_fp8_flat + b = b.T.contiguous() + # Ensure scales are in the right format and contiguous + a_scale, b_scale = a_scale.contiguous(), b_scale.contiguous() + M, _ = a.shape + _, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.bfloat16) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_group_triton( + (a, a_scale), + (b, b_scale), + c, + num_groups, + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + print(f"Time: {ms*1000:.2f} ms, TFLOPS: {tflops:.2f}") + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/fp8_group_gemm/", + help="Path to save deepgemm fp8 group gemm benchmark results", + ) + parser.add_argument( + "--run_correctness", + action="store_true", + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Enable TF32, adapted from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py#L148 + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(8192, 7168, 4096, 4) + calculate_diff(8192, 2048, 7168, 4) + calculate_diff(4096, 7168, 4096, 8) + calculate_diff(4096, 2048, 7168, 8) + calculate_diff(4096, 576, 7168, 8) + + # Get the benchmark function with the specified tp_size + benchmark = get_benchmark(args.tp_size) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/fused_moe_triton/README.md b/benchmark/kernels/fused_moe_triton/README.md new file mode 100644 index 0000000..2a3e37f --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/README.md @@ -0,0 +1,49 @@ +## Benchmark Kernels + +This directory contains benchmarking tools for MoE (Mixture of Experts) kernels. + +### Tuning Tool + +- `tuning_fused_moe_triton.py`: A tool for tuning the `fused_moe_triton` kernel. Adapted from [vllm's benchmark_moe.py](https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py), with added support for various model architectures. + +Example usage: +```bash +# Tune Qwen2-57B with FP8 and TP=4 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --tp-size 4 \ + --dtype fp8_w8a8 \ + --tune + +# Tune Mixtral-8x7B with default settings +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --tune +``` + +After tuning, a configuration file (e.g., `E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json`) will be generated in the current directory. You can move this file to `sglang/srt/layers/fused_moe_triton/configs/` to use it in `sglang`. + +### Performance Comparison Tool + +- `benchmark_vllm_vs_sglang_fused_moe_triton.py`: A tool for comparing the performance of fused MoE kernels between vllm and sglang implementations. Supports various model architectures and data types. + +Example usage: +```bash +# Compare with default settings (Mixtral model) +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py + +# Compare with FP8 mode for Qwen2-57B +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --use-fp8 + +# Compare with custom TP size +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --tp-size 4 +``` + +The benchmark results will be saved as plots and data files in the specified output directory (default: `./configs/benchmark_ops/vllm_sglang_fused_moe/`). + +- `benchmark_torch_compile_fused_moe.py`: A tool for benchmarking the performance of the fused MoE kernel with `torch.compile` and original fused MoE kernel. + +Usage is the same as `benchmark_vllm_vs_sglang_fused_moe_triton.py`. diff --git a/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py b/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py new file mode 100644 index 0000000..8206684 --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py @@ -0,0 +1,294 @@ +import argparse + +import torch +import triton +from torch.nn import functional as F +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_triton, +) +from sglang.srt.model_executor.cuda_graph_runner import set_torch_compile_config + + +def get_model_config(model_name: str, tp_size: int): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + shape_configs = { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + } + print(f"{shape_configs=}") + return shape_configs + + +def fused_topk_native( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + M, _ = hidden_states.shape + topk_weights = torch.empty( + M, topk, dtype=torch.float32, device=hidden_states.device + ) + topk_ids = torch.empty(M, topk, dtype=torch.int32, device=hidden_states.device) + topk_weights = F.softmax(gating_output.float(), dim=-1) + topk_weights, topk_ids = torch.topk(topk_weights, topk, dim=-1) + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + return topk_weights, topk_ids + + +@torch.compile(dynamic=False) +def fused_moe_torch( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +) -> torch.Tensor: + assert not use_fp8_w8a8, "Not supported" + + topk_weights, topk_ids = fused_topk_native( + hidden_states=x, + gating_output=input_gating, + topk=topk, + renormalize=True, + ) + w13_weights = w1[topk_ids] + w1_weights, w3_weights = torch.chunk(w13_weights, 2, dim=2) + w2_weights = w2[topk_ids] + x1 = torch.einsum("ti,taoi -> tao", x, w1_weights) + x1 = F.silu(x1) + x3 = torch.einsum("ti, taoi -> tao", x, w3_weights) + expert_outs = torch.einsum("tao, taio -> tai", (x1 * x3), w2_weights) + return torch.einsum("tai,ta -> ti", expert_outs, topk_weights.to(expert_outs.dtype)) + + +def fused_moe_torch_compile( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +): + return fused_moe_torch( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +): + return fused_moe_triton( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list(range(1, 5)), + line_arg="provider", + line_vals=[ + "fused_moe_triton", + "fused_moe_torch_compile", + ], + line_names=[ + "fused_moe_triton", + "fused_moe_torch_compile", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark(batch_size, provider, model_config, use_fp8=False): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + set_torch_compile_config() + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + + if use_fp8: + init_dtype = dtype + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + w1_scale = w2_scale = a1_scale = a2_scale = None + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + # Warmup + api_func = ( + fused_moe_torch_compile + if provider == "fused_moe_torch_compile" + else fused_moe_sglang_api + ) + for _ in range(10): + y = api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + torch.cuda.synchronize() + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + )[0], + quantiles=quantiles, + ) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", type=int, default=2) + parser.add_argument("--use-fp8", action="store_true") + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/fused_moe_torch_compile/", + ) + args = parser.parse_args() + + model_config = get_model_config(args.model, args.tp_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8=args.use_fp8, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py new file mode 100644 index 0000000..0760116 --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py @@ -0,0 +1,299 @@ +import argparse + +import torch +import triton +import vllm +from transformers import AutoConfig +from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe as fused_moe_vllm + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_sglang, +) + + +def get_model_config(model_name: str, tp_size: int): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + vllm_version_num = ( + vllm.__version_tuple__[0] * 100 + + vllm.__version_tuple__[1] * 10 + + vllm.__version_tuple__[2] + ) + block_shape = None + if ( + hasattr(config, "quantization_config") + and "weight_block_size" in config.quantization_config + ): + block_shape = config.quantization_config["weight_block_size"] + assert len(block_shape) == 2 + assert ( + vllm_version_num >= 66 + ), "Block-wise quantized fp8 fused_moe is only supported for VLLM>=0.6.6.post1" + + shape_configs = { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + "block_shape": block_shape, + } + print(f"{shape_configs=}") + return shape_configs + + +def fused_moe_vllm_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + if block_shape is not None: + return fused_moe_vllm( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + else: + return fused_moe_vllm( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + return fused_moe_sglang( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list(range(1, 513)), + line_arg="provider", + line_vals=[ + "vllm_fused_moe_triton", + "sglang_fused_moe_triton", + ], + line_names=[ + "vllm_fused_moe_triton", + "sglang_fused_moe_triton", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark(batch_size, provider, model_config, use_fp8=False): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + block_shape = getattr(model_config, "block_shape", None) + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + w1_scale = w2_scale = a1_scale = a2_scale = None + + if use_fp8: + init_dtype = dtype + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + + if block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + else: + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + # Warmup + api_func = ( + fused_moe_vllm_api + if provider == "vllm_fused_moe_triton" + else fused_moe_sglang_api + ) + for _ in range(10): + y = api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + torch.cuda.synchronize() + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + )[0], + quantiles=quantiles, + ) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", type=int, default=2) + parser.add_argument("--use-fp8", action="store_true") + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/vllm_sglang_fused_moe/", + ) + args = parser.parse_args() + + model_config = get_model_config(args.model, args.tp_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8=args.use_fp8, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py new file mode 100644 index 0000000..89309ac --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py @@ -0,0 +1,564 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py +import argparse +import json +import time +from datetime import datetime +from typing import Any, Dict, List, Tuple, TypedDict + +import ray +import torch +import triton +from ray.experimental.tqdm_ray import tqdm +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe, + get_config_dtype_str, + get_config_file_name, + get_default_config, + get_moe_configs, +) +from sglang.srt.utils import is_hip + +_is_hip_ = is_hip() + + +class BenchmarkConfig(TypedDict): + BLOCK_SIZE_M: int + BLOCK_SIZE_N: int + BLOCK_SIZE_K: int + GROUP_SIZE_M: int + num_warps: int + num_stages: int + + +def benchmark_config( + config: BenchmarkConfig, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int] = None, + num_iters: int = 100, +) -> float: + init_dtype = torch.float16 if use_fp8_w8a8 else dtype + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + if use_int8_w8a16 or use_int8_w8a8: + w1 = torch.randint( + -127, + 127, + ( + num_experts, + shard_intermediate_size, + hidden_size, + ), + dtype=torch.int8, + ) + w2 = torch.randint( + -127, + 127, + ( + num_experts, + hidden_size, + shard_intermediate_size // 2, + ), + dtype=torch.int8, + ) + else: + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + gating_output = torch.randn(num_iters, num_tokens, num_experts, dtype=torch.float32) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + if use_int8_w8a16: + w1_scale = torch.randn( + (num_experts, 2 * shard_intermediate_size), dtype=torch.float32 + ) + w2_scale = torch.randn((hidden_size, num_experts), dtype=torch.float32) + if use_fp8_w8a8 or use_int8_w8a8: + if use_int8_w8a8 and block_shape is None: + w1_scale = torch.randn( + num_experts, shard_intermediate_size, dtype=torch.float32 + ) + w2_scale = torch.randn(num_experts, hidden_size, dtype=torch.float32) + elif block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + + if use_fp8_w8a8: + w1 = w1.to(torch.float8_e4m3fnuz if _is_hip_ else torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fnuz if _is_hip_ else torch.float8_e4m3fn) + + input_gating = torch.empty(num_tokens, num_experts, dtype=torch.float32) + + def prepare(i: int): + input_gating.copy_(gating_output[i]) + + def run(): + from sglang.srt.layers.moe.fused_moe_triton import override_config + + with override_config(config): + fused_moe( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + # JIT compilation & warmup + run() + torch.cuda.synchronize() + + # Capture 10 invocations with CUDA graph + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(10): + run() + torch.cuda.synchronize() + + # Warmup + for _ in range(5): + graph.replay() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for i in range(num_iters): + prepare(i) + torch.cuda.synchronize() + + start_event.record() + graph.replay() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + avg = sum(latencies) / (num_iters * 10) * 1000 # us + graph.reset() + return avg + + +def get_rocm_configs_compute_bound() -> List[Dict[str, int]]: + configs: List[BenchmarkConfig] = [] + waves_per_eu_range = 0 + for num_stages in [2]: + for block_m in [32, 64, 128, 256]: + for block_k in [32, 64, 128, 256]: + for block_n in [16, 32, 64, 128, 256]: + for num_warps in [1, 2, 4, 8]: + for group_size in [1, 4, 8, 16, 32]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu_range, + } + ) + return configs + + +def get_configs_compute_bound() -> List[Dict[str, int]]: + # Reduced search space for faster tuning. + # TODO(woosuk): Increase the search space and use a performance model to + # prune the search space. + configs: List[BenchmarkConfig] = [] + if _is_hip_: + configs = get_rocm_configs_compute_bound() + else: + for num_stages in [2, 3, 4, 5]: + for block_m in [16, 32, 64, 128, 256]: + for block_k in [64, 128, 256]: + for block_n in [32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 16, 32, 64]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + } + ) + return configs + + +@ray.remote(num_gpus=1) +class BenchmarkWorker: + + def __init__(self, seed: int) -> None: + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + self.seed = seed + + def benchmark( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + ) -> Tuple[Dict[str, int], float]: + torch.cuda.manual_seed_all(0) + dtype_str = get_config_dtype_str( + dtype, use_int8_w8a16=use_int8_w8a16, use_fp8_w8a8=use_fp8_w8a8 + ) + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + block_n = block_shape[0] if block_shape else 0 + block_k = block_shape[1] if block_shape else 0 + op_config = get_moe_configs( + num_experts, shard_intermediate_size // 2, dtype_str, block_n, block_k + ) + if op_config is None: + config = get_default_config( + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype_str, + False, + block_shape, + ) + else: + config = op_config[min(op_config.keys(), key=lambda x: abs(x - num_tokens))] + kernel_time = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + return config, kernel_time + + def tune( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + search_space: List[Dict[str, int]], + ) -> Dict[str, int]: + best_config = None + best_time = float("inf") + for config in tqdm(search_space): + try: + kernel_time = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + num_iters=10, + ) + except triton.runtime.autotuner.OutOfResources: + # Some configurations may be invalid and fail to compile. + continue + + if kernel_time < best_time: + best_time = kernel_time + best_config = config + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}") + assert best_config is not None + return best_config + + +def sort_config(config: BenchmarkConfig) -> BenchmarkConfig: + return { + "BLOCK_SIZE_M": config["BLOCK_SIZE_M"], + "BLOCK_SIZE_N": config["BLOCK_SIZE_N"], + "BLOCK_SIZE_K": config["BLOCK_SIZE_K"], + "GROUP_SIZE_M": config["GROUP_SIZE_M"], + "num_warps": config["num_warps"], + "num_stages": config["num_stages"], + **( + {"waves_per_eu": config["waves_per_eu"]} if "waves_per_eu" in config else {} + ), + } + + +def save_configs( + configs: Dict[int, BenchmarkConfig], + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], +) -> None: + dtype_str = get_config_dtype_str( + dtype, + use_int8_w8a16=use_int8_w8a16, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + ) + + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + filename = get_config_file_name( + num_experts, + shard_intermediate_size // 2, + dtype_str, + block_shape, + ) + + print(f"Writing best config to {filename}...") + with open(filename, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def main(args: argparse.Namespace): + print(args) + + config = AutoConfig.from_pretrained(args.model, trust_remote_code=True) + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + + hidden_size = config.hidden_size + dtype = config.torch_dtype + use_fp8_w8a8 = args.dtype == "fp8_w8a8" + use_int8_w8a8 = args.dtype == "int8_w8a8" + use_int8_w8a16 = args.dtype == "int8_w8a16" + block_shape = None + if ( + hasattr(config, "quantization_config") + and "weight_block_size" in config.quantization_config + ): + block_shape = config.quantization_config["weight_block_size"] + assert len(block_shape) == 2 + + if args.batch_size is None: + batch_sizes = [ + 1, + 2, + 4, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 256, + 512, + 1024, + 1536, + 2048, + 3072, + 4096, + ] + else: + batch_sizes = [args.batch_size] + + ray.init() + num_gpus = int(ray.available_resources()["GPU"]) + workers = [BenchmarkWorker.remote(args.seed) for _ in range(num_gpus)] + + def _distribute(method: str, inputs: List[Any]) -> List[Any]: + outputs = [] + worker_idx = 0 + for input_args in inputs: + worker = workers[worker_idx] + worker_method = getattr(worker, method) + output = worker_method.remote(*input_args) + outputs.append(output) + worker_idx = (worker_idx + 1) % num_gpus + return ray.get(outputs) + + if args.tune: + search_space = get_configs_compute_bound() + if block_shape is not None: + block_n, block_k = block_shape[0], block_shape[1] + search_space = [ + config + for config in search_space + if block_k % config["BLOCK_SIZE_K"] == 0 + ] + print(f"Start tuning over {len(search_space)} configurations...") + + start = time.time() + configs = _distribute( + "tune", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + search_space, + ) + for batch_size in batch_sizes + ], + ) + best_configs = { + M: sort_config(config) for M, config in zip(batch_sizes, configs) + } + save_configs( + best_configs, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + end = time.time() + print(f"Tuning took {end - start:.2f} seconds") + else: + outputs = _distribute( + "benchmark", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + for batch_size in batch_sizes + ], + ) + + for batch_size, (config, kernel_time) in zip(batch_sizes, outputs): + print(f"Batch size: {batch_size}, config: {config}") + print(f"Kernel time: {kernel_time:.2f} us") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", "-tp", type=int, default=2) + parser.add_argument( + "--dtype", + type=str, + choices=["auto", "fp8_w8a8", "int8_w8a16", "int8_w8a8"], + default="auto", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument("--tune", action="store_true") + args = parser.parse_args() + + main(args) diff --git a/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py new file mode 100644 index 0000000..78d8149 --- /dev/null +++ b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py @@ -0,0 +1,576 @@ +import itertools +import math +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +import triton +import triton.language as tl +from einops import rearrange +from sgl_kernel import lightning_attention_decode as sgl_lightning_attention_decode + + +@triton.jit +def _decode_kernel( + Q, + K, + V, + KV, + Out, + S, + b: tl.constexpr, + h: tl.constexpr, + n: tl.constexpr, + d: tl.constexpr, + d_original: tl.constexpr, + e: tl.constexpr, + e_original: tl.constexpr, +): + off_bh = tl.program_id(0) + off_h = off_bh % h + + qk_offset = off_bh * n * d + v_offset = off_bh * n * e + o_offset = off_bh * n * e + kv_offset = off_bh * d * e + + s = tl.load(S + off_h) + ratio = tl.exp(-s) + + d_idx = tl.arange(0, d) + e_idx = tl.arange(0, e) + + # Create masks for original dimensions + d_mask = d_idx < d_original + e_mask = e_idx < e_original + + # Load with masking + q = tl.load(Q + qk_offset + d_idx, mask=d_mask, other=0.0) + k = tl.load(K + qk_offset + d_idx, mask=d_mask, other=0.0) + v = tl.load(V + v_offset + e_idx, mask=e_mask, other=0.0) + + # Load KV with 2D masking + kv = tl.load( + KV + kv_offset + d_idx[:, None] * e + e_idx[None, :], + mask=(d_mask[:, None] & e_mask[None, :]), + other=0.0, + ) + + # Compute outer product using element-wise operations + k_v_prod = k[:, None] * v[None, :] + kv = ratio * kv + k_v_prod + + # Store KV with 2D masking + tl.store( + KV + kv_offset + d_idx[:, None] * e + e_idx[None, :], + kv.to(KV.dtype.element_ty), + mask=(d_mask[:, None] & e_mask[None, :]), + ) + + # Compute matrix-vector multiplication using element-wise operations and reduction + o = tl.sum(q[:, None] * kv, axis=0) + + # Store output with masking + tl.store(Out + o_offset + e_idx, o.to(Out.dtype.element_ty), mask=e_mask) + + +def lightning_attn_decode(q, k, v, kv, s): + """Triton implementation of Lightning Attention decode operation""" + b, h, n, d = q.shape + e = v.shape[-1] + assert n == 1, "Sequence length must be 1 in decode mode" + + # Get padded dimensions (power of 2) + d_padded = next_power_of_2(d) + e_padded = next_power_of_2(e) + + # Create output tensor (padded) + o_padded = torch.empty(b, h, n, e_padded, dtype=v.dtype, device=v.device) + + # Create padded tensors without actually padding the data + q_padded = torch.empty(b, h, n, d_padded, dtype=q.dtype, device=q.device) + k_padded = torch.empty(b, h, n, d_padded, dtype=k.dtype, device=k.device) + v_padded = torch.empty(b, h, n, e_padded, dtype=v.dtype, device=v.device) + kv_padded = torch.empty( + b, h, d_padded, e_padded, dtype=torch.float32, device=kv.device + ) + + # Copy data to padded tensors + q_padded[..., :d] = q + k_padded[..., :d] = k + v_padded[..., :e] = v + kv_padded[..., :d, :e] = kv + + # Launch kernel + grid = (b * h, 1) + _decode_kernel[grid]( + q_padded, + k_padded, + v_padded, + kv_padded, + o_padded, + s, + b=b, + h=h, + n=n, + d=d_padded, + d_original=d, + e=e_padded, + e_original=e, + ) + + # Get unpadded outputs + o = o_padded[..., :e] + kv_out = kv_padded[..., :d, :e] + + return o, kv_out + + +def next_power_of_2(n): + return 2 ** (int(math.ceil(math.log(n, 2)))) + + +class MiniMaxText01LightningAttention(nn.Module): + def __init__(self, config=None, layer_idx: Optional[int] = None, **kwargs): + super().__init__() + if config is None: + config = type("Config", (), kwargs) + + bias = False + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = getattr(config, "head_dim", self.hidden_size // self.num_heads) + + self.out_proj = nn.Linear( + self.head_dim * self.num_heads, self.hidden_size, bias=bias + ) + self.act = get_activation_fn(config.hidden_act) + self.norm = MiniMaxText01RMSNorm(self.head_dim * self.num_heads) + + self.qkv_proj = nn.Linear( + self.hidden_size, 3 * self.head_dim * self.num_heads, bias=bias + ) + self.output_gate = nn.Linear( + self.hidden_size, self.head_dim * self.num_heads, bias=bias + ) + + # for inference only + self.offset = 0 + self.layer_idx = layer_idx + + def forward( + self, + hidden_states, + attn_mask: Optional[torch.Tensor] = None, # (b, h, n, m) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, + **kwargs, + ): + if (not self.training) and (not do_eval): + return self.inference( + hidden_states, + attn_mask, + output_attentions, + past_key_value, + use_cache, + slope_rate, + ) + + def inference( + self, + x, + attn_mask: Optional[torch.Tensor] = None, # (b, n) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, # (h, 1, 1) + ): + # x: b n d + b, n, d = x.shape + # linear map + qkv = self.act(self.qkv_proj(x)) + new_shape = qkv.size()[:-1] + (self.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [self.head_dim] * 3, dim=3) + q = q.transpose(1, 2) # [b, n, h, d] -> [b, h, n, d] + k = k.transpose(1, 2) # [b, n, h, d] -> [b, h, n, d] + v = v.transpose(1, 2) # [b, n, h, d] -> [b, h, n, e] + + self.offset += 1 + ratio = torch.exp(-slope_rate) # [h, 1, 1] + + # decode mode + kv = past_key_value # [b, h, d, e] + output = [] + for i in range(n): + # kv: [b, h, d, e] + # ratio: [h, 1, 1] + # k: [b, h, n, d] + # v: [b, h, n, e] + # k[:, :, i : i + 1]: [b, h, 1, d] + # v[:, :, i : i + 1]: [b, h, 1, e] + # ratio * kv: [b, h, d, e] + # torch.einsum( + # "... n d, ... n e -> ... d e", + # k[:, :, i : i + 1], + # v[:, :, i : i + 1], + # ) + # [b, h, d, e] + [b, h, d, e] -> [b, h, d, e] + kv = ratio * kv + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + # q[:, :, i : i + 1]: [b, h, 1, d] + # kv.to(q.dtype): [b, h, d, e] + # torch.einsum( + # "... n e, ... e d -> ... n d", q[:, :, i : i + 1], kv.to(q.dtype) + # ) + # [b, h, 1, d] * [b, h, d, e] -> [b, h, 1, e] + qkv = torch.einsum( + "... n e, ... e d -> ... n d", q[:, :, i : i + 1], kv.to(q.dtype) + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + + # reshape + output = rearrange(output, "b h n d -> b n (h d)") + # normalize + output = self.norm(output) + # gate + output = F.sigmoid(self.output_gate(x)) * output + # outproj + output = self.out_proj(output) + + attn_weights = None + + return output, attn_weights, kv + + +def get_activation_fn(activation): + if activation == "gelu": + return F.gelu + elif activation == "relu": + return F.relu + elif activation == "elu": + return F.elu + elif activation == "sigmoid": + return F.sigmoid + elif activation == "exp": + + def f(x): + with torch.no_grad(): + x_max = torch.max(x, dim=-1, keepdims=True).values + y = torch.exp(x - x_max) + return y + + return f + elif activation == "leak": + return F.leaky_relu + elif activation == "1+elu": + + def f(x): + return 1 + F.elu(x) + + return f + elif activation == "2+elu": + + def f(x): + return 2 + F.elu(x) + + return f + elif activation == "silu" or activation == "swish": + return F.silu + elif activation == "sine": + return torch.sin + else: + return lambda x: x + + +class MiniMaxText01RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + MiniMaxText01RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +def test_lightning_attention_implementations(model_params): + torch.manual_seed(42) + + batch_size = 64 + seq_len = 1 + dtype = torch.bfloat16 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + hidden_states = torch.randn( + batch_size, seq_len, model_params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(model_params["num_attention_heads"]).to(device) + + model_attn = MiniMaxText01LightningAttention(**model_params).to(dtype).to(device) + model_attn.eval() + + d = model_params["head_dim"] + past_kv = torch.randn( + batch_size, + model_params["num_attention_heads"], + d, + d, + device=device, + ) + with torch.no_grad(): + model_output, _, new_kv = model_attn.inference( + hidden_states, + attn_mask=attention_mask, + slope_rate=slope_rate, + past_key_value=past_kv, + ) + + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + past_kv = past_kv.contiguous() + slope_rate = slope_rate.contiguous() + + # Test Triton implementation + triton_output, triton_new_kv = lightning_attn_decode(q, k, v, past_kv, slope_rate) + triton_output = triton_output.transpose(1, 2).contiguous() + triton_output = triton_output.view(batch_size, seq_len, -1) + triton_output = model_attn.norm(triton_output) + triton_output = torch.sigmoid(model_attn.output_gate(hidden_states)) * triton_output + triton_output = model_attn.out_proj(triton_output) + + # Test SGL implementation + sgl_output = torch.empty_like(v) + sgl_new_kv = torch.empty_like(past_kv) + sgl_lightning_attention_decode(q, k, v, past_kv, slope_rate, sgl_output, sgl_new_kv) + + sgl_output = sgl_output.transpose(1, 2).contiguous() + sgl_output = sgl_output.view(batch_size, seq_len, -1) + sgl_output = model_attn.norm(sgl_output) + sgl_output = torch.sigmoid(model_attn.output_gate(hidden_states)) * sgl_output + sgl_output = model_attn.out_proj(sgl_output) + + # Verify Triton implementation results + torch.testing.assert_close( + model_output, + triton_output, + rtol=1e-3, + atol=1e-2, + msg="Triton lightning attention implementation produces different output results", + ) + torch.testing.assert_close( + new_kv, + triton_new_kv, + rtol=1e-3, + atol=1e-2, + msg="Triton lightning attention implementation produces different kv results", + ) + + # Verify SGL implementation results + torch.testing.assert_close( + model_output, + sgl_output, + rtol=1e-3, + atol=1e-2, + msg="SGL lightning attention implementation produces different output results", + ) + torch.testing.assert_close( + new_kv, + sgl_new_kv, + rtol=1e-3, + atol=1e-2, + msg="SGL lightning attention implementation produces different kv results", + ) + + print("✅ All implementations match") + + +def _build_slope_tensor(n_attention_heads: int): + def get_slopes(n): + def get_slopes_power_of_2(n): + start = 2 ** (-(2 ** -(math.log2(n) - 3))) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(n).is_integer(): + return get_slopes_power_of_2(n) + else: + closest_power_of_2 = 2 ** math.floor(math.log2(n)) + return ( + get_slopes_power_of_2(closest_power_of_2) + + get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2] + ) + + slopes = torch.tensor(get_slopes(n_attention_heads)).reshape( + n_attention_heads, 1, 1 + ) + return slopes + + +def get_benchmark(): + batch_size_range = [i for i in range(1, 33)] # max 32 + seq_length_range = [1] # decode mode sequence length is fixed to 1 + configs = list(itertools.product(batch_size_range, seq_length_range)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["Original", "Triton", "SGL"], + line_names=[ + "Original PyTorch Implementation", + "Triton Implementation", + "SGL Implementation", + ], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name="lightning-attention-decode-performance", + args={}, + ) + ) + def benchmark(batch_size, seq_len, provider): + dtype = torch.bfloat16 + device = torch.device("cuda") + + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "gelu", + } + + hidden_states = torch.randn( + batch_size, seq_len, params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(params["num_attention_heads"]).to(device) + model_attn = MiniMaxText01LightningAttention(**params).to(dtype).to(device) + model_attn.eval() + + d = params["head_dim"] + past_kv = torch.randn( + batch_size, + params["num_attention_heads"], + d, + d, + device=device, + ) + + quantiles = [0.5, 0.2, 0.8] + if provider == "Original": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: model_attn.inference( + hidden_states, + attn_mask=attention_mask, + slope_rate=slope_rate, + past_key_value=past_kv, + ), + quantiles=quantiles, + ) + elif provider == "Triton": + + def run_triton(): + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + output, new_kv = lightning_attn_decode(q, k, v, past_kv, slope_rate) + output = output.transpose(1, 2).contiguous() + output = output.view(batch_size, seq_len, -1) + output = model_attn.norm(output) + output = torch.sigmoid(model_attn.output_gate(hidden_states)) * output + return model_attn.out_proj(output) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_triton, + quantiles=quantiles, + ) + else: # SGL + + def run_sgl(): + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2).contiguous() + k = k.transpose(1, 2).contiguous() + v = v.transpose(1, 2).contiguous() + + output = torch.empty_like(v) + new_kv = torch.empty_like(past_kv) + sgl_lightning_attention_decode( + q, k, v, past_kv, slope_rate, output, new_kv + ) + + output = output.transpose(1, 2).contiguous() + output = output.view(batch_size, seq_len, -1) + output = model_attn.norm(output) + output = torch.sigmoid(model_attn.output_gate(hidden_states)) * output + return model_attn.out_proj(output) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_sgl, + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/lightning_attention_decode/", + help="Path to save lightning attention decode benchmark results", + ) + args = parser.parse_args() + + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "silu", + } + # Run correctness test first + # Adapted from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/config.json + test_lightning_attention_implementations(params) + + # Run performance benchmark + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py new file mode 100644 index 0000000..3bf9054 --- /dev/null +++ b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py @@ -0,0 +1,603 @@ +import itertools +import math +import os +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +import triton +import triton.language as tl +from einops import rearrange + + +# Adapted from https://github.com/OpenNLPLab/lightning-attention/blob/main/lightning_attn/ops/triton/lightning_attn2.py +@triton.jit +def _fwd_kernel( + Q, + K, + V, + Out, + S, # log lambda + b: tl.constexpr, + h: tl.constexpr, + n: tl.constexpr, + d: tl.constexpr, + e: tl.constexpr, + BLOCK: tl.constexpr, + NUM_BLOCK: tl.constexpr, + BLOCK_MODEL: tl.constexpr, +): + ##### get offset + off_bh = tl.program_id(0) + off_h = off_bh % h + off_e = tl.program_id(1) + qk_offset = off_bh * n * d + v_offset = off_bh * n * e + o_offset = off_bh * n * e + # channel offset + e_offset = off_e * BLOCK_MODEL + + ##### get block ptr + Q_block_ptr = Q + qk_offset + tl.arange(0, d)[None, :] + K_trans_block_ptr = K + qk_offset + tl.arange(0, d)[:, None] + V_block_ptr = V + v_offset + e_offset + tl.arange(0, BLOCK_MODEL)[None, :] + O_block_ptr = Out + o_offset + e_offset + tl.arange(0, BLOCK_MODEL)[None, :] + S_block_ptr = S + off_h + + ##### init diag decay(Lambda); q, k decay; kv + s = tl.load(S_block_ptr) + # q, k decay + off_block = tl.arange( + 0, BLOCK + ) # Not bug, this is a bit different from algorithm 1, but is mathematically equivalent + q_decay = tl.exp(-s.to(tl.float32) * off_block[:, None]) + k_trans_decay = tl.exp(-s.to(tl.float32) * (BLOCK - off_block[None, :])) + block_decay = tl.exp(-s.to(tl.float32) * BLOCK) + # diag decay + index = off_block[:, None] - off_block[None, :] + s_index = s * index + s_index = tl.where(index >= 0, -s_index, float("-inf")) + diag_decay = tl.exp(s_index) + kv = tl.zeros([d, BLOCK_MODEL], dtype=tl.float32) + + ##### compute + for i in range(NUM_BLOCK): + # load + q = tl.load( + Q_block_ptr + off_block[:, None] * d, mask=off_block[:, None] < n, other=0.0 + ).to(tl.float32) + k_trans = tl.load( + K_trans_block_ptr + off_block[None, :] * d, + mask=off_block[None, :] < n, + other=0.0, + ).to(tl.float32) + v = tl.load( + V_block_ptr + off_block[:, None] * e, mask=off_block[:, None] < n, other=0.0 + ).to(tl.float32) + + # compute + qk = tl.dot(q, k_trans) * diag_decay + o_intra = tl.dot(qk, v) + o_inter = tl.dot(q, kv) * q_decay + o = o_intra + o_inter + + # save and update + tl.store( + O_block_ptr + off_block[:, None] * e, + o.to(O_block_ptr.dtype.element_ty), + mask=off_block[:, None] < n, + ) + kv = block_decay * kv + tl.dot(k_trans * k_trans_decay, v) + off_block += BLOCK + + +def lightning_attn2(q, k, v, s): + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + s = s.contiguous() + + b, h, n, d = q.shape + e = v.shape[-1] + + # Pad d to next power of 2 + d_padded = next_power_of_2(d) + if d_padded != d: + q_padded = F.pad(q, (0, d_padded - d)) + k_padded = F.pad(k, (0, d_padded - d)) + else: + q_padded = q + k_padded = k + + # Pad e to next power of 2 + e_padded = next_power_of_2(e) + if e_padded != e: + v_padded = F.pad(v, (0, e_padded - e)) + else: + v_padded = v + + o_padded = torch.empty((b, h, n, e_padded), dtype=q.dtype, device=q.device) + + BLOCK = 64 + NUM_BLOCK = triton.cdiv(q.shape[2], BLOCK) + # parallel over channel + BLOCK_MODEL = min(triton.next_power_of_2(e_padded), 32) + grid = (b * h, triton.cdiv(e_padded, BLOCK_MODEL)) + + _fwd_kernel[grid]( + q_padded, + k_padded, + v_padded, + o_padded, + s, + b, + h, + n, + d_padded, + e_padded, + BLOCK=BLOCK, + NUM_BLOCK=NUM_BLOCK, + BLOCK_MODEL=BLOCK_MODEL, + ) + + # Remove padding from output + if e_padded != e: + o = o_padded[..., :e] + else: + o = o_padded + + return o + + +def is_support(dim): + return 16 % dim + + +def next_power_of_2(n): + return 2 ** (int(math.ceil(math.log(n, 2)))) + + +def lightning_attn_func(q, k, v, s): + b, h, n, d = q.shape + e = v.shape[-1] + assert is_support(d) and is_support(e) + + # pad v's feature dim to power of 2 + e_pad = next_power_of_2(e) + need_pad = e_pad != e + if need_pad: + v = F.pad(v, (0, e_pad - e)) + + if d > 128: + # split over head + if 64 % d: + m = 64 + elif 32 % d: + m = 32 + elif 16 % d: + m = 16 + arr = [m * i for i in range(d // m + 1)] + if arr[-1] != d: + arr.append(d) + n = len(arr) + o = 0 + for i in range(n - 1): + start = arr[i] + end = arr[i + 1] + q1 = q[..., start:end] + k1 = k[..., start:end] + o += lightning_attn2(q1, k1, v, s) + else: + o = lightning_attn2(q, k, v, s) + + if need_pad: + o = o[:, :, :, :e] + + return o + + +debug = eval(os.environ.get("debug", default="False")) + +BLOCK = 256 + + +# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->MiniMaxText01 +class MiniMaxText01RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + MiniMaxText01RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +# Copied from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/modeling_minimax_text_01.py +def get_activation_fn(activation): + if debug: + logger.info(f"activation: {activation}") + if activation == "gelu": + return F.gelu + elif activation == "relu": + return F.relu + elif activation == "elu": + return F.elu + elif activation == "sigmoid": + return F.sigmoid + elif activation == "exp": + + def f(x): + with torch.no_grad(): + x_max = torch.max(x, dim=-1, keepdims=True).values + y = torch.exp(x - x_max) + + return y + + return f + elif activation == "leak": + return F.leaky_relu + elif activation == "1+elu": + + def f(x): + return 1 + F.elu(x) + + return f + elif activation == "2+elu": + + def f(x): + return 2 + F.elu(x) + + return f + elif activation == "silu" or activation == "swish": + return F.silu + elif activation == "sine": + return torch.sin + else: + logger.info(f"activation: does not support {activation}, use Identity!!!") + return lambda x: x + + +# Copied from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/modeling_minimax_text_01.py +class MiniMaxText01LightningAttention(nn.Module): + def __init__(self, config=None, layer_idx: Optional[int] = None, **kwargs): + super().__init__() + if config is None: + config = type("Config", (), kwargs) + + bias = False + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = getattr(config, "head_dim", self.hidden_size // self.num_heads) + + self.out_proj = nn.Linear( + self.head_dim * self.num_heads, self.hidden_size, bias=bias + ) + self.act = get_activation_fn(config.hidden_act) + self.norm = MiniMaxText01RMSNorm(self.head_dim * self.num_heads) + + self.qkv_proj = nn.Linear( + self.hidden_size, 3 * self.head_dim * self.num_heads, bias=bias + ) + self.output_gate = nn.Linear( + self.hidden_size, self.head_dim * self.num_heads, bias=bias + ) + + # for inference only + self.offset = 0 + self.layer_idx = layer_idx + + def forward( + self, + hidden_states, + attn_mask: Optional[torch.Tensor] = None, # (b, h, n, m) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, + **kwargs, + ): + if (not self.training) and (not do_eval): + return self.inference( + hidden_states, + attn_mask, + output_attentions, + past_key_value, + use_cache, + slope_rate, + ) + + def inference( + self, + x, + attn_mask: Optional[torch.Tensor] = None, # (b, n) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, # (h, 1, 1) + ): + # x: b n d + b, n, d = x.shape + # linear map + qkv = self.act(self.qkv_proj(x)) + new_shape = qkv.size()[:-1] + (self.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [self.head_dim] * 3, dim=3) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + if past_key_value is None: + self.offset = q.shape[-2] + else: + self.offset += 1 + + # for align with metaseq + ratio = torch.exp(-slope_rate) + + # only use for the first time + if past_key_value is None: + slope_rate = slope_rate.to(torch.float32) + if attn_mask is not None: + v = v.masked_fill( + (1 - attn_mask).unsqueeze(1).unsqueeze(-1).to(torch.bool), 0 + ) + NUM_BLOCK = (n + BLOCK - 1) // BLOCK + b, h, n, d = q.shape + e = v.shape[-1] + # other + array = torch.arange(BLOCK).to(q) + 1 + q_decay = torch.exp(-slope_rate * array.reshape(-1, 1)) + k_decay = torch.exp(-slope_rate * (BLOCK - array.reshape(-1, 1))) + index = array[:, None] - array[None, :] + s_index = ( + slope_rate + * index[ + None, + None, + ] + ) + s_index = torch.where(index >= 0, -s_index, float("-inf")) + diag_decay = torch.exp(s_index) + + kv = torch.zeros(b, h, d, e).to(torch.float32).to(q.device) + output = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device) + for i in range(NUM_BLOCK): + si = i * BLOCK + ei = min(si + BLOCK, n) + m = ei - si + qi = q[:, :, si:ei].contiguous() + ki = k[:, :, si:ei].contiguous() + vi = v[:, :, si:ei].contiguous() + qkv_none_diag = torch.matmul(qi * q_decay[:, :m], kv).to(torch.float32) + + # diag + qk = ( + torch.matmul(qi, ki.transpose(-1, -2)).to(torch.float32) + * diag_decay[:, :, :m, :m] + ) + qkv_diag = torch.matmul(qk, vi.to(torch.float32)) + block_decay = torch.exp(-slope_rate * m) + output[:, :, si:ei] = qkv_none_diag + qkv_diag + kv = block_decay * kv + torch.matmul( + (ki * k_decay[:, -m:]).transpose(-1, -2).to(vi.dtype), vi + ) + + else: + kv = past_key_value + output = [] + for i in range(n): + kv = ratio * kv + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + qkv = torch.einsum( + "... n e, ... e d -> ... n d", q[:, :, i : i + 1], kv.to(q.dtype) + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + # reshape + output = rearrange(output, "b h n d -> b n (h d)") + # normalize + output = self.norm(output) + # gate + output = F.sigmoid(self.output_gate(x)) * output + # outproj + output = self.out_proj(output) + + attn_weights = None + + return output, attn_weights, kv + + +def _build_slope_tensor(n_attention_heads: int): + def get_slopes(n): + def get_slopes_power_of_2(n): + start = 2 ** (-(2 ** -(math.log2(n) - 3))) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(n).is_integer(): + return get_slopes_power_of_2( + n + ) # In the paper, we only train models that have 2^a heads for some a. This function has + else: # some good properties that only occur when the input is a power of 2. To maintain that even + closest_power_of_2 = 2 ** math.floor( + math.log2(n) + ) # when the number of heads is not a power of 2, we use this workaround. + return ( + get_slopes_power_of_2(closest_power_of_2) + + get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2] + ) + + # h, 1, 1 + slopes = torch.tensor(get_slopes(n_attention_heads)).reshape( + n_attention_heads, 1, 1 + ) + + return slopes + + +def test_lightning_attention_implementations(model_params): + torch.manual_seed(42) + + batch_size = 2 + seq_len = 1024 + dtype = torch.bfloat16 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + hidden_states = torch.randn( + batch_size, seq_len, model_params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(model_params["num_attention_heads"]).to(device) + + model_attn = MiniMaxText01LightningAttention(**model_params).to(dtype).to(device) + model_attn.eval() + + with torch.no_grad(): + model_output, _, _ = model_attn.inference( + hidden_states, attn_mask=attention_mask, slope_rate=slope_rate + ) + + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + lib_output = lightning_attn_func(q, k, v, slope_rate) + lib_output = lib_output.transpose(1, 2).contiguous() + lib_output = lib_output.view(batch_size, seq_len, -1) + lib_output = model_attn.norm(lib_output) + lib_output = torch.sigmoid(model_attn.output_gate(hidden_states)) * lib_output + lib_output = model_attn.out_proj(lib_output) + + torch.testing.assert_close( + model_output, + lib_output, + rtol=1e-3, + atol=1e-2, + msg="Lightning attention implementations produce different results", + ) + + print("✅ Two implementations match") + + +def get_benchmark(): + batch_size_range = [2**i for i in range(0, 7)] # max 64 + seq_length_range = [256, 512, 1024, 2048, 4096] # max 4096 + configs = list(itertools.product(batch_size_range, seq_length_range)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["MiniMax-Text-01", "OpenNLPLab"], + line_names=[ + "MiniMax-Text-01 Model Implementation", + "OpenNLPLab Library Implementation", + ], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="lightning-attention-prefill-performance", + args={}, + ) + ) + def benchmark(batch_size, seq_len, provider): + dtype = torch.bfloat16 + device = torch.device("cuda") + + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "gelu", + } + + hidden_states = torch.randn( + batch_size, seq_len, params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(params["num_attention_heads"]).to(device) + model_attn = MiniMaxText01LightningAttention(**params).to(dtype).to(device) + model_attn.eval() + + quantiles = [0.5, 0.2, 0.8] + if provider == "MiniMax-Text-01": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: model_attn.inference( + hidden_states, attn_mask=attention_mask, slope_rate=slope_rate + ), + quantiles=quantiles, + ) + else: + + def run_lib(): + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + lib_output = lightning_attn_func(q, k, v, slope_rate) + lib_output = lib_output.transpose(1, 2).contiguous() + lib_output = lib_output.view(batch_size, seq_len, -1) + lib_output = model_attn.norm(lib_output) + lib_output = ( + torch.sigmoid(model_attn.output_gate(hidden_states)) * lib_output + ) + return model_attn.out_proj(lib_output) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_lib, + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/lightning_attention_prefill/", + help="Path to save lightning attention prefill benchmark results", + ) + args = parser.parse_args() + + # Run correctness test first + # Adapted from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/config.json + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "silu", + } + test_lightning_attention_implementations(params) + + # Run performance benchmark + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/quantization/bench_int8_quant.py b/benchmark/kernels/quantization/bench_int8_quant.py new file mode 100644 index 0000000..94b7956 --- /dev/null +++ b/benchmark/kernels/quantization/bench_int8_quant.py @@ -0,0 +1,94 @@ +import argparse + +import torch +import triton +from vllm._custom_ops import scaled_int8_quant as vllm_scaled_int8_quant + +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 + + +@torch.compile(backend="inductor") +def torch_int8_quant(x): + int8_max = torch.iinfo(torch.int8).max + + abs_max = x.abs().max(dim=-1, keepdim=True).values + scales = abs_max.to(torch.float32) / float(int8_max) + + q_x = (x / scales).round().to(torch.int8) + + return q_x, scales + + +def _test_accuracy_once(M, K, input_dtype, device): + x = torch.randn(M, K, dtype=input_dtype, device=device) * 5000 + out, scales, _ = vllm_scaled_int8_quant(x, symmetric=True) + out1, scales1 = per_token_quant_int8(x) + out2, scales2 = torch_int8_quant(x) + torch.testing.assert_close(out, out2, atol=1, rtol=0) + torch.testing.assert_close(out, out1, atol=1, rtol=0) + torch.testing.assert_close(scales, scales2) + torch.testing.assert_close(scales1, scales2) + print(f"M: {M}, K: {K}, type: {input_dtype} OK") + + +def test_accuracy(): + Ms = [1, 13, 128, 1024, 2048, 4096] + Ks = [512, 1024, 2048, 8192] + input_dtypes = [torch.float16, torch.bfloat16] + for M in Ms: + for K in Ks: + for input_dtype in input_dtypes: + _test_accuracy_once(M, K, input_dtype, "cuda") + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 32, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=["vllm op", "triton", "torch.compile"], + line_names=["vllm op", "triton", "torch.compile"], + styles=[("blue", "-"), ("orange", "-"), ("red", "-")], + ylabel="ms", + plot_name="int8 per token quant", + args={}, + ) +) +def benchmark(batch_size, provider): + M, K = batch_size, 16384 + x = torch.randn(M, K, dtype=torch.float16, device="cuda") * 1000 + + quantiles = [0.5, 0.2, 0.8] + if provider == "vllm op": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: vllm_scaled_int8_quant(x, symmetric=True), + quantiles=quantiles, + ) + if provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: per_token_quant_int8(x), + quantiles=quantiles, + ) + if provider == "torch.compile": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: torch_int8_quant(x), + quantiles=quantiles, + ) + + return ms, min_ms, max_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./bench_int8_quant_res", + help="Path to save int8 quant benchmark results", + ) + args = parser.parse_args() + + test_accuracy() + + benchmark.run(print_data=True, show_plots=True, save_path=args.save_path) diff --git a/benchmark/kernels/quantization/tuning_block_wise_kernel.py b/benchmark/kernels/quantization/tuning_block_wise_kernel.py new file mode 100644 index 0000000..197939f --- /dev/null +++ b/benchmark/kernels/quantization/tuning_block_wise_kernel.py @@ -0,0 +1,474 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import json +import multiprocessing as mp +import os +import time +from datetime import datetime +from typing import Any, Dict, List + +import torch +import triton +from tqdm import tqdm + +mp.set_start_method("spawn", force=True) + +from sglang.srt.layers.quantization.fp8_kernel import ( + _w8a8_block_fp8_matmul, + _w8a8_block_fp8_matmul_unrolledx4, +) +from sglang.srt.layers.quantization.int8_kernel import _w8a8_block_int8_matmul +from sglang.srt.utils import get_device_core_count, get_device_name, is_hip + +is_hip_ = is_hip() + +DTYPE_MAP = { + "float32": torch.float32, + "float16": torch.float16, + "half": torch.half, + "bfloat16": torch.bfloat16, +} + + +def w8a8_block_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + config: Dict[str, Any], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + N, K = B.shape + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + def grid(META): + return ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + # Use manually unrolledx4 kernel on AMD GPU when the grid size is small. + # Empirical testing shows the sweet spot lies when it's less than the # of + # compute units available on the device. + num_workgroups = triton.cdiv(M, config["BLOCK_SIZE_M"]) * triton.cdiv( + N, config["BLOCK_SIZE_N"] + ) + + if A.dtype == torch.float8_e4m3fnuz or A.dtype == torch.float8_e4m3fn: + kernel = ( + _w8a8_block_fp8_matmul_unrolledx4 + if (is_hip_ == True and num_workgroups <= get_device_core_count()) + else _w8a8_block_fp8_matmul + ) + else: + kernel = _w8a8_block_int8_matmul + + kernel[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + ) + + return C + + +def get_rocm_configs_compute_bound(): + configs = [] + waves_per_eu_range = 0 + for num_stages in [2]: + for block_m in [32, 64, 128, 256]: + for block_k in [32, 64, 128, 256]: + for block_n in [16, 32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 4, 8, 16, 32]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu_range, + } + ) + return configs + + +def get_configs_compute_bound(): + configs = [] + if is_hip_: + configs = get_rocm_configs_compute_bound() + else: + for num_stages in [2, 3, 4, 5]: + for block_m in [16, 32, 64, 128, 256]: + for block_k in [64, 128]: + for block_n in [32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 16, 32, 64]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + } + ) + return configs + + +def get_weight_shapes(tp_size): + # NOTE(HandH1998): The weight shapes only works for DeepSeek-V3. Modify them, if you tune for another different model. + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + return weight_shapes + + +def benchmark_config( + A, B, As, Bs, block_size, config, out_dtype=torch.float16, num_iters=10 +): + def run(): + w8a8_block_matmul(A, B, As, Bs, block_size, config, out_dtype) + + torch.cuda.synchronize() + # JIT complication & warmup + for _ in range(5): + run() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for i in range(num_iters): + torch.cuda.synchronize() + start_event.record() + run() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + avg = sum(latencies) / (num_iters * 10) * 1000 # us + return avg + + +def tune(M, N, K, block_size, out_dtype, search_space, input_type): + factor_for_scale = 1e-2 + + if input_type == "fp8": + fp8_info = torch.finfo( + torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + ) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + ) + A = A_fp32.clamp(min=fp8_min, max=fp8_max).to( + torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + ) + + B_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + ) + B = B_fp32.clamp(min=fp8_min, max=fp8_max).to( + torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + ) + else: + int8_info = torch.iinfo(torch.int8) + int8_max, int8_min = int8_info.max, int8_info.min + + A_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * int8_max + ) + A = A_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + B_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * int8_max + ) + B = B_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + + As = torch.rand(M, k_tiles, dtype=torch.float32, device="cuda") * factor_for_scale + Bs = ( + torch.rand(n_tiles, k_tiles, dtype=torch.float32, device="cuda") + * factor_for_scale + ) + + best_config = None + best_time = float("inf") + for config in tqdm(search_space): + try: + kernel_time = benchmark_config( + A, + B, + As, + Bs, + block_size, + config, + out_dtype, + num_iters=10, + ) + except triton.runtime.autotuner.OutOfResources: + # Some configurations may be invalid and fail to compile. + continue + + if kernel_time < best_time: + best_time = kernel_time + best_config = config + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={M}") + assert best_config is not None + return best_config + + +def save_configs( + N, + K, + block_n, + block_k, + configs, + save_path, + input_type="fp8", +) -> None: + os.makedirs(save_path, exist_ok=True) + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype={input_type}_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join(save_path, json_file_name) + print(f"Writing best config to {config_file_path}...") + + with open(config_file_path, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def get_available_gpu_count(): + """Get the number of available GPUs.""" + return torch.cuda.device_count() + + +def tune_on_gpu(args_dict): + """Run tuning on a specific GPU.""" + gpu_id = args_dict["gpu_id"] + batch_sizes = args_dict["batch_sizes"] + weight_shapes = args_dict["weight_shapes"] + args = args_dict["args"] + + torch.cuda.set_device(gpu_id) + print(f"Starting tuning on GPU {gpu_id} with batch sizes {batch_sizes}") + + block_n = args.block_n + block_k = args.block_k + out_dtype = DTYPE_MAP[args.out_dtype] + save_path = args.save_path + input_type = args.input_type + + search_space = get_configs_compute_bound() + search_space = [ + config for config in search_space if block_k % config["BLOCK_SIZE_K"] == 0 + ] + + start = time.time() + results = {} + for shape in tqdm(weight_shapes, desc=f"GPU {gpu_id} - Shapes"): + N, K = shape[0], shape[1] + print(f"[GPU {gpu_id}] Tune for weight shape of `N: {N}, K: {K}`") + benchmark_results = [ + tune( + batch_size, + N, + K, + [block_n, block_k], + out_dtype, + search_space, + input_type, + ) + for batch_size in tqdm(batch_sizes, desc=f"GPU {gpu_id} - Batch sizes") + ] + best_configs = {M: config for M, config in zip(batch_sizes, benchmark_results)} + save_configs(N, K, block_n, block_k, best_configs, save_path, input_type) + + end = time.time() + print(f"Tuning on GPU {gpu_id} took {end - start:.2f} seconds") + + +def distribute_batch_sizes(batch_sizes, num_gpus): + """Distribute batch sizes across available GPUs.""" + batches_per_gpu = [] + for i in range(num_gpus): + start_idx = i * len(batch_sizes) // num_gpus + end_idx = (i + 1) * len(batch_sizes) // num_gpus + batches_per_gpu.append(batch_sizes[start_idx:end_idx]) + return batches_per_gpu + + +def main(args): + print(args) + + num_gpus = get_available_gpu_count() + if num_gpus == 0: + raise RuntimeError("No GPU available for tuning") + print(f"Found {num_gpus} GPUs for parallel tuning") + + torch.cuda.init() + + if args.batch_size is None: + batch_sizes = [ + 1, + 2, + 4, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 256, + 512, + 1024, + 1536, + 2048, + 3072, + 4096, + ] + else: + batch_sizes = [args.batch_size] + num_gpus = 1 # If only one batch size, use only one GPU + + weight_shapes = get_weight_shapes(args.tp_size) + + batches_per_gpu = distribute_batch_sizes(batch_sizes, num_gpus) + + process_args = [] + for gpu_id in range(num_gpus): + process_args.append( + { + "gpu_id": gpu_id, + "batch_sizes": batches_per_gpu[gpu_id], + "weight_shapes": weight_shapes, # Each GPU processes all weight shapes + "args": args, + } + ) + + ctx = mp.get_context("spawn") + with ctx.Pool(num_gpus) as pool: + pool.map(tune_on_gpu, process_args) + + print("Multi-GPU tuning completed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--tp-size", "-tp", type=int, default=8) + parser.add_argument( + "--input-type", type=str, choices=["fp8", "int8"], default="fp8" + ) + parser.add_argument( + "--out-dtype", + type=str, + choices=["float32", "float16", "bfloat16", "half"], + default="float16", + ) + parser.add_argument("--block-n", type=int, default=128) + parser.add_argument("--block-k", type=int, default=128) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument( + "--save-path", type=str, default="python/sglang/srt/layers/quantization/configs" + ) + args = parser.parse_args() + + main(args) diff --git a/benchmark/kernels/rmsnorm/benchmark_rmsnorm.py b/benchmark/kernels/rmsnorm/benchmark_rmsnorm.py new file mode 100644 index 0000000..aeeea62 --- /dev/null +++ b/benchmark/kernels/rmsnorm/benchmark_rmsnorm.py @@ -0,0 +1,230 @@ +import itertools +from typing import Optional, Tuple, Union + +import torch +import triton +from flashinfer.norm import fused_add_rmsnorm, rmsnorm +from torch import nn +from vllm import _custom_ops as vllm_ops + + +class HuggingFaceRMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = x.to(orig_dtype) * self.weight + if residual is None: + return x + else: + return x, residual + + +def rmsnorm_naive( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + naive_norm = HuggingFaceRMSNorm(x.shape[-1], eps=eps) + naive_norm.weight = nn.Parameter(weight) + naive_norm = naive_norm.to(x.device) + + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + output = naive_norm(x, residual) + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_flashinfer( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + fused_add_rmsnorm(x, residual, weight, eps) + output = (x, residual) + else: + output = rmsnorm(x, weight, eps) + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_vllm( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + vllm_ops.fused_add_rms_norm(x, residual, weight, eps) + output = (x, residual) + else: + out = torch.empty_like(x) + vllm_ops.rms_norm(out, x, weight, eps) + output = out + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def calculate_diff(batch_size, seq_len, hidden_size, use_residual=True): + dtype = torch.bfloat16 + x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") + weight = torch.ones(hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) if use_residual else None + + output_naive = rmsnorm_naive( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_flashinfer = rmsnorm_flashinfer( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_vllm = rmsnorm_vllm( + x.clone(), weight, residual.clone() if residual is not None else None + ) + + if use_residual: + output_naive = output_naive[0] + output_flashinfer = output_flashinfer[0] + output_vllm = output_vllm[0] + + print(f"Naive output={output_naive}") + print(f"FlashInfer output={output_flashinfer}") + print(f"VLLM output={output_vllm}") + + if torch.allclose( + output_naive, output_flashinfer, atol=1e-2, rtol=1e-2 + ) and torch.allclose(output_naive, output_vllm, atol=1e-2, rtol=1e-2): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +batch_size_range = [2**i for i in range(0, 7, 2)] +seq_length_range = [2**i for i in range(6, 11, 1)] +head_num_range = [32, 48] +configs = list(itertools.product(head_num_range, batch_size_range, seq_length_range)) + + +def get_benchmark(use_residual): + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["head_num", "batch_size", "seq_len"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["huggingface", "flashinfer", "vllm"], + line_names=["HuggingFace", "FlashInfer", "vLLM"], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name=f"rmsnorm-performance-{'with' if use_residual else 'without'}-residual", + args={}, + ) + ) + def benchmark(head_num, batch_size, seq_len, provider): + dtype = torch.bfloat16 + hidden_size = head_num * 128 # assuming head_dim = 128 + + x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") + weight = torch.ones(hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) if use_residual else None + + quantiles = [0.5, 0.2, 0.8] + + if provider == "huggingface": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: rmsnorm_naive( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ), + quantiles=quantiles, + ) + elif provider == "flashinfer": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: rmsnorm_flashinfer( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ), + quantiles=quantiles, + ) + else: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: rmsnorm_vllm( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--use_residual", action="store_true", help="Whether to use residual connection" + ) + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/rmsnorm/", + help="Path to save rmsnorm benchmark results", + ) + args = parser.parse_args() + + # Run correctness test + calculate_diff( + batch_size=4, seq_len=128, hidden_size=4096, use_residual=args.use_residual + ) + + # Get the benchmark function with proper use_residual setting + benchmark = get_benchmark(args.use_residual) + # Run performance benchmark + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py b/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py new file mode 100644 index 0000000..1ce43c8 --- /dev/null +++ b/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py @@ -0,0 +1,342 @@ +import itertools +import os + +import torch +import triton +import triton.language as tl + + +@triton.jit +def write_req_to_token_pool_triton( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + req_to_token_ptr_stride: tl.constexpr, +): + BLOCK_SIZE: tl.constexpr = 512 + pid = tl.program_id(0) + + req_pool_index = tl.load(req_pool_indices + pid) + pre_len = tl.load(pre_lens + pid) + seq_len = tl.load(seq_lens + pid) + + # TODO: optimize this? + cumsum_start = 0 + for i in range(pid): + cumsum_start += tl.load(extend_lens + i) + + num_loop = tl.cdiv(seq_len - pre_len, BLOCK_SIZE) + for i in range(num_loop): + offset = tl.arange(0, BLOCK_SIZE) + i * BLOCK_SIZE + mask = offset < (seq_len - pre_len) + value = tl.load(out_cache_loc + cumsum_start + offset, mask=mask) + tl.store( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + offset + + pre_len, + value, + mask=mask, + ) + + +@triton.jit +def write_req_to_token_pool_triton_optimize( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + req_to_token_ptr_stride: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid_batch = tl.program_id(0) + pid_token = tl.program_id(1) + + req_pool_index = tl.load(req_pool_indices + pid_batch) + pre_len = tl.load(pre_lens + pid_batch) + seq_len = tl.load(seq_lens + pid_batch) + extend_len = seq_len - pre_len + + cumsum_start = 0 + for i in range(pid_batch): + cumsum_start += tl.load(extend_lens + i) + + token_start = pid_token * BLOCK_SIZE + + offset = tl.arange(0, BLOCK_SIZE) + actual_offset = token_start + offset + mask = actual_offset < extend_len + + src_ptr = out_cache_loc + cumsum_start + actual_offset + src_ptr = tl.max_contiguous(tl.multiple_of(src_ptr, BLOCK_SIZE), BLOCK_SIZE) + value = tl.load(src_ptr, mask=mask) + dst_ptr = ( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + actual_offset + + pre_len + ) + dst_ptr = tl.max_contiguous(tl.multiple_of(dst_ptr, BLOCK_SIZE), BLOCK_SIZE) + + tl.store(dst_ptr, value, mask=mask) + + +def write_req_to_token_pool_reference( + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + pre_lens: torch.Tensor, + seq_lens: torch.Tensor, + extend_lens: torch.Tensor, + out_cache_loc: torch.Tensor, +) -> None: + """Reference implementation using PyTorch""" + for i in range(len(req_pool_indices)): + req_pool_idx = req_pool_indices[i].item() + pre_len = pre_lens[i].item() + seq_len = seq_lens[i].item() + extend_len = extend_lens[i].item() + + cumsum_start = sum(extend_lens[:i].tolist()) + + # Copy values from out_cache_loc to req_to_token + req_to_token[req_pool_idx, pre_len:seq_len] = out_cache_loc[ + cumsum_start : cumsum_start + extend_len + ] + + +def test_write_req_to_token_pool(): + max_batch = 4097 + max_context_len = 6148 + batch_size = 1 + extend_len = 14 + + # Initialize input tensors + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.tensor([42], dtype=torch.int32, device="cuda") + pre_lens = torch.tensor([8], dtype=torch.int32, device="cuda") + seq_lens = torch.tensor([22], dtype=torch.int32, device="cuda") + extend_lens = torch.tensor([extend_len], dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(extend_len, dtype=torch.int32, device="cuda") + + # Create copies for reference implementation + req_to_token_ref = req_to_token.clone() + req_to_token_opt = req_to_token.clone() + + # Run original triton kernel + write_req_to_token_pool_triton[(batch_size,)]( + req_to_token, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ) + + # Run optimized triton kernel + def grid(batch_size, extend_len): + num_token_blocks = triton.cdiv(extend_len, 512) + return (batch_size, num_token_blocks) + + write_req_to_token_pool_triton_optimize[grid(batch_size, extend_len)]( + req_to_token_opt, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=512, + ) + + # Run reference implementation + write_req_to_token_pool_reference( + req_to_token_ref, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ) + + # Compare results + torch.testing.assert_close(req_to_token, req_to_token_ref) + torch.testing.assert_close(req_to_token_opt, req_to_token_ref) + + # Test case 2: batch size > 1 + batch_size = 3 + extend_lens_list = [14, 20, 30] + total_extend_len = sum(extend_lens_list) + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.tensor([42, 100, 200], dtype=torch.int32, device="cuda") + pre_lens = torch.tensor([8, 10, 15], dtype=torch.int32, device="cuda") + seq_lens = torch.tensor([22, 30, 45], dtype=torch.int32, device="cuda") + extend_lens = torch.tensor(extend_lens_list, dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(total_extend_len, dtype=torch.int32, device="cuda") + + req_to_token_ref = req_to_token.clone() + req_to_token_opt = req_to_token.clone() + + # Run original triton kernel + write_req_to_token_pool_triton[(batch_size,)]( + req_to_token, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ) + + # Run optimized triton kernel + max_extend_len = max(extend_lens_list) + write_req_to_token_pool_triton_optimize[grid(batch_size, max_extend_len)]( + req_to_token_opt, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=512, + ) + + # Run reference implementation + write_req_to_token_pool_reference( + req_to_token_ref, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ) + + # Compare results + torch.testing.assert_close(req_to_token, req_to_token_ref) + torch.testing.assert_close(req_to_token_opt, req_to_token_ref) + + +def get_benchmark(): + batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128] + extend_lens = [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + configs = list(itertools.product(batch_sizes, extend_lens)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "extend_len"], + x_vals=configs, + line_arg="provider", + line_vals=["reference", "triton", "triton_optimize"], + line_names=["PyTorch", "Triton", "Triton Optimized"], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name="write-req-to-token-pool-performance", + args={}, + ) + ) + def benchmark(batch_size, extend_len, provider): + max_batch = 256 + max_context_len = 16384 + + extend_lens_list = [extend_len] * batch_size + total_extend_len = sum(extend_lens_list) + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int32, device="cuda") + pre_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda") * 8 + seq_lens = pre_lens + extend_len + extend_lens = torch.tensor(extend_lens_list, dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(total_extend_len, dtype=torch.int32, device="cuda") + + quantiles = [0.5, 0.2, 0.8] + + if provider == "reference": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: write_req_to_token_pool_reference( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: write_req_to_token_pool_triton[(batch_size,)]( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ), + quantiles=quantiles, + ) + else: + + def run_optimized(): + block_size = 128 if extend_len <= 1024 else 512 + grid_config = (batch_size, triton.cdiv(extend_len, block_size)) + write_req_to_token_pool_triton_optimize[grid_config]( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=block_size, + ) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_optimized, quantiles=quantiles + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def run_benchmark(save_path: str = "./configs/benchmark_ops/write_req_to_token_pool/"): + """Run benchmark and save results""" + + # Ensure save path exists + os.makedirs(save_path, exist_ok=True) + + # Run correctness test + test_write_req_to_token_pool() + print("Correctness test passed!") + + # Run performance test + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=save_path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/write_req_to_token_pool/", + help="Path to save benchmark results", + ) + args = parser.parse_args() + + run_benchmark(args.save_path) diff --git a/benchmark/line_retrieval/README.md b/benchmark/line_retrieval/README.md new file mode 100644 index 0000000..3c81a63 --- /dev/null +++ b/benchmark/line_retrieval/README.md @@ -0,0 +1,37 @@ +## Download data + +``` +wget https://raw.githubusercontent.com/merrymercy/merrymercy.github.io/master/files/random_words.json +python3 gen_data.py --number 1000 +``` + +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-hf --port 30000 +``` + +``` +python3 bench_sglang.py --src-index 600 --num-q 50 --parallel 1 +``` + + +### + +``` +# original +Accuracy: 0.940, latency: 332.83 s + +# parallel encoding (no_adjust, offset = 1000) +Accuracy: 0.760, latency: 238.46 s + +# parallel encoding (no_adjust, offset = 3000) +Accuracy: 0.760, latency: 238.46 s + +# parallel encoding (no_adjust, offset = 0) +Accuracy: 0.520, latency: 238.46 s + +# parallel encoding (adjust_cache) +Accuracy: 0.460, latency: 257.66 s +``` diff --git a/benchmark/line_retrieval/bench_sglang.py b/benchmark/line_retrieval/bench_sglang.py new file mode 100644 index 0000000..922d500 --- /dev/null +++ b/benchmark/line_retrieval/bench_sglang.py @@ -0,0 +1,149 @@ +import argparse +import json +import re +import time + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def line_retrieval(s, prefix, suffix, body_0, body_1, body_2, body_3): + s += prefix + "\n" + + contexts = [body_0, body_1, body_2, body_3] + position_ids_offset = [i * 1000 for i in range(len(contexts))] + forks = s.fork(len(contexts), position_ids_offset) + forks += lambda i: contexts[i] + "\n" + forks.join(mode="concate_and_append") + + s += "\n" + suffix + s += sgl.gen("answer", max_tokens=16) + + +def eval_model(args, line_obj, num_hoops, src_indices, dst_percents): + arguments = [] + labels = [] + sum_src_indices = [] + sum_dst_indices = [] + + for i in range(len(src_indices)): + for j in range(len(dst_percents)): + src_index = src_indices[i] + dst_percent = dst_percents[j] + + query_indices = line_obj["group_by_num_hoops"][str(num_hoops)] + query_indices = [ + q + for q in query_indices + if all(l <= src_index for l in line_obj["links"][q]) and q < src_index + ] + dst_index = query_indices[ + min(int(len(query_indices) * dst_percent), len(query_indices) - 1) + ] + label = line_obj["values"][dst_index] + + body = line_obj["lines"][: src_index + 1] + suffix = line_obj["suffix"].replace("???", line_obj["indices"][dst_index]) + body_part_len = len(body) // 4 + + arguments.append( + { + "prefix": line_obj["prefix"], + "body_0": "\n".join(body[:body_part_len]), + "body_1": "\n".join(body[body_part_len : 2 * body_part_len]), + "body_2": "\n".join(body[2 * body_part_len : 3 * body_part_len]), + "body_3": "\n".join(body[3 * body_part_len :]), + "suffix": suffix, + } + ) + labels.append(label) + sum_src_indices.append(src_index) + sum_dst_indices.append(dst_index) + + # Select backend + backend = select_sglang_backend(args) + + tic = time.time() + states = line_retrieval.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + corrects = [] + for i in range(len(arguments)): + output = states[i]["answer"] + prompt_len = states[i].get_meta_info("answer").get("prompt_length", -1) + label = labels[i] + + # Try all numbers + findall = re.findall("\d+", output) + if not findall: + response_number = output + else: + for response_number in findall: + if response_number == label: + break + + correct = response_number == label + corrects.append(correct) + + # Log results + summary = ( + f"Line index: {sum_src_indices[i]} -> {sum_dst_indices[i]}, " + f"Prompt len: {prompt_len}, " + f"Correct: {correct}, " + f"Label: {label}, Predicted: {response_number}, " + ) + print(summary) + + accuracy = np.mean(corrects) + print(f"Accuracy: {accuracy:.3f}, latency: {latency:.2f} s") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "line_retrieval", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "num_questions": len(arguments), + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +def main(args): + line_obj = json.load(open(args.data_path, "r")) + + num_hoops = args.num_hoops + for src_index in args.src_index: + src_indices = [src_index] + num_queries = args.num_queries_per_src + dst_percents = [i * (1 / (num_queries)) for i in range(num_queries)] + eval_model(args, line_obj, num_hoops, src_indices, dst_percents) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="lines_1000_0.0.json") + parser.add_argument("--src-index", type=int, nargs="+", default=[100]) + parser.add_argument("--num-queries-per-src", type=int, default=10) + parser.add_argument("--num-hoops", type=int, default=1) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/line_retrieval/gen_data.py b/benchmark/line_retrieval/gen_data.py new file mode 100644 index 0000000..c88ecba --- /dev/null +++ b/benchmark/line_retrieval/gen_data.py @@ -0,0 +1,139 @@ +""" +Generate line data for line retrieval task. + +Usage: +python3 gen_data.py --number 1000 +""" + +import argparse +import json +from collections import defaultdict + +import numpy as np +from tqdm import tqdm + + +def generate_lines(random_words, num_lines, redirect_ratio): + prefix = "Here is a list of lines, each with its corresponding REGISTER_CONTENT value. Please memorize them. Be prepared to provide the REGISTER_CONTENT value for a specific line index when I ask." + suffix = "The list has ended. Please give the final REGISTER_CONTENT value for a specific line after resovling the redirections and references. For example, the REGISTER_CONTENT of Line __idx0__ is __val0__. The REGISTER_CONTENT of Line __idx1__ is __val1__. The REGISTER_CONTENT of Line __idx2__ is __val2__. The REGISTER_CONTENT of Line ??? is" + + # Raw lines + visited_indices = set([None]) + visited_values = set([None]) + + lines = [] + redirects = [] + indices = [] + values = [] + for i in tqdm(range(num_lines)): + line_index = None + while line_index in visited_indices: + line_index = "-".join(np.random.choice(random_words, size=(2,))) + visited_indices.add(line_index) + + line_value = np.random.randint(low=0, high=999999) + line_value = f"{line_value:06}" + + line = f"Line {line_index}: The REGISTER_CONTENT is {line_value}." + lines.append(line) + redirects.append(None) + indices.append(line_index) + values.append(line_value) + + # Add redirect + if redirect_ratio > 0: + num_redirect_lines = int(len(lines) * redirect_ratio) + redirect_indices = np.random.choice( + np.arange(len(lines)), size=(num_redirect_lines,), replace=False + ) + for i in redirect_indices: + target_idx = np.random.choice(min(i * 2 + 100, num_lines)) + lines[i] = ( + f"Line {indices[i]}: The REGISTER_CONTENT is the same as Line {indices[target_idx]}." + ) + redirects[i] = target_idx + + # Build links and find sources + links = [[] for _ in range(num_lines)] + contains_ring = set() + for i in range(num_lines): + if redirects[i] is None: + continue + + tmp_link = [] + cur = i + visited = set() + while redirects[cur] is not None: + visited.add(cur) + tmp_link.append(redirects[cur]) + cur = redirects[cur] + + if cur in visited: + contains_ring.add(i) + tmp_link = None + break + values[i] = values[cur] + links[i] = tmp_link + + # Group by num_links + group_by_num_hoops = defaultdict(list) + for i in range(num_lines): + if i in contains_ring: + continue + group_by_num_hoops[len(links[i]) + 1].append(i) + + keys = sorted(list(group_by_num_hoops.keys())) + for num_links in keys: + print(f"#links: {num_links}, #lines: {len(group_by_num_hoops[num_links])}") + + # Append few-shot examples + hoop1_candidates = list(group_by_num_hoops[1]) + hoop1_candidate_keys = {c: max([c] + links[c]) for c in hoop1_candidates} + hoop1_candidates.sort(key=lambda c: hoop1_candidate_keys[c]) + hoop2_candidates = list(group_by_num_hoops[2]) + hoop2_candidate_keys = {c: max([c] + links[c]) for c in hoop2_candidates} + hoop2_candidates.sort(key=lambda c: hoop2_candidate_keys[c]) + + i = hoop1_candidates[5] + suffix = suffix.replace("__idx0__", indices[i]).replace("__val0__", values[i]) + if len(hoop2_candidates): + i = hoop2_candidates[0] + suffix = suffix.replace("__idx1__", indices[i]).replace("__val1__", values[i]) + i = hoop2_candidates[1] + suffix = suffix.replace("__idx2__", indices[i]).replace("__val2__", values[i]) + else: + i = hoop1_candidates[1] + suffix = suffix.replace("__idx1__", indices[i]).replace("__val1__", values[i]) + i = hoop1_candidates[10] + suffix = suffix.replace("__idx2__", indices[i]).replace("__val2__", values[i]) + + obj = { + "prefix": prefix, + "suffix": suffix, + "lines": lines, + "indices": indices, + "values": values, + "links": links, + "group_by_num_hoops": group_by_num_hoops, + "contains_ring": sorted(list(contains_ring)), + } + return obj + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--number", type=int) + parser.add_argument("--redirect-ratio", type=float, default=0.0) + args = parser.parse_args() + + num_lines = args.number + + random_words_filename = "random_words.json" + random_words = json.load(open(random_words_filename, "r")) + + np.random.seed(42) + obj = generate_lines(random_words, num_lines, args.redirect_ratio) + + fout = f"lines_{num_lines}_{args.redirect_ratio:.1f}.json" + with open(fout, "w") as fout: + json.dump(obj, fout, indent=2) diff --git a/benchmark/llava_bench/README.md b/benchmark/llava_bench/README.md new file mode 100644 index 0000000..a71f339 --- /dev/null +++ b/benchmark/llava_bench/README.md @@ -0,0 +1,61 @@ +## Download benchmark images + +``` +python3 download_images.py +``` + +image benchmark source: https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild + +### Other Dependency +``` +pip3 install "sglang[all]" +pip3 install "torch>=2.1.2" "transformers>=4.36" pillow +``` + +## Run benchmark + +### Benchmark sglang +Launch a server +``` +python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000 +``` + +Run benchmark +``` +# Run with local models +python3 bench_sglang.py --num-questions 60 + +# Run with OpenAI models +python3 bench_sglang.py --num-questions 60 --backend gpt-4-vision-preview +``` + +### Bench LLaVA original code +``` +git clone git@github.com:haotian-liu/LLaVA.git +cd LLaVA +git reset --hard 9a26bd1435b4ac42c282757f2c16d34226575e96 +pip3 install -e . + +cd ~/sglang/benchmark/llava_bench +CUDA_VISIBLE_DEVICES=0 bash bench_hf_llava_bench.sh +``` + + +### Benchmark llama.cpp + +``` +# Install +CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +pip install sse_starlette starlette_context pydantic_settings + +# Download weights +mkdir -p ~/model_weights/llava-v1.5-7b/ +wget https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-f16.gguf -O ~/model_weights/llava-v1.5-7b/ggml-model-f16.gguf +wget https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf -O ~/model_weights/llava-v1.5-7b/mmproj-model-f16.gguf +``` + +``` +python3 -m llama_cpp.server --model ~/model_weights/llava-v1.5-7b/ggml-model-f16.gguf --clip_model_path ~/model_weights/llava-v1.5-7b/mmproj-model-f16.gguf --chat_format llava-1-5 --port 23000 + +OPENAI_BASE_URL=http://localhost:23000/v1 python3 bench_sglang.py --backend gpt-4-vision-preview --num-q 1 +``` diff --git a/benchmark/llava_bench/bench_hf_llava_bench.sh b/benchmark/llava_bench/bench_hf_llava_bench.sh new file mode 100644 index 0000000..a51a715 --- /dev/null +++ b/benchmark/llava_bench/bench_hf_llava_bench.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m llava.eval.model_vqa \ + --model-path liuhaotian/llava-v1.5-7b \ + --question-file ./questions.jsonl \ + --image-folder ./images \ + --answers-file ./answers_hf.jsonl \ + --temperature 0 \ + --conv-mode vicuna_v1 diff --git a/benchmark/llava_bench/bench_hf_mme.sh b/benchmark/llava_bench/bench_hf_mme.sh new file mode 100644 index 0000000..6ed332f --- /dev/null +++ b/benchmark/llava_bench/bench_hf_mme.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m llava.eval.model_vqa_loader \ + --model-path liuhaotian/llava-v1.5-7b \ + --question-file ./mme_pack/llava_mme_bench_replace.jsonl \ + --image-folder ./mme_pack/MME_Benchmark_release_version \ + --answers-file ./answers_hf_mme.jsonl \ + --temperature 0 \ + --conv-mode vicuna_v1 diff --git a/benchmark/llava_bench/bench_sglang.py b/benchmark/llava_bench/bench_sglang.py new file mode 100644 index 0000000..f84c8a9 --- /dev/null +++ b/benchmark/llava_bench/bench_sglang.py @@ -0,0 +1,96 @@ +import argparse +import json +import os +import time + +import tqdm + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def image_qa(s, image_file, question): + s += sgl.user(sgl.image(image_file) + question) + s += sgl.assistant(sgl.gen("answer", max_tokens=args.max_tokens)) + + +def main(args): + lines = list(read_jsonl(args.question_file))[: args.num_questions] + arguments = [ + { + "image_file": os.path.abspath(args.image_folder + "/" + l["image"]), + "question": l["text"], + } + for l in lines + ] + # arguments = [ + # {"image_file": + # Image.open(os.path.abspath(args.image_folder + "/" + l["image"])), + # "question": l["text"]} for l in lines + # ] + + states = [None] * len(lines) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + if args.parallel == 1: + for i in tqdm.tqdm(range(len(lines))): + image_file = arguments[i]["image_file"] + question = arguments[i]["question"] + ret = image_qa.run(image_file=image_file, question=question, temperature=0) + states[i] = ret + else: + states = image_qa.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.time() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + print(f"Write output to {args.answer_file}") + with open(args.answer_file, "w") as fout: + for i in range(len(lines)): + value = { + "question_id": lines[i]["question_id"], + "prompt": lines[i]["text"], + "text": states[i]["answer"].strip(), + "model_id": backend.model_info["model_path"], + "answer_id": i, + "metadata": {}, + } + fout.write(json.dumps(value) + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "llava_bench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(lines), + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="questions.jsonl") + parser.add_argument("--answer-file", type=str, default="answers.jsonl") + parser.add_argument("--image-folder", type=str, default="./images") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--num-questions", type=int, default=None) + parser.add_argument("--max-tokens", type=int, default=768) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/llava_bench/bench_sglang_mme.sh b/benchmark/llava_bench/bench_sglang_mme.sh new file mode 100644 index 0000000..8c8c09c --- /dev/null +++ b/benchmark/llava_bench/bench_sglang_mme.sh @@ -0,0 +1,2 @@ +MME_FOLDER=./mme_pack +python3 bench_sglang.py --num-questions 5000 --question-file $MME_FOLDER/llava_mme_bench_replace.jsonl --answer-file answer_mme.jsonl --image-folder $MME_FOLDER/MME_Benchmark_release_version --max-tokens 4 diff --git a/benchmark/llava_bench/download_images.py b/benchmark/llava_bench/download_images.py new file mode 100644 index 0000000..701190a --- /dev/null +++ b/benchmark/llava_bench/download_images.py @@ -0,0 +1,20 @@ +import os + +# Create the 'images' directory if it doesn't exist +if not os.path.exists("images"): + os.makedirs("images") + +# Base URL +base_url = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/" + +# Loop through image numbers +for i in range(1, 25): + # Format the image number with leading zeros + image_number = str(i).zfill(3) + image_url = base_url + image_number + ".jpg" + image_path = "images/" + image_number + ".jpg" + + # Download the image using wget + os.system(f"wget -O {image_path} {image_url}") + +print("Download complete.") diff --git a/benchmark/llm_judge/README.md b/benchmark/llm_judge/README.md new file mode 100644 index 0000000..164b19a --- /dev/null +++ b/benchmark/llm_judge/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 25 --parallel 8 +python3 bench_sglang.py --num-questions 16 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 25 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 25 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --backend lmql --num-questions 25 --parallel 1 +``` diff --git a/benchmark/llm_judge/bench_other.py b/benchmark/llm_judge/bench_other.py new file mode 100644 index 0000000..2231bcd --- /dev/null +++ b/benchmark/llm_judge/bench_other.py @@ -0,0 +1,151 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +system_prompt = "Please serve as an impartial judge and rigorously evaluate the quality of the following article. Apply the most stringent standards possible, showing no leniency." + +dimension_prompts = [ + "Content: This refers to the essences of the essay. The substance should be well researched, accurate, relevant to the topic and should show a thorough understanding of the subject. The essay should also reflect a clear goal or purpose.", + "Organization and Structure: An essay needs to be properly structured with a clear introduction, body, and conclusion. The essay should flow naturally, with one paragraph leading seamlessly into the next.", + "Argument and Analysis: The argument made in the essay should be logical, coherent and clearly articulated. Each point made should be backed up by solid evidence and thorough analysis.", + "Clarity and Precision: The essay should be written in a clear and concise manner. The points made should be easily understood by the reader. The language used should also be precise and unambiguous.", + "Grammar and Punctuation: Proper use of grammar and punctuation is vital in an academic essay. Errors in grammar and punctuation not only distract the reader but can also negatively impact the meaning and interpretation of the content.", + "Referencing and Citation: An essay should contain proper citations and references for all sources used. This not only prevents accusations of plagiarism but also gives credit to the authors of the works that have contributed to the essay. The citation should adhere to a specific format as required by the academic institution or specified by the professor.", +] + + +def multi_dimension_judge(article, generate): + s = system_prompt + s += "\n```\n" + article + "\n```\n\n" + + judges = [] + for i in range(len(dimension_prompts)): + comp = generate( + s + + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:', + max_tokens=256, + stop="END", + ) + judges.append(comp) + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += dimension_prompts[i].split(":")[0] + ": " + judges[i].strip() + "\n" + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += generate(s, max_tokens=2, stop=None) + + return s + + +async def multi_dimension_judge_async(article, generate): + s = system_prompt + s += "\n```\n" + article + "\n```\n\n" + + judges = [] + for i in range(len(dimension_prompts)): + comp = await generate( + s + + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:', + max_tokens=256, + stop="END", + ) + judges.append(comp) + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += dimension_prompts[i].split(":")[0] + ": " + judges[i].strip() + "\n" + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += await generate(s, max_tokens=2, stop=None) + + return s + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + states = [None] * len(lines) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + tic = time.time() + + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = multi_dimension_judge(lines[i], call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(lines))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(lines)))), + total=len(lines), + ) + ) + + else: + import asyncio + + async def get_one_answer_async(i): + states[i] = await multi_dimension_judge_async(lines[i], call_generate) + + batches = [] + for i in range(0, len(lines), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(lines))))) + + loop = asyncio.get_event_loop() + for bt in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in bt]) + ) + + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "llm_judge", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="articles.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/llm_judge/bench_sglang.py b/benchmark/llm_judge/bench_sglang.py new file mode 100644 index 0000000..38c9597 --- /dev/null +++ b/benchmark/llm_judge/bench_sglang.py @@ -0,0 +1,97 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +system_prompt = "Please serve as an impartial judge and rigorously evaluate the quality of the following article. Apply the most stringent standards possible, showing no leniency." + +dimension_prompts = [ + "Content: This refers to the essences of the essay. The substance should be well researched, accurate, relevant to the topic and should show a thorough understanding of the subject. The essay should also reflect a clear goal or purpose.", + "Organization and Structure: An essay needs to be properly structured with a clear introduction, body, and conclusion. The essay should flow naturally, with one paragraph leading seamlessly into the next.", + "Argument and Analysis: The argument made in the essay should be logical, coherent and clearly articulated. Each point made should be backed up by solid evidence and thorough analysis.", + "Clarity and Precision: The essay should be written in a clear and concise manner. The points made should be easily understood by the reader. The language used should also be precise and unambiguous.", + "Grammar and Punctuation: Proper use of grammar and punctuation is vital in an academic essay. Errors in grammar and punctuation not only distract the reader but can also negatively impact the meaning and interpretation of the content.", + "Referencing and Citation: An essay should contain proper citations and references for all sources used. This not only prevents accusations of plagiarism but also gives credit to the authors of the works that have contributed to the essay. The citation should adhere to a specific format as required by the academic institution or specified by the professor.", +] + + +@sgl.function +def multi_dimension_judge(s, article): + s += system_prompt + s += "\n```\n" + article + "\n```\n\n" + + forks = s.fork(len(dimension_prompts)) + for i in range(len(dimension_prompts)): + forks[i] += ( + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:' + ) + forks[i] += sgl.gen("judgement", max_tokens=256, stop="END") + forks.join() + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += ( + dimension_prompts[i].split(":")[0] + + ": " + + forks[i]["judgement"].strip() + + "\n" + ) + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += sgl.gen("score", max_tokens=2) + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"article": l} for l in lines] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.time() + states = multi_dimension_judge.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "llm_judge", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="articles.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/long_json_decode/README.md b/benchmark/long_json_decode/README.md new file mode 100644 index 0000000..37fceee --- /dev/null +++ b/benchmark/long_json_decode/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-instruct-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 5 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model codellama/CodeLlama-7b-instruct-hf --disable-log-requests --port 21000 --gpu 0.97 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 5 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 5 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf +``` + + +### Build dataset +``` +pip install wikipedia +python3 build_dataset.py +``` diff --git a/benchmark/long_json_decode/bench_other.py b/benchmark/long_json_decode/bench_other.py new file mode 100644 index 0000000..a83c797 --- /dev/null +++ b/benchmark/long_json_decode/bench_other.py @@ -0,0 +1,89 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def json_decode(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "country": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "air port code": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "top 3 landmarks": "' + s += generate(s, max_tokens=24, stop='"') + '",\n' + s += "}\n" + return s + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = json_decode(generate=call_generate, **arguments[i]) + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "long_json_decode", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/long_json_decode/bench_sglang.py b/benchmark/long_json_decode/bench_sglang.py new file mode 100644 index 0000000..6e19a73 --- /dev/null +++ b/benchmark/long_json_decode/bench_sglang.py @@ -0,0 +1,81 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def json_decode(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": "' + sgl.gen("name", max_tokens=8, stop='"') + '",\n' + s += ' "country": "' + sgl.gen("country", max_tokens=8, stop='"') + '",\n' + s += ( + ' "air port code": "' + + sgl.gen("air port code", max_tokens=8, stop='"') + + '",\n' + ) + s += ( + ' "top 3 landmarks": "' + + sgl.gen("landmarks", max_tokens=24, stop='"') + + '",\n' + ) + s += "}\n" + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + states = json_decode.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "long_json_decode", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/long_json_decode/build_dataset.py b/benchmark/long_json_decode/build_dataset.py new file mode 100644 index 0000000..7b15d39 --- /dev/null +++ b/benchmark/long_json_decode/build_dataset.py @@ -0,0 +1,27 @@ +import json + +import transformers +import wikipedia + +name = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(name) +city_names = ["los angles", "london", "tokyo", "beijing", "singapore"] + + +for city_name in city_names: + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + truncate_len = int((10000 / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + with open("questions.jsonl", "a") as fout: + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/benchmark/lora/launch_server.py b/benchmark/lora/launch_server.py new file mode 100644 index 0000000..33ad848 --- /dev/null +++ b/benchmark/lora/launch_server.py @@ -0,0 +1,68 @@ +import argparse +import os + +NUM_LORAS = 4 +LORA_PATH = { + "base": "meta-llama/Llama-2-7b-hf", + "lora": "winddude/wizardLM-LlaMA-LoRA-7B", +} + + +def launch_server(args): + base_path = LORA_PATH["base"] + lora_path = LORA_PATH["lora"] + + if args.base_only: + cmd = f"python3 -m sglang.launch_server --model {base_path} " + else: + cmd = f"python3 -m sglang.launch_server --model {base_path} --lora-paths " + for i in range(NUM_LORAS): + lora_name = f"lora{i}" + cmd += f"{lora_name}={lora_path} " + cmd += f"--disable-radix --disable-cuda-graph " + cmd += f"--max-loras-per-batch {args.max_loras_per_batch} " + cmd += f"--max-running-requests {args.max_running_requests} " + cmd += f"--lora-backend {args.lora_backend} " + cmd += f"--tp-size {args.tp_size} " + if args.disable_custom_all_reduce: + cmd += "--disable-custom-all-reduce" + print(cmd) + os.system(cmd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-only", + action="store_true", + ) + parser.add_argument( + "--max-loras-per-batch", + type=int, + default=8, + ) + parser.add_argument( + "--max-running-requests", + type=int, + default=8, + ) + parser.add_argument( + "--lora-backend", + type=str, + default="triton", + ) + parser.add_argument( + "--tp-size", + type=int, + default=1, + help="Tensor parallel size for distributed inference", + ) + # disable_custom_all_reduce + parser.add_argument( + "--disable-custom-all-reduce", + action="store_true", + help="Disable custom all reduce when device does not support p2p communication", + ) + args = parser.parse_args() + + launch_server(args) diff --git a/benchmark/lora/lora_bench.py b/benchmark/lora/lora_bench.py new file mode 100644 index 0000000..1d93942 --- /dev/null +++ b/benchmark/lora/lora_bench.py @@ -0,0 +1,475 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import asyncio +import json +import random +import resource +import sys +import time +import traceback +from argparse import ArgumentParser +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +from launch_server import LORA_PATH, NUM_LORAS +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +from sglang.bench_serving import ( + AIOHTTP_TIMEOUT, + RequestFuncInput, + RequestFuncOutput, + calculate_metrics, + get_request, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +global args + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + # assert api_url.endswith( + # "completions" + # ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + # payload = { + # "model": request_func_input.model, + # "prompt": prompt, + # "temperature": 0.0, + # "best_of": 1, + # "max_tokens": request_func_input.output_len, + # "stream": not args.disable_stream, + # "ignore_eos": not args.disable_ignore_eos, + # **request_func_input.extra_request_body, + # } + # headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"} + if args.base_only: + payload = { + "text": prompt, + "sampling_params": {"max_new_tokens": request_func_input.output_len}, + } + else: + payload = { + "text": prompt, + "sampling_params": {"max_new_tokens": request_func_input.output_len}, + "lora_path": f"lora{random.randint(0, NUM_LORAS - 1)}", + } + headers = {"Authorization": ""} + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["text"]: + # if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + # generated_text += data["choices"][0]["text"] + generated_text += data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_openai_completions, +} + + +async def benchmark( + backend: str, + api_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[Tuple[str, int, int]], + request_rate: float, + disable_tqdm: bool, + extra_request_body: Dict[str, Any], +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + print("Starting initial single prompt test run...") + test_prompt, test_prompt_len, test_output_len = input_requests[0] + test_input = RequestFuncInput( + model=model_id, + prompt=test_prompt, + api_url=api_url, + prompt_len=test_prompt_len, + output_len=test_output_len, + lora_name="dummy", # the lora_name argument will not be used + extra_request_body=extra_request_body, + ) + test_output = await request_func(request_func_input=test_input) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + pbar = None if disable_tqdm else tqdm(total=len(input_requests)) + + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_request(input_requests, request_rate): + prompt, prompt_len, output_len = request + request_func_input = RequestFuncInput( + model=model_id, + prompt=prompt, + api_url=api_url, + prompt_len=prompt_len, + output_len=output_len, + lora_name="dummy", + extra_request_body=extra_request_body, + ) + tasks.append( + asyncio.create_task( + request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + if pbar is not None: + pbar.close() + + benchmark_duration = time.perf_counter() - benchmark_start_time + + metrics, output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format("Total throughput (tok/s):", metrics.total_throughput) + ) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + "backend": args.backend, + "request_rate": request_rate, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "median_itl_ms": metrics.median_itl_ms, + "output_throughput": metrics.output_throughput, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + "duration": benchmark_duration, + "completed": metrics.completed, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + } + return result + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + }.get(args.backend, 30000) + + # api_url = ( + # f"{args.base_url}/v1/completions" + # if args.base_url + # else f"http://{args.host}:{args.port}/v1/completions" + # ) + api_url = ( + f"{args.base_url}/generate" + if args.base_url + else f"http://{args.host}:{args.port}/generate" + ) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model = LORA_PATH["base"] + tokenizer_id = args.model + + tokenizer = get_tokenizer(tokenizer_id) + + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path="", + ) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + disable_tqdm=False, + extra_request_body={}, + ) + ) + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online lora serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=50, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + type=int, + default=128, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--base-only", + action="store_true", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + args = parser.parse_args() + run_benchmark(args) diff --git a/benchmark/mmlu/README.md b/benchmark/mmlu/README.md new file mode 100644 index 0000000..16de20c --- /dev/null +++ b/benchmark/mmlu/README.md @@ -0,0 +1,59 @@ +## Download data +``` +bash download_data.sh +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --nsub 10 +``` + +``` +# OpenAI models +python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8 +``` + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --nsub 10 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 + +# V100 +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 4500 --port 22000 +``` + +``` +python3 bench_other.py --nsub 10 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --nsub 10 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +CUDA_VISIBLE_DEVICES=0,1 lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --nsub 10 --backend lmql --parallel 2 +``` diff --git a/benchmark/mmlu/bench_other.py b/benchmark/mmlu/bench_other.py new file mode 100644 index 0000000..c5d48da --- /dev/null +++ b/benchmark/mmlu/bench_other.py @@ -0,0 +1,173 @@ +import argparse +import asyncio +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +import tiktoken +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate + +choices = ["A", "B", "C", "D"] + +tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") + + +def format_subject(subject): + l = subject.split("_") + s = "" + for entry in l: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( + format_subject(subject) + ) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def evaluate(args, subject, dev_df, test_df, call_generate): + prompts = [] + labels = [] + + # Construct prompts + k = args.ntrain + train_prompt = gen_prompt(dev_df, subject, k) + while len(tokenizer.encode(train_prompt)) > 1536: + k -= 1 + train_prompt = gen_prompt(dev_df, subject, k) + + for i in range(test_df.shape[0]): + prompt_end = format_example(test_df, i, include_answer=False) + prompt = train_prompt + prompt_end + prompts.append(prompt) + + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + preds = [None] * len(prompts) + max_tokens = 1 + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + pred = call_generate(prompts[i], temperature=0, max_tokens=max_tokens) + preds[i] = pred.strip()[0] + + tic = time.time() + if args.parallel == 1: + for i in range(len(prompts)): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + executor.map(get_one_answer, list(range(len(prompts)))) + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(prompts), batch_size): + tasks = [] + for p in prompts[i : i + batch_size]: + tasks.append(call_generate(p, temperature=0, max_tokens=max_tokens)) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + preds[i + j] = rets[j].strip()[0] + + tic = time.time() + asyncio.run(batched_call(batch_size=args.parallel)) + latency = time.time() - tic + + # Compute accuracy + cors = [pred == label for pred, label in zip(preds, labels)] + acc = np.mean(cors) + cors = np.array(cors) + + print( + "Average accuracy {:.3f}, latency {:.2f}, #q: {} - {}".format( + acc, latency, len(prompts), subject + ) + ) + + return cors, acc, latency + + +def main(args): + subjects = sorted( + [ + f.split("_test.csv")[0] + for f in os.listdir(os.path.join(args.data_dir, "test")) + if "_test.csv" in f + ] + ) + + all_cors = [] + all_latencies = [] + num_requests = 0 + + # Select backend + call_generate = get_call_generate(args) + + for subject in tqdm(subjects[: args.nsub]): + dev_df = pd.read_csv( + os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None + )[: args.ntrain] + test_df = pd.read_csv( + os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None + ) + + cors, acc, latency = evaluate(args, subject, dev_df, test_df, call_generate) + all_cors.append(cors) + all_latencies.append(latency) + num_requests += len(test_df) + + total_latency = np.sum(all_latencies) + print("Total latency: {:.3f}".format(total_latency)) + + weighted_acc = np.mean(np.concatenate(all_cors)) + print("Average accuracy: {:.3f}".format(weighted_acc)) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "mmlu", + "backend": args.backend, + "num_gpus": 1, + "latency": round(total_latency, 3), + "accuracy": round(weighted_acc, 3), + "num_requests": num_requests, + "other": { + "nsub": args.nsub, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ntrain", type=int, default=5) + parser.add_argument("--data_dir", type=str, default="data") + parser.add_argument("--nsub", type=int, default=60) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/mmlu/bench_sglang.py b/benchmark/mmlu/bench_sglang.py new file mode 100644 index 0000000..210b611 --- /dev/null +++ b/benchmark/mmlu/bench_sglang.py @@ -0,0 +1,173 @@ +import argparse +import json +import os +import time + +import numpy as np +import pandas as pd +import tiktoken + +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +choices = ["A", "B", "C", "D"] + +tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") + + +def format_subject(subject): + l = subject.split("_") + s = "" + for entry in l: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( + format_subject(subject) + ) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def main(args): + subjects = sorted( + [ + f.split("_test.csv")[0] + for f in os.listdir(os.path.join(args.data_dir, "test")) + if "_test.csv" in f + ] + ) + + # Build prompts + arguments = [] + labels = [] + num_questions = [] + + for subject in subjects[: args.nsub]: + dev_df = pd.read_csv( + os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None + )[: args.ntrain] + test_df = pd.read_csv( + os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None + ) + num_questions.append(test_df.shape[0]) + + k = args.ntrain + few_shot_examples = gen_prompt(dev_df, subject, k) + while len(tokenizer.encode(few_shot_examples)) > 1536: + k -= 1 + few_shot_examples = gen_prompt(dev_df, subject, k) + + for i in range(test_df.shape[0]): + prompt_end = format_example(test_df, i, include_answer=False) + + arguments.append( + { + "examples": few_shot_examples, + "question": prompt_end, + } + ) + + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + if args.backend.startswith("gpt-"): + + @sgl.function + def few_shot_mmlu(s, examples, question): + s += sgl.user(examples + question) + s += sgl.assistant(sgl.gen("answer")) + + else: + + @sgl.function + def few_shot_mmlu(s, examples, question): + s += examples + question + sgl.gen("answer") + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Select backend + backend = select_sglang_backend(args) + + # Run + tic = time.time() + states = few_shot_mmlu.run_batch( + arguments, + temperature=0, + max_new_tokens=1, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + preds = [ + s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states + ] + latency = time.time() - tic + + # Compute accuracy + cors = [pred == label for pred, label in zip(preds, labels)] + + pt = 0 + for subject, num_qs in zip(subjects[: args.nsub], num_questions): + print( + f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt: pt + num_qs]):.3f}" + ) + pt += num_qs + assert pt == len(cors) + weighted_acc = np.mean(cors) + + # Print results + print("Total latency: {:.3f}".format(latency)) + print("Average accuracy: {:.3f}".format(weighted_acc)) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "mmlu", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(weighted_acc, 3), + "num_requests": len(arguments), + "other": { + "nsub": args.nsub, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ntrain", "-k", type=int, default=5) + parser.add_argument("--data_dir", "-d", type=str, default="data") + parser.add_argument("--save_dir", "-s", type=str, default="results") + parser.add_argument("--nsub", type=int, default=60) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/mmlu/download_data.sh b/benchmark/mmlu/download_data.sh new file mode 100644 index 0000000..fb22416 --- /dev/null +++ b/benchmark/mmlu/download_data.sh @@ -0,0 +1,2 @@ +wget https://people.eecs.berkeley.edu/~hendrycks/data.tar +tar xf data.tar diff --git a/benchmark/mmmu/README.md b/benchmark/mmmu/README.md new file mode 100644 index 0000000..00cdf16 --- /dev/null +++ b/benchmark/mmmu/README.md @@ -0,0 +1,30 @@ +## Run evaluation + +### Evaluate sglang + +Host the VLM: + +``` +python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template qwen2-vl --port 30000 +``` + +Benchmark: + +``` +python benchmark/mmmu/bench_sglang.py --port 30000 +``` + +It's recommended to reduce the memory usage by appending something ike `--mem-fraction-static 0.6` to the command above. + +### Evaluate hf + +``` +python benchmark/mmmu/bench_hf.py --model-path Qwen/Qwen2-VL-7B-Instruct +``` + +Some popular model results: + +1. Qwen/Qwen2-VL-2B-Instruct: 0.241 +2. Qwen/Qwen2-VL-7B-Instruct: 0.255 +3. Qwen/Qwen2.5-VL-3B-Instruct: 0.245 +4. Qwen/Qwen2.5-VL-7B-Instruct: 0.242 diff --git a/benchmark/mmmu/bench_hf.py b/benchmark/mmmu/bench_hf.py new file mode 100644 index 0000000..c6588c7 --- /dev/null +++ b/benchmark/mmmu/bench_hf.py @@ -0,0 +1,105 @@ +import argparse + +import torch +from data_utils import save_json +from eval_utils import ( + EvalArgs, + eval_result, + get_sampling_params, + prepare_samples, + process_result, +) +from tqdm import tqdm +from transformers import AutoModel, AutoProcessor, GenerationConfig + + +@torch.no_grad() +def eval_mmmu(args): + eval_args = EvalArgs.from_cli_args(args) + try: + from transformers import AutoModelForImageTextToText + + model = AutoModelForImageTextToText.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + ) + except Exception as first_exception: + try: + model = AutoModel.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + init_tts=False, + ) + except Exception as second_exception: + raise RuntimeError( + f"Failed to load model: First attempt failed with {first_exception}, " + f"second attempt failed with {second_exception}" + ) from second_exception + + model = model.eval().cuda() + + processor = AutoProcessor.from_pretrained( + args.model_path, torch_dtype="auto", device_map="auto", trust_remote_code=True + ) + + samples = prepare_samples(eval_args) + out_samples = dict() + + sampling_params = get_sampling_params(eval_args) + generation_config = GenerationConfig( + max_new_tokens=sampling_params["max_new_tokens"], + do_sample=False, + ) + + answer_dict = {} + for sample in tqdm(samples): + prompt = sample["final_input_prompt"] + image = sample["image"] + prefix = prompt.split("<")[0] + suffix = prompt.split(">")[1] + assert image is not None + contents = [] + if prefix: + contents += [{"type": "text", "text": prefix}] + contents += [ + { + "type": "image", + "image": sample["image_path"], + } + ] + if suffix: + contents += [{"type": "text", "text": suffix}] + messages = [{"role": "user", "content": contents}] + model_inputs = processor.apply_chat_template( + messages, + tokenize=True, + return_dict=True, + add_generation_prompt=True, + return_tensors="pt", + ).to(model.device) + input_len = model_inputs["input_ids"].shape[-1] + generation = model.generate(**model_inputs, generation_config=generation_config) + generation = generation[0][input_len:] + response = processor.decode(generation, skip_special_tokens=True) + print(f"response: {response}") + process_result(response, sample, answer_dict, out_samples) + + args.output_path = f"{args.model_path}_val_hf.json" + save_json(args.output_path, out_samples) + eval_result(model_answer_path=args.output_path, answer_dict=answer_dict) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-path", + type=str, + help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.", + required=True, + ) + EvalArgs.add_cli_args(parser) + args = parser.parse_args() + + eval_mmmu(args) diff --git a/benchmark/mmmu/bench_sglang.py b/benchmark/mmmu/bench_sglang.py new file mode 100644 index 0000000..19f60d6 --- /dev/null +++ b/benchmark/mmmu/bench_sglang.py @@ -0,0 +1,93 @@ +""" +Bench the sglang-hosted vLM with benchmark MMMU + +Usage: + Host the VLM: python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template qwen2-vl --port 30000 + + Benchmark: python benchmark/mmmu/bench_sglang.py --port 30000 + +The eval output will be logged +""" + +import argparse +import time + +import openai +from data_utils import save_json +from eval_utils import ( + EvalArgs, + eval_result, + get_sampling_params, + prepare_samples, + process_result, +) +from tqdm import tqdm + +from sglang.test.test_utils import add_common_sglang_args_and_parse + + +def eval_mmmu(args): + eval_args = EvalArgs.from_cli_args(args) + + out_samples = dict() + + sampling_params = get_sampling_params(eval_args) + + samples = prepare_samples(eval_args) + + answer_dict = {} + + # had to use an openai server, since SglImage doesn't support image data + client = openai.Client(api_key="sk", base_url=f"http://127.0.0.1:{args.port}/v1") + + start = time.time() + for i, sample in enumerate(tqdm(samples)): + prompt = sample["final_input_prompt"] + prefix = prompt.split("<")[0] + suffix = prompt.split(">")[1] + image = sample["image"] + assert image is not None + image_path = sample["image_path"] + # TODO: batch + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": prefix, + }, + { + "type": "image_url", + "image_url": {"url": image_path}, + }, + { + "type": "text", + "text": suffix, + }, + ], + } + ], + temperature=0, + max_completion_tokens=sampling_params["max_new_tokens"], + max_tokens=sampling_params["max_new_tokens"], + ) + response = response.choices[0].message.content + process_result(response, sample, answer_dict, out_samples) + + print(f"Benchmark time: {time.time() - start}") + + args.output_path = f"./val_sglang.json" + save_json(args.output_path, out_samples) + eval_result(model_answer_path=args.output_path, answer_dict=answer_dict) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args = add_common_sglang_args_and_parse(parser) + EvalArgs.add_cli_args(parser) + args = parser.parse_args() + + eval_mmmu(args) diff --git a/benchmark/mmmu/data_utils.py b/benchmark/mmmu/data_utils.py new file mode 100644 index 0000000..cf89169 --- /dev/null +++ b/benchmark/mmmu/data_utils.py @@ -0,0 +1,221 @@ +"""Utils for data load, save, and process (e.g., prompt construction)""" + +import json +import os +import re + +import yaml + +DOMAIN_CAT2SUB_CAT = { + "Art and Design": ["Art", "Art_Theory", "Design", "Music"], + "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], + "Science": [ + "Biology", + "Chemistry", + "Geography", + "Math", + "Physics", + ], + "Health and Medicine": [ + "Basic_Medical_Science", + "Clinical_Medicine", + "Diagnostics_and_Laboratory_Medicine", + "Pharmacy", + "Public_Health", + ], + "Humanities and Social Science": [ + "History", + "Literature", + "Sociology", + "Psychology", + ], + "Tech and Engineering": [ + "Agriculture", + "Architecture_and_Engineering", + "Computer_Science", + "Electronics", + "Energy_and_Power", + "Materials", + "Mechanical_Engineering", + ], +} + + +CAT_SHORT2LONG = { + "acc": "Accounting", + "agri": "Agriculture", + "arch": "Architecture_and_Engineering", + "art": "Art", + "art_theory": "Art_Theory", + "bas_med": "Basic_Medical_Science", + "bio": "Biology", + "chem": "Chemistry", + "cli_med": "Clinical_Medicine", + "cs": "Computer_Science", + "design": "Design", + "diag_med": "Diagnostics_and_Laboratory_Medicine", + "econ": "Economics", + "elec": "Electronics", + "ep": "Energy_and_Power", + "fin": "Finance", + "geo": "Geography", + "his": "History", + "liter": "Literature", + "manage": "Manage", + "mark": "Marketing", + "mate": "Materials", + "math": "Math", + "mech": "Mechanical_Engineering", + "music": "Music", + "phar": "Pharmacy", + "phys": "Physics", + "psy": "Psychology", + "pub_health": "Public_Health", + "socio": "Sociology", +} + + +# DATA SAVING +def save_json(filename, ds): + with open(filename, "w") as f: + json.dump(ds, f, indent=4) + + +def get_multi_choice_info(options): + """ + Given the list of options for multiple choice question + Return the index2ans and all_choices + """ + + start_chr = "A" + all_choices = [] + index2ans = {} + for i, option in enumerate(options): + index2ans[chr(ord(start_chr) + i)] = option + all_choices.append(chr(ord(start_chr) + i)) + + return index2ans, all_choices + + +def load_yaml(file_path): + with open(file_path, "r") as stream: + try: + yaml_dict = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + return yaml_dict + + +def parse_img_path(text): + matches = re.findall("", text) + return matches + + +def process_single_sample(data): + question = data["question"] + o_imgs_paths = [] + for option in data["options"]: + current_o_imgs_paths = parse_img_path(option) + for img_path in current_o_imgs_paths: + o_imgs_paths.append(img_path) + + if len(o_imgs_paths) > 1: # multiple images in options, used for random selection + return { + "id": data["id"], + "question": question, + "options": data["options"], + "answer": data["answer"], + "image": None, + "question_type": data["question_type"], + } + else: + return { + "id": data["id"], + "question": question, + "options": data["options"], + "answer": data["answer"], + "image": data["image_1"], + "question_type": data["question_type"], + } + + +# DATA SAVING +def save_json(filename, ds): + print(f"answers saved to: {filename}") + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as f: + json.dump(ds, f, indent=4) + + +def save_jsonl(filename, data): + """ + Save a dictionary of data to a JSON Lines file with the filename as key and caption as value. + + Args: + filename (str): The path to the file where the data should be saved. + data (dict): The dictionary containing the data to save where key is the image path and value is the caption. + """ + with open(filename, "w", encoding="utf-8") as f: + for img_path, caption in data.items(): + # Extract the base filename without the extension + base_filename = os.path.basename(img_path) + # Create a JSON object with the filename as the key and caption as the value + json_record = json.dumps({base_filename: caption}, ensure_ascii=False) + # Write the JSON object to the file, one per line + f.write(json_record + "\n") + + +def save_args(args, path_dir): + argsDict = args.__dict__ + with open(path_dir + "setting.txt", "w") as f: + f.writelines("------------------ start ------------------" + "\n") + for eachArg, value in argsDict.items(): + f.writelines(eachArg + " : " + str(value) + "\n") + f.writelines("------------------- end -------------------") + + +# DATA PROCESSING +def construct_prompt(sample, config): + question = sample["question"] + options = eval(sample["options"]) + example = "" + if sample["question_type"] == "multiple-choice": + start_chr = "A" + prediction_range = [] + index2ans = {} + for option in options: + prediction_range.append(start_chr) + example += f"({start_chr}) {option}\n" + index2ans[start_chr] = option + start_chr = chr(ord(start_chr) + 1) + empty_prompt_sample_structure = config["multi_choice_example_format"] + empty_prompt = empty_prompt_sample_structure.format(question, example) + res_dict = {} + res_dict["index2ans"] = index2ans + res_dict["correct_choice"] = sample["answer"] + res_dict["all_choices"] = prediction_range + res_dict["empty_prompt"] = empty_prompt + if config["task_instructions"]: + res_dict["final_input_prompt"] = ( + config["task_instructions"].strip() + "\n\n" + empty_prompt + ) + else: + res_dict["final_input_prompt"] = empty_prompt + + res_dict["gt_content"] = options[ord(sample["answer"].upper()) - ord("A")] + else: + empty_prompt_sample_structure = config["short_ans_example_format"] + empty_prompt = empty_prompt_sample_structure.format(question) + res_dict = {} + res_dict["empty_prompt"] = empty_prompt + if config["task_instructions"]: + res_dict["final_input_prompt"] = ( + config["task_instructions"].strip() + "\n\n" + empty_prompt + ) + else: + res_dict["final_input_prompt"] = empty_prompt + res_dict["gt_content"] = sample["answer"] + + res_dict.update(sample) + return res_dict diff --git a/benchmark/mmmu/eval_utils.py b/benchmark/mmmu/eval_utils.py new file mode 100644 index 0000000..046bed6 --- /dev/null +++ b/benchmark/mmmu/eval_utils.py @@ -0,0 +1,561 @@ +"""Response Parsing and Evaluation for various models""" + +import argparse +import dataclasses +import json +import os +import pprint +import random +import re +from typing import Dict, Optional + +import numpy as np +import torch +from data_utils import ( + CAT_SHORT2LONG, + DOMAIN_CAT2SUB_CAT, + construct_prompt, + load_yaml, + process_single_sample, +) +from datasets import concatenate_datasets, load_dataset +from tqdm import tqdm + + +@dataclasses.dataclass +class EvalArgs: + seed: int = 42 + split: str = "validation" + # Default setting to make the benchmark available on A100 for most 7B models + image_pixels_limit: int = 4300000 + result_filename: str = "" + prompt_format_file: str = "prompt_format.yaml" + dataset_path: str = "MMMU/MMMU" + extra_request_body: Optional[str] = None + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument( + "--result-filename", type=str, default=EvalArgs.result_filename + ) + + parser.add_argument( + "--image-pixels-limit", type=int, default=EvalArgs.image_pixels_limit + ) + + parser.add_argument( + "--dataset-path", + type=str, + default=EvalArgs.dataset_path, + help="path to the dataset", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--prompt-format-file", + type=str, + help="The path to the prompt format of mmmu. If not, a default format llava_config.yaml will be used", + ) + parser.add_argument("--split", type=str, default=EvalArgs.split) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + default=EvalArgs.extra_request_body, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def set_seed(seed_value): + """ + Set the seed for PyTorch (both CPU and CUDA), Python, and NumPy for reproducible results. + + :param seed_value: An integer value to be used as the seed. + """ + torch.manual_seed(seed_value) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed_value) + torch.cuda.manual_seed_all(seed_value) # For multi-GPU setups + random.seed(seed_value) + np.random.seed(seed_value) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def prepare_samples(eval_args: EvalArgs): + print("preparing samples...") + # Build prompts + set_seed(eval_args.seed) + + prompt_format_file = ( + eval_args.prompt_format_file + if eval_args.prompt_format_file is not None + else os.path.join(os.path.dirname(__file__), "prompt_format.yaml") + ) + # load config and process to one value + eval_args.config = load_yaml(prompt_format_file) + for key, value in eval_args.config.items(): + if key != "eval_params" and type(value) == list: + assert len(value) == 1, "key {} has more than one value".format(key) + eval_args.config[key] = value[0] + + # run for each subject + sub_dataset_list = [] + + for subject in tqdm(CAT_SHORT2LONG.values()): + sub_dataset = load_dataset( + eval_args.dataset_path, subject, split=eval_args.split + ) + sub_dataset_list.append(sub_dataset) + # break + + # merge all dataset + dataset = concatenate_datasets(sub_dataset_list) + + ## prepare images + samples = [] + skip_count = 0 + + # use image file as input to ensure the consistency between sglang and hf + images_path = os.path.expanduser("~/.cache/mmmu/images") + os.makedirs(images_path, exist_ok=True) + print(f"Saving images to: {images_path}") + + for i, sample in enumerate(tqdm(dataset)): + sample = process_single_sample(sample) + sample = construct_prompt(sample, eval_args.config) + image = sample["image"] + + width, height = image.size + if width * height >= eval_args.image_pixels_limit: + skip_count += 1 + continue + image_path = f"{images_path}/image_{i}.png" + if not os.path.exists(image_path): + image.save(image_path) + sample["image_path"] = image_path + samples.append(sample) + + print( + f"skipping {skip_count} samples with large images, {round((float(skip_count) / len(dataset)) * 100, 2)}% of dataset" + ) + print("samples have been prepared") + return samples + + +def get_sampling_params(eval_args): + max_new_tokens = 30 + temperature = 0.001 + + extra_request_body = {} + if eval_args.extra_request_body: + extra_request_body = json.loads(eval_args.extra_request_body) + + return { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + **extra_request_body, + } + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct + + +# ----------- Batch Evaluation ------------- +def evaluate(samples): + """ + Batch evaluation for multiple choice and open questions. + """ + pred_correct = 0 + judge_dict = dict() + for sample in samples: + gold_i = sample["answer"] + pred_i = sample["parsed_pred"] + if sample["question_type"] == "multiple-choice": + correct = eval_multi_choice(gold_i, pred_i) + else: # open question + correct = eval_open(gold_i, pred_i) + + if correct: + judge_dict[sample["id"]] = "Correct" + pred_correct += 1 + else: + # print(f"Wrong! expected {pred_i}, answered with {gold_i}") + judge_dict[sample["id"]] = "Wrong" + + if len(samples) == 0: + return {"acc": 0} + return judge_dict, {"acc": pred_correct / len(samples)} + + +# ----------- Calculate Accuracy ------------- +def calculate_ins_level_acc(results: Dict): + """Calculate the instruction level accuracy for given Subject results""" + acc = 0 + ins_num = 0 + for cat_results in results.values(): + acc += cat_results["acc"] * cat_results["num_example"] + ins_num += cat_results["num_example"] + if ins_num == 0: + return 0 + return acc / ins_num + + +def process_result(response, sample, answer_dict, out_samples): + if sample["question_type"] == "multiple-choice": + pred_ans = parse_multi_choice_response( + response, sample["all_choices"], sample["index2ans"] + ) + else: # open question + pred_ans = response + + out_samples[sample["id"]] = pred_ans + + # set ground truth answer + answer_dict[sample["id"]] = { + "question_type": sample["question_type"], + "ground_truth": sample["answer"], + } + + +def eval_result(model_answer_path, answer_dict): + print("Evaluating...") + output_dict = json.load(open(model_answer_path)) + # answer_dict = json.load(open(answer_path)) + + # group by category + output_dict_w_cat = {} + for data_id, parsed_pred in output_dict.items(): + category = "_".join(data_id.split("_")[1:-1]) + if category not in output_dict_w_cat: + output_dict_w_cat.update({category: {}}) + output_dict_w_cat[category].update({data_id: parsed_pred}) + + # group by category + answer_dict_w_cat = {} + for data_id, parsed_pred in answer_dict.items(): + category = "_".join(data_id.split("_")[1:-1]) + if category not in answer_dict_w_cat: + answer_dict_w_cat.update({category: {}}) + answer_dict_w_cat[category].update({data_id: parsed_pred}) + + evaluation_result = {} + + for category in CAT_SHORT2LONG.values(): + # print("Evaluating: {}".format(category)) + # get cat_outputs and cat_answers + try: + cat_outputs = output_dict_w_cat[category] + cat_answers = answer_dict_w_cat[category] + except KeyError: + # print("Skipping {} for not found".format(category)) + continue + + exampels_to_eval = [] + for data_id, parsed_pred in cat_outputs.items(): + question_type = cat_answers[data_id]["question_type"] + if question_type != "multiple-choice": + parsed_pred = parse_open_response( + parsed_pred + ) # mainly for type consistency (make it number, etc.) + else: + parsed_pred = parsed_pred + + exampels_to_eval.append( + { + "id": data_id, + "question_type": question_type, + "answer": cat_answers[data_id]["ground_truth"], + "parsed_pred": parsed_pred, + } + ) + + judge_dict, metric_dict = evaluate(exampels_to_eval) + metric_dict.update({"num_example": len(exampels_to_eval)}) + + evaluation_result[category] = metric_dict + + printable_results = {} + # pdb.set_trace() + # add domain Subject + for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): + in_domain_cat_results = {} + for cat_name in in_domain_cats: # use the order in DOMAIN_CAT2SUB_CAT + if cat_name in evaluation_result.keys(): + in_domain_cat_results[cat_name] = evaluation_result[cat_name] + else: + pass + in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) + in_domain_data_num = sum( + [ + cat_results["num_example"] + for cat_results in in_domain_cat_results.values() + ] + ) + printable_results["Overall-" + domain] = { + "num": int(in_domain_data_num), + "acc": round(in_domain_ins_acc, 3), + } + # add sub category + for cat_name, cat_results in in_domain_cat_results.items(): + printable_results[cat_name] = { + "num": int(cat_results["num_example"]), + "acc": round(cat_results["acc"], 3), + } + + # table.append(["-----------------------------", "-----", "----"]) + all_ins_acc = calculate_ins_level_acc(evaluation_result) + overall_acc = round(all_ins_acc, 3) + printable_results["Overall"] = { + "num": sum( + [cat_results["num_example"] for cat_results in evaluation_result.values()] + ), + "acc": overall_acc, + } + pprint.pprint(printable_results) + out = model_answer_path + with open(out, "w", encoding="utf-8") as outfile: + json.dump(printable_results, outfile) + print(f"eval out saved to {out}") + + print(f"Overall accuracy: {overall_acc}") diff --git a/benchmark/mmmu/prompt_format.yaml b/benchmark/mmmu/prompt_format.yaml new file mode 100644 index 0000000..1a0f721 --- /dev/null +++ b/benchmark/mmmu/prompt_format.yaml @@ -0,0 +1,15 @@ +task_instructions: +- "" +multi_choice_example_format: +- "{} + +{} + +Answer with the option's letter from the given choices directly." + +short_ans_example_format: +- "{} + +Answer the question using a single word or phrase." +temperature: +- 0 diff --git a/benchmark/mtbench/README.md b/benchmark/mtbench/README.md new file mode 100644 index 0000000..e6babf9 --- /dev/null +++ b/benchmark/mtbench/README.md @@ -0,0 +1,48 @@ +## Download Dataset + +```sh +wget -O question.jsonl https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/llm_judge/data/mt_bench/question.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 80 +``` + +### Benchmark sglang EAGLE +``` +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algo EAGLE \ + --speculative-draft lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \ + --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --dtype float16 --port 30000 +``` + +``` +python3 bench_sglang_eagle.py --num-questions 80 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 80 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 80 --backend lightllm +``` diff --git a/benchmark/mtbench/bench_other.py b/benchmark/mtbench/bench_other.py new file mode 100644 index 0000000..2c321e8 --- /dev/null +++ b/benchmark/mtbench/bench_other.py @@ -0,0 +1,111 @@ +import argparse +import json +import os +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from fastchat.model import get_conversation_template +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +def main(args): + questions = load_questions(args.question_file) + questions = (questions * 10)[: args.num_questions] + max_tokens = 256 + model_id = "llama-2-chat" + + conv_main = get_conversation_template(model_id) + + # Select backend + call_generate = get_call_generate(args) + + answers = [None] * len(questions) + + def get_answer(i): + conv = conv_main.copy() + cur_answers = [] + for j in range(2): + q = questions[i]["turns"][j] + conv.append_message(conv.roles[0], q) + conv.append_message(conv.roles[1], None) + + prompt = conv.get_prompt() + output = call_generate(prompt, temperature=0, max_tokens=max_tokens).strip() + + cur_answers.append(output) + conv.update_last_message(output) + + answers[i] = cur_answers + + # Run requests + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + latency = time.time() - tic + + print(f"#questions: {len(questions)}, Latency: {latency:.2f}") + + # Write results + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/mtbench/bench_sglang.py b/benchmark/mtbench/bench_sglang.py new file mode 100644 index 0000000..b57d164 --- /dev/null +++ b/benchmark/mtbench/bench_sglang.py @@ -0,0 +1,99 @@ +import argparse +import json +import os +import time +import uuid + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +@sgl.function +def answer_mt_bench(s, question_1, question_2): + s += sgl.system() + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def main(args): + # Construct prompts + questions = load_questions(args.question_file)[: args.num_questions] + arguments = [ + {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions + ] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + rets = answer_mt_bench.run_batch( + arguments, + temperature=0, + max_new_tokens=256, + num_threads=args.parallel, + progress_bar=True, + ) + answers = [[s["answer_1"], s["answer_2"]] for s in rets] + latency = time.time() - tic + + print(f"#questions: {len(questions)}, Latency: {latency:.2f}") + + # Write results + model_id = backend.model_info["model_path"] + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/mtbench/bench_sglang_eagle.py b/benchmark/mtbench/bench_sglang_eagle.py new file mode 100644 index 0000000..e1207af --- /dev/null +++ b/benchmark/mtbench/bench_sglang_eagle.py @@ -0,0 +1,137 @@ +""" +Adapted from https://github.com/chromecast56/sglang/blob/6f145d2eadb93a116134f703358ce76f15381045/benchmark/mtbench/bench_sglang.py + +Benchmark SGLang EAGLE/EAGLE3 Speculative Decoding + +Usage: +python3 benchmark/mtbench/bench_sglang_eagle.py --num-questions 80 --parallel 1 +""" + +import argparse +import json +import os +import time +import uuid + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +@sgl.function +def answer_mt_bench(s, question_1, question_2): + s += sgl.system( + "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information." + ) + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def main(args): + # Construct prompts + questions = load_questions(args.question_file)[: args.num_questions] + arguments = [ + {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions + ] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + rets = answer_mt_bench.run_batch( + arguments, + temperature=0, + max_new_tokens=2048, + num_threads=args.parallel, + progress_bar=True, + ) + answers = [[s["answer_1"], s["answer_2"]] for s in rets] + + latency = time.time() - tic + num_output_tokens = sum( + s.get_meta_info("answer_1")["completion_tokens"] + + s.get_meta_info("answer_2")["completion_tokens"] + for s in rets + ) + + # NOTE: acceptance length is just completion_tokens / spec_verify_ct + # {'id': '3bb9c5ead109488d8ed5ee9cbecaec29', 'finish_reason': {'type': 'length', 'length': 256}, 'prompt_tokens': 37, 'spec_verify_ct': 101, 'completion_tokens': 256, 'cached_tokens': 0} + + output_throughput = num_output_tokens / latency + + has_verify = "spec_verify_ct" in rets[0].get_meta_info("answer_1") + if has_verify: + num_verify_tokens = sum( + s.get_meta_info("answer_1")["spec_verify_ct"] + + s.get_meta_info("answer_2")["spec_verify_ct"] + for s in rets + ) + + accept_length = num_output_tokens / num_verify_tokens + else: + accept_length = 1.0 + + print( + f"#questions: {len(questions)}, Throughput: {output_throughput:.2f} token/s, Acceptance length: {accept_length:.2f}" + ) + + # Write results + model_id = backend.model_info["model_path"] + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "throughput": round(output_throughput, 3), + "accept_length": round(accept_length, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_chain_reasoning/README.md b/benchmark/multi_chain_reasoning/README.md new file mode 100644 index 0000000..4c9f740 --- /dev/null +++ b/benchmark/multi_chain_reasoning/README.md @@ -0,0 +1,49 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --schedule-conservativeness 1.3 +``` + +``` +python3 bench_sglang.py --num-questions 64 +python3 bench_sglang.py --num-questions 32 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 64 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 64 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 64 --backend lmql --parallel 1 +``` diff --git a/benchmark/multi_chain_reasoning/bench_other.py b/benchmark/multi_chain_reasoning/bench_other.py new file mode 100644 index 0000000..e0ff2be --- /dev/null +++ b/benchmark/multi_chain_reasoning/bench_other.py @@ -0,0 +1,186 @@ +import argparse +import ast +import asyncio +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +prompt_lib = [ + "Let us think step by step.", + "Approach this methodically. Let's dissect the problem into smaller, more manageable parts.", + "It's important to proceed step by step, ensuring accuracy at each stage.", + "Take a deep breath and break this down.", + "A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.", + "I am extremely good at math.", +] + + +def multi_chain_gsm8k(question, num_chains, call_generate): + s = "Question: " + question + "\n" + # s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256, + # stop="Question", temperature=0) + # return s + + comps = [] + for i in range(num_chains): + comps.append( + call_generate( + s + "Answer: " + prompt_lib[i % num_chains], + max_tokens=256, + temperature=0.3, + stop="Question", + ) + ) + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + comps[i].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += call_generate(s, max_tokens=16, temperature=0, stop=None) + return s + + +async def multi_chain_gsm8k_async(question, num_chains, call_generate): + s = "Question: " + question + "\n" + # s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256, + # stop="Question", temperature=0) + # return s + + comps = [] + for i in range(num_chains): + comps.append( + await call_generate( + s + "Answer: " + prompt_lib[i % num_chains], + max_tokens=256, + temperature=0.3, + stop="Question", + ) + ) + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + comps[i].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += await call_generate(s, max_tokens=16, temperature=0, stop=None) + return s + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + k = args.num_shot + + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + + states = [None] * len(labels) + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + answer = multi_chain_gsm8k(questions[i], args.num_chains, call_generate) + states[i] = answer + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + # Use asyncio + async def get_one_answer_asyncio(i): + answer = await multi_chain_gsm8k_async( + questions[i], args.num_chains, call_generate + ) + states[i] = answer + + tic = time.time() + loop = asyncio.get_event_loop() + batches = [ + list(range(i, min(i + args.parallel, len(questions)))) + for i in range(0, len(questions), args.parallel) + ] + for bt in tqdm(batches): + tasks = [get_one_answer_asyncio(k) for k in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.time() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_chain_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shot", type=int, default=0) + parser.add_argument("--num-chains", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=50) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_chain_reasoning/bench_sglang.py b/benchmark/multi_chain_reasoning/bench_sglang.py new file mode 100644 index 0000000..98a6b51 --- /dev/null +++ b/benchmark/multi_chain_reasoning/bench_sglang.py @@ -0,0 +1,140 @@ +import argparse +import ast +import json +import re +import time + +import numpy as np + +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +prompt_lib = [ + "Let us think step by step.", + "Approach this methodically. Let's dissect the problem into smaller, more manageable parts.", + "It's important to proceed step by step, ensuring accuracy at each stage.", + "Take a deep breath and break this down.", + "A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.", + "I am extremely good at math.", +] + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + # k = args.num_shot + # few_shot_examples = get_few_shot_examples(lines, k) + + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + num_chains = args.num_chains + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def multi_chain_gsm8k(s, question): + s += "Question: " + question + "\n" + # s += "Answer: " + prompt_lib[0] + sgl.gen("answer", max_tokens=256, stop="Question", + # temperature=0) + # return + + forks = s.fork(num_chains) + for i in range(num_chains): + forks[i] += ( + "Answer: " + + prompt_lib[i % num_chains] + + sgl.gen("chain", max_tokens=256, temperature=0.3, stop="Question") + ) + forks.join() + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + forks[i]["chain"].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += sgl.gen("answer", max_tokens=16) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.time() + states = multi_chain_gsm8k.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i]["answer"])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_chain_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shot", type=int, default=0) + parser.add_argument("--num-chains", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=50) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_document_qa/README.md b/benchmark/multi_document_qa/README.md new file mode 100644 index 0000000..09f49c7 --- /dev/null +++ b/benchmark/multi_document_qa/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-instruct-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 10 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model codellama/CodeLlama-7b-instruct-hf --disable-log-requests --port 21000 --gpu 0.97 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 64 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf +``` + + + +### Build dataset + +``` +pip install PyPDF2 +python3 build_dataset.py +``` + +```python +import PyPDF2 + +with open('llama2.pdf', 'rb') as file: + reader = PyPDF2.PdfReader(file) + text = '' + for page_num in range(len(reader.pages)): + text += reader.pages[page_num].extract_text() + with open('output.txt', 'w') as text_file: + text_file.write(text) +``` diff --git a/benchmark/multi_document_qa/bench_other.py b/benchmark/multi_document_qa/bench_other.py new file mode 100644 index 0000000..97ff416 --- /dev/null +++ b/benchmark/multi_document_qa/bench_other.py @@ -0,0 +1,114 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + + +def multi_document_qa(docs, question, generate): + s = USER_PREFIX + s += "Pleaes answer a question according to given documents.\n" + s += "Question:" + question + "Documents begin.\n" + + s += "".join(docs) + + s += "\nDocuments end." + s += ( + "\n\nBased on the above documents, please answer this question:\n" + + question + + "\nAnswer in three words or fewer." + ) + s += USER_SUFFIX + s += ASSISTANT_PREFIX + answer = generate(s, max_tokens=16, stop=None) + return answer + + +def main(args): + lines = read_jsonl(args.data_path) + l = lines[0] + arguments = [] + labels = [] + + num_docs = 10 + if args.backend == "guidance": + num_docs = 7 # due to OOM + + for i in range(len(l["questions"][: args.num_questions])): + arguments.append( + { + "docs": l["documents"][:num_docs], + "question": l["questions"][i], + } + ) + labels.append(l["answers"][i]) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = multi_document_qa(generate=call_generate, **arguments[i]) + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(labels))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(labels)))), + total=len(labels), + ) + ) + + latency = time.time() - tic + + # Compute accuracy + print(states) + correct = 0 + for s, label in zip(states, labels): + answer = s.lower() + if all(x in answer for x in label.lower().split(" ")): + correct += 1 + accuracy = correct / len(labels) + print(f"Accuracy: {accuracy:.3f}") + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_document_qa", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "accuracy": accuracy, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_document_qa/bench_sglang.py b/benchmark/multi_document_qa/bench_sglang.py new file mode 100644 index 0000000..2c2db4f --- /dev/null +++ b/benchmark/multi_document_qa/bench_sglang.py @@ -0,0 +1,93 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def multi_document_qa(s, docs, question): + s += sgl.user_begin() + s += "Pleaes answer a question according to given documents.\n" + s += "Question:" + question + "Documents begin.\n" + + forks = s.fork(len(docs)) + forks += lambda i: docs[i] + forks.join("concate_and_append") + + s += "\nDocuments end." + s += ( + "\n\nBased on the above documents, please answer this question:\n" + + question + + "\nAnswer in three words or fewer." + ) + s += sgl.user_end() + s += sgl.assistant(sgl.gen("answer", max_tokens=16)) + + +def main(args): + lines = read_jsonl(args.data_path) + l = lines[0] + arguments = [] + labels = [] + for i in range(len(l["questions"][: args.num_questions])): + arguments.append( + { + "docs": l["documents"][:10], + "question": l["questions"][i], + } + ) + labels.append(l["answers"][i]) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.time() + states = multi_document_qa.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.time() - tic + + # Compute accuracy + print([s["answer"] for s in states]) + correct = 0 + for s, label in zip(states, labels): + answer = s["answer"].lower() + if all(x in answer for x in label.lower().split(" ")): + correct += 1 + accuracy = correct / len(labels) + print(f"Accuracy: {accuracy:.3f}") + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_document_qa", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "accuracy": accuracy, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_document_qa/build_dataset.py b/benchmark/multi_document_qa/build_dataset.py new file mode 100644 index 0000000..27df947 --- /dev/null +++ b/benchmark/multi_document_qa/build_dataset.py @@ -0,0 +1,70 @@ +import json + +import transformers + +content = "\n".join( + open("llama2.txt", "r", encoding="utf-8", errors="ignore").readlines() +) +content = content.replace("\n\n", "\n") + +# Count token +name = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(name) +print(f"num tokens: {len(t.encode(content))}") + +# Segment +SEP = "\n\n" +parts = content.split(SEP) +print(f"num segments: {len(parts)}") + +segment_len = 1100 + +segments = [] +tmp = [] +tmp_len = 0 +for i in range(len(parts)): + tmp.append(parts[i]) + tmp_len += len(t.encode(parts[i])) + + if tmp_len > segment_len: + segments.append(SEP.join(tmp)) + tmp = [] + tmp_len = 0 + +for i, s in enumerate(segments): + print(i, len(t.encode(segments[i]))) + +# Dump +with open("questions.jsonl", "w") as fout: + fout.write( + json.dumps( + { + "documents": segments[:30], + "questions": [ + "What is the name of the fine-tuned LLMs?", + "Which figure shows the helpfulness human evaluation results for Llama 2-Chat?", + "What is the number of parameters in the largest Llama 2 model?", + "What is the batch size of fine-tuning?", + "Where can we find the details of potential data contamination?", + "What is the full name of MPT?", + "What is the power consumption of RSC in Watt?", + "How many tokens of data do they train on?", + "Which model's release is delayed due to a lack of time to sufficiently red team?", + "Which activation function is used in Llama?", + ], + "answers": [ + "Llama 2 Chat", + "1", + "70 B", + "64", + "A 6", + "MosaicML", + "400", + "2 trillion", + "34 B", + "SwiGLU", + ], + } + ) + + "\n" + ) diff --git a/benchmark/multi_turn_chat/README.md b/benchmark/multi_turn_chat/README.md new file mode 100644 index 0000000..0fb5b21 --- /dev/null +++ b/benchmark/multi_turn_chat/README.md @@ -0,0 +1,66 @@ +### Benchmark sglang + +Run Llama-7B + +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Run Mixtral-8x7B +(When there is a CUDA out-of-memory error, try to reduce the `--mem-fraction-static`) + +``` +python3 -m sglang.launch_server --model-path mistralai/Mixtral-8x7B-Instruct-v0.1 --port 30000 --tp-size 8 +``` + +Benchmark(short output) + +``` +python3 bench_sglang.py --tokenizer meta-llama/Llama-2-7b-chat-hf +``` + +Benchmark(long output) + +``` +python3 bench_sglang.py --tokenizer meta-llama/Llama-2-7b-chat-hf --long +``` + +### Benchmark vLLM + +Run Llama-7B + +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Run Mixtral-8x7B + +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model mistralai/Mixtral-8x7B-Instruct-v0.1 --disable-log-requests --port 21000 --tensor-parallel-size 8 +``` + +Benchmark(short output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend vllm +``` + +Benchmark(long output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend vllm --long +``` + +### Benchmark guidance + +Benchmark Llama-7B (short output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +Benchmark Llama-7B (long output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf --long +``` diff --git a/benchmark/multi_turn_chat/bench_other.py b/benchmark/multi_turn_chat/bench_other.py new file mode 100644 index 0000000..81d67ab --- /dev/null +++ b/benchmark/multi_turn_chat/bench_other.py @@ -0,0 +1,93 @@ +import json +import time +from argparse import ArgumentParser +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from data_gen import gen_arguments +from tqdm import tqdm +from vllm.transformers_utils.tokenizer import get_tokenizer + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text + + +def multi_turns(generate, qas): + s = "" + for qa in qas: + s += qa["prompt"] + s += generate(s, max_tokens=qa["new_tokens"]) + + return s + + +def main(args): + print(args) + + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + states = [None] * args.num_qa + + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = multi_turns(generate=call_generate, **multi_qas[i]) + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(multi_qas))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(multi_qas)))), + total=len(multi_qas), + ) + ) + for _ in rets: + pass + + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_chat", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + "output_mode": "long" if args.long else "short", + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=4) + parser.add_argument("--num-qa", type=int, default=20) + parser.add_argument("--min-len-q", type=int, default=256) + parser.add_argument("--max-len-q", type=int, default=512) + parser.add_argument("--min-len-a", type=int, default=4) + parser.add_argument("--max-len-a", type=int, default=8) + parser.add_argument("--tokenizer", type=str, required=True) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--long", action="store_true") + args = add_common_other_args_and_parse(parser) + + if args.long: + args.min_len_a = 256 + args.max_len_a = 512 + args.num_qa = 20 + main(args) diff --git a/benchmark/multi_turn_chat/bench_sglang.py b/benchmark/multi_turn_chat/bench_sglang.py new file mode 100644 index 0000000..7feaced --- /dev/null +++ b/benchmark/multi_turn_chat/bench_sglang.py @@ -0,0 +1,79 @@ +import json +import time +from argparse import ArgumentParser + +from data_gen import gen_arguments +from vllm.transformers_utils.tokenizer import get_tokenizer + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def multi_turns(s, qas): + for qa in qas: + s += qa["prompt"] + s += sgl.gen(max_tokens=qa["new_tokens"], ignore_eos=True) + + +def main(args): + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + backend = select_sglang_backend(args) + + tic = time.time() + states = multi_turns.run_batch( + multi_qas, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_chat", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + "output_mode": "long" if args.long else "short", + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=4) + parser.add_argument("--num-qa", type=int, default=20) + parser.add_argument("--min-len-q", type=int, default=256) + parser.add_argument("--max-len-q", type=int, default=512) + parser.add_argument("--min-len-a", type=int, default=4) + parser.add_argument("--max-len-a", type=int, default=8) + parser.add_argument("--tokenizer", type=str, required=True) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--long", action="store_true") + args = add_common_sglang_args_and_parse(parser) + + if args.long: + args.min_len_a = 256 + args.max_len_a = 512 + args.num_qa = 20 + + print(args) + main(args) diff --git a/benchmark/multi_turn_chat/data_gen.py b/benchmark/multi_turn_chat/data_gen.py new file mode 100644 index 0000000..043c07a --- /dev/null +++ b/benchmark/multi_turn_chat/data_gen.py @@ -0,0 +1,29 @@ +import random +import string + +random.seed(42) + + +def gen_prompt(tokenizer, token_num): + cha_set = string.ascii_letters + string.digits + ret = "".join(random.choices(cha_set, k=token_num)) + while len(tokenizer(ret).input_ids) < token_num: + ret += random.choice(cha_set) + return ret + + +def gen_arguments(args, tokenizer): + multi_qas = [{"qas": []} for _ in range(args.num_qa)] + for i in range(args.num_qa): + qas = multi_qas[i]["qas"] + for _ in range(args.turns): + prompt_len = random.randint(args.min_len_q, args.max_len_q) + new_tokens = random.randint(args.min_len_a, args.max_len_a) + qas.append( + { + "prompt": gen_prompt(tokenizer, prompt_len), + "new_tokens": new_tokens, + } + ) + + return multi_qas diff --git a/benchmark/multi_turn_chat/long_prompt_multi_turn.py b/benchmark/multi_turn_chat/long_prompt_multi_turn.py new file mode 100644 index 0000000..20f6dd5 --- /dev/null +++ b/benchmark/multi_turn_chat/long_prompt_multi_turn.py @@ -0,0 +1,129 @@ +import json +import random +import time +from argparse import ArgumentParser +from pathlib import Path + +from tqdm import tqdm + +import sglang as sgl +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +def gen_prompt(tokenizer, token_num): + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + ret = tokenizer.decode(selected_tokens) + return ret + + +def get_cache_path(args): + # Create cache directory under ~/.cache/sglang + cache_dir = Path.home() / ".cache" / "sglang" + + # Create a unique cache filename based on the arguments that affect generation + cache_key = f"qa_{args.num_qa}_{args.turns}_{args.system_prompt_len}_{args.len_q}_{args.len_a}_{args.tokenizer.replace('/', '_')}.json" + return cache_dir / cache_key + + +def gen_arguments(args, tokenizer): + cache_path = get_cache_path(args) + + # Try to load from cache first + if cache_path.exists(): + print(f"Loading cached arguments from {cache_path}") + with open(cache_path, "r") as f: + return json.load(f) + + print("Generating new arguments...") + # First progress bar for system prompts + multi_qas = [] + for _ in tqdm(range(args.num_qa), desc="Generating system prompts"): + multi_qas.append( + {"system_prompt": gen_prompt(tokenizer, args.system_prompt_len), "qas": []} + ) + + # Nested progress bars for QA pairs + for i in tqdm(range(args.num_qa), desc="Generating QA pairs"): + qas = multi_qas[i]["qas"] + for j in range(args.turns): + qas.append( + { + "prompt": gen_prompt(tokenizer, args.len_q), + "new_tokens": args.len_a, + } + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + with open(cache_path, "w") as f: + json.dump(multi_qas, f) + print(f"Cached arguments saved to {cache_path}") + + return multi_qas + + +@sgl.function +def multi_turns(s, system_prompt, qas): + s += system_prompt + + for i, qa in enumerate(qas): + s += qa["prompt"] + s += sgl.gen(max_tokens=qa["new_tokens"], ignore_eos=True) + + +def main(args): + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + backend = select_sglang_backend(args) + + tic = time.time() + states = multi_turns.run_batch( + multi_qas, + temperature=0, + backend=backend, + num_threads="auto", + progress_bar=True, + ) + latency = time.time() - tic + + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_system_prompt_chat", + "backend": args.backend, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=8) + parser.add_argument("--num-qa", type=int, default=128) + parser.add_argument("--system-prompt-len", type=int, default=2048) + parser.add_argument("--len-q", type=int, default=32) + parser.add_argument("--len-a", type=int, default=128) + parser.add_argument( + "--tokenizer", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct" + ) + parser.add_argument("--trust-remote-code", action="store_true") + args = add_common_sglang_args_and_parse(parser) + + print(args) + main(args) diff --git a/benchmark/react/README.md b/benchmark/react/README.md new file mode 100644 index 0000000..51bcaa4 --- /dev/null +++ b/benchmark/react/README.md @@ -0,0 +1,34 @@ +## Run benchmark + +NOTE: This is an implementation for replaying a given trace for throughput/latency benchmark purposes. It is not an actual ReAct agent implementation. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 100 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 100 --backend vllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 100 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 100 --backend lmql --parallel 1 +``` diff --git a/benchmark/react/bench_other.py b/benchmark/react/bench_other.py new file mode 100644 index 0000000..91c5546 --- /dev/null +++ b/benchmark/react/bench_other.py @@ -0,0 +1,202 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def get_prompt(question): + prompt = ( + """Solve a question answering task with interleaving Thought, Action, Observation steps. Thought can reason about the current situation, and Action can be three types: +(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it will return some similar entities to search. +(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage. +(3) Finish[answer], which returns the answer and finishes the task. +Here are some examples. +Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area. +Action 1: Search[Colorado orogeny] +Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas. +Thought 2: It does not mention the eastern sector. So I need to look up eastern sector. +Action 2: Lookup[eastern sector] +Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny. +Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range. +Action 3: Search[High Plains] +Observation 3: High Plains refers to one of two distinct land regions: +Thought 4: I need to instead search High Plains (United States). +Action 4: Search[High Plains (United States)] +Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3] +Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Action 5: Finish[1,800 to 7,000 ft] +Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who? +Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after. +Action 1: Search[Milhouse] +Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening. +Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after". +Action 2: Lookup[named after] +Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous. +Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon. +Action 3: Finish[Richard Nixon] +Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture? +Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups. +Action 1: Search[Adam Clayton Powell] +Observation 1: Could not find [Adam Clayton Powell]. Similar: ['Adam Clayton Powell III', 'Seventh Avenue (Manhattan)', 'Adam Clayton Powell Jr. State Office Building', 'Isabel Washington Powell', 'Adam Powell', 'Adam Clayton Powell (film)', 'Giancarlo Esposito']. +Thought 2: To find the documentary, I can search Adam Clayton Powell (film). +Action 2: Search[Adam Clayton Powell (film)] +Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. +The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience. +Thought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture. +Action 3: Finish[The Saimaa Gesture] +Question: What profession does Nicholas Ray and Elia Kazan have in common? +Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Action 1: Search[Nicholas Ray] +Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 – June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Action 2: Search[Elia Kazan] +Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Action 3: Finish[director, screenwriter, actor] +Question: Which magazine was started first Arthur's Magazine or First for Women? +Thought 1: I need to search Arthur's Magazine and First for Women, and find which was started first. +Action 1: Search[Arthur's Magazine] +Observation 1: Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century. +Thought 2: Arthur's Magazine was started in 1844. I need to search First for Women next. +Action 2: Search[First for Women] +Observation 2: First for Women is a woman's magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989. +Thought 3: First for Women was started in 1989. 1844 (Arthur's Magazine) < 1989 (First for Women), so Arthur's Magazine was started first. +Action 3: Finish[Arthur's Magazine] +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same. +Action 1: Search[Pavel Urysohn] +Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 â August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory. +Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work. +Action 2: Search[Leonid Levin] +Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist. +Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work. +Action 3: Finish[yes] +""" + + question + ) + return prompt + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"question": k, "triplets": v} for l in lines for k, v in l.items()] + + states = [] + + # Select backend + call_generate = get_call_generate(args) + + def run_single_agent(argument): + question = argument["question"] + triplets = argument["triplets"] + prompt = get_prompt(question) + for i in range(1, len(triplets) + 2): + prompt += "Thought " + str(i) + ":" + states.append(prompt) + answer = call_generate( + prompt, max_tokens=200, temperature=0, stop="Observation" + ) + if i > len(triplets): + break + prompt += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + states.append(answer) + + async def run_single_agent_async(argument): + question = argument["question"] + triplets = argument["triplets"] + prompt = get_prompt(question) + for i in range(1, len(triplets) + 2): + prompt += "Thought " + str(i) + ":" + states.append(prompt) + answer = await call_generate( + prompt, max_tokens=200, temperature=0, stop="Observation", max_len=4096 + ) + if i > len(triplets): + break + prompt += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + states.append(answer) + + tic = time.time() + + if args.backend != "lmql": + if args.parallel == 1: + for arg in tqdm(arguments): + run_single_agent(arg) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(run_single_agent, arguments), total=len(arguments) + ) + ) + + else: + import asyncio + + loop = asyncio.get_event_loop() + batches = [ + [] for _ in range((len(arguments) + args.parallel - 1) // args.parallel) + ] + for i, arg in enumerate(arguments): + batches[i // args.parallel].append(arg) + for bt in tqdm(batches): + tasks = [run_single_agent_async(arg) for arg in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.time() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "ReAct Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="hotpotqa_100.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/react/bench_sglang.py b/benchmark/react/bench_sglang.py new file mode 100644 index 0000000..b07105e --- /dev/null +++ b/benchmark/react/bench_sglang.py @@ -0,0 +1,153 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def webthink(s, question, triplets): + s += ( + """Solve a question answering task with interleaving Thought, Action, Observation steps. Thought can reason about the current situation, and Action can be three types: +(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it will return some similar entities to search. +(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage. +(3) Finish[answer], which returns the answer and finishes the task. +Here are some examples. +Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area. +Action 1: Search[Colorado orogeny] +Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas. +Thought 2: It does not mention the eastern sector. So I need to look up eastern sector. +Action 2: Lookup[eastern sector] +Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny. +Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range. +Action 3: Search[High Plains] +Observation 3: High Plains refers to one of two distinct land regions: +Thought 4: I need to instead search High Plains (United States). +Action 4: Search[High Plains (United States)] +Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3] +Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Action 5: Finish[1,800 to 7,000 ft] +Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who? +Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after. +Action 1: Search[Milhouse] +Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening. +Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after". +Action 2: Lookup[named after] +Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous. +Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon. +Action 3: Finish[Richard Nixon] +Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture? +Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups. +Action 1: Search[Adam Clayton Powell] +Observation 1: Could not find [Adam Clayton Powell]. Similar: ['Adam Clayton Powell III', 'Seventh Avenue (Manhattan)', 'Adam Clayton Powell Jr. State Office Building', 'Isabel Washington Powell', 'Adam Powell', 'Adam Clayton Powell (film)', 'Giancarlo Esposito']. +Thought 2: To find the documentary, I can search Adam Clayton Powell (film). +Action 2: Search[Adam Clayton Powell (film)] +Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. +The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience. +Thought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture. +Action 3: Finish[The Saimaa Gesture] +Question: What profession does Nicholas Ray and Elia Kazan have in common? +Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Action 1: Search[Nicholas Ray] +Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 – June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Action 2: Search[Elia Kazan] +Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Action 3: Finish[director, screenwriter, actor] +Question: Which magazine was started first Arthur's Magazine or First for Women? +Thought 1: I need to search Arthur's Magazine and First for Women, and find which was started first. +Action 1: Search[Arthur's Magazine] +Observation 1: Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century. +Thought 2: Arthur's Magazine was started in 1844. I need to search First for Women next. +Action 2: Search[First for Women] +Observation 2: First for Women is a woman's magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989. +Thought 3: First for Women was started in 1989. 1844 (Arthur's Magazine) < 1989 (First for Women), so Arthur's Magazine was started first. +Action 3: Finish[Arthur's Magazine] +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same. +Action 1: Search[Pavel Urysohn] +Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 â August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory. +Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work. +Action 2: Search[Leonid Levin] +Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist. +Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work. +Action 3: Finish[yes] +""" + + question + ) + for i in range(1, len(triplets) + 2): + s += "Thought " + str(i) + ":" + # NOTE: This is an implementation for replaying a given trace for benchmark purposes. It is not an actual ReAct agent implementation. + ss = s.fork(1) + ss[0] += sgl.gen(name="thought_action", max_tokens=200, stop="Observation") + ss.join() + # to verify the correctness of output, this should be collected + # print(ss[0]["thought_action"]) + if i > len(triplets): + break + s += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"question": k, "triplets": v} for l in lines for k, v in l.items()] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + states = [] + tic = time.time() + states = webthink.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "ReAct Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="hotpotqa_100.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/reasoning_benchmark/README.md b/benchmark/reasoning_benchmark/README.md new file mode 100644 index 0000000..f9d26e0 --- /dev/null +++ b/benchmark/reasoning_benchmark/README.md @@ -0,0 +1,77 @@ +# Run benchmark + +This benchmark is primarily intended to be used with reasoning models like `DeepSeek-R1` and its distilled models like `DeepSeek-R1-Distill-Qwen-1.5B`. Please use + +```bash +pip install antlr4-python3-runtime +``` + +for `parse_latex` which we use for symbolic equality check. + +## Benchmark sglang + +1. Launch the Server +```bash +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 +``` + +Note that depending on the GPU this benchmark will take quiet some time. To employ data parallelism please use: + +```bash +python3 -m sglang_router.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 --dp-size 4 +``` + + +2. Benchmarking + +We use [suggested](https://github.com/deepseek-ai/DeepSeek-R1) parameters of `temperature=0.6`, `top_p=.95`, `max_new_tokens=32768`. The command line argument `num-tries` can be used to evaluate the model multiple times on the same question. We use the suggested `64` from the repo for AIME 2024. For LIMO, we use `8` as the number of tries due to the size of the dataset. + +By default evaluate on LIMO dataset. + +```bash +python3 bench_sglang.py --parallel 256 --num-tries 64 --port 30000 +``` + +Evaluate on AIME 2024 dataset. + +```bash +python3 bench_sglang.py --parallel 256 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries 64 +``` + +Evaluate on [AIME 2025 I dataset](https://huggingface.co/datasets/opencompass/AIME2025). For benchmark result see [here](https://matharena.ai/). + +```bash +python3 bench_sglang.py --parallel 256 --port 30000 --data-path opencompass/AIME2025 --question-key question --answer-key answer --num-tries 64 +``` +## Results + +### Evaluation Results +| Dataset | Num Tries | Accuracy | Reference | Standard Error | +|------------|-----------|----------|-----------|-----------| +| LIMO | 8 | 47.7% | ? | ? | +| AIME 2024 | 64 | 33.2% | 28.9% | 3.4% | +| AIME 2025 I| 64 | 29.9% | 25.0% | ? | + +### Statistic Analysis Results +Set up SGLang engine for statistic analysis, for high efficiency we use `--dp-size 8` for data parallelism: +```bash +python3 -m sglang_router.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 --dp-size 8 +``` +**Experiment 1**: +We fixed the number of attempts (num_tries) and conducted multiple runs to assess the consistency of the model's performance. The results show that all recorded accuracies lie within ± one standard error deviation from the mean. This suggests that **our metric serves as an effective upper bound for the deviation of reported accuracy**. + +To collect the accuracy, run the following command 30 times: +```bash +python3 bench_sglang.py --parallel 64 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries 64 +``` + +![acc_hist](figure/Acc_histplot.png) + + +**Experiment 2**: We explored the relationship between the number of attempts (num_tries) and the standard error (SE) by varying num_tries across a range (e.g., 8, 16, 32, ..., 256) and performing a single run for each value. The results demonstrate that as the number of attempts increases, the standard error decreases, leading to **greater stability in answer accuracy**. + +To reveal the relationship, run the command 6 times and adjust the parameter `--num-tries` for each run: +```bash +python3 bench_sglang.py --parallel 64 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries +``` +![SE_num_tries](figure/SE_numtries.png) diff --git a/benchmark/reasoning_benchmark/answer_extraction.py b/benchmark/reasoning_benchmark/answer_extraction.py new file mode 100644 index 0000000..45ce59a --- /dev/null +++ b/benchmark/reasoning_benchmark/answer_extraction.py @@ -0,0 +1,269 @@ +# Adapted from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/data_processing/answer_extraction.py + +import re + +import regex + + +def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if len(substr) > 0 and substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + +def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + if "sqrt" not in a: + a = int(a) + if "sqrt" not in b: + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except: + return string + + +def _fix_sqrt(string): + _string = re.sub(r"\\sqrt(-?[0-9.a-zA-Z]+)", r"\\sqrt{\1}", string) + _string = re.sub(r"\\sqrt\s+(\w+)$", r"\\sqrt{\1}", _string) + return _string + + +def _fix_tan(string): + _string = re.sub(r"\\tan(-?[0-9.a-zA-Z]+)", r"\\tan{\1}", string) + _string = re.sub(r"\\tan\s+(\w+)$", r"\\tan{\1}", _string) + return _string + + +def strip_string(string): + string = str(string).strip() + # linebreaks + string = string.replace("\n", "") + + # right "." + string = string.rstrip(".") + + # remove inverse spaces + string = string.replace("\\!", "") + # string = string.replace("\\ ", "") + + # replace \\ with \ + # string = string.replace("\\\\", "\\") + # string = string.replace("\\\\", "\\") + + if string.startswith("\\text{") and string.endswith("}"): + string = string.split("{", 1)[1][:-1] + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + string = string.replace("cfrac", "frac") + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + + # Remove unit: miles, dollars if after is not none + _string = re.sub(r"\\text{.*?}$", "", string).strip() + if _string != "" and _string != string: + # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) + string = _string + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "").strip() + string = string.replace("^\\circ", "").strip() + + string = regex.sub(r"\{(c|m)?m\}(\^(2|3))?", "", string).strip() + string = regex.sub(r"p\.m\.$", "", string).strip() + string = regex.sub(r"(\d)\s*t$", r"\1", string).strip() + + # remove dollar signs + string = string.replace("\\$", "") + string = string.replace("$", "") + + # string = string.replace("\\text", "") + string = string.replace("x\\in", "") + + # remove percentage + string = string.replace("\\%", "%") + string = string.replace("\%", "%") + # string = string.replace("%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + + # cdot + string = string.replace("\\cdot", "") + + # inf + string = string.replace("infinity", "\\infty") + if "\\infty" not in string: + string = string.replace("inf", "\\infty") + string = string.replace("+\\inity", "\\infty") + + # and + # string = string.replace("and", "") + string = string.replace("\\mathbf", "") + string = string.replace("\\mathrm", "") + + # use regex to remove \mbox{...} + string = re.sub(r"\\mbox{.*?}", "", string) + + # quote + string.replace("'", "") + string.replace('"', "") + + # i, j + if "j" in string and "i" not in string: + string = string.replace("j", "i") + + # replace a.000b where b is not number or b is end, with ab, use regex + string = re.sub(r"(\d+)\.0+([^\d])", r"\1\2", string) + string = re.sub(r"(\d+)\.0+$", r"\1", string) + + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + # if len(string.split("=")) == 2: + # if len(string.split("=")[0]) <= 2: + # string = string.split("=")[1] + + string = _fix_sqrt(string) + string = _fix_tan(string) + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + string = regex.sub(r"(\\|,|\.)+$", "", string) + + return string + + +def extract_boxed_answers(text): + answers = [] + for piece in text.split("boxed{")[1:]: + n = 0 + for i in range(len(piece)): + if piece[i] == "{": + n += 1 + elif piece[i] == "}": + n -= 1 + if n < 0: + if i + 1 < len(piece) and piece[i + 1] == "%": + answers.append(piece[: i + 1]) + else: + answers.append(piece[:i]) + break + return answers + + +def extract_program_output(pred_str): + """ + extract output between the last ```output\n...\n``` + """ + if "```output" not in pred_str: + return "" + if "```output" in pred_str: + pred_str = pred_str.split("```output")[-1] + if "```" in pred_str: + pred_str = pred_str.split("```")[0] + output = pred_str.strip() + return output + + +def extract_answer(pred_str, exhaust=False): + pred = [] + if "final answer is $" in pred_str and "$. I hope" in pred_str: + tmp = pred_str.split("final answer is $", 1)[1] + pred = [tmp.split("$. I hope", 1)[0].strip()] + elif "boxed" in pred_str: + pred = extract_boxed_answers(pred_str) + elif "he answer is" in pred_str: + pred = [pred_str.split("he answer is")[-1].strip()] + else: + program_output = extract_program_output(pred_str) + if program_output != "": + # fall back to program + pred.append(program_output) + else: # use the last number + pattern = "-?\d*\.?\d+" + ans = re.findall(pattern, pred_str.replace(",", "")) + if len(ans) >= 1: + ans = ans[-1] + else: + ans = "" + if ans: + pred.append(ans) + + # multiple line + _pred = [] + for ans in pred: + ans = ans.strip().split("\n")[0] + ans = ans.lstrip(":") + ans = ans.rstrip(".") + ans = ans.rstrip("/") + ans = strip_string(ans) + _pred.append(ans) + if exhaust: + return _pred + else: + return _pred[-1] if _pred else "" + + +def extract_math_answer(question, reasoning, task): + answer = [] + for ans in extract_answer(reasoning, exhaust=True): + if "separated by commas" in question and all(ch not in ans for ch in "()[]"): + answer.extend([a.strip() for a in ans.split(",")]) + elif regex.search(r"\\text\{\s*and\s*\}", ans): + answer.extend( + [ + a.strip() + for a in regex.sub(r"\\text\{\s*and\s*\}", "[SEP]", ans).split( + "[SEP]" + ) + ] + ) + else: + answer.append(ans.strip()) + return answer diff --git a/benchmark/reasoning_benchmark/bench_sglang.py b/benchmark/reasoning_benchmark/bench_sglang.py new file mode 100644 index 0000000..c832049 --- /dev/null +++ b/benchmark/reasoning_benchmark/bench_sglang.py @@ -0,0 +1,135 @@ +import argparse +import json +import time + +import answer_extraction +import eval_utils +import numpy as np +from datasets import load_dataset + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def reasoning_gen(s, question: str): + s += sgl.user( + question + + "\nPlease reason step by step, and put your final answer within \boxed{}." + ) + s += sgl.assistant( + sgl.gen( + "answer", + ) + ) + + +def convert_dataset(path: str, question_key: str, answer_key: str, num_tries: int): + raw_dataset = load_dataset(path) + questions = [] + answers = [] + for data in raw_dataset["train"]: + question = data[question_key] + answer = data[answer_key] + for _ in range(num_tries): + questions.append({"question": question}) + answers.append({"answer": answer}) + return questions, answers + + +def main(args): + # Select backend + sgl.set_default_backend(select_sglang_backend(args)) + + # Get dataset + questions, answers = convert_dataset( + args.data_path, args.question_key, args.answer_key, args.num_tries + ) + + # Run requests + tic = time.time() + states = reasoning_gen.run_batch( + questions, + num_threads=args.parallel, + progress_bar=True, + temperature=0.6, + max_new_tokens=32768, + top_p=0.95, + ) + latency = time.time() - tic + + # Extract results and record outcomes in a list. + outcomes = [] + for i, state in enumerate(states): + try: + pred_answer = answer_extraction.extract_math_answer( + questions[i]["question"], state["answer"], "limo" + ) + gt_answer = str(answers[i]["answer"]) + pred_answer = ( + pred_answer[-1] if isinstance(pred_answer, list) else pred_answer + ) + is_correct = 1 if eval_utils.math_equal(pred_answer, gt_answer) else 0 + except Exception as e: + print(f"Error extracting answer: {e}") + is_correct = 0 + + outcomes.append(is_correct) + + # Calculate overall accuracy using numpy + overall_accuracy = np.mean(outcomes) + print(f"Overall Accuracy: {overall_accuracy}") + + # Calculate mean standard error over questions if num_tries >= 2 + if args.num_tries > 1: + outcomes_np = np.array(outcomes).reshape(-1, args.num_tries) + # Using sample standard deviation with ddof=1 + std_per_question = np.std(outcomes_np, axis=1, ddof=1) + # Compute the standard error for each question: std / sqrt(num_tries) + se_per_question = std_per_question / np.sqrt(args.num_tries) + mean_se = se_per_question.mean() + print(f"Mean Standard Error of Accuracy across questions: {mean_se}") + else: + mean_se = None + print("Not enough samples per question to compute standard error.") + + # Calculate output throughput + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + print(f"Output throughput: {output_throughput} token/s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "limo", + "backend": args.backend, + "latency": round(latency, 3), + "overall_accuracy": round(overall_accuracy, 3), + "mean_se_accuracy": round(mean_se, 3) if mean_se is not None else None, + "num_requests": len(questions), + "other": { + "num_questions": len(questions), + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="GAIR/LIMO") + parser.add_argument("--question-key", type=str, default="question") + parser.add_argument("--answer-key", type=str, default="answer") + parser.add_argument("--num-tries", type=int, default=1) + add_common_sglang_args_and_parse(parser) + args = parser.parse_args() + main(args) diff --git a/benchmark/reasoning_benchmark/eval_utils.py b/benchmark/reasoning_benchmark/eval_utils.py new file mode 100644 index 0000000..ab73695 --- /dev/null +++ b/benchmark/reasoning_benchmark/eval_utils.py @@ -0,0 +1,206 @@ +# Adapted from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/eval/eval_utils.py + +from math import isclose + +import regex +from sympy import N, simplify +from sympy.parsing.latex import parse_latex +from sympy.parsing.sympy_parser import parse_expr + + +def parse_digits(num): + # format: 234.23 || 23% + num = regex.sub(",", "", str(num)) + try: + return float(num) + except: + if num.endswith("%"): + num = num[:-1] + if num.endswith("\\"): + num = num[:-1] + try: + return float(num) / 100 + except: + pass + return None + + +def is_digit(num): + # paired with parse_digits + return parse_digits(num) is not None + + +def symbolic_equal(a, b): + def _parse(s): + for f in [parse_latex, parse_expr]: + try: + return f(s) + except: + pass + return s + + a = _parse(a) + b = _parse(b) + + try: + if simplify(a - b) == 0: + return True + except: + pass + + try: + if isclose(N(a), N(b), abs_tol=1e-3): + return True + except: + pass + return False + + +def math_equal(prediction, reference, include_percentage=True, is_close=True): + """ + Exact match of math if and only if: + 1. numerical equal: both can convert to float and are equal + 2. symbolic equal: both can convert to sympy expression and are equal + """ + if str(prediction) == str(reference): + return True + + try: # 1. numerical equal + if is_digit(prediction) and is_digit(reference): + prediction = parse_digits(prediction) + reference = parse_digits(reference) + # number questions + if include_percentage: + gt_result = [reference / 100, reference, reference * 100] + else: + gt_result = [reference] + for item in gt_result: + try: + if is_close: + if isclose(item, prediction, abs_tol=1e-3): + return True + else: + if item == prediction: + return True + except Exception: + continue + return False + except: + pass + + if not prediction and prediction not in [0, False]: + return False + + # 2. symbolic equal + reference = str(reference).strip() + prediction = str(prediction).strip() + + if ( + regex.match(r"(\(|\[).+(\)|\])", prediction) is not None + and regex.match(r"(\(|\[).+(\)|\])", reference) is not None + ): + pred_parts = prediction[1:-1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts): + if all( + [ + math_equal( + pred_parts[i], ref_parts[i], include_percentage, is_close + ) + for i in range(len(pred_parts)) + ] + ): + return True + + # Add back matrix comparison + if ( + ( + prediction.startswith("\\begin{pmatrix}") + or prediction.startswith("\\begin{bmatrix}") + ) + and ( + prediction.endswith("\\end{pmatrix}") + or prediction.endswith("\\end{bmatrix}") + ) + and ( + reference.startswith("\\begin{pmatrix}") + or reference.startswith("\\begin{bmatrix}") + ) + and ( + reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}") + ) + ): + pred_lines = [ + line.strip() + for line in prediction[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + ref_lines = [ + line.strip() + for line in reference[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + matched = True + if len(pred_lines) == len(ref_lines): + for pred_line, ref_line in zip(pred_lines, ref_lines): + pred_parts = pred_line.split("&") + ref_parts = ref_line.split("&") + if len(pred_parts) == len(ref_parts): + if not all( + [ + math_equal( + pred_parts[i], + ref_parts[i], + include_percentage, + is_close, + ) + for i in range(len(pred_parts)) + ] + ): + matched = False + break + else: + matched = False + if not matched: + break + else: + matched = False + if matched: + return True + + # Add back equation comparison + if prediction.count("=") == 1 and reference.count("=") == 1: + pred = prediction.split("=") + pred = f"{pred[0].strip()} - ({pred[1].strip()})" + ref = reference.split("=") + ref = f"{ref[0].strip()} - ({ref[1].strip()})" + if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref): + return True + elif ( + prediction.count("=") == 1 + and len(prediction.split("=")[0].strip()) <= 2 + and "=" not in reference + ): + if math_equal( + prediction.split("=")[1], reference, include_percentage, is_close + ): + return True + elif ( + reference.count("=") == 1 + and len(reference.split("=")[0].strip()) <= 2 + and "=" not in prediction + ): + if math_equal( + prediction, reference.split("=")[1], include_percentage, is_close + ): + return True + + # symbolic equal with sympy + if symbolic_equal(prediction, reference): + return True + + return False diff --git a/benchmark/tip_suggestion/.gitignore b/benchmark/tip_suggestion/.gitignore new file mode 100644 index 0000000..3322de9 --- /dev/null +++ b/benchmark/tip_suggestion/.gitignore @@ -0,0 +1 @@ +!topic.jsonl diff --git a/benchmark/tip_suggestion/README.md b/benchmark/tip_suggestion/README.md new file mode 100644 index 0000000..97ebe1d --- /dev/null +++ b/benchmark/tip_suggestion/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 64 +python3 bench_sglang.py --num-questions 32 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 64 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --backend lmql --num-questions 32 --parallel 1 +``` diff --git a/benchmark/tip_suggestion/bench_other.py b/benchmark/tip_suggestion/bench_other.py new file mode 100644 index 0000000..fcc4fd6 --- /dev/null +++ b/benchmark/tip_suggestion/bench_other.py @@ -0,0 +1,133 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +number = 5 + + +def expand_tip(topic, tip, generate): + s = ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + return generate(s, max_tokens=128, stop=["\n\n"]) + + +def suggest_tips(topic, generate): + s = "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + tips = [] + for i in range(1, 1 + number): + s += f"{i}." + tip = generate(s, max_tokens=24, stop=[".", "\n"]) + s += tip + ".\n" + tips.append(tip) + + paragraphs = [expand_tip(topic, tip, generate=generate) for tip in tips] + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1] + "\n" + return s + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + states = [None] * len(lines) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + tic = time.time() + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = suggest_tips(lines[i]["topic"], call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(lines))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(lines)))), + total=len(lines), + ) + ) + + else: + import asyncio + + from lmql_funcs import suggest_tips_async + + async def get_one_answer_async(i): + states[i] = await suggest_tips_async(lines[i]["topic"], call_generate) + + batches = [] + for i in range(0, len(lines), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(lines))))) + loop = asyncio.get_event_loop() + for batch in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in batch]) + ) + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "tip_suggestion", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="topic.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/tip_suggestion/bench_sglang.py b/benchmark/tip_suggestion/bench_sglang.py new file mode 100644 index 0000000..6d17821 --- /dev/null +++ b/benchmark/tip_suggestion/bench_sglang.py @@ -0,0 +1,100 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +number = 5 + + +@sgl.function +def expand_tip(s, topic, tip): + s += ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + s += sgl.gen("paragraph", max_tokens=128, stop=["\n\n"], temperature=0) + + +@sgl.function +def suggest_tips(s, topic): + s += "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + paragraphs = [] + for i in range(1, 1 + number): + s += f"{i}." + sgl.gen(f"tip_{i}", max_tokens=24, stop=[".", "\n"]) + ".\n" + paragraphs.append(expand_tip(topic=topic, tip=s[f"tip_{i}"])) + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1]["paragraph"] + "\n" + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"topic": l["topic"]} for l in lines] + + # Select backend + sgl.set_default_backend(select_sglang_backend(args)) + + # Run requests + tic = time.time() + states = suggest_tips.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.time() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "tip_suggestion", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="topic.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/tip_suggestion/lmql_funcs.py b/benchmark/tip_suggestion/lmql_funcs.py new file mode 100644 index 0000000..7790bbe --- /dev/null +++ b/benchmark/tip_suggestion/lmql_funcs.py @@ -0,0 +1,50 @@ +number = 5 + + +async def expand_tip_async(topic, tip, generate): + s = ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + return await generate(s, max_tokens=128, stop="\n\n") + + +async def suggest_tips_async(topic, generate): + s = "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + tips = [] + for i in range(1, 1 + number): + s += f"{i}." + # NOTE: stop is different due to lmql does not support a list of stop tokens + tip = await generate(s, max_tokens=24, stop=".\n") + s += tip + ".\n" + tips.append(tip) + + paragraphs = [await expand_tip_async(topic, tip, generate=generate) for tip in tips] + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1] + "\n" + return s diff --git a/benchmark/tip_suggestion/topic.jsonl b/benchmark/tip_suggestion/topic.jsonl new file mode 100644 index 0000000..c8ac6d1 --- /dev/null +++ b/benchmark/tip_suggestion/topic.jsonl @@ -0,0 +1,50 @@ +{"topic": "organizing a successful charity event", "number": 6} +{"topic": "improving personal credit scores", "number": 7} +{"topic": "staying motivated during job searches", "number": 5} +{"topic": "maintaining a work-life balance", "number": 9} +{"topic": "reducing carbon footprint at home", "number": 8} +{"topic": "starting a book club", "number": 5} +{"topic": "learning to play a musical instrument", "number": 7} +{"topic": "getting into freelance writing", "number": 6} +{"topic": "beginner yoga poses", "number": 8} +{"topic": "preparing for graduate school exams", "number": 5} +{"topic": "exploring minimalist living", "number": 9} +{"topic": "effective grocery shopping", "number": 7} +{"topic": "winter camping", "number": 5} +{"topic": "starting a podcast on a budget", "number": 8} +{"topic": "creating a capsule wardrobe", "number": 6} +{"topic": "improving your writing skills", "number": 7} +{"topic": "learning a new software quickly", "number": 9} +{"topic": "reducing anxiety before public speaking", "number": 5} +{"topic": "planning a solo travel adventure", "number": 8} +{"topic": "beginner skateboarders", "number": 6} +{"topic": "studying abroad", "number": 7} +{"topic": "planting a vegetable garden", "number": 5} +{"topic": "adopting a shelter pet", "number": 9} +{"topic": "learning to cook ethnic cuisines", "number": 8} +{"topic": "effective conflict resolution", "number": 5} +{"topic": "starting a vlog", "number": 7} +{"topic": "keeping a daily journal", "number": 6} +{"topic": "improving sleep hygiene", "number": 8} +{"topic": "beginner mountain climbers", "number": 5} +{"topic": "creating a mobile app", "number": 9} +{"topic": "maintaining a saltwater aquarium", "number": 7} +{"topic": "preparing for a baby's arrival", "number": 6} +{"topic": "writing a fantasy novel", "number": 5} +{"topic": "effective team leadership", "number": 8} +{"topic": "making a documentary film", "number": 9} +{"topic": "learning about historical events", "number": 7} +{"topic": "baking gluten-free treats", "number": 6} +{"topic": "improving mental arithmetic skills", "number": 5} +{"topic": "building a treehouse", "number": 8} +{"topic": "getting started with watercolor painting", "number": 9} +{"topic": "creating a YouTube tutorial series", "number": 7} +{"topic": "landscape photography", "number": 5} +{"topic": "navigating cultural differences", "number": 6} +{"topic": "preparing for a marathon", "number": 8} +{"topic": "building an online business", "number": 9} +{"topic": "learning to dance at home", "number": 5} +{"topic": "self-publishing a book", "number": 7} +{"topic": "starting an urban farm", "number": 6} +{"topic": "improving your memory", "number": 8} +{"topic": "creating a personal brand online", "number": 9} diff --git a/benchmark/tree_of_thought_deep/README.md b/benchmark/tree_of_thought_deep/README.md new file mode 100644 index 0000000..bf5ab16 --- /dev/null +++ b/benchmark/tree_of_thought_deep/README.md @@ -0,0 +1,51 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +NOTE: This is an implementation for throughput/latency benchmark purposes. The prompts are not tuned to achieve good accuracy on the GSM-8K tasks. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 32 +python3 bench_sglang.py --num-questions 16 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 8 --backend lmql --parallel 1 +``` diff --git a/benchmark/tree_of_thought_deep/bench_other.py b/benchmark/tree_of_thought_deep/bench_other.py new file mode 100644 index 0000000..21c7df3 --- /dev/null +++ b/benchmark/tree_of_thought_deep/bench_other.py @@ -0,0 +1,222 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.001 + + +def propose_plan(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def execute_plan(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def reflect_solution(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def get_final_answer(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def tree_search(question, num_branches, call_generate): + plan_forks = propose_plan("", question, num_branches, call_generate) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches, call_generate) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = reflect_solution(sol, num_branches, call_generate) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + ans = get_final_answer(sol, num_branches, call_generate) + solutions.append(ans) + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 2 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + states = [None] * len(questions) + + tic = time.time() + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = tree_search(**arguments[i], call_generate=call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + import asyncio + + from lmql_funcs import tree_search_async + + async def get_one_answer_async(i): + states[i] = await tree_search_async( + **arguments[i], call_generate=call_generate + ) + + batches = [ + [] for _ in range((len(questions) + args.parallel - 1) // args.parallel) + ] + for i in range(len(questions)): + batches[i // args.parallel].append(i) + + loop = asyncio.get_event_loop() + for bt in tqdm(batches): + tasks = [get_one_answer_async(k) for k in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.time() - tic + + answers_text = [] + for s in states: + answers_text.append([x for xs in s for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/tree_of_thought_deep/bench_sglang.py b/benchmark/tree_of_thought_deep/bench_sglang.py new file mode 100644 index 0000000..bfb2a41 --- /dev/null +++ b/benchmark/tree_of_thought_deep/bench_sglang.py @@ -0,0 +1,171 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.001 + + +def propose_plan(s, question, num_branches): + s += sgl.user( + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("plan", max_tokens=256, temperature=temp)) + return forks + + +def execute_plan(s, num_branches): + s += sgl.user( + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=temp)) + return forks + + +def reflect_solution(s, num_branches): + s += sgl.user( + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("score", max_tokens=256, temperature=temp)) + return forks + + +def get_final_answer(s, num_branches): + s += sgl.user( + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("final_answer", max_tokens=256, temperature=temp)) + return forks + + +@sgl.function +def tree_search(s, question, num_branches): + plan_forks = propose_plan(s, question, num_branches) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = reflect_solution(sol, num_branches) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + forks = get_final_answer(sol, num_branches) + solutions.append(forks) + solutions = [[s.text() for s in forks] for forks in solutions] + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + lines = list(lines) + + # Construct prompts + num_branches = 2 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.time() + states = tree_search.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + answers_text = [] + for s in states: + answers_text.append([x for xs in s.ret_value for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/tree_of_thought_deep/lmql_funcs.py b/benchmark/tree_of_thought_deep/lmql_funcs.py new file mode 100644 index 0000000..c783cdb --- /dev/null +++ b/benchmark/tree_of_thought_deep/lmql_funcs.py @@ -0,0 +1,82 @@ +from bench_other import ( + ASSISTANT_PREFIX, + ASSISTANT_SUFFIX, + USER_PREFIX, + USER_SUFFIX, + temp, +) + + +async def propose_plan_async(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def execute_plan_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def reflect_solution_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def get_final_answer_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def tree_search_async(question, num_branches, call_generate): + plan_forks = await propose_plan_async("", question, num_branches, call_generate) + + sol_states = [] + for plan in plan_forks: + forks = await execute_plan_async(plan, num_branches, call_generate) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = await reflect_solution_async(sol, num_branches, call_generate) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + ans = await get_final_answer_async(sol, num_branches, call_generate) + solutions.append(ans) + + return solutions diff --git a/benchmark/tree_of_thought_v0/README.md b/benchmark/tree_of_thought_v0/README.md new file mode 100644 index 0000000..821bb20 --- /dev/null +++ b/benchmark/tree_of_thought_v0/README.md @@ -0,0 +1,43 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 32 --parallel 16 +python3 bench_sglang.py --num-questions 10 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 32 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` diff --git a/benchmark/tree_of_thought_v0/bench_other.py b/benchmark/tree_of_thought_v0/bench_other.py new file mode 100644 index 0000000..86e1335 --- /dev/null +++ b/benchmark/tree_of_thought_v0/bench_other.py @@ -0,0 +1,179 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.3 + + +def propose_plan(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def execute_plan(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def reflect_solution(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now you evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def tree_search(question, num_branches, call_generate): + s = "" + solutions = [] + + plan_forks = propose_plan(s, question, num_branches, call_generate) + for plan in plan_forks: + sol_forks = execute_plan(plan, num_branches, call_generate) + for sol in sol_forks: + score_forks = reflect_solution(sol, num_branches, call_generate) + solutions.append(sol_forks) + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 3 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + states = [None] * len(questions) + + def get_one_answer(i): + states[i] = tree_search(**arguments[i], call_generate=call_generate) + + tic = time.time() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + latency = time.time() - tic + + answers_text = [] + for s in states: + answers_text.append([x for xs in s for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/tree_of_thought_v0/bench_sglang.py b/benchmark/tree_of_thought_v0/bench_sglang.py new file mode 100644 index 0000000..f0d1307 --- /dev/null +++ b/benchmark/tree_of_thought_v0/bench_sglang.py @@ -0,0 +1,159 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.3 + + +def propose_plan(s, question, num_branches): + s += sgl.user( + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("plan", max_tokens=256, temperature=temp)) + return forks + + +def execute_plan(s, num_branches): + s += sgl.user( + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=temp)) + return forks + + +def reflect_solution(s, num_branches): + s += sgl.user( + """Okay. Now you evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("score", max_tokens=256, temperature=temp)) + return forks + + +@sgl.function +def tree_search(s, question, num_branches): + forks_to_join = [] + + plan_forks = propose_plan(s, question, num_branches) + forks_to_join.append(plan_forks) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches) + forks_to_join.append(forks) + sol_states.extend(forks) + + for sol in sol_states: + forks = reflect_solution(sol, num_branches) + forks_to_join.append(forks) + + for f in reversed(forks_to_join): + f.join() + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 3 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.time() + states = tree_search.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.time() - tic + answers_text = [] + for s in states: + answers_text.append([x for xs in s["answer"] for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..60352a4 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,77 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +# 安装依赖（强制 IPv4） +RUN echo 'tzdata tzdata/Areas select Asia' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/Asia select Shanghai' | debconf-set-selections \ + && apt -o Acquire::ForceIPv4=true update -y \ + && apt -o Acquire::ForceIPv4=true install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt -o Acquire::ForceIPv4=true update \ + && apt -o Acquire::ForceIPv4=true install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 \ + && apt -o Acquire::ForceIPv4=true install python3.10-distutils -y \ + && apt -o Acquire::ForceIPv4=true install curl git sudo libibverbs-dev -y \ + && apt -o Acquire::ForceIPv4=true install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# 安装 datamodel_code_generator（用于 MiniCPM 模型） +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +# 拷贝 sglang 源代码并构建包 +COPY ./sglang /sgl-workspace/sglang + +# 拷贝模型文件（修正方式） +#COPY ./Alibaba/QwQ-32B /root/.cradle/Alibaba/QwQ-32B +COPY ./Llama-3.1-8B-Instruct /root/.cradle/Meta/Llama-3.1-8B-Instruct + +ARG CUDA_VERSION + +# 安装依赖、安装 sglang、安装 transformers，并清理源码 +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + CUINDEX=121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + CUINDEX=118; \ + python3 -m pip install --no-cache-dir sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && python3 -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m pip install --no-cache-dir psutil pyzmq pynvml \ + && cd /sgl-workspace/sglang/python \ + && python3 -m pip install --no-cache-dir '.[srt,openai]' --find-links https://flashinfer.ai/whl/cu${CUINDEX}/torch2.5/flashinfer-python \ + && cd / && rm -rf /sgl-workspace/sglang \ + && python3 -m pip install --no-cache-dir transformers==4.48.3 \ + && python3 -c "import sglang; print('✅ sglang module installed successfully')" + +# 暴露端口 +EXPOSE 30000 + + +# 启动命令（QwQ 模型为例） +CMD ["python3", "-m", "sglang.launch_server", \ + "--host","0.0.0.0", \ + "--port","30000", \ + "--model-path", "/root/.cradle/Meta/Llama-3.1-8B-Instruct/", \ + "--tp", "1", \ + "--api-key", "token-abc123"] + +# 设置回默认交互模式 +ENV DEBIAN_FRONTEND=interactive diff --git a/docker/Dockerfile.bak b/docker/Dockerfile.bak new file mode 100644 index 0000000..075b1e8 --- /dev/null +++ b/docker/Dockerfile.bak @@ -0,0 +1,48 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt update -y \ + && apt install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y && apt update \ + && apt install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 && apt install python3.10-distutils -y \ + && apt install curl git sudo libibverbs-dev -y \ + && apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# For openbmb/MiniCPM models +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +ARG CUDA_VERSION +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && git clone --depth=1 https://github.com/sgl-project/sglang.git \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + export CUINDEX=121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + export CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + export CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + export CUINDEX=118; \ + python3 -m pip install --no-cache-dir sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && python3 -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && cd sglang \ + && python3 -m pip --no-cache-dir install -e "python[${BUILD_TYPE}]" --find-links https://flashinfer.ai/whl/cu${CUINDEX}/torch2.5/flashinfer-python + +ENV DEBIAN_FRONTEND=interactive diff --git a/docker/Dockerfile.deepep b/docker/Dockerfile.deepep new file mode 100644 index 0000000..bd32e37 --- /dev/null +++ b/docker/Dockerfile.deepep @@ -0,0 +1,77 @@ +FROM lmsysorg/sglang:latest + +# CMake +RUN apt-get update \ +&& apt-get install -y --no-install-recommends \ +build-essential \ +wget \ +libssl-dev \ +&& wget https://github.com/Kitware/CMake/releases/download/v3.27.4/cmake-3.27.4-linux-x86_64.sh \ +&& chmod +x cmake-3.27.4-linux-x86_64.sh \ +&& ./cmake-3.27.4-linux-x86_64.sh --skip-license --prefix=/usr/local \ +&& rm cmake-3.27.4-linux-x86_64.sh + +# Python +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 \ + python3-pip \ + && ln -s /usr/bin/python3 /usr/bin/python + +# GDRCopy +WORKDIR /tmp +RUN git clone https://github.com/NVIDIA/gdrcopy.git +WORKDIR /tmp/gdrcopy +RUN git checkout v2.4.4 + +RUN apt update +RUN apt install -y nvidia-dkms-535 +RUN apt install -y build-essential devscripts debhelper fakeroot pkg-config dkms +RUN apt install -y check libsubunit0 libsubunit-dev + +WORKDIR /tmp/gdrcopy/packages +RUN CUDA=/usr/local/cuda ./build-deb-packages.sh +RUN dpkg -i gdrdrv-dkms_*.deb +RUN dpkg -i libgdrapi_*.deb +RUN dpkg -i gdrcopy-tests_*.deb +RUN dpkg -i gdrcopy_*.deb + +ENV GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ + +# IBGDA dependency +RUN ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so +RUN apt-get install -y libfabric-dev + +# DeepEP +WORKDIR /sgl-workspace +RUN git clone https://github.com/deepseek-ai/DeepEP.git + +# NVSHMEM +WORKDIR /sgl-workspace +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.2.5/source/nvshmem_src_3.2.5-1.txz +RUN tar -xf nvshmem_src_3.2.5-1.txz \ + && mv nvshmem_src nvshmem + +WORKDIR /sgl-workspace/nvshmem +RUN git apply /sgl-workspace/DeepEP/third-party/nvshmem.patch + +WORKDIR /sgl-workspace/nvshmem +ENV CUDA_HOME=/usr/local/cuda +RUN NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/sgl-workspace/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 \ + && cd build \ + && make install -j + +WORKDIR /sgl-workspace/DeepEP +ENV NVSHMEM_DIR=/sgl-workspace/nvshmem/install +RUN NVSHMEM_DIR=/sgl-workspace/nvshmem/install python setup.py install + +# Set workspace +WORKDIR /sgl-workspace diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 0000000..6faa6ff --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,224 @@ +# Extend from the base sglang image +FROM lmsysorg/sglang:latest + +# Install development tools and utilities +RUN apt-get update && apt-get install -y \ + gdb \ + ninja-build \ + vim \ + tmux \ + htop \ + wget \ + curl \ + locales \ + git \ + git-lfs \ + zsh \ + tree \ + silversearcher-ag \ + cloc \ + unzip \ + pkg-config \ + libssl-dev \ + bear \ + && apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +RUN apt update -y \ + && apt install -y --no-install-recommends gnupg \ + && echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu2004/amd64 /" | tee /etc/apt/sources.list.d/nvidia-devtools.list \ + && apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub \ + && apt update -y \ + && apt install nsight-systems-cli -y + +# Set up locale +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +# Install minimal Python packages +RUN python3 -m pip install --no-cache-dir \ + pytest \ + black \ + isort \ + icdiff \ + pre-commit + +# Install diff-so-fancy +RUN curl -LSso /usr/local/bin/diff-so-fancy https://github.com/so-fancy/diff-so-fancy/releases/download/v1.4.4/diff-so-fancy \ + && chmod +x /usr/local/bin/diff-so-fancy + +# Install clang-format +RUN curl -LSso /usr/local/bin/clang-format https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-32d3ac78/clang-format-16_linux-amd64 \ + && chmod +x /usr/local/bin/clang-format + +# Install clangd +RUN curl -L https://github.com/clangd/clangd/releases/download/18.1.3/clangd-linux-18.1.3.zip -o clangd.zip \ + && unzip clangd.zip \ + && cp -r clangd_18.1.3/bin/* /usr/local/bin/ \ + && cp -r clangd_18.1.3/lib/* /usr/local/lib/ \ + && rm -rf clangd_18.1.3 clangd.zip + +# Install CMake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.1/cmake-3.31.1-linux-x86_64.tar.gz \ + && tar -xzf cmake-3.31.1-linux-x86_64.tar.gz \ + && cp -r cmake-3.31.1-linux-x86_64/bin/* /usr/local/bin/ \ + && cp -r cmake-3.31.1-linux-x86_64/share/* /usr/local/share/ \ + && rm -rf cmake-3.31.1-linux-x86_64 cmake-3.31.1-linux-x86_64.tar.gz + +# Add yank script +COPY --chown=root:root <<-"EOF" /usr/local/bin/yank +#!/bin/bash +put() { + esc=$1 + test -n "$TMUX" -o -z "${TERM##screen*}" && esc="\033Ptmux;\033$esc\033\\" + printf "$esc" +} +put "\033]52;c;!\a" +buf=$( cat "$@" ) +len=$( printf %s "$buf" | wc -c ) max=74994 +test $len -gt $max && echo "$0: input is $(( len - max )) bytes too long" >&2 +put "\033]52;c;$( printf %s "$buf" | head -c $max | base64 | tr -d '\r\n' )\a" +test -n "$TMUX" && tmux set-buffer "$buf" ||: +EOF + +RUN chmod +x /usr/local/bin/yank + +# Install oh-my-zsh and plugins +RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \ + && git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \ + && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting + +# Configure Vim +COPY --chown=root:root <<-"EOF" /root/.vimrc +function! Yank(text) abort + let escape = system('yank', a:text) + if v:shell_error + echoerr escape + else + call writefile([escape], '/dev/tty', 'b') + endif +endfunction + +noremap y y:call Yank(@0) + +" automatically run yank(1) whenever yanking in Vim +function! CopyYank() abort + call Yank(join(v:event.regcontents, "\n")) +endfunction + +autocmd TextYankPost * call CopyYank() + +" Basic settings +set number +syntax on +set mouse=a +filetype indent on + +" Indentation +set autoindent nosmartindent +set smarttab +set expandtab +set shiftwidth=4 +set softtabstop=4 + +" Visual guides +set colorcolumn=120 +highlight ColorColumn ctermbg=5 + +" Status line +set laststatus=2 +set statusline=%<%f\ %h%m%r%=%{\"[\".(&fenc==\"\"?&enc:&fenc).((exists(\"+bomb\")\ &&\ &bomb)?\",B\":\"\").\"]\ \"}%k\ %-14.(%l,%c%V%)\ %P + +" Backspace behavior +set backspace=2 + +" Encoding +set encoding=utf-8 +set fileencoding=utf-8 +EOF + +# Configure tmux +COPY --chown=root:root <<-"EOF" /root/.tmux.conf +# Pane border styling +set -g pane-border-style fg='#742727',bg=black +set -g pane-active-border-style fg=red,bg=black + +# Status bar styling +set -g status-style bg='#0C8A92',fg=black + +# Change prefix key to backtick +set-option -g prefix ` +unbind C-b +bind-key ` send-prefix + +# Split panes using - and = with current path +unbind '"' +bind - splitw -v -c '#{pane_current_path}' +unbind '%' +bind = splitw -h -c '#{pane_current_path}' + +# Vi mode settings +bind-key -T copy-mode-vi Y send-keys -X copy-pipe 'yank > #{pane_tty}' +set-window-option -g mode-keys vi + +# Other settings +set-option -g escape-time 0 +set-option -g base-index 1 +set-window-option -g mouse on +EOF + +# Configure Git +RUN git config --global core.editor "vim" \ + && git config --global core.whitespace "fix,-indent-with-non-tab,trailing-space,cr-at-eol" \ + && git config --global core.pager "diff-so-fancy | less --tabs=4 -RFX" \ + && git config --global color.ui true \ + && git config --global color."diff-highlight".oldNormal "red bold" \ + && git config --global color."diff-highlight".oldHighlight "red bold 52" \ + && git config --global color."diff-highlight".newNormal "green bold" \ + && git config --global color."diff-highlight".newHighlight "green bold 22" \ + && git config --global color.diff.meta "11" \ + && git config --global color.diff.frag "magenta bold" \ + && git config --global color.diff.commit "yellow bold" \ + && git config --global color.diff.old "red bold" \ + && git config --global color.diff.new "green bold" \ + && git config --global color.diff.whitespace "red reverse" \ + && git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset - %s %Cgreen(%cr) %C(bold blue)<%an>%Creset%C(auto)%d%Creset' --abbrev-commit --" \ + && git config --global http.sslVerify false \ + && git config --global pull.rebase true + +# Configure zsh +COPY --chown=root:root <<-"EOF" /root/.zshrc +export ZSH="/root/.oh-my-zsh" + +# Theme +ZSH_THEME="robbyrussell" + +# Plugins +plugins=( + git + z + zsh-autosuggestions + zsh-syntax-highlighting +) + +source $ZSH/oh-my-zsh.sh + +# Aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' +alias vi='vim' + +# Enhanced history +HISTSIZE=10000 +SAVEHIST=10000 +setopt HIST_IGNORE_ALL_DUPS +setopt HIST_FIND_NO_DUPS +setopt INC_APPEND_HISTORY +EOF + +# Set workspace directory +WORKDIR /sgl-workspace/sglang diff --git a/docker/Dockerfile.ds-14b b/docker/Dockerfile.ds-14b new file mode 100644 index 0000000..6674347 --- /dev/null +++ b/docker/Dockerfile.ds-14b @@ -0,0 +1,77 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +# 安装依赖（强制 IPv4） +RUN echo 'tzdata tzdata/Areas select Asia' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/Asia select Shanghai' | debconf-set-selections \ + && apt -o Acquire::ForceIPv4=true update -y \ + && apt -o Acquire::ForceIPv4=true install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt -o Acquire::ForceIPv4=true update \ + && apt -o Acquire::ForceIPv4=true install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 \ + && apt -o Acquire::ForceIPv4=true install python3.10-distutils -y \ + && apt -o Acquire::ForceIPv4=true install curl git sudo libibverbs-dev -y \ + && apt -o Acquire::ForceIPv4=true install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# 安装 datamodel_code_generator（用于 MiniCPM 模型） +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +# 拷贝 sglang 源代码并构建包 +COPY ./sglang /sgl-workspace/sglang + +# 拷贝模型文件（修正方式） +#COPY ./Alibaba/QwQ-32B /root/.cradle/Alibaba/QwQ-32B +COPY ./DeepSeek/DeepSeek-R1-Distill-Qwen-14B /root/.cradle/DeepSeek/DeepSeek-R1-Distill-Qwen-14B + +ARG CUDA_VERSION + +# 安装依赖、安装 sglang、安装 transformers，并清理源码 +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + CUINDEX=121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + CUINDEX=118; \ + python3 -m pip install --no-cache-dir sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && python3 -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m pip install --no-cache-dir psutil pyzmq pynvml \ + && cd /sgl-workspace/sglang/python \ + && python3 -m pip install --no-cache-dir '.[srt,openai]' --find-links https://flashinfer.ai/whl/cu${CUINDEX}/torch2.5/flashinfer-python \ + && cd / && rm -rf /sgl-workspace/sglang \ + && python3 -m pip install --no-cache-dir transformers==4.48.3 \ + && python3 -c "import sglang; print('✅ sglang module installed successfully')" + +# 暴露端口 +EXPOSE 30000 + + +# 启动命令（QwQ 模型为例） +CMD ["python3", "-m", "sglang.launch_server", \ + "--host","0.0.0.0", \ + "--port","30000", \ + "--model-path", "/root/.cradle/DeepSeek/DeepSeek-R1-Distill-Qwen-14B/", \ + "--tp", "2", \ + "--api-key", "token-abc123"] + +# 设置回默认交互模式 +ENV DEBIAN_FRONTEND=interactive diff --git a/docker/Dockerfile.ds-8b b/docker/Dockerfile.ds-8b new file mode 100644 index 0000000..a6c93be --- /dev/null +++ b/docker/Dockerfile.ds-8b @@ -0,0 +1,77 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +# 安装依赖（强制 IPv4） +RUN echo 'tzdata tzdata/Areas select Asia' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/Asia select Shanghai' | debconf-set-selections \ + && apt -o Acquire::ForceIPv4=true update -y \ + && apt -o Acquire::ForceIPv4=true install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt -o Acquire::ForceIPv4=true update \ + && apt -o Acquire::ForceIPv4=true install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 \ + && apt -o Acquire::ForceIPv4=true install python3.10-distutils -y \ + && apt -o Acquire::ForceIPv4=true install curl git sudo libibverbs-dev -y \ + && apt -o Acquire::ForceIPv4=true install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# 安装 datamodel_code_generator（用于 MiniCPM 模型） +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +# 拷贝 sglang 源代码并构建包 +COPY ./sglang /sgl-workspace/sglang + +# 拷贝模型文件（修正方式） +#COPY ./Alibaba/QwQ-32B /root/.cradle/Alibaba/QwQ-32B +COPY ./DeepSeek/DeepSeek-R1-Distill-Llama-8B /root/.cradle/DeepSeek/DeepSeek-R1-Distill-Llama-8B + +ARG CUDA_VERSION + +# 安装依赖、安装 sglang、安装 transformers，并清理源码 +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + CUINDEX=121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + CUINDEX=118; \ + python3 -m pip install --no-cache-dir sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && python3 -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m pip install --no-cache-dir psutil pyzmq pynvml \ + && cd /sgl-workspace/sglang/python \ + && python3 -m pip install --no-cache-dir '.[srt,openai]' --find-links https://flashinfer.ai/whl/cu${CUINDEX}/torch2.5/flashinfer-python \ + && cd / && rm -rf /sgl-workspace/sglang \ + && python3 -m pip install --no-cache-dir transformers==4.48.3 \ + && python3 -c "import sglang; print('✅ sglang module installed successfully')" + +# 暴露端口 +EXPOSE 30000 + + +# 启动命令（QwQ 模型为例） +CMD ["python3", "-m", "sglang.launch_server", \ + "--host","0.0.0.0", \ + "--port","30000", \ + "--model-path", "/root/.cradle/DeepSeek/DeepSeek-R1-Distill-Llama-8B/", \ + "--tp", "1", \ + "--api-key", "token-abc123"] + +# 设置回默认交互模式 +ENV DEBIAN_FRONTEND=interactive diff --git a/docker/Dockerfile.llama3.1-8b b/docker/Dockerfile.llama3.1-8b new file mode 100644 index 0000000..60352a4 --- /dev/null +++ b/docker/Dockerfile.llama3.1-8b @@ -0,0 +1,77 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +# 安装依赖（强制 IPv4） +RUN echo 'tzdata tzdata/Areas select Asia' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/Asia select Shanghai' | debconf-set-selections \ + && apt -o Acquire::ForceIPv4=true update -y \ + && apt -o Acquire::ForceIPv4=true install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt -o Acquire::ForceIPv4=true update \ + && apt -o Acquire::ForceIPv4=true install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 \ + && apt -o Acquire::ForceIPv4=true install python3.10-distutils -y \ + && apt -o Acquire::ForceIPv4=true install curl git sudo libibverbs-dev -y \ + && apt -o Acquire::ForceIPv4=true install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# 安装 datamodel_code_generator（用于 MiniCPM 模型） +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +# 拷贝 sglang 源代码并构建包 +COPY ./sglang /sgl-workspace/sglang + +# 拷贝模型文件（修正方式） +#COPY ./Alibaba/QwQ-32B /root/.cradle/Alibaba/QwQ-32B +COPY ./Llama-3.1-8B-Instruct /root/.cradle/Meta/Llama-3.1-8B-Instruct + +ARG CUDA_VERSION + +# 安装依赖、安装 sglang、安装 transformers，并清理源码 +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + CUINDEX=121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + CUINDEX=118; \ + python3 -m pip install --no-cache-dir sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && python3 -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m pip install --no-cache-dir psutil pyzmq pynvml \ + && cd /sgl-workspace/sglang/python \ + && python3 -m pip install --no-cache-dir '.[srt,openai]' --find-links https://flashinfer.ai/whl/cu${CUINDEX}/torch2.5/flashinfer-python \ + && cd / && rm -rf /sgl-workspace/sglang \ + && python3 -m pip install --no-cache-dir transformers==4.48.3 \ + && python3 -c "import sglang; print('✅ sglang module installed successfully')" + +# 暴露端口 +EXPOSE 30000 + + +# 启动命令（QwQ 模型为例） +CMD ["python3", "-m", "sglang.launch_server", \ + "--host","0.0.0.0", \ + "--port","30000", \ + "--model-path", "/root/.cradle/Meta/Llama-3.1-8B-Instruct/", \ + "--tp", "1", \ + "--api-key", "token-abc123"] + +# 设置回默认交互模式 +ENV DEBIAN_FRONTEND=interactive diff --git a/docker/Dockerfile.qwq-32b b/docker/Dockerfile.qwq-32b new file mode 100644 index 0000000..1e10283 --- /dev/null +++ b/docker/Dockerfile.qwq-32b @@ -0,0 +1,77 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +# 安装依赖（强制 IPv4） +RUN echo 'tzdata tzdata/Areas select Asia' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/Asia select Shanghai' | debconf-set-selections \ + && apt -o Acquire::ForceIPv4=true update -y \ + && apt -o Acquire::ForceIPv4=true install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt -o Acquire::ForceIPv4=true update \ + && apt -o Acquire::ForceIPv4=true install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 \ + && apt -o Acquire::ForceIPv4=true install python3.10-distutils -y \ + && apt -o Acquire::ForceIPv4=true install curl git sudo libibverbs-dev -y \ + && apt -o Acquire::ForceIPv4=true install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \ + && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# 安装 datamodel_code_generator（用于 MiniCPM 模型） +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +# 拷贝 sglang 源代码并构建包 +COPY ./sglang /sgl-workspace/sglang + +# 拷贝模型文件（修正方式） +#COPY ./Alibaba/QwQ-32B /root/.cradle/Alibaba/QwQ-32B +COPY ./Alibaba/QwQ-32B /root/.cradle/Alibaba/QwQ-32B + +ARG CUDA_VERSION + +# 安装依赖、安装 sglang、安装 transformers，并清理源码 +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + CUINDEX=121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + CUINDEX=124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + CUINDEX=118; \ + python3 -m pip install --no-cache-dir sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && python3 -m pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m pip install --no-cache-dir psutil pyzmq pynvml \ + && cd /sgl-workspace/sglang/python \ + && python3 -m pip install --no-cache-dir '.[srt,openai]' --find-links https://flashinfer.ai/whl/cu${CUINDEX}/torch2.5/flashinfer-python \ + && cd / && rm -rf /sgl-workspace/sglang \ + && python3 -m pip install --no-cache-dir transformers==4.48.3 \ + && python3 -c "import sglang; print('✅ sglang module installed successfully')" + +# 暴露端口 +EXPOSE 30000 + + +# 启动命令（QwQ 模型为例） +CMD ["python3", "-m", "sglang.launch_server", \ + "--host","0.0.0.0", \ + "--port","30000", \ + "--model-path", "/root/.cradle/Alibaba/QwQ-32B/", \ + "--tp", "4", \ + "--api-key", "token-abc123"] + +# 设置回默认交互模式 +ENV DEBIAN_FRONTEND=interactive diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm new file mode 100644 index 0000000..0dde646 --- /dev/null +++ b/docker/Dockerfile.rocm @@ -0,0 +1,85 @@ +# Usage (to build SGLang ROCm docker image): +# docker build --build-arg SGL_BRANCH=v0.4.4.post3 -t v0.4.4.post3-rocm630 -f Dockerfile.rocm . + +# default base image +ARG BASE_IMAGE="rocm/sgl-dev:vllm20250114" + +FROM $BASE_IMAGE AS base +USER root + +WORKDIR /sgl-workspace +ARG BUILD_TYPE=all +ARG SGL_REPO="https://github.com/sgl-project/sglang" +ENV SGL_DEFAULT="main" +ARG SGL_BRANCH=${SGL_DEFAULT} + +ARG TRITON_REPO="https://github.com/ROCm/triton.git" +ARG TRITON_COMMIT="improve_fa_decode_3.0.0" + + +ARG AITER_REPO="https://github.com/ROCm/aiter.git" +ARG AITER_COMMIT="testx" + +RUN git clone ${SGL_REPO} \ + && cd sglang \ + && if [ "${SGL_BRANCH}" = ${SGL_DEFAULT} ]; then \ + echo "Using ${SGL_DEFAULT}, default branch."; \ + else \ + echo "Using ${SGL_BRANCH} branch."; \ + git checkout ${SGL_BRANCH}; \ + fi \ + && cd sgl-kernel \ + && rm -f pyproject.toml \ + && mv pyproject_rocm.toml pyproject.toml \ + && python setup_rocm.py install \ + && cd .. \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + python -m pip --no-cache-dir install -e "python[srt_hip]"; \ + else \ + python -m pip --no-cache-dir install -e "python[all_hip]"; \ + fi + +RUN cp -r /sgl-workspace/sglang /sglang +RUN python -m pip cache purge + +RUN pip install IPython \ + && pip install orjson \ + && pip install python-multipart \ + && pip install torchao \ + && pip install pybind11 + +RUN pip uninstall -y triton +RUN git clone ${TRITON_REPO} \ + && cd triton \ + && git checkout ${TRITON_COMMIT} \ + && cd python \ + && python3 setup.py install + +RUN git clone ${AITER_REPO} \ + && cd aiter \ + && git checkout ${AITER_COMMIT} \ + && git submodule update --init --recursive \ + && PREBUILD_KERNELS=1 GPU_ARCHS=gfx942 python3 setup.py develop + +# Copy config files to support MI300X in virtualized environments (MI300X_VF). Symlinks will not be created in image build. +RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ + /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ + -type f -name '*MI300X*' | xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} + +# Performance environment variable. + +ENV HIP_FORCE_DEV_KERNARG=1 +ENV HSA_NO_SCRATCH_RECLAIM=1 +ENV SGLANG_SET_CPU_AFFINITY=1 +ENV SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 +ENV NCCL_MIN_NCHANNELS=112 + +ENV MOE_PADDING=1 +ENV VLLM_FP8_PADDING=1 +ENV VLLM_FP8_ACT_PADDING=1 +ENV VLLM_FP8_WEIGHT_PADDING=1 +ENV VLLM_FP8_REDUCE_CONV=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1 + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.sagemaker b/docker/Dockerfile.sagemaker new file mode 100644 index 0000000..fde8d55 --- /dev/null +++ b/docker/Dockerfile.sagemaker @@ -0,0 +1,78 @@ +ARG CUDA_VERSION=12.5.1 + +FROM nvcr.io/nvidia/tritonserver:24.04-py3-min + +ARG BUILD_TYPE=all +ENV DEBIAN_FRONTEND=noninteractive + +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt update -y \ + && apt install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y && apt update \ + && apt install python3.10 python3.10-dev -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --set python3 /usr/bin/python3.10 && apt install python3.10-distutils -y \ + && apt install curl git sudo libibverbs-dev -y \ + && apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py \ + && python3 --version \ + && python3 -m pip --version \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# For openbmb/MiniCPM models +RUN pip3 install datamodel_code_generator + +WORKDIR /sgl-workspace + +ARG CUDA_VERSION +RUN python3 -m pip install --upgrade pip setuptools wheel html5lib six \ + && git clone --depth=1 https://github.com/sgl-project/sglang.git \ + && if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + python3 -m pip install torch --index-url https://download.pytorch.org/whl/cu121; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + python3 -m pip install torch --index-url https://download.pytorch.org/whl/cu124; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + python3 -m pip install torch --index-url https://download.pytorch.org/whl/cu124; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + python3 -m pip install torch --index-url https://download.pytorch.org/whl/cu118; \ + python3 -m pip install sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi \ + && cd sglang \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + python3 -m pip --no-cache-dir install -e "python[srt]" --find-links https://flashinfer.ai/whl/cu121/torch2.5/flashinfer-python; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + python3 -m pip --no-cache-dir install -e "python[srt]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + python3 -m pip --no-cache-dir install -e "python[srt]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + python3 -m pip --no-cache-dir install -e "python[srt]" --find-links https://flashinfer.ai/whl/cu118/torch2.5/flashinfer-python; \ + python3 -m pip install sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi; \ + else \ + if [ "$CUDA_VERSION" = "12.1.1" ]; then \ + python3 -m pip --no-cache-dir install -e "python[all]" --find-links https://flashinfer.ai/whl/cu121/torch2.5/flashinfer-python; \ + elif [ "$CUDA_VERSION" = "12.4.1" ]; then \ + python3 -m pip --no-cache-dir install -e "python[all]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python; \ + elif [ "$CUDA_VERSION" = "12.5.1" ]; then \ + python3 -m pip --no-cache-dir install -e "python[all]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python; \ + elif [ "$CUDA_VERSION" = "11.8.0" ]; then \ + python3 -m pip --no-cache-dir install -e "python[all]" --find-links https://flashinfer.ai/whl/cu118/torch2.5/flashinfer-python; \ + python3 -m pip install sgl-kernel -i https://docs.sglang.ai/whl/cu118; \ + else \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1; \ + fi; \ + fi + +ENV DEBIAN_FRONTEND=interactive + +COPY serve /usr/bin/serve +RUN chmod 777 /usr/bin/serve + +ENTRYPOINT [ "/usr/bin/serve" ] diff --git a/docker/compose.yaml b/docker/compose.yaml new file mode 100644 index 0000000..f7ff1fb --- /dev/null +++ b/docker/compose.yaml @@ -0,0 +1,35 @@ +services: + sglang: + image: lmsysorg/sglang:latest + container_name: sglang + volumes: + - ${HOME}/.cache/huggingface:/root/.cache/huggingface + # If you use modelscope, you need mount this directory + # - ${HOME}/.cache/modelscope:/root/.cache/modelscope + restart: always + network_mode: host # required by RDMA + privileged: true # required by RDMA + # Or you can only publish port 30000 + # ports: + # - 30000:30000 + environment: + HF_TOKEN: + # if you use modelscope to download model, you need set this environment + # - SGLANG_USE_MODELSCOPE: true + entrypoint: python3 -m sglang.launch_server + command: --model-path meta-llama/Llama-3.1-8B-Instruct + --host 0.0.0.0 + --port 30000 + ulimits: + memlock: -1 + stack: 67108864 + ipc: host + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["0"] + capabilities: [gpu] diff --git a/docker/k8s-sglang-distributed-sts.yaml b/docker/k8s-sglang-distributed-sts.yaml new file mode 100644 index 0000000..6b81d9b --- /dev/null +++ b/docker/k8s-sglang-distributed-sts.yaml @@ -0,0 +1,104 @@ +# Two Nodes Sglang example + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: distributed-sglang +spec: + replicas: 2 # number of nodes/pods to run distributed sglang + selector: + matchLabels: + app: distributed-sglang + serviceName: "" + template: + metadata: + labels: + app: distributed-sglang + spec: + containers: + - name: sglang-container + image: docker.io/lmsysorg/sglang:latest + imagePullPolicy: Always # image may be replaced by official CI versioned image + command: + - /bin/bash + - -c + # please modify the sglang serving arguments below, as necessary. + # NOTE: the --expert-parallel-size and --enable-ep-moe are for MoE model like DeepSeek-R1 + args: + - | + python3 -m sglang.launch_server \ + --model /llm-folder \ + --dist-init-addr sglang-master-pod:5000 \ + --tensor-parallel-size 16 \ + --nnodes 2 \ + --node-rank $POD_INDEX \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 8000 \ + --enable-metrics \ + --enable-ep-moe \ + --expert-parallel-size 16 + env: + - name: POD_INDEX # reflects the node-rank + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: NCCL_DEBUG + value: INFO + resources: + limits: + nvidia.com/gpu: "8" + requests: + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /llm-folder + name: llm + securityContext: + privileged: true # to leverage RDMA/InfiniBand device, co-work with HostNetwork=true + hostNetwork: true + volumes: + - emptyDir: + medium: Memory + sizeLimit: 10Gi + name: dshm + - hostPath: + path: /llm-folder # replace with PVC or hostPath with your model weights + type: DirectoryOrCreate + name: llm + #- persistentVolumeClaim: + # claimName: llm-pvc + # name: llm +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-master-pod +spec: + type: ClusterIP + selector: + app: distributed-sglang + apps.kubernetes.io/pod-index: "0" + ports: + - name: dist-port + port: 5000 + targetPort: 5000 +--- +# the serving service +apiVersion: v1 +kind: Service +metadata: + name: sglang-serving-on-master +spec: + type: NodePort + selector: + app: distributed-sglang + apps.kubernetes.io/pod-index: "0" + ports: + - name: serving + port: 8000 + targetPort: 8000 + - name: metrics + port: 8080 + targetPort: 8080 diff --git a/docker/k8s-sglang-service.yaml b/docker/k8s-sglang-service.yaml new file mode 100644 index 0000000..6f6f9af --- /dev/null +++ b/docker/k8s-sglang-service.yaml @@ -0,0 +1,81 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: meta-llama-31-8b-instruct-sglang +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: meta-llama-31-8b-instruct-sglang + template: + metadata: + labels: + app: meta-llama-31-8b-instruct-sglang + model: meta-llama-31-8b-instruct + engine: sglang + spec: + restartPolicy: Always + runtimeClassName: nvidia + containers: + - name: meta-llama-31-8b-instruct-sglang + image: docker.io/lmsysorg/sglang:latest + imagePullPolicy: Always # IfNotPresent or Never + ports: + - containerPort: 30000 + command: ["python3", "-m", "sglang.launch_server"] + args: ["--model-path", "meta-llama/Llama-3.1-8B-Instruct", "--host", "0.0.0.0", "--port", "30000"] + env: + - name: HF_TOKEN + value: + resources: + limits: + nvidia.com/gpu: 1 + volumeMounts: + - name: shm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + readOnly: true + - name: localtime + mountPath: /etc/localtime + readOnly: true + livenessProbe: + httpGet: + path: /health + port: 30000 + initialDelaySeconds: 30 + periodSeconds: 10 + volumes: + - name: shm + emptyDir: + medium: Memory + sizeLimit: 10Gi + - name: hf-cache + hostPath: + path: /root/.cache/huggingface + type: Directory + - name: localtime + hostPath: + path: /etc/localtime + type: File +--- +apiVersion: v1 +kind: Service +metadata: + name: meta-llama-31-8b-instruct-sglang +spec: + selector: + app: meta-llama-31-8b-instruct-sglang + ports: + - protocol: TCP + port: 30000 # port on host + targetPort: 30000 # port in container + type: LoadBalancer diff --git a/docker/serve b/docker/serve new file mode 100644 index 0000000..493ecbd --- /dev/null +++ b/docker/serve @@ -0,0 +1,31 @@ +#!/bin/bash + +echo "Starting server" + +SERVER_ARGS="--host 0.0.0.0 --port 8080" + +if [ -n "$TENSOR_PARALLEL_DEGREE" ]; then + SERVER_ARGS="${SERVER_ARGS} --tp-size ${TENSOR_PARALLEL_DEGREE}" +fi + +if [ -n "$DATA_PARALLEL_DEGREE" ]; then + SERVER_ARGS="${SERVER_ARGS} --dp-size ${DATA_PARALLEL_DEGREE}" +fi + +if [ -n "$EXPERT_PARALLEL_DEGREE" ]; then + SERVER_ARGS="${SERVER_ARGS} --ep-size ${EXPERT_PARALLEL_DEGREE}" +fi + +if [ -n "$MEM_FRACTION_STATIC" ]; then + SERVER_ARGS="${SERVER_ARGS} --mem-fraction-static ${MEM_FRACTION_STATIC}" +fi + +if [ -n "$QUANTIZATION" ]; then + SERVER_ARGS="${SERVER_ARGS} --quantization ${QUANTIZATION}" +fi + +if [ -n "$CHUNKED_PREFILL_SIZE" ]; then + SERVER_ARGS="${SERVER_ARGS} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE}" +fi + +python3 -m sglang.launch_server --model-path /opt/ml/model $SERVER_ARGS diff --git a/docker/start.sh b/docker/start.sh new file mode 100755 index 0000000..492b40a --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,6 @@ +docker run -d \ + -p 30000:30000 \ + --gpus all \ + --shm-size=2g \ + --name alibaba_qwq-32b \ + alibaba_qwq-32b diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..0d6cad1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,57 @@ +# Minimal Makefile for Sphinx documentation +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD ?= sphinx-autobuild +SOURCEDIR = . +BUILDDIR = _build +PORT ?= 8003 + +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @echo "" + @echo "Additional targets:" + @echo " serve to build and serve documentation with auto-build and live reload" + +# Compile Notebook files and record execution time +compile: + @set -e; \ + echo "Starting Notebook compilation..."; \ + mkdir -p logs; \ + echo "Notebook execution timings:" > logs/timing.log; \ + START_TOTAL=$$(date +%s); \ + find $(SOURCEDIR) -path "*/_build/*" -prune -o -name "*.ipynb" -print0 | \ + parallel -0 -j3 --halt soon,fail=1 ' \ + NB_NAME=$$(basename {}); \ + START_TIME=$$(date +%s); \ + retry --delay=0 --times=3 -- \ + jupyter nbconvert --to notebook --execute --inplace "{}" \ + --ExecutePreprocessor.timeout=600 \ + --ExecutePreprocessor.kernel_name=python3; \ + RET_CODE=$$?; \ + END_TIME=$$(date +%s); \ + ELAPSED_TIME=$$((END_TIME - START_TIME)); \ + echo "$${NB_NAME}: $${ELAPSED_TIME}s" >> logs/timing.log; \ + exit $$RET_CODE' || exit 1; \ + END_TOTAL=$$(date +%s); \ + TOTAL_ELAPSED=$$((END_TOTAL - START_TOTAL)); \ + echo "---------------------------------" >> logs/timing.log; \ + echo "Total execution time: $${TOTAL_ELAPSED}s" >> logs/timing.log; \ + echo "All Notebook execution timings:" && cat logs/timing.log + +# Serve documentation with auto-build and live reload +serve: + @echo "Starting auto-build server at http://localhost:$(PORT)" + @$(SPHINXAUTOBUILD) "$(SOURCEDIR)" "$(BUILDDIR)/html" \ + --port $(PORT) \ + --watch $(SOURCEDIR) \ + --re-ignore ".*\.(ipynb_checkpoints|pyc|pyo|pyd|git)" + +.PHONY: help Makefile compile clean serve + +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + find . -name "*.ipynb" -exec nbstripout {} \; + rm -rf $(BUILDDIR) + rm -rf logs diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..0dfadb1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,110 @@ +# SGLang Documentation + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. Most documentation files are located under the `docs/` folder. We prefer **Jupyter Notebooks** over Markdown so that all examples can be executed and validated by our docs CI pipeline. + +## Docs Workflow + +### Install Dependency + +```bash +pip install -r requirements.txt +``` + +### Update Documentation + +Update your Jupyter notebooks in the appropriate subdirectories under `docs/`. If you add new files, remember to update `index.rst` (or relevant `.rst` files) accordingly. + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. +- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. + +```bash +# 1) Compile all Jupyter notebooks +make compile + +# 2) Compile and Preview documentation locally with auto-build +# This will automatically rebuild docs when files change +# Open your browser at the displayed port to view the docs +bash serve.sh + +# 2a) Alternative ways to serve documentation +# Directly use make serve +make serve +# With custom port +PORT=8080 make serve + +# 3) Clean notebook outputs +# nbstripout removes notebook outputs so your PR stays clean +pip install nbstripout +find . -name '*.ipynb' -exec nbstripout {} \; + +# 4) Pre-commit checks and create a PR +# After these checks pass, push your changes and open a PR on your branch +pre-commit run --all-files +``` +--- + +### **Port Allocation and CI Efficiency** + +**To launch and kill the server:** + +```python +from sglang.test.test_utils import is_in_ci +from sglang.utils import wait_for_server, print_highlight, terminate_process + +if is_in_ci(): + from patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + +server_process, port = launch_server_cmd( + """ +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --host 0.0.0.0 +""" +) + +wait_for_server(f"http://localhost:{port}") + +# Terminate Server +terminate_process(server_process) +``` + +**To launch and kill the engine:** + +```python +# Launch Engine +import sglang as sgl +import asyncio +from sglang.test.test_utils import is_in_ci + +if is_in_ci(): + import patch + +llm = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + +# Terminalte Engine +llm.shutdown() +``` + +### **Why this approach?** + +- **Dynamic Port Allocation**: Avoids port conflicts by selecting an available port at runtime, enabling multiple server instances to run in parallel. +- **Optimized for CI**: The `patch` version of `launch_server_cmd` and `sgl.Engine()` in CI environments helps manage GPU memory dynamically, preventing conflicts and improving test parallelism. +- **Better Parallel Execution**: Ensures smooth concurrent tests by avoiding fixed port collisions and optimizing memory usage. + +### **Model Selection** + +For demonstrations in the docs, **prefer smaller models** to reduce memory consumption and speed up inference. Running larger models in CI can lead to instability due to memory constraints. + +### **Prompt Alignment Example** + +When designing prompts, ensure they align with SGLang's structured formatting. For example: + +```python +prompt = """You are an AI assistant. Answer concisely and accurately. + +User: What is the capital of France? +Assistant: The capital of France is Paris.""" +``` + +This keeps responses aligned with expected behavior and improves reliability across different files. diff --git a/docs/_static/css/custom_log.css b/docs/_static/css/custom_log.css new file mode 100644 index 0000000..61f65d0 --- /dev/null +++ b/docs/_static/css/custom_log.css @@ -0,0 +1,29 @@ +.output_area { + color: #615656; +} + +table.autosummary td { + width: 50% + } + + img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +.output_area.stderr { + color: #d3d3d3 !important; +} + +.output_area.stdout { + color: #d3d3d3 !important; +} + +div.output_area.stderr { + color: #d3d3d3 !important; +} + +div.output_area.stdout { + color: #d3d3d3 !important; +} diff --git a/docs/_static/css/readthedocs.css b/docs/_static/css/readthedocs.css new file mode 100644 index 0000000..aca6649 --- /dev/null +++ b/docs/_static/css/readthedocs.css @@ -0,0 +1,9 @@ +table.autosummary td { + width: 50% +} + +img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} diff --git a/docs/_static/image/logo.ico b/docs/_static/image/logo.ico new file mode 100644 index 0000000000000000000000000000000000000000..7a0b5b2f00e4b626e250e5c4a03de4136d96943a GIT binary patch literal 38078 zcmeHQcX(B0o{l?zIQz`A&+L)}#lGwm-JNk~$604reROtoq=YJnVxbC{kuFFRK|rbj z1nJET1Of;eq=uB6p4^*q(|d1(^j`7Z_jkfg&MkrDdgVUP?)i9L&bjA)<(&8XeZTgd zBrY!Z@!zy*F8u!q7vC>kTn4+ixI9Vll8cUz>q^<#eYn&A09XYon0?iId+a#2#zKISlx4Sjp>X#W@TZIBdOhhSo+U z1Cm4T(wJ}>1Cmb`Uq~y?VLz)zsA3F8F23Cn*3}0JC_0Hj;_hS#=mnJaPH`u{A>I>mjUOF zuF3yh`2U`}BR>>(7{V_8Z}En@ZkdB0?^>ivb4w?R%9>zEs>IdMQk=V1h~s{FICef4 zCobgUoPQC5^%Y3VYrxIg4z#x4!ma*vt{PqH->7UsjPWME3oM4ug**A$5tsWZ^GfzS@`mJ`hc%V12eMnzStGLIJH z42XYuuE_YC_`*9K+YYCiVfXP27*cD{*=74#s%zVD_)HewUK)v)W`yE56NB;S$Ur3 z*MjiS$l#v-9~`OUm;w0B>%sWbj1YLPkHg{f1&GURwAp_`S~cEW6oySlvatO`9+BHG z?C{CMg3U&ZnG=XLUnJnv#XKaWR-vQgmikuZd_eqH*R|oVYh&>7muXnNHyvyCW#F^J zS=ewW3-dlpz`Tv|s3K3citzHsHv{nV5m9iRmUGGBFgac%n-PZ2*IH7@k&27`+uGNkBfurh;Tgg z>NU(-mw?Oqdfec8Q7uJzs*YCkx@_TZD6AHTayz9VhcKYmEVARnnz(5xp~UVFU(^ zjm6-}Y37`B4&2}9hB;qpaF@9l6@{n91z_c#Oz4stQBA#JsWNmi<@nFZ!Eha8FzegY zpSz4a;TNmZm2=7Z}3h!~b9S|6D+F{6Uhu(xNo~t0= zUDelN3i)5M$$)()GY}GU1EK>g%7FQwzrldj2eROGx(K^D54+BlVEMi*tlpoA*=u9_ zx1tAzr9i&|68ZmJvN7u0iJT*Q58^6j& zL~0Y7T6@>B$i)QCt+Z3Oai7F-#^~WbA`}mK=oGk(42L_%Hq~>RXvDw25sY15mm)T+ z*-FRl^~r$O@m$2`brKzjOl!sZgPHg%b=SPl;<0dZ0zTYk#L~|bd+?E05*BVs!0;IX znDcQY^>rGqgp{C+_i0fE#J||Ix$6w@xmJP07t8StWq$|f!TV|jcKH_Iy-y4%ueL4! zGVZZ+Ie284zoOs82E4mckLBAFvD+sLC%(&v53%E`ba;Mh!0*V3`&e^6T}MS?m}dlz zU#~(|No$YY6#e?>D4m%;6OJGA2*N9~!!e(_c(+eB4xP=$^3N0SxG}!om$P*tjndzTf2`D5~5F zI?BBG8Cm)D5E*G{wbIQ4;=h^gH||TutkrtV-4KVV%cC)UWsGt@Wmy!wyfe|*()a6h z8F}?MN)7^JDv*@Z*r$IZ>poO}1Ai7>K92h8w%(9+Gjwzy)*Z=1NK%vH8%bT6FIURB zy53_Rf$-dzgm1&EQC8i7PB9QHHT7+nuqYC)v=65GV)y55O~J*Odek(UbCi)=M>&r} zXiBrfzv!QpU!>rzMIqRJG!>V0#mFYVc9sG0FJsi!w<9I17V)Xon7>|+PxqxFj($&S zb{*;(I~@8l(JuN~!i|Y1WZGAopPCfu%Ze_syn>uWwJzcUn78gQpm@YZ|xc5W`{Zr}{mECmrh``KG zjW{1shl;vRS$Jz!sX!iY!@toI5-TtrJ1S6iF7=)c?OJJ~JgAbg_#O9;fiu}tO@Q-c4BFR=!PtNU_Pktfj>C8XRZL=>1JDVyR!6Rd%V1G1e}|cWd{iqEY3C^k=|3 z)sfgOr=T7?j-}z*w`hOInA>0B>tV)cOLt|!KfXcnfu#QLCky|*^6x$@Ske8W&&5}j zSY7mZw6PKqlvR1IUyb&M`|UPjgCAkPpB&7wX@f5X7h}n`MB3nP|5JR;CG>gbuZw}i zxxvxpC@Qt4mi_MH%Uzn=HgC3#uPfhx~H$D^J^@XV+ICSv_lJmv?0sV?*yfBSE z>ubgy`FEv{IdyfsqN60H5*s~_Tn+W`$G?oz;fc{fkU0{2Zn6PlXBK*;b=zRhnYxIr zo4Y<15%fJxF~DkaGkaA8Htna44=7T6obN4z5_KoG&#m~6OR2(pA4e=r4nh14K|IyUvR`RmT!y0vG2O$-~`%$Z$qnT zgTt`$^X~kD*c2Q2g}c#U#sA;F$-$@OU+j!&tw}t+XiFkK;2MliujBog&$Gx^40Y>l z%82)cLd7q?+rTx`za-@Gf&6y0~hq{ETHoKkCfKM~gfro9P>O<^v>O@LT#LGuBYHu9hLAsD$r(KE5rE0d4e-8PhD_$e zn(1HN&8~?Jc>5#fRaG&t_>{jI8>r-q#9mnD6>QztovtNk)+jzn7~^2k3!)#^>`lYu zg(28_i1A=h5z^=j_a!5J^linz)a#@jk@3!6DS$8cEaK^l%05Vc%Jqm+jPcY#pIT?O zhShCljy>1KD!IoVokN{Ggt_|J8;tOyZ(m;1F`#ET%{xbj!c!p`oLh-}EkEuBAJ-bEokv zzpJcn#iYfN7&Kay2V$QO(B_I?;}cMcyz=&b&w-_?6JKM&20e~mC{=tDk^i$nm3VJu zB-ZUVDn7f!jFJ}^Kye8FqECDnGkAU)i}N8>2uW_n>42MlsH)QPX~e$8u9vqP{v`=%HCd3 zR2f7L22dQre@$JxlA}69d64{+=tnuqzQr`=M{`Q7^4|29tMMO`f|$4Ijho!*Yd*@H z@v!&9v0_(>;?G_UE5lXVwv){9Zrsn@>nD1Q_6)&czhdY!o2l1(V?T+rWbaY-9M!Rv ztS!++zZeq;j}IcSc&ia#9M7cRlBM)He`6d*c!uGZD;wCoXwaQ=$R24sUB!oSGQJo={B4E2g#GTEn#k+pc2YalVZ z!Ac~=26!D!BUjABa6iyw2ZT3?pJXjOP9IU?)uDv!&$x{U>$VxM1}ga-B?oE|t4e-$ zKJFJOFZ{* z-(PEB{MT~Z26tf1o-};0CJI{*CPRF>Oy=0{<}axFbO`?q)CH0YJ>pjiW5Mk`QtC&E z5odExvbe%_ZmNp8-48d%De|hsSxTN$_dmA1S$u>1z`3byzJ*xo@B@H>q3128&#Y{L32O#h75?++bxN=FH`M z?z2}9#0EQrf4N6lo6}cBVJG*2zYea%mSZ`Xyd)Ce1eKwK_O`DgahT+=o_T|Mo3c2F zc?37E2elv5vo<&%vN!)v9?^J_IiI84vz2_Y+6I`$X`?Q>$h_Hv1>tyPWRPO(B}d&o zpB8f={mPy^_YnT^UF16@+>aMolew6-EJDde`!dEh zBvlP04t5y-;?so3RbbWjcbp z520RooH44zS(2mkTpv$;SBJuz?Kb&0&4b9SUqB)ChF*E^MbnL(8>p}L{2dhgav1=7UH6=6j6!X`z>rl z75!R?sU)}DpCUd^QCTDR6)KcC1(<5L|*tV->QCnowQOd*?Is)o)r( zos#boU+4g1O*u-;DesX*#8*J}dIzu<<8c1_!^QuFIx_2O*!fHv|8^4Rpk^BX4kFUd zZZ!VwB+fz2H2xh#q@CSp{M$*KgPLjlJBUa-yV3Z!lQ;)8)A)A~k#=^Y@oy(_4r-?H z?;s-W>_+3?PU0NYOyl1{MB3Sn#=o7!IjEV&zk`Uhvm1?nJBf2pGmU=-5ou>P8vk|@ z=b&a9{|+M3&TfQ%^Os!g>_}^77X$LLmi5a~c6FsSvV(zgw#l@koz}!L48*aUi;n3^ z>%|TReA#9dVfzI^J37>w^qT?sfz^4$ABlWIEBBUx3=a1+@ejnIdpoo?8X4F@$lv{y zzkw=$bG($$%DrJAkHb7qnEv+l&v?F_=-&@{v7%P&W+rMqMdF)Ee2`CQaWYA`OKIizImBM#CBo$6_wcXg-140Dd+MpqB}S7t`C~%zMOlRW z;xSD>b?5IThb%Tz{+9bP;tzz#t`(P-eAo3}_<2F^`Te@y^PjBkJ-@H_R6Elc&=}Ad XFlS(G@5lRez2`qE=smw*t#ketBIbCo literal 0 HcmV?d00001 diff --git a/docs/_static/image/logo.png b/docs/_static/image/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2a8bc258f6663112f02b40f04843789585d86d68 GIT binary patch literal 402692 zcmeFZcQ~AD7e2}uj7}It9a5B0qm2@b5JX9onbB*4h~7J+1S3I;lBkgoMkjjjqKoLn z=$#;XJF>I)_uJn&-}&?WcU+h2E#`f%=ULBs*1GR|Efe`jQ<;L4i4+eHk3v;NQ5z4B zL=O)SFa{#Jx)X9pCVzF%e4?jnrJ;exdvy-N1L8B`5ni3)U;WDBGymuOK0Y@d!C&_Q zS1+Lb`~3H({`$dxaCNx4`g6QNbf|!bCy%G92-o$(->7>QdvnzLmWN0ww%WJamyLyz zh;dHI0u;fhpvVZ)=Y(gBUM%Zr6%sz-3~rM1u-PA?@yLs(8yBGrKfiaO5#BNzyDEmK zL(LKStI&9oDU8mo^)>a5EuIKo@jXCuNXRUCR?X*{TwW3XzTR3yMM-I@@wBnAzra9T zVbY;hYFCbwdxLAg^$QF&H9$cg1j6Tl07HZR|BL^RY=BmULlWPT4Z8oo7}__w@#YJ% zPrd=8Ymff57;hpQ1Dr&hi@*BOyCME-nP{Ik2E^`@gvck{_+RUWYlIB!d!CtNUbqu- z_%c(`(h)B;4Tq8bz08T)!0K?q~?=JWB$=U>7j_9Xvy0`g#ab~YImTI~OP zzdVRMvax(-9-~MRiKE0i5;RQ5|9CD=Z$Jz9Gfby3>aqJYisbE=AUgudb%jSU?6zgx z{~oH46|mZAT=+0!>3`asg%T(gVEs`2Q)f*q8H0L zv2G*&{q%XyUj@ouyOcyEwfyX6kHYJIkr5;hkWzcpS`s8zhVb4p-aNTc64E`K^Uq^h zgy<2}&s4vij5jTPy+lhE`JPsXY-I`zzPq4}AJf@S{=L7@s`t|y?{Xb$yW9Ws{C_6$ zd#DFMbo5MD&9%%`nnc4Nv;49(cTB>uTpm7mrDbO1&Ldq@TjB!EBKFCQ*9_pu4(XZ9 zIZBTN(FSPDD5Zeb=l`toW1xX)rsXBzE@aA%&g5W&ab_^ZrMPa%JfHY>i`hm@FXIP< z&VP-77!M6-kikRcsnG%c>koe}gxG)^j&sFI2JLi->s+x+qD3A%YhGcdbWV|ND z?YuPxd6rw;+44A>?RS0G{(grLC&0J4KR&z?<}6}KG;4A0#mK?_JSxt)$eDC*Gh`%y zvi;yq`t+xf|FS|B1sV{ZTs}E;{y#TOXXt(D==X0y*+dk5?qoggv}zQb(FVISV}A3; zo*jZ%pVn>b&i&=^JUNwp<-P@r+^EfoAGW^M=J_bIB9y$-YXG`7vsu0`ntdIhKJm|V zi%_7k!Y{0WuU>ZcQ}RkM$0uwOGSt9uv_mhW+~>_hoQlY9`3t78t0$Ve*gi|>sPH(8 zC}$e}uQ1w(R&Jb+B)%|lBlsUSbgHDWId5l`ymm(+wt!C_bvt?O)_a4wQ!CRK*A;KJ z;U;oNqYRQ=tb4Ctm-VZ@5}nN&GO;Z2&I=Y-8M`+0Vdg zU-j}X89?9$Xw1jOc>5!NofT25^Opd(_`kzBnuFTL|L)7PMbVpb(h*a34c_CLm5knu zY2on7F-hFj1F6UNxRDwxH&Zb4qrUDNpqnhYPU&))i+R_QT^S*}B&5cq& z$2xz_G_0kZUx(k5^oniF=w3?H4@pNK;#vd|NljGQ9p@29Mjkvk=!rs}79gnhjO@o< zCwHT#P6kwA<5eH;&0Z_x0se8C^oT~qz0zBrWqPY^{Yz4z83k+5o%#N2Lmn2&Fjx2?b$wP9U%-wd=Nc&OA|dAfpZxGOjewJXJsM#7Va?=p^= z#y$xNvb)SUm)p*9Egq7G7}gAoe;SV@BS_+v0kS62SAWa;XMyhkH&9;(Od+t@=CFg& z0T=!5W7)KiFGJGG!gNBMMNHb$&+K#^#8qqUnAw@pQ7@ZY)-naMt`kpovX)?CR0U~} zazBQq~9k#7(i2 z>N{9TS1m6n;SN5+P-d75HhNEV5vUU&RB-R+BFV=tfa7Op@6qZ=tkcE`lF5&yJG#VP z+2dC`@-uAxafj;scqP(i=}7&v3fBFoW#ql_>{8^r^k*9fH(QbHZmAX(MnOgD*^6Xf zl3}@w4A6)#EL5&6FTCP+gu0tZgO4^9^0qkh z8~}L%$+^rk)s1Am-Hf5AIXy~s{(l+BDTw-Kdk^IRVqN=RcmMGkv-i}DQf3l)iP*H* zwd2#tIX+^$cP((`pn@nbEMxMXlFk!&m4iYK2*BgZpUREFugExB{pe#t75A_*sy{)3 zdDC*q>omQ>7FoA9DRy9GUpzvuIsLQdmv}UC(p_({8nvD1DeEa>GsZF1MG!izXwVx> zi&T5NNC+^dj@_)0HyedOpx)?42Y92yDoG6?V0;4>jJC7Zokf`AslwsGIh?;YQKAFz z`I5D|;(BsUe2v+B)HGLc+G6lQh8b3y)56hNBgv4L@`f!wkhSe0h_e5|sO|3}_Zi{j+icXdEY?^gnjvRnpvCOLXuY>o`wWS1;qQ zo6!#Bh}*uu-?wVzROoq^POIv>e;*ploxYnwVX%7SGON3gIb~^9a8H!5jP7FFDVmow zQvvkJ&zz3HF+ip-GWE>A!mCfggLdz@&d$m7!!IdhwylK3oG7eW>{J{1v1l7v-rHs* zIyNbr5cPS4OvBlAvRKRr!&UAA;Ft62^k>Ca*ifl;9dj@;-UJzFB`yY|r-Aw*5ygHS z@f3iP(y5nCRpxP$!bb42A82`TQ7@s`vh$#HcJhVd0DSn!xcQ@44L7qM*1ugxNDD+R zkS!H-*2D1dSW~Co2Fq=8NT{P_S={!~U~WIz?`2X_oO5{zS7uxHRMVXRaR7s*>w-~D zbKA+!q7N{=NX7u>!?l*00-T9|-!PAtAVW=pq>1~y{yA)<=nT70jX~`QBWV6lj%%#v z#5tGx*HO9){GDq4O_zqk8qKh;ThOTLY zr?UOHuy<$>IW71=-8Q@w!Bi>AMcs@{6SipE|>_ZPLW)8m?9B1`T2V6+RBA@nga_%?sBa z{O`?yCh*XFq=Xkw#$6dSP5a}*q^a7%r@_R6S4umMB*%}7cijyanT??&4hZV=6my!( zxihQ#&Zdh&olwNSNeA@dZH+tWr({?c;S8hGGvW!Wd@tK>2F2OXbeut1+~ZA#7N>oq zpv99UXQ)FY?OMR3!)0vuds>9IaDBc4b6GG96Rz)n+m3jwQZSP&F6!3~Y@$!Rw4Dd% zf>z)faT!igiR$ooMph4RedO-Sh-!J&yKw}u$P0kWi=vo7d|55m6@$w787m{qa7i2Z z+mp@eP@yq3ud+8mkkhKhgjyB1#n=Ju-%}TI27PH*@iCr=4aQOiJoiXzXy`R(YdT1` zI5b|gcD+vQa+Y@wCYD50uhM0D*rr_{;h#K}*V%nEax|6qBH@Yq1kBYpR=d5;E49DF zBPl9wS19%lSF9sRk(04QU1yV@@R^2P<*&iMxGxEK(A2L($PhQl*_yO~frochcEHp3 zE1dJSXNNt|rsr_n60az1!DXvtP=A4p77D65SV%^V=c(cn1QFO{xc*$hd z!Z>7Mc?-Wx{bFQhrE$L!LNAQ;YyEr3)3i66Urolh$|0$y`0M)u%k)@xyOHL;>lduV z#k0XaPw?-ul`hvj@Q_xy`;xvDkuq8KT-Dfp)Jp49ocM(BQY&eLRG5A}cQ+w`cKNN? z5(Kgn@hX0xktB+TBuYR}oO-M!tm&cLN{vvFF#~mNu=~tDHtnY^7e4%;+l9lBJjG<- zBBZ;Ib>YUXZ6&fl7Fh|FKP4+vBiv#OOWUc|XX=v$ZOU+~ItO|R=aETKu8p;5&#Z;kq1LG4B%e%tMB zc@bp9{Na0{dpTV3g7fA|&N5*;N|Z~jKQ+DDo|R@CO;L0+G;z7zXrttMZz}&_CFapg zWMOoH^`&U?Y23Eou$tGRYyO2yu94fY#+Yn&_pne7VO#Fe+iVsO;6-msM*^iZQT62x zhtuiDn3sjc90t|ruLQ}pd}Rdd1n=Kb z0GtW}Mjp`Xl{RzN9?e173c1Qgw*5z*Rvvg2wI|g6qR|xF+rt)IP;=|Xf6Lv&k z9({t3dE%nZdLk$N1D41pQ=GaYCsJ&%ZDlUSx^fQf&%QXIjNl(up4vEw0|YHiL!3l9 z+VleHWW`At_PiNS_mvD(`$$-+pqP@*FUh_+^JbdV+o5{0m%GNQDWGm=L4l~+T<&_8 zxrg-=G=CQPfUu=S8w%{$sKP){A!|wuI}X}CRq41!F9-_Oxmo7+^Ox)g&Ga# zPDZss>LP%Lcm2*}*be7Lr$uKbviaFWxY)dG7<{=M-=DJu;z}2ZJ)b&STAs(L z1zw~$ZfNfrCs8!xCQznF!!gy!fvYG82YJ=n1{XX?_$!6|-?Yv*q4I$YK=NS`vyP|( zW#Lo8jcO8Xvh*-_Q?(7$H7A&jEB!_st@h3YW47Tv`K^@nRjEIhsx7F^e%riU zs2gUVPoM5Mr<1K}G!U6F?Imzi(t0c1x2NZonybTa&)9$%d=ON!eepdYMqA8~ah6JOZ7Lohj*vmQE@d#rMMVOxWgNAGAKx`QU~&VJ7{ z{T?XdoXqEDDr~<64oaipN=x!z>zinBBpLl$Bhx3!D>2DnA`WEVtD@g=^2_C7aNP`v zI_BiZEV3?7G-GHxrBk28z!|q;iGr;Y!6#jjvr%VJoO$CmLs^jmv}Te&hBf>af!ooJ zk_AtU3uQhJIy6H&N{!w(6*!>0G4L5Jw?~5rIe5)HGGrH#*B6SYq zx7I26ldC3s<{b;kPLt(Y9(Vbo9O^E&L>%uPei!i_wH(p0%jh=sU2ReGoi03~1$>0y zcCxiJMC+`2@dt9m zsckQ}>Q1vzoyku#Uw*hCEN2bfeCtP^A-3MMe2-1u8D1u;8YnV36!4O^)ZX-T<)Q$= zIsY)$(OA9g+J)t_8>W)lQ6>J5T&LZ$N;bWU0|=!J0Q}n!0tXGpno-+ZeZi8Nu*K1Q zpx-a1BSEM@Q3wB)>vDOyy}!eU${;r0YFv`^Olp!emk@X7`Gu@+0RzYlE-!BQ%*w%l z99Z25IsAydVKbU;X}+FvxL$P#$6 zD-s~G?~=nyiuF0Rw_QtKFccx`?1;aJDrisEL4ou z%XQv)T1zf-W8UIIX*WZQ^QW+<;8d!!(Tk3shhIhFmhuVhuhN?c3xCD%+Paudk^%H_@%9s$7s*}DLZsNPP`w4yNku*5%7l4iWawg5 z%t>-L$#hf!YC05EKOZ-*uZfc-@8I>UUhB<&4lWf;>o4m25xG_P!_sypX-mvq5yV0c zObs|1o0&5=kzz#U<|exVZx&6Qd#V6(r#tlSzM#U7k$73CRd(}lQh}r995~UOj(Qb= zw7x#zJGjW=M+o90+B7@E`tH*7@AuYhovmoPm?WN?+kBmioA?}I+N$Ix26b|z0 z?$51C&JLIKyw(AGZ5W`l$PxGY|xL zC$uxzmH{GQDZQATD?$-&SV=89CZ(X{x#EW#sic6LUd{fOBHNyIBlaM)&M#(}*6>XtwTTZKz4W=z^) zxcjb~H>@en3yelc#)4$_e+u7iAl^+$>WPv)U$H!OF|W9L5_Wg{FmC&7JmIib$L(lA zZ8Wfw{T%y{Lx>c>&nesWRZ979X4A-_n10!~1Ppo|*t8zL0IX%G2L28;jrj8E+n$E+ zbcTzbJ~3+SlZ3c|uvk3m@6&<@ud1(cfA4oLjaMnC!? zG51~C3Ls2oxUBb{0Jk1JwOcda!72cKkUaG?m%AO?`VM;UAqRb!p;Id_rYM;}dGnef=S`9{!t>I$bXXh(aNd4UzTWS28wQJmhG%Dfd;zwTYWES9NG;9e=r zp^ERyjg)v*~WN**m6p*nbje8NSXy zx-2MYY4-7Bg+f@fl?ft*gV5;9UqU$S(^dH*%*2%RUL-ai7yQ%VApg6d87d?QlPF^5 z$h!3p*9jnaLsrxeb5rTWQF{CFAZ9>-Oc)~Ni1cCdG>XouSaVc9omOfQzkxJ3 zNMD|66;CZVNEailhq~QtD=7@&)>C4|1;K6j2FFv?b+#C)_>xF5=e0+ zk6w%_@Y%`Lp*E^pCn<>V?^gQKx@gO2f>_YC^QAUv^WmFAU8nR(F#@Lg936T48S+2e z0Y|I~orquv z!)T2ky*e%RxRbzlMTFLuS6vMH^ym=Y_;Sk;h}+E%Tz42wEwC*PX!&ZPF+8(3lw^}3 z=mRP4dIhK0W?s2XGr>`)9KAs06@eacX*mO{=cw@j-=#BY%hhSb)MFgG3G4kvQh(B(lR`*z!eo~Bf7~LA9*wSFj<1{aoN?T$>Gl5{ml?mKgkZ1r+fx$`cGJ2RNFz)I z%~n{Lk1`p$q7ya2c-phI>V3!%P`A6&LWK7*?PsVx!*e6Iws`@_N~nH-P!SlUh$onv zB>4m!5+q(pjMgm<5C(OM;|^X)^QCKC8C5o-={L4e$>`!;^Z^bFH|eqWPvL zEMEEw(B>-{BDC@G4Ixpv`OVmm6wUo--+Sh)-<(5OT61q~1D3l&QPg!a-o9-GL;7!7 za@`*;-0=P&Br55_~*IqUMgme~#Fh;k%c)dCgxIqc7dwmhsl@cOW!W*j@o@=m|<**R%fvK*}NcLjhA3uZw*@nNSz!&4n*cQL65@CFZFwR z-z98U-*s)+4M7VGbdY%pkSYnse+@Ix)VuNmn9Za@dh|{UlC(*HlaFhxB$}e0ES>@M z2@guatWQnd)R;jQLPkWNQiye`&X|v*04)`u$nhqg3DB5~f0xWl`^=QAM4Ai~An$x` z>BsfFBpresqYHVm>L``WXoYJndO!Hn1Pw1AH42>^PHHD-V&t2zXBg0|csnsyqp1HI zcKFMi*#xvdW6{-B8a$1eIUP=bq*FBCytv&Dmb*d9>l-GYd9C(NNCV!K&6dg$sgB%? zyJ2626U~S@9jSA7Qm3bUqJe;JZKqKzqP=ND(q9rhmFt%pA63Vl?lPB~v65nuOFz>U zrZ*)(Ya&^{pt!Z5&*O!B$;rQ)H??W2HwbP+DsgFHuG|6KGQPIr6Kg`O?pB3q$|VP) z+^VEVW>1L6OmA0m=C$ndrp{M_NBxDRkHKmZ(Q&N3Ca(2uFHPjs&UL;B0NaBbb*R{f zL?q~M4ufc5zriP<)jlS`y#LgHQ9AHo@$NPqtL*W&efBNohBckvA)&+hzP-ZoKRrS) zYS81sEhXPW%{JtI4MDO&ib5gEFCJBu7xYI8$m`j*7UG~|ERE|2lIB+5SJU(tM z!-~EgFYz;BJis+~=p@OPjpyCVB_d@E!Hqg1C6|nJ_DtY!aEH&~Bt*S(;UB&(NK11_ zHgUeu(ZLq33q;2*Ra`Y8xD(IA2N0CeL01(beb>@iC^-@5Qn)-sW?YXEsJPRnccpx= zT=(NZYB?=xPYD&RDC!XjklMY0%S|$bU`!yf06vEsBi6iOqOdlzbg;oFcsvL)nY&zoYzu-!4?_wpY~c)$JbgHQNEm$W^Yx8{i>wv5vPTp{F+IvzC^PD5 zCQQM767rL$j~FT~N+Yz^F6*j8AxHKF7J>!?hX)v+qlx z5WEkk&@zJ}g(h%);Lr_f(-cXBciT=XwO{yDWLGYI8i2Ue56j)knFfDKl5@4!JBPk@ zk>JM5=Z)W0^3s}?c{3i-tVuUXhneu#D_ty*6jnMMGgi?PZp%ckXTxZGJzk2(;l!P5 z5-fzivMOyJOEx^XtK|#WT9l4F5kTcuE+1<|6hPWDx`#OxkrEDgC2UU#m1hyH9s|Nk zBe(i3^kviP1jmubxs@nMfeGn=E9%N5mla;6M3U!mIi1jGdnk;gnpc7>i_Y8Y=Ktdp zi4b5l<>LdFEt^b*=9e;oavg!mw8N4$Kdxzkfx6rI5+OCaNtb3K#qRx6uQn^IX{gTY zx|^s|&$oQ_nbkixizR5^nw(61*={c}ba|x@@-OGk;Z%fww(bew3MYe90%pq{NB%C& zp(7O<1JpfFW4CT&=U9c>)YqjQzb1aZTw*k_KaTg0gCZ!y>BGVJM8MV7#?!PkD${y` zrp6Q?z9IY`5%Kj;AthK8WPoUI*Nh{jfV24ZCMgZ_NbgHyqg%+*Nw#uttBO-idTf|+>>srm5)o@)*$cym&$1(htKS8%$GhBM} z<$SGi7;?FS`0^+I#O6Hh-R!?`=>7ZY>v}*#IyvU+s1%qID9Ad$M>hm4Wy1Q745=An z{R+B6zbe{xtAgi?MX>{F%w~s&boOhY7kM+6KnmWE-CN0Onk{=#bGhPawpM;!k)=-2 zxzD^A%A=Zd@1(XJB$s9jI&gKY-LB@nlKmaT$hfrF9@c*ET|n_DB9V3o5}joiL_G2* z)7J43@6^3nCnddVlJPGk( z2iBL3TJTX766qYD|jr%k`cCo88hOcVBB3HuB_A1mD zcooROzVRQm`gC6zK1mc$;qh-YM-8A2kv~pQ4)CrEZOz;Lp41V~$+xp;)~4%}4O7g`aPR$2>~3958fol2#r}lLstLmU2QX+B2fCV?0`_wL~|k4!>JCcjD3#YBeUr;X_l2BS^X93z)tzb-h z_Q;yLxMY#uVKVI`g?tIk4Vc@=lD8qS&cIvT3wLQMb&w64L5sg}XNmjQ7E-_5gr8V{ z^4hPol_%af0R=78D7+@lwQ8Yw$Lza5Zq%7Yh`fxTr^@0#PXdXaYz3`BrG9YSe!Bst>1T5x$VTwdwJcakserD*f_ z^NBpq_YoT{&5AGE>}RT;^@_r}LcFUY7Xd^{Q{{%$?`%Bc`7PdYMAqRI)ox z75YydVsDbkQwR?Qz9Z1It1{m0wI7`*_#pOYC}T>%@<9EC6mVGU)TVZi%$^8v{bwZQ z_*TVHl>mHLp#Gli$pB}O5W@wbW2z>(^>#r(KOufqzoc*_B;ECQr_z^Y1>msDGj0;1 zJB=#B^aL1iL?PpVWn=VmiCo1dS>ei2V-O_FfX+ntXzYPlr~C(s3yOq`=ja9lj&llv z5I0M1DdAsaTr5X^qmlW31?!>6TxtWFVC@gX$9?i_a`gJBzp|Zo?1LV;A##Zdckjpa5JRnH+4A*1(3MZhV0(S_ z3Oj2sbI^4^!nxLSw>qZ#UP}02?}SCff*MX^!OSk|9GL-Gy;WV(>;W^B?0{k>pmFyV z*a$7GQgSP=DinTcY1QLZMrExV`x`7-RO%<70;2~_oQ0fLFPXI-Ec?xf%`BtxhI*%M zsf)q=CbBb0j!$b}ap!CR(Vy964=RpiO(s;CrBEQ)l@Gb7Z2Zu^l2BWa{M&2*Nz?H< z%I_urXeD>$(?th1q&dV1?ObiB+(>&Q&wwRFk#4DjxDpYrGEbH{Gq}2$M zDz$>p6Z%30X2hu=* z9?|lvo+-~pF8k(X1GeoI9wBD$Th#Xw%4V}KYyZXzbxP@JSL&Ei93enBMa0!o9wK=E z7rz2^;Yz@hX*$M%drkwS)yb6@{CjW9E3q{{G5AfdhGx7p$*+p|pJjwpByz74am;oV z@e8eFHgGh&Z1RuNB&RdFB{^zuF`ZerV7hzWAPQE7XyodEn{N zaxGj%a)VKa&;apU+tm)1oYdJ0PGUM@?^f1SDqWTyf3Xnk*BTX6IX;rMyq_mF=aMEJ%dsl-*{cPY=p)h;W%0uPu|axU|-yIgay-m4efSd0uD z0UXqI0|q)}l=SIT#bP*lNwc{NTsX02w`sM*G=LSB5ubOHe&l5~(NwUVS`wyY_gKWs z9SW6^TA#T3hpe+G!hc3it|DWlcEBNz=*)zN9{{YOT(uc{2tTrnT$x_X)dKe&399Db z>3I=oLaP!M(0C2>SBqk!7Y+A$L8&oLXOwE{?s?06Ob260q5Pw;doeMGcfSATY4eKd zS9O=HH*l^i4NcOl>>CcE(y?w+L0_rKQ>AHRqK!O%xR^N29G8V!K(ns#xlA4TBx>6zb%dRqiZtGE zb%^WilD+P8m3z!&KyHiHEHP#b-QIHiT5-VbdALbF2+an1oRLeNk=yFGwzX!eEfeOJ zbo-aKK!T*HWc^~)boW)Mq;q?$=wDNTR+0!y_q8NiNSF+(b!t}lbrgas z_*+Sk>7jC77Y1P4KJ?0F{!ojj8?OYJA74^gW?#Vz$eM8=Jk(2q@A$u^&N>c zb-@m;O=1sq%gR$juUdz8bU`%3QmwjL0$Fpiq&dkEyg^>Pp*FUCJ@5JXLTqoBG*c@- z3h%Uiekc(I4>!^aLl~Aj@tKy95hYiaL^d#mA@#;f*5O_L4kMT3(e^k9u{vfIX$R>_ zJ5D3cNXFTk%c@VwM>5|uhhXS&hKx67?L;Y~WxOnI`Lmrqr!iQR_z0)MW&P>M;s6j3 zZgFEt=iIKd*T!rY$)#NNs5Q#oxB@gPJP=Zova0}~{o!q7A$L$o2S$K=O!XCgzjFcV zNi&pa3loj9cLt|0Qde5aUppU=>@G*U%W=b0PM8mLa)2Z$jJC4}jb>#`x9gG_ZAHw! zYud%R8e#I6-s8R`KWcC7`_CCYpCQC8N^?i_ z@8)P8TMIvTVO-KV@C8UFSY z+o+D7NMbgo-Mkc(bwdaH(MBILQ$|>O`w2AR;!Jvmaqexg?thvP+P@_-(ExlS`N{Vk}O5rfQtk0(7*66SH+NW?u-c?@<-X$B0+&7f6TO^CuGpE>Knu?^f0kChX! zFCSd{ma=9ekUjCD^5rZIC_^D4C)i7T4ErwKs4lRG=35fhw4oq3f9=LTA&y?N&Irt1~G*KNg!s;=1IN>`uWe2>_Mbk}uz8Y`@M z{QEMB@cCi87wKRk6^Ese+xcd6Q~5T8Tf`qVdzGG6=wAvzh6U?L3-2~m*)?LM{{U)olI|8s7a>BtT91 zIg^zVET9G{vD%~F?r5E)r9eFT_=am)HPMIRW{bB%8scF{_Q|~4BfZ#|&aUKsp`xQX zLL;Qi2>L_jRqQmbx-Q4Fa+^0GEBvZD%L1fd%2)>cd{=d&=~)^~U%!|VwID%lqMzDg zxKoLIHxCX&$@cF~7aZNozokxWT9UzEt?NFEdY=51!C-M+9coj^wfxMj1*7dGu*+M% zzE<@m*t=`VwhUha%EI0_Yug^pkWs+(_>gt?Yaw^J#rq`-DYy&2!jrH~Mm5oG^E}$A zbxJ{Fkr?YOh?B0m$!@H%QQ!+}&6cZW_zJ(_oRGaQ|0X;DCqqr?i=8@xn$X5G{#&USC zNi$Y?;5b)>K*bzE*o-}Nslcx7z?@buN09Te8Q7TN7@`XCxmW>2QquTqN`w~h>or9! zcfNZdEO&;j=}Ff z@#MP|%qcIU=+nv&kEDqlzrYkV%FYF$1p=;|Wx%5roazgG8wctz2Rw3>ThvCwyXP`n z+W9?woDjl6grx(&v&iLn5S{hMs-{<23~%RdeH(tD0OkDfSG;2+102+R5Pct$@=(sl zFEW_-Ahfrm=NtPIqOk8y*Z)fuP5*=V@S1O<>ZYRZFk%9Ciy!%AEo_R7pb z;6b#!(|)d@gwFa{HB**hM@DG!!NOOAxxI1DgeQ=Gs=39`yDj^TvOLvJSJc2GqOn{U zG|JY&$&I}x=Ne(d1EdrH3}d|QvzY42tx_0S8vCh^)jUe|3Ro(e}ikSSm7V5Pi9JOz=FB{Pf%ox?s98Yf6*~zaF(woMiZk!U2$0 z%>OEAMn4%MnNUrtssRI0%5}QE^)+BFhkCMNunx&Lp0r?Smi@ui8RG53o*QAJ!Q<)x zON<3H?cr=B`bayEAlH0Adsn2vSa?faVyU@6Tgc0wL4?kTn*rkRmTaaLN;hNcV?xsnnRr0=|mY-XCUU`A5gw~W^B!yf4&79%GYxS zTV;u|zFKkUS~svmGvmdru*xrQ*rrlJma)eL3jfVA&`GE+akL5`d4&+IJLBp0x!P}G%4i`K2DB_P14 zhrOvXanE9~BRtz}G@}PTX17m($_X)gYmzUNk<&G2c2x@>)s{c^Sx;1vKP1%TOdsHV z-q__l;d4VB=8f~Z@!-dGjmNjp;51?|htm-?_o8*^?5gseL1|f*)_%$v*ALRNH$jh_ z#K@Y!j)(QWSn;}au$p;|u2&q{)YmkY4wnq*mZfz43T8`BV=GjFHkpeStMgrx1H8Vx zE*6n1RT;tvKevE#eW`9_!hH}vD97>AG);4{0w-LE&EPD{TPW@76Y zk2UDbaj9U8$5LP4UwC~=pG?XSv_S=6DM$S!c3cT&LOCEeO1oXIYa53N#aGIVvH+eA!mU{8k$-111!d_uAJ% zEjOLoUOpSbV4LasP(&dz9=BPRir0K0Hd)HaQAh+UGt!Z~mTE(O z56z@V*5G~v4B`2D-$28>Iqs7oBMd!;;Xxu_O~!UxSqi+9X1cZi5|S5a#~*n`KO5L5 zn_kh1C7SY!4t4p7ZpCiy{>THQ`#Op7Q;K+;B%pf=eLyivD`(~2E`Wn$uP*9kRk2M^N{7a7Uzv~Bu4r)Fi-Y1R$DX!s zy(P3vP2rx>R2ePRRTO%vobGzY(O}p;$R`()O?p4@C_?u)8Ntxli1u1D^&W}TwlSE* z1g3R|9j_ENYJNN2DuC2MLeaj zPNx^QXtJQVprMs08Tl;*^F3;cm$0Jb={w=R1ZTcvi#CM4iBcbM?yWP|_m4w}_FV3< zU>vDGva_!7CCVCDp)cL8elJCA0(4eEc&jg z>`n?K;LGh&SiNL|Nej659=RN<$)o!HA$}|EKi>u5(5NogzJr*$(SCiXi|>8#m@ONl zaZUWE&!b7JlVon?w$0;fb_TvfKnFjXx4qHACbb}E@bUHbfN9tbs;7Ccxc`XDfAN^p zmw3ccf1@&akUA4!l1Mz|SsX3WIJC%?rkU5|h(!l~DH<_K4Rd~qhkEAhg-s<|AHM14 zfJi8u^2tMUg!ShCYG^+qQbN!gL#lm>krbvlrPMl!vzfeGl@iZh!lBY*7Q~N*xaai4 zlzH-6U6Y(9LbwtaP1{TXzCSgx^qYKMyoCzK2a&?DJ>!@hbcjtF-Jd-lSCq4;hFxWr1A27ywgeZpdF;4C7+3$LA|d0))wE#h zAih4Oeqly0`rzDnG6DjZ2yz$cp*7b?1!p?414AyN@8&-RU30pkDb`!5z0b--gdz^c z9yUed!)2hAC4TYv_wC;FPmll?ia;O(lc;?g@fWn!dK|oZV&>`+&}zM5dZ_cqs`#Fk zuh?SnEp>*Y;$B+?O5bPxvgSVTm2vU%=r~tZZWq+qSZ7gD(8_PAhDWh6nuqT4!<&zh zX&SRN`G(g3mhz3H1mK%@@U-ykDm0^l;k9tJb@r2f`~m(1@>h2YuR$ zF;f_Ki-qGG%5#3rb>r@8^kC;rBgu;i!3{m zR*BEurI9MHz#G5peU^Y~H5BN0SE|e!Fl!+!AW>dV$rnxFTaD^L(!OT+EpXL7UEYUFyD6vi&6|9@uJK`oD-{!g)4;QEv^RJG0zxavjEMoySt*(5 z_%s47mGo@F(XwYxR*CO&5jp9mj*1A`5ct{)OqxmKiYtpmRo_P@$u#CMi&(U!bP)9Q2>x?}k?r2zK;(FW8Y&F3zfgMGLvqBcXNOoihh9h6n)R?UM>1@}b}5<=xtJw%Ds;G}XnMV$;=om%9&p?gjqhx;uFuc+nmBSc^<&wDeI%lKO6~1 zJtp4xw9hk(z^wpLSddH>t#1z2#%KBjc)>h7Wq0j6wU+*~o3qT0#HOt<#h()+y@7&Pj^o0Xe|o47+x!b-`* z{_T50zqk0+bBtgtJ3hB83@*wV9PpSOJCoI4^RXaKC^);r|D{>^h?pAzerPaA&XGzK zoiTbcVt#{57; zD{ZH}I69DMPjoY5cdky54W!owQ#{bO-v?S!%#`l~{77J^Q zEv<>e4AbYy00n);Y^f!K-aM?;P~ zW{M4Bm!u0-W*!9)1!Fa~`e>bsjC36>7fUg+N&yy2!M~bdEv9ZJwfjk2iC!sCse``N z80QAZkOIt0yq$|uhjbrC2IYsk;Vwg7i|DM|$#KB*urgu2-x)I)myy(zN)GuXz?d}T z&}-T(CuVoXNl=gIkYHeM`N77ut9aPEAT^dgvcS_2_F$MY!$rH^DxY~~t6w)C>Mz9w z9J7ZV&{}GvK6ZZzm}C2a1tzp_U%wN*KZ>gA`)-$RQ}XyB7i;^=I7a_8HPd?{<6nt6 zxUF42+dQ@^%|0=}ej2WfeKB}>>SFbsYRQ`!=U;+mMy;*0epu+7_C%<5!Y?E);0diH z$iuh$#BFU-X=CU`mZ*^ze=r6bmX-b=7&t@@%+-TN<`t5E_g#BFtdh4FQB4BN5}tP&IAT<@`dh-KT! zK3x3e0>Hq+FK?`@LHpzLmhCmiAAd_{4Q}s`=b!Hu@SxRu*k8{&R=ja-vgQyMO#}^T z57|SLuCO%l@kdTy_^qQzp+v@@P*na^*RYb`YHCvcyUQPq4pSE&KfY663C3M2r%U`o zyuwTl?Q1fQ|7SUjYb+il94ZtpFRo*1OeVPg9`8i~j-b|(e*)w+mXj%6JpBC(qL<)s z<}ncOx0z64OsSJ9=&xM1(8d^N$+mmM=RFF5c=q6Guc!FxsKRNp z!J|e}6g0)PahjrkAzpT|yF9&#D2ix~U?Y8{_0h7fKYGjK_k_sxQiLI@=eBATKCmR; z|1E-0M^E31yM6%ki0b?&P<ko6~FOCJg@vj|AtV9jjGMgY+cnQD`6haxN zVxU-vQ!N(``e*<)HGO4MiBmD{@a19zruN&x2nk~nPs4fDs4wKCz}ZVtw`x-pFcoml z(~L8#Wp1RD)I2p3tYsdhARF?aIIq}~_V5G^H%Q+!dL42bub6U+e-t6EZ#>RzxY)b5 z+clEBoTPcSXz`)V?}p*teN_A#?aj;kq|ts2qA3{lqh2cId7*1IRI6&c!I(a+9l zA~{4vxXnfQ-#=7d{x*)6fJHSyeeK&a-;Gc(>)$SA@;8>E1CD%x??$mMl^*CrgY2i{ zCU@F@qEdj(BhmWtsLokGdOzE$xI6n`8^Zje0YBy$k$8?-k)Z?~Kd6Y(Mu#>1g~jjt zHtl-BE(bw{Z)idr?XDo~s(WJVpPGc$H}b}qrjWo_Dv4&?qUe;Opgg9YkD5;$yK2Ck zux;p6kULs1tV&>Rgbuj@MwB?0hPqc*f%DqQu~elR8eV*f8>VaP!UhR3@GGyk|;F5FTRRZBVqv$)QDHe-@Y#{dToA17n3 z4^ukDPevJk-W+N2T(9zo>=l^cq~H;`8U^@QP#Ux`M4I}sgDw&lKf=_eQ(R|+*tRM- z<-hGh_wsv}f(m;0A^uQaP3_aDi+Rnq=&HAI3H6S~tiR8qt5$BE;ET@)277%zv(eYC z9BjMX%E$h-Kealxm)t2VbG}6h4^cAsGlS@c-ztpCYST$g;V4N_5!NQ>G0h?8aluVk z!TQS<@1k|6C817AEL~xc!O_yAx)D1@Ba;OBb0!dJ*X#6jy^0&|7aV=-TBO%oU!g8_ zf%Uk#CE`nna^SrgSG$4k%ui~gzm5xS)IB7#YYxz~UW+j#YkKiqMc?0V@92#}O#EDG zI!ZA`O%)mPZ7qdWM$?Pfk4_70Glj97VE!gOl=XV!kMGH9`J5aw`!W!_G5a@#^q-8b zbr^*IiG1yVrKQc~PtHBpj|aVf99(&~&YeijuU%~n^|FlA|EpPqph`rKA65UiZ_qnR z#vY6F+P$O1(uwg|Z!w(x{hcrAQs_!IcpHe~?G44-)YY`_!v>EicEB*EU&oX$wNT}Q zndisTSuOhaT(3ZPSt!RBpUvX!NXp_hQH)SCKHBw2q{+x~wJdjdiObN)->?-C>w9_a z5a|dcwJ66Y>uoiLIdc8>eXews%|B#d2Oo#Wx;Bib0}k;^UEu@0u(j2-5RXzT z-*WPN9EVK)9Qzz$>wMuA&PYK)7a&W|B5;Zk^tc@Y@J%N7H=(nWvFH{I5M~6mq++>< zl^c>{YjWV`q7lLzGM=LNyLx3HPR>$9E)}S@7JPyiZ!W4Y5?RzhuV7&7vO#EGYUqqr z$QQ6KTTnalf?GX%@gISY4GADLg*;GW9%~tY#R3f|q-mtb2n}i_9}#h{oBT9h_j3+) z9LFi`LrV2c;#5Xgc|z|?DCz)d084;u7P~(`B5tr2%Q7E(x*@wwuSmn-PVIy)gK2*tt zm2?TJ>B9x%)y4JawF6hZr}-M7Wd%Y!_F!uyOSH+9a)L!qo`5D)C`}d>=GAFely!o< z#zg9O3TMmYbrnUp^9UrpsHaXanB7MT@C`fWm^prl$4!UeDWBle>askU(!G4>)_NV$ z*6MX}G2ztw`)yh31a`rwV}|cLzYcX&Cv4GZV)Q*9yQ~qdAN>Sw`s}aryP^`Bccb6 zx3J@^6QV%)f$P}sJ8!#}A{AZrIK58s>;P@B(!BdnT2(2OQo5bQZF$u$WG$-RsaHOi z(n&{CUx+V@piss}Q3+1{s;o;O{bT65ff8R43Hxb}er1TBwFq(FLj6}(6xNx*%2Xe^ zo_DD7b%NDv5Clh^Frn&ds04}{K-<;H`qMIMiPx3yB}=_T&;n-6!V6zIzF6Q<@IZiE z+jr{>*|t47o*!TE&KhJ)ywmSGecerp?|(=rI5Hl^!2F$qiU22m~H{c-B97Ctm0u!V(yQ8LiAFkm=OFm$yP zpYPQ^#FPt55$2kaSfa+A92(7<9%IEHmraia-F<$-bo_IvNYdIFc`7?k$QfK&TF}8>UlwIz;W3yx9M`LqG9?7i85n-+ktWq^@8|jmNEUT%&Zzg6TDJ${m49kI z$`ND5yeIrh*pJ*6#seXI4w^!qNvt$Gq6JJFNTW^Cy8YoOzgp$+lir{|dLgcUS8Dvt zrL3{lIKwnXBta8ZYmj|?`$>Y8sxR@a*C4 z^{@$jM~TkL(hYkkZcomQ$8_3f-#&<9w5HeMe0#ss(r0TIhg6S&Wz(d#9#xEJ| zN-=3D0SgL@_QbXr)+-_#boF}69=C>OkH3$$OB_xj!&Mxa9IcksP#W64`ha__& z0@E_PFlR&(RxxgnrNV2;Y56#Lq=2_0b33OEoQ)50&!mHxQQF)El0_3Ef7~*4*OPpR zmlZo`??+KbC%4>!gAzCQP!hWwI7$^N&;K1{eNi;44#5tsswa2e**7>u`77j2yCT_Q%IbGL zm%4#;@l9K!et=ao^Bqupy`?uj^~$d_vT3cQeGFi}UHaIiFHBKgO}pR!J4ZkP+~| z&;$2q8o2UOG3$VJz2vx~(p;uOn5y1$J9eoTkVKX6Q51#2DT9iIV^zm2Y%#up<>0J1 zTUIq75jYD+wCO=f z%l55CHHHZ8)ai3orE8`!->)~Ri;5^se4jpv$))a|&+V4aMG#s$FV;oh6hdAuZQ2uJ z6xXMv`5W*MzH#V(dOD2Yl$Kl3l~QJZ1LMu#&tdG0r7&en!)ryoP53WBwt!6+slo3TP2G?OQi6~b@YA~U zyW`%0_EFVZtb|X^C88Qr*YStqXXr1(&z^KrUpx*zXUuIg7{l*1kY;bPibIk%HallW zeI$JdnsIUB`8^>l?!V;7^iNS+NMEVM$|(tdnGcUF273!TERfgSS$6b8_=#9&)soHJiBI{|lfZaFSD&LlQ$5}Dkh zwfZ@lpHqSAua8@U6jY}KgDacfcsZ)GX6vGjyjjsVfq)mKL8K8lcen&D+D5VErds^ z-8LLtBjEa_P%D9puq4<$nZSB$Tvl#(GO@1FXfVwACYN}K0q2dFTjlXv+(;+;W6JkSZ)u(MJ@27U^6!)RHc%Fgq^RzHst;$_=+Z8QMBnxf zWcf8WYwKpG=$XpBLGQ58=U+;I`yNYECL;xC^PfHf{4%d*VoVvOEj~<5%FN|%8#>u? z=$;C6@DJqjCR|iS{8*{|Lt6^wskVExGuY;Dy#Ep!@D2JkTuA1#{VyVgvlL(n_w%*b z*`5)+Y&R%@U%5F!CvIxMV|$DOJe7~aOA*^D=H&Y&_O^$vQNbvRV|9qG$6-1eWu*yo zS;RR~KhLz|1hUhiZ^A-i!Y%K#$)p$WK8s0x{U~k4Fz>W-wy?kg#8S$p*?h~)&Cx=# zLQOmAE3~72@-G7-Gy3iaDZN)_Q?8iaSZ#E4RmlY(JtNiTsHDe#VA@Qq z5c?@B;V~&21Q%(Q%D-X3o^9nSP>PAu!^UmJEj*_cCJv7q!7gb0L81p`sw_ctc5e%Q zF3altXu?MvVm`y{=3oi-%XOqGA6{v=!cf(2G(I>TYa)3MdQ2i_ru{}l9SZk zn^YfC>B-jmO`Q)Q2}6UInYsodFA*!MSttef_IWC$TF!G^c`(SVn@JM=QMuVyNPSpc z2#i2ff7m2E#7hr==p@b7gNtY}TpHw38)9{s^h{x>>EopiH&~2l|0Jl_8*X2qxu>bu z%O4ruj=Nq_8=}GwM7^qciLXiPOpW9?oT;KH5Gm}tsNy4HAfN8XgV>&K)CYMa5l$)uD^TL{Xgf9yF zJNc8ppgU1CSP(|Xa9Xi@d1E6nw$hQP5B0Tl2SeoT$9Uv66a`*zWcu=FhK*07AVdExS#Tu$PXTC>RWWur^`LkbA5=>f`8hV=M}cwBc``@Kz7+e15&tSjS&~oznI9$Xa;h9~@pOc6 zor`%d_|RCSzj(0@aB?SBuWMNhFtE*hGUtFpE&VIVRT=lP-3gyc`XaH6fduHDCiPI! zA6i+eIwd8NthWBx@c9sam20pv$mTm{yf}C_l?0w6KwV=C1lrU8^(YkuHb%fT+G_@Q zsDi%uGrb06N-pEywNg0W7$|76BHnK8t;Ng1{d^pEK80K!&ZAPdJ3xC0u#*;)UlRhO zB$H*uyW;GrVQjH;A)&#N zES3>wQx+)K)6xLd$%Sk6@SW}16*UIQF!cCHkC_y8DqXOi?aVyA6cNfSjur+atG91G z;Z@ExF`^+wsxtug#SSR&e`PRi_^b$d6xH{ z&n#IW_j4&j?#p;Bk&*I6v-)JI%p&7 zaSb4uF~UbWVUg3crEskfH)xPqsbG=jA3s_@vM8Y_FCt!0p;~t<1+C@-D2oRlUZ&ht zSOGkJ{T|ceuzF%`QFF|DDIKr&|)1tdfO{*WuTt}C3Tef<5cqsF(5w&8eBrk^l9?H zU{j7uRA=kUTnEVq^U(}vAd3=X7_(Kiu%Bpp{FdY1#j|gY!hNq#`oW9RGG}swT>)lb zuqCOoGh9%df?4(6;hE!1e0B>Jux6OG2q7FKG85&>n@7hB%kYnPw<0pw^mSCn0c4-P zc>R{UF`9{3_wt8lnB+IV(?u+>?)TH6!fO#hi)dc}oXGe1T)uX$RotS+Z?=ql2_75e z-khHjOrjf2@v5el2-U(`LzE4FsDK5xG0A2Z)<>X_nvodARBZfuFN5JN>?-qegQ?vz z*EvBBF5*XwZGG@5GsFN5OKL%m8*HbK89WR;Dnm;_C+Lxi_37S5q|OnhF->CiQ1TQj z4qG)R2CHZjSmpASP7LD)AREO<958oW={kkdKyUdiSH)jpWj5YA9B&V()xW=2wTpcU z)__?ub4$<=q1l;O%m6p<%678%Hm zwUcfReS>6cG}Fdgio1!1*LG#D;>tcNIY)M*cf*)UCW6q%g&o?^Bl)F3#gCdg82PG= z!f_XBDS6e}G_=HINn}NJm-#uDTs)prg1lv7#xDUbgnsRqnlQ|4G$Tu-QNt|Z*V>F~>G$h#RtA@PKy zPE0Sl;&E1}a*HpWPR!xGVia4gAz}m-<(hj41HuQwLFwY>hvC-Sf)8trQU&0n*4~}+ zhvo9Of1v$<_z^@W1wuO35b?95VP`WiY2|}6r%JqZciF|0{adz@UMCP@IpV$>udMqX z@W-U({*QNVYuhyji_3`Ww|q!51^_>q@e8CKiHpJQQW;?m?B*-P&(yJvQnjsWQ6wNo zAl{0=>$ULgtRn4rZiY?E^GLFi^}@c%ctENSXii3a1itTGH3XmiAk$m(s$k|03Lw65 zMfA3qxb%G{vb25(2Yq&&BNFi znF`|wwVST2G<-9Ib03=v|GFB154A1WBsG!zxOnr ze;FMDKUj~jQv^KyEQGwVLphD9W!^l!&Ys(T)CDuq+u)f1#hdd#6ENID=HaLf z6yW5o;%>7)<#w>{bT+i6$upT*#=X2|%4lJ-%W_el0lZ@c1ljPb0O!SC~UIVA=-OZlGY<(IDfR4@^ zTJJQ6!yM_Wgv`A+t$*K+u79Jy`Q>}mhlk$?J(oDENs?+7 zTu2dr0a7D;uuK^7&f)$CG1Tw?bf|4@zFS5c4(I*wvxLI$Y^3hU90k8gZ%WHnBJNFo zM_;sKAEeznuPJutBmjF>C{M!8f)RXofMw`pFp3%GVs$Rh@H?vw`S}z^$t0 z+pdt=5jTOSCgefP(WgVhb^>}~C~%@}%|BQ5`CY)tP7bM_>)AX>ukuKawWJb>p88>D z?h8@?Haufw|g&^B;|+!_)strqlt16 zUVDdW`(^X(G>IYqSTBntkud0eyYM&1IStNDC44X8D%Z}*{OV%h&S zTG8?*-sIw!#7n!)mx*Q>))n7)M0F5_j;NE@6Y2j1v3)%1*-WRQZ2!}m^!WMKQ+DzI z+uu6gI2Xf)4yQ9uQa4^JGuU_;@n#;s7D{-}Hs_qU3$B;YYf_(&!Yeq&-Gxsr<8I1O zIlJNOyC#wvKvUTHWIwcneDg#DsnQ&%mgRII?FFcNe=ihun#58a~X^4S7rvRe2rWDQR{ z((eBr98R9UFyDa^>`?=v3oeHulZ3PM(qmfu&PundD9=_APc@7?SCzxxQ($Q#+93&5 zCN#FdZw0c{ZA-&k8vL#JcdIUyB|^3`nn5u#&%&+2*JePT6p>!|j#bUL%)(bz-R7^X zN*ZQ<`xd_=5xklRPbnI~wQLD`UMb@++Z(z%_=Ed|2KX=pDq*|Hm_BF{Jkd`Z?5R2{ zIt)_t)+~S2ET=YfT*)*3@@#}W%=}|JL^K~cx=L^V+9o7@68!2nGdHcJ0r-L?JuE6< zO5c@^7QFWqnf;h{F5-{pbpDWT#Ddg&47eKubL7yT)(63oyy2QN{)d+C@;cDP(k|ak z`T98kPb}47#h`0C4J>*A`f>z6mmZ>2+IWvipHoQ?aJ^r0izb20rlW^ZW?>Kw&XK;; z(B-IDr$@+ZD`rl=ty4vNQpZ`#+*qE`5za^=WS-H;#ru?A8E;(8A;d9?<}JC^*EK3h z!k=72<%rnYd_MYUGf_QL1!7!;p+#Z=|3bSBivmn5*emTH5u1>v_3RxlOeCj1%Rq2r z`#G(TLlsWajjDK$+9{bV38Fd_YJqov7Sn)WX_RTB5i<3_{FKyHrO z;}94fGk2FNWxf0xyGG_Yz}>t^@Ptgp&kzR_;};i-A&ENMcmnG&1Mm0jAF`7@5G^@M zGURzPgmhMt_eRCkIZl9+*{AE;^M>hbhNtuC5$i_3l)qG&oBknbNP*Nz9CxvSD0e0$ zrXq5(Kgd1%e3h@^Fb!n+wpD^!NGADe=k9}FF0@o#l+3g$EJZ&^e}VA%G;Nx4L~wtd z?gBry)o|gbU96M%xpan|aS?3NxU}YO;W?V&a**O-C5j83_cR9y2GlXBubv!~0^vFp z>)IL+1J6GFvp3!6JYW98RSDv?A>!eqAzV4yVKBrue=n~+iZog!GxZr;+=0^S-PC_e*nlh80IciRw={B9u zz;IuylIHi}a9>Sl7p!X#<99zT@^ds@LgIP_<%eFZZ$tk>gC?Ax7~jogtMhY%%T`dY zSZJV%^{FTViUNO~jMG*E;mM8N+ClYNVPjkbR$M1moe252At^S>6k7@~X}@cS4~w_X z>?8dHBXa8}?ou>-?QTiYMSo*z8Wtann*-znWhAEYv6sYK@(Zxgv37M?OHxzov_;-3 z6K}CKif?J)w`}2N>Ds>d6@0$%2}#*z?oTMky%B|OpuiV13XuL;4zQHIA<3TX#VcZn zaX`f}Uzf(GWIE84jsgG%^mVWCm$xfCxvr31djz!`nJr*t@u}6`cf+Yxq>y>Ky`8U( zc&mcyzClxPn-jAOWj)mqsr2G-p|T@hKs)%b0piC^=g~eret8&{t@+jYH>mcNPy#Ac z<#E3w{4J=eL$qm{YAJm7mH(-Wz;4y%?_%J`3W(i;U?8Jgk=)m$O8&F9PYoAUh+*M2 zVLMq&4P95@oAAy=ve*2R)V$f_Ma(rO2}vZuHG&t@&q+d+g~9~`+Le@1eWGsCb83O! zo9Z71jhgG4UC=dPlj7jjtn~8++BN;DYcxDK#)lBC)R0*kZkwsMI^G1_F+~Dt0Xso?yUQ+2R%!)*H zmhg}7n&ccy;G!DJ+&BIwf+<>j{%BI69I%j3*f3cGfMqJ?$d7u4`C@gl8#h5Pi2aVS zbirW=OX>5Jx;b4V?_m>Zed&4agvl8pBKD-1)(klD+jZr}G3$1zK26)({Q2tkK%}`7 zzy1FI19a;+`2FS-``LfMn~l*lDIg&VZ7ckLlXwEgiQCHdXJXvaTB6l||19yGjd`5H zfQ>E8mQAX06qaZp?e6rKL67ym<#0Qx?44809q;aKiZa#3b&{WH;6v&7seiY~Mpe6B zn-m@;T|k_OlKE~Q&hyu1_@luUbRHGs$m~zWHs!=JjWVM6$u&jP!CKFf+q3(G6goGC zd{D|_j!JQyxalWMmB$%~N0nT8czFqs#s>laDj&;#oSEWI^KGyBTx2D1N(fITk>1j~ zpj{;sms;vKKOQeR`a97s5pLBgKOapcvTk5h^hbST2fu z8{&6cA6z~NtF3e$3*S6@izOM4{UCCnqybFwB2fMPi{ElXgWFmdSJ{!G{MUkbrR1L# zk~m?7CIe4eRtNGOdXB!%MPRgyEOSXv$!11aTHWtWFp{x^zorQl0OAiOJ6YKQY;MUvYsPKj3*lX4t3}IKokH zq{P38Ls4|-HwF8TOYM^VL_XK|&pl~=5k#hyaot(A_)UWNmAfrH^JCna1ppGeb)xi7 zmkzedJJhLeJ>OXDN&jATe9NaGh&LJ(ZDU#w3@Bx6|0VF=Yw@0skyQI9D@v*e4O%Vv zWD!uHhZC}S8upco0=6c)#r6urYO@UsomevA?Sv!PkvW)x+5%fJaog?#Pxyr55VpP#Liw1oo8#SzhLx@S1U_wWN5CqMJk=ANfd-(*}jOHnPRcKifrvzpKd|8%SL? zPnqb#g|2CZn94mux`0j}4OV{OA*6k!rs$fOXl^PQw`lSfy%J@f=Sr;rrIo*?p_E4O zTwTaSP9H4TYe_F_EW-YN0D&1-9!XA^H|$`y9-mfJe|wG+Qa!W;8S`E| z7#4c9mNPcjZ)W7m`}{CojPac={J+@4RWPH!@z@!*kaNvc!$1U_ohE#lT${k{T*K$L zf==08x98rU1m61GzHMvg!uF5n+O08409Ri8G6kHkloUAppulnz+ZR6NAIi+5(6Y)5 z1`^XEUWw|3KrBeQ#9%g10Z2GPSU6TZ_Vo@fGdzK#V&|%%>l31snY`A}rBaSSBD1sV zAE)jMyg~aRxP%P!m^|d&rVnoW!LkbQU*Es{hsQkAEcfEhF40@e=y2hOi=72`=8Q2r zh?f(+(#f>`?}T5w43q6+yQs(*MR{nwB^Oi^*AcM&$gRA|Z94FgcKUmWbOF93CLSF7 z;OU=kcn^?6>{_TbUA;BesQXTacNZ zu=7V(C@KTLBT7}lWJWy;Q_(;dls#G2pk8CLZG}Aq<|~*$4wLjhw!QgiG5hyzGPtd( z;$8B-&iTS~li%Jej?PiGd$s65nDg!GAO6!Br-H1u?Vzlik4?wPGl!fvp0}xi-qW|Y z+xXti55(O+5XJ+LVoRBeoC^7mbyQmyQ(M+ht*GmCl``IYRFl__`>l(*BbrNt{foTE z8ASu0@u}7JzfSAtpSG;piTFxJt^d$*W^28!s-xLR@PuV`#u$j|SvKgE(=UQ3d4O`F zI^*$ZKVa-pKku`?*Imc04^&||`&d{(BD;xRTc||SpYGNUIf*ES3u8W`+PcDCs>LiK zpya!9&FTH9`@ROxGw!y$>1AfCEG7PT{r0D@XEFA0;CCE58GXlLe6dQ#7Z|V7;~VFU z5Tip;O`A*MX^_OBD8#@(Ure7{eqX(-bJwjXasSot+YC|Z$sRm^S5Q0@?*%`j8QU;r z*+fkstjUR&mp)Z`x5nj1l`70DK2dgzTGF#u8Rq1UZQj;gQe7Y&v6v?uERzo|ipO}V z(WLq;H8l#g6$%ynr9^eZ3u_nC10j-bhm(49X(_mY6&1GggLqUV6;&`KtgTh^bnBX(iJ@OtX zIUaFc*n}gTwkOaNC+#|LHn`c*x+|0yh3$^O9dw~f8%jXFm&g`(m)6p5q@-L%DRc+kxduC;!h@ze+4{`=M2GCEHlyerg>M z5e8M4g+AEg%LKn85o6k$zdbLa@1x1MZTy>{q@nn~j#a}w7a55@)3Rcr9JK>oF#mAz zlq!Im0c#Ns2oHx2>U1;Y7Yq0Ia~@n6I`)GeRSk|l^=*!i^o$0-AdXqN>+pZgI4-@# zr*ORamKyfT3-xqXlmR88hs~3p>jJ(0J#AYxlO}w8fZ91%8~`OQe@XhxZAiivsN-bd zveeC}E~G`CFc~8`EKT;By*(A=m72?^EMKHU6^r0ednsHWe3QVzc=&A@KB{zvmQzeL zDuOj18MHlL9S=@^0W2{~x=h>L6PUN6C*DI5bz5O*_ZgyopKCZz3URk#=>)R}^&FSa zMs2ATY6EXKIxt^N(^sIY6b3>!7rr%qUw^45I3V2cCfu!35(E=%QqTAHSV|qK86czgH{nmNDoVP&35BMAXCC=%N=-$n=%HUl334)>axb@ol zzVRSmf~a-h!1HT_&W{`812UugANGSfXXlY4l%@MCH?1qn4Ew4^Y`_8{bYeuoL8gtu zztt;#&Uj$DT1LsGbEJN7PdWgdd|ujkROV}`T$<`pQ!=@9_)Rmg^{G_UYIQ7<&|ssG z2DO$??zeiQ@99+6-qY4+0#3Pj(#Hn7z1X#@BFbX}XGa$vMZ`JR#VEtP*x=+Hn8K?U zS1n#P+ckMmqJ{k}7+Xs#J?Pmi zwkxwdUi=r#9!X)6%?gh32*(R;rPq&1_D3_t*QLE*KWOL!RG7;fj{Yb#`WWg>;9vj1p$8@-|@DX2NE`HN4~J1 zR48Sx7zmr&iK+`s*%#Bm7>{6ux89XuJ1?X4M>rQNzPC8^pl&w#lt9>)i*M1Y(x4jn zs6!PeA6{e^BG8nG*w(BRA5w-@7wPz!ksXN?oT;_%Iz()bTF2>v%cBT8G{)49}jF%;TcS7IB~nPT(VGx^l@+ z6-f#|yrt(HMhckRCD8USK+mjw@rXkG!%_#jYx&sXN8PbqJ(-KW{EhP+wc(DIOw|-g z=HB%f2FKX-9(aoc^U>X&ay4>o$$E8zH_yI&+=v@{gL-jK!F{BqKBDbUK}Q@^*EQVZ zDJmyk9ndG0kWa6v`UeO&Rg7!jNv|2v?Fz;hKIMgun$~HmUY9OXzc4w_x{L_+Qp!~t zvL@L`!|sXY^TXsi|30xmt=~8ae{Knt@wafk;i0_LZk%0E`Vl~#t_p;`xh*86n*E)% zMC;uOrZi;mF5Ch~EgW!+mtBzoP!nMTp?ZBKMi2L&%qe&)7VcvsQ)no}_N#;|7@hVk z?$6zkh;44->q_SSZEYYkpn(epU!x>KHw)L9ExtN-pLzYp&h@vTqD^Zp$cQS zU5jY6t@%SJP(b{IDjw0&J3!cpx$kf2e){LdC$9@d6Jb|?3d|*{Ve>>|Q%5*!as2-0 z^KYuCBl{&%<_%rWQs#iI(dVYOE>Jk?XS7QG8ck0pVvG^@wNV}~r`d;|$Ct?<-hL9u zH&tQ-im!q##i3=m2ISB4kvWLpa==%#=&anqAcyDI!jj=UKCbMgj*F497M}8y-|)Q- zF>6ow7-uV5BMAQR)(R!PxE`onzzJwjY9obPT^2$d1=vX#e0=wv5ojnGR`xsK#Q{Ml z&W8!eH1gF1R7}{=SI#onr>AK7itp0_tCUx+&u0tB7|adN^~PGHj(Ib0e{%4oQ|-h; zx1ya#cIZd3L4)G6NAZtPttUmIHznVybVx z6ap~os-NzSul@^PuY+W+b|9Tczqm?tgk(!QUT*k_{Lw1C+Zg47V7cf@J{ynl4khAQ z$L>|>-B5-R5&pPRx-Y-;3pSK&SB>o6anLLcOyJ=&14c1dokKqTVb{ad<&&j@ zfYi#z%EQz{y=>?8eSQdc&^g%Fx4q}AHTF4qbrH5>%)0oV!>&#sk^6S%+Jc!7R5t(#Hvq*NK2@WE`V44iBX%-lzfT=^1okjMgU=ifw%P+SrD zd=x+CGf--L>uM=e5Rv^x@f8GLZAy0DzgH6hz!>DzXDh-RUwuWp@PRG|pi*i_eL@f_ zt1RNDSarH%wj5*idrY-94?60Ke$8 zJ{7t59XZfA_~TtUou>jb_FcY8y1$J?aTW@dmBceISJB}C+S7zAlMXfN{?nwSb%*6O zj^AO}RB&5!ypTX?nf?K0Lq>Almb$RBfe01yIw#7u544 z`vRLupWJ&|UnvowlQk#u^e*!(aYk#QuAT%&&CrE*m8;)kytIrdmwLEsCM;DE#RR#& z_`S5d;tH6-@AIx5<|aodQ`w+gS}Vm~z=%AZ3+mF3(pd4-&>y;=>ZylHZj`xR;1Ij#VOy9szYN z2W;&|<2WKuw|pl!l92LD>_j&(`giy#8J*=L8tUCz^WF0sN9Ju!wTe4t{=9C@im9y)IOHz_1YT!5o>^NkAkz& zIzSz-$$R;=u@}f$xtozQwd#)u=JE)Y=5jub(7axecFszrYqCi_^9Bt$83(Xl%tZe> z+YYrXl94KOUP9ORe6@!m)%&g|14}xC8wC_Dz{!dnnaa?0Z>gz-hmcNuaVF7xCjC>p zU_rJuCterR^fd`$qD!*x^%RdKF%(f0FB1}smInuUFn0pPf1}QRT)3hEkwr0undj)}yXs=cShld}#*%afo#P`q(8G#NA%1zByGAM9z} z3@9qogBY<=SM5*1>r_OB&v`)S=FaDmiZP}Lj|3mn*Ag4WP>TV zas(HigSPw?yx#Mwh~a^cy((IqiHMI=BJz05(~G-9#(XhI+NPgmD_?PFvKo2f!ivN) zn{BC5nx{M6qnBe>*x4KpP9=` z^i>EF57z9~B25hcPGoS4X=%*bdO43laW3{w1J9{p!IaDpxlepV2i2x^Hq*I__ zqWY9{-Ms0hS0<1UWm*ji_yN+a!EHGrDxYawxQ_b6mA}59Phsz5{zl~i5-|cp<49I<*1AyZLJtyHdx>1~+}L z?5Qm7c=*$!M(v*fH>d*is``)5RnPS334TA6)`frrxR*YAs=$nMPUUAlr`qLjJ?K6i zFPZd?_e2Jfe~t@N*9m*p>nM`eA$&Fny7VbVkN-Zg#4PjlJypW_IOBkgfuqI~0C(Bl zna6{;fKY$dGH0`0)Buk0<3XI*hB0=ZW9scDwnT`5~C|^KX#(R5r0;5-uqSHDG z4!P2JyXpCZhWi}RPvrrE+}dkT1adINJ~#UiN^u(7tI%7oJ+Ka5eVfjK0KE z*T6)|NBWqt4R$}(YeOCdaLF#RgD7;%IKGm9;C5iPVVn?ok|J$*&ph8lGQs`@F~dl( zZ)mj@KUzfz1FEQi)UW|wA;EtrwT>R;g=7Bq0e@vyXM8Ux)h>jZ6BUh`O^RISRYiDPW*#Iwld|WQ)E{ zCrV1{SP$Ag2?U0j5crJRp@|$90}L;(dd#;;g}cDFE0BcmssT#-%#`(5PU_)QP=k*z zjsCV%@XpS-EYn$h;rfL3Wp7AXtp_(0e^`)4;D_l`+0FFYmc(FY-3)TU4Czx)xQWndoifJ1+45>w=&@QCiz4ct6J!3v zPj~~YE&cW*dus}h<2kUS;iDyI^0X>2gZtBa(KF6?i<(QLV-cM!ynAuh{s?Y%FbV@k zloe}rqoEr+faSC$KY{Ns$+%ER_<5TTfj0M|jp=y4lX)74W&0)z z&6AqIZX``WLyIY_F3LRzaL`yBZ`|Frxv*&zAcZ=D25pm-Dy=GJhTb@4|wH^*-x~QpF=zeNNUa^cmIHcnmPcOCX&cd2WO>Ln<_K zr~Yde+nN%hZC)|N9;tagE+s=hEQ1r~clUC{u9yTl1yQi$5kr6;OOv;P0^(TN<_QY; zLcvAasQ4dqW7ubn=y8{mPEdGhmMGh|31Z4RYGM1e7w!0Sfm-=5f1kE`vquWwFE@d< z$gkz=nqSjVpPdxbVw8Oqpk5Y&F~lrwHE;h9o4}!nRuK7rrs!CMw=T$h!fR$Il6(m~ zyIZ|ChT`#HekazeQm=WNWXTz0A52zdEn1(p*|u}OPA(c@+N{cG^Up! zhyJEf^#OAed1M@VLXKGbdZ|<9@^Bk zAT1+@`hmgM1?^`cET5m7mzZH5LpuBYwY%3J*=Z5Bvm+6-IhN#4Yl2u`OYp4(=+;oq zQ+ze^e%~_gTC26^-(O6Zr}q&zK$T^6kKe~qs*S?b9mWpl^RIeaD5!zD_cjOu()-Vb zrt~)Dp0TW-k(It&sp_g+iF>95fJyxlkjH3@9j%u_54baoi5)ez{Jwk_P=R}y9>7o0 zG+xGGUhZ08HF;yPKG^9;v-S*?>~MkPu7S%lsLVJ$U#9b;EZH?7XSP|Rlf{Rg36>lnUB>oU>KOJiJLPr86^*~;7q zNS68i2sOR^!`5Nb7P^Dbd4GLf6*6 z5$U5VBfiCh-v_e8W;E79n~p=6ePJ5HKMp{KX-fD{pr>_w@*uXu_FEF(04T%uo|`a8 z9cTQVV<=qp?3rCxgWv?DD4LM(Ryesl`*VwmOQE%?|2Ay4D(DO3*vpHvQJOPda&3UP z} z2y)$&-bf+>>ib#W?bTcz_mm(97_NihXr1jy4?53`H&pxG>6-N$_XEe_o&rXI&ns>88h1g>Vw#^J{ zg{0+FTSeFWia2p-$2L5lz1FF?islSMNY+BW*5_oj`!Y7+mNu6LJpQqL>__!zSdq_E zt9~LTXvN?@FS4_gR$};v56|g`mGV`#Wt2*8N9HSN3brbrYAia+c-j_jhEjosKY;S+ z)z~g5_Ywc;K~$ragxa266g9C-`wvI!Qo=n*@<4C@{1)0KgEmDG{n}u(lEQ8F?y!zJ z3~ba>LmbS@8M|zwM+a>7SQ=E#B-x{L+lxjnmIq39K5y2NZym=q>#%4UXHJ-7DYXJHD20;WwY;gD})VXzvh+>W^xKWT)uMH@ab>_WmNXh&4wlF)@jNLD%}g+ z?aB&W#RW1>3aj41bQ1RjR}UB06DVyI2-PENJswmV1q8h^!vitxwOQ)PT1>GY!5e-7 zbUf-|+Y(v{jOobp1(rDk5DO4u-W`=Zi0= zyBQeuR?*)-Z&Jac}N* z4v)R*59CW^8#&2kw*xm=4spr~1ZAp{%U>U^wXA3WJu9BYlPvM;5a~|vJ40N{yGu)Z zO>N_;iWCD?xnfhk-Ebn7f_BuPomd<4yw`B&igBSc;f&!7tV zE}TL;swZ;jnxTUSH-H`6A;+PYnlRGl`0Qo57h!Y4NY$F0es-}%Wx@!5!Cei5REJ(i zs;_~9+s%c#y&G&C{u<4- zRkP&74um~$$fsRJD|Zv!$)Ayd@+DwriBhlzP`)-xyz3a4%zM^6P(ca1src_JYM2Yy z&(>6{gkdVrS|k8TW5~rz^>79T=m#d}H$QaZizfVkwul!XPUs@ap*P zn5>oDvx8o#PC0~D^|6LPFvW^iVW+Uxfm9p|5s3rj)WV%(wU%e7r#pKfGUPrRA4%Q? z{97zXj&Wtae37)0^^cq1aVRML&dsmn5(UHmBMK^0SJ3?x&11Hr%gxQC!gc|*xRe#x zuE@P?XZ6b+l1I_|hssigr2P2nEL8sn#Cu-Krh9sR&3RFhW zGG2wyeWfYy6za&%1F3}HK+UL(^hc7u|3%VCs4g5ql-MgH5D6F#!_c4h0>8q!shy22_8>CAWLTPCNTl+#4noN_T}B6>R~g7bmZlk<|+^F01Q;+lO!J2~j&_r-KrYe@b=dv|O{B4zwR+l8kMC zeg6mtY9IcB8pE~!J*c560iy}+9#3$2zZnpdV1crs&xFL-gMOb%1@RCe<%dMY(M7ExRAZ+@dhLO9r*yPHjN(A_ES?r}-kFe;w>DKAI6+7uJMd>6*ixgw)@@w9 z@Yjt3(25MiTSlD5v@JU84jitpMqH#P-!3^j}*^I_2;Qv?p z$%uE6l(jim^6qjlih6k1S$%(Tg6Yce?7jHnwJ%1R731;gwz1r%vK1bv=%H1fdygiz z60oEouyX8`Z$eJ#ioFr7axabP;X<)x$kg}x0&&0}MZ|$4BF;V)ZE_tj7ucp7=9Z_8 zM9+VpbyfX7SMp%8W;LwtXFVT2yroq&)ku+U11=VaPeX&V8SkZs@TR zCwT{}Q>KH{tWQJaw4yNhUwdOv-0g%+S~Pf@H$NsVdBF6wvb?CSd*h zk%>i7p%d+QGuY+WU1iY8vnmN##WdYq^k%mN4Knde2w(SX#ni9e*j>ozOVUcjDb1N?kX@(KCqwtNSNh4`^$(b z#@g3x)aUmgJMcdSKf|vfa2kdlBS_=s(dV5~?@xH%u>R z+?#FI-bV#YUCVn%DIx@^6+%Hs(j$poYy%^2HeT1Oin+sVx^2#GdY#?0{to^!DEY?1 z!)RH(YV^lB8vx*D&i!tkW6GJ{9r#jP3@Kk-_;8r5aw}&9$_L10ij<&*JY%2Xrf;hq zOp;%Ks{fbu-s-+2>Nq8{yXVdL(SIL7DBd;da$_ z3FB6`56&O2S~A=m+?HaUNPBL*n2~$U7Fwafpk<`*`pitQ!kJ1HBwJJ9ht#V(LRnK0; z7sj?Ij^*J{xKrL z5(^MXVj((>gb6<%dBq%H8}@5#s@<94v+rz*{YIn0cgOcsf|pNwtrQD4-#&64tS6qA zyzB6Ya^G)ejLxy8ioH5!IiuP5lJrAR&YX(L<@dbSnO!6`jZthPQ}QlSQD*WB>0n+~ zKjv6jrqS&1P)B_NCqDC@-xM*(Sjw!wKAenA?}nNS>1o?rX7gIR!5y}xSpgTx&ox$W z^Ge%9AhC}wHfIMa5d>Li9eeMUT7%*SuU08wJkG_)?ah<%qt~Hx^MOBo7h7r(O9ID7 z`QH|UT~GUeKN@4`ngaA=MSndl2!a~#qQ87SbP9XgKzDB>C9rR~S8f|NCCKVfB|Y7Za4{YZ7RBB+>g{+9}oyQRWae3?#VH7OFBt ztRr>9MBIdu)vYlUJ4_nUS-G7(w-u0|7g$-7eQ(ae!d;j6c2lc`eH-ryA zj*3Xdbu=zQ$CBf6f?pLlK~C?Ozp&HMGGgy>q-M8SC_`3fA6w2mi3m-2pG7gskKflH z8M@GT97b%Ecf^rnZrba3c$Mk1(0%!b^sTKLpX^9Nn{6F{k}h2dEFsw+xETx(JRWj@ z1VSSrp92wO@3z)oyO-0=|H``P_EtOUnr{F7y+IjFMv`6eItZ*<_3D>-As(zGntWug$CF5C?u`I=hg|7UI z6s{!|QQhY()7R*pAz^KX&D#G+#p@(^j&BpWb!EwjNCJn8xtjZ@s+nKceyb|IG7Y6Y zcmk{6wC);vGvFelk>;;mq!YQo|4}Pl7dR_ntA`;Rx7cde%n%Hs7oToEMyl63d-qeC z1lF6hqp{0$iJ1N^YQeOlnpr_-NeF%-CgzI4{2&+Zm|s-B3lIoA4k0lRi*aYuB<2Tj zDQ~bilrG6$Ey?23CdS?DZSUs?qvcP`q!u6IH^;#EeolR6ylbmWQC>w$t%B_@A9=4= zubtoUqJVxComM6b#h8)WP7R+5ndv;4E%K{6-FftLG5Zb8IT_qY;m*>0tI4oFPDW)1 zEtE;_zZcG8_l*$7D|Z8SBG6Kvy&H5bY>ez-k~3jeM#p9v&%Ns~(|4!pMffu7rAAjH z4NF{>GCh3WH``HRDEMRmb*i;YbQr^8fttnZ>IVEXqgX*PWSeh zV{c4oD^p_CBZ^*S=-}Rf;j?`G%ix-}2K%4e-WFTPB3*5kl(;KeUdHa9;q^JUc42R? zpx~POF>3y$ih6lJA2B^O0$AjlRRGr*@hV)SN%>9Wxxv#l2DHfz4@W9Cd)NAH4j57W zEBYpb3Y3qH+hgaai)w>RTy$JSm`d0x9GmJFYRQcsXI!2n(`mnNTnSV=cW6;1t$}=x zYU~)0d&7SScwW25M*wnw)lS%v>!Z;b{4zM18E+qPPc@IJetS$F-n+JqM?}l^rG6dA z`JS1UDKfo682CO2PuHV|&fa&Hfd942?C#TUK$h#^=g=c~=WKBpZCGvBC><=YDBNq5 zB{Ji~FcKr2J1_3qivTLzl!~7YTZL7NW-v3B*%W)|%+*<6c9XvWnN;%AbL5*!i$^=d zW9eSNDT;anNojvG-)M*Y`@(ko`v&)r0C(qE6^({DwrWiH9R(n;WRI6)X*R4@&Rn@Q zb8RD2X*JNC)iH3RO@;pBvKXYjv)~oaE;VxS{>!RZCr;6H6}?vU^>?3!XM?Eh`!d!cDJwY_ak_(y_iz`1Y(UK8 zHG1c^@+6z9f1&~(l`X;Q+7c_2vkDjz>_dlM@TA|Pks%90jgPIdKmlqDTMN`issh~e z9LGK3AY_@et6PXse~6JAT{Pr#%b!5r3#*n#weGF|ifoNWd{g`Q_GI=THUF2*w=`O^ zT-j0>82vm#@_?DSW<=paE#49sN z4Pu%KxJr>u+mlT$&t;3e$_l@W3onUs?CTCLL?@VYK z!7xEY0*Pi)=}}dBts|qit5uk9NVX=$U53`F55K<^P+;g*9X+dbl6m3za#kt&^V+w1 zI#9A=M>P07ezVN8-_L&`p00EHATF7m6a&bCH_57%>cxEVq3A9}g(gAxrZo=H5#qALhFt%Us&FA+^j9D zUAk~d)yAEJ!T@wC2speCR|YEt>XE%e-%wo|F7SdFNiK#wY=^x6AE1*6cW^wLD^$9K zPW9(uXAUvkJ$Y93zg|DSHTw=zU#1ua!_=f=e}9vVhCm^bj992(yEQWEwt!N^6%^hN zVofaZ6CqeLTv!8c#IfLC5Q8Lrl4@Ly-f8T5>@iF*ml1<(n zi|0184?qX3S)L%KI_?7+`7CN)x6jC2`ut7*TA5#hEW!wK2Z|%K39u9MTduv_uhAuH z;eT6Z2|>ed1qU}=C1BXuqK3=?M;nm-#zS5?=XU(J!CEneyky8ftG^Ho-jgKD4!JTHt`(OW>NIIgFVf-Y*~#?B0q54zd3I9O z3OGp9H&sy)jM4tAepeX>k{)od%+14;ax0_+-Ugy}x{pgE?3U_4tg$W>^K=_MSx%4- zUGNT%fj%=rRC_Vqi?Cz;CI&N;hGIsa_@=rcs)5L1Y@T&JQx170>n!_%?vsEL=wuEn z_-*jhdyKES1N^(d4|^!LLUiNkzG*@bY6n-jxBVuiqq|EdYf@2(^F5j2@?GoTFTPo zArfzQZmJ@cL0Go3_kyW=L4Hd|nnTDFEZ5AU8~kx3sgLZ1UpioUeo8F$2MVwVMPG^= zUB%`KbidbN6hM9>kh8sU<-ww98%G~Sm`*57Od*RcbVmw+!O^f@9`=a|Kz-e0<*o^L zCE~~#qPC}afHv-RytP-_o2`1T5s!tcN z2b;{ehtSc7W2n9d3nPBqX&W-P@6Uq+8GZP@*D_do7Ban{qA>luHkBH$K;}lhVStv} zN1njTY*Qmc+`{dr+`z}nTy}44nc9;`o2^Pqu~$$c_vi(`8X=MvfTgcp{>sdl48r70_XiXw48!7*QFrQNP9@}gU>0gV zZ8qMkd~lY(F6FXvDT@>Un#&31ic$<4u?1XibTNdXlVmhH_mliBLjAdft{oenq-**( zN-0ueS`x>qabUEx4M|a}0OGQ{IiWs04>=rb{LY_( z8k}DlZt@SLu=#FedVCTH!WK%c&yO0PG~Iz> zy6zt&_q=ugBUWj9)t{jhVF@=CFc}IjOQ|y(j;AAqb6(nYLkSXM?PZY2&htB{G_>NEfOPwA` zk8;(Pk?j7m{bwJMdVI8ra8U#rF&~p>#~G1~x6(hhvl;eWzdHd1Lr#_XjYjzPep^a0 zkC=VOM#;6Wtg{p53xrm)VrNo~{3hvxvM?e*r3b>f($?e0#*eHxTQ%nu&FpB?jfhs7)1X@0AGy0&xH#lL^--urbQuT{(EPweR6jC+R{f5zrXFNGt@7Wgd81C!QKHufykVZ7LD?|t^JOuGf&yi^V0D`C@sC# zNwP0Hw7p2lK@kKtBwex6cJ1v=WSirwWQtxcAt$fY%)ryuJRHeawthtE0SJgP)}0H$ zNy0^@3O5*ti%hnWQqoLK`iqX*xqjAVen2KX?IG;qCwyQLFv}6Ly+~d@F!z%yDxf5b z$Y@VrPW-Q9WlwkI74;J@zppw)I=7ku|ETYzmdbuHB+n35^g@FQDydNe{G>-@F4wk@ zzBAsRliE0)R+Ff+0lr`D*J8{HaZM@3kr0BXrGuJpdFj zBUkWb`~M9iaqHRtS^KW)B}RII`4?H>ys27syiUE&eqhLcw3BHbD7egISzJo6&wA}T zDSoNxyPob|;Z1@e_Lp#Vo=_pIuT>B5OE zm;yt@p+^jwz-5n+yp6^$fI>%)1B|BE>5fj|sP5mG7Z}hq!WoFjz-VS7<4xgbNC@>) zJrSZwGC%(ps7o={QYwQ_Pu64Ep0#AzelX~l;*HEZG2se4;bG)NQUa@R%6Xnc=2Bze z*j)#vmAJHM-AFnh^W#fuso9<$e^Ve34VC={)snRjtoZ5>Q6^HhAMug%c)TzBey7}USn(g@L^By;mD+8*O;3+Wf*?2uX=QaO%5gMh5v$r9SLw3 zv1WUWaR`w4al9#K*Y{VMm9ZlqvC5uV?Q*;w|5CL$^dg||Q@okhjVDu|Y!D=(VPOy> zQIBLX)k|!63dFk6``-0+`(Z)&_Nx95310IiTYNP9K)i*dlsKnrU{ImZ;CPD2SW=oS z_JXF}&#$7D=mNtq>hxkRRgE9X{GFxL(>r?DO)uuP`Sj8|7($lBRz)v2sj;@yq3YzN zVrflAQEDs2upohhIX}59k5TTsNOF;J|itsSh%UC_*apauLnEmH#Gb0 z&z_gNq;i41)cvp-QfZi_ggTfW?!iwqO`<~JrNylU5NlQJK1y<9>9zN7=~83;cl}YW z6cw}463#3$rJsBX(N(l|ixy-d=^eLx!+{8t@eBQQMZzeB^(!bX*`HQH)%UX>4`2P0 ziC-tKcKN`3IEC&m+sB@C;=vPEvy?VE`=m5jnB6=&5)r9zj=ImDPdr5U7kk7>PCZ0g ztElq)JbZ;~NO=w$rGG+oxCx_b>{I|UqMAEUUk#}Bi3kjFhkh#VH_BB7_j4Kx+*CWc z&Fp70Ax$mlKFsjuqBmirZa(uBl3Z;(9URQRpr4r1k3_&M6?-=%dXXbHk_1a)3LDu9 zV~_?gfi?jsnz?<)XJ<{g{GRbw!PsCpE{;3C*1z+G0)=baDO$Y!(f!!fe`EP$=(k}5 z%2yu$QvzoT(3X%#d)4=E-~>j%05hm$WSTUb(>I050_cCv>&yh(4V82@<`~fhdSt@= z=nteIUxXgK7INv)nSw{e0FivM0(JTZ3eGBx^o~V&RS&NSq7Z-l9XJt+m`g~@E7kw2 z7H?iBd3rH0f?AOm$j>FbZtxI6)EwwY$$Sfu%J3s0M98rB!AW>j94%OcbEtvzUxAo! z|2qa?$tRam00Fy8W3&9Hex-+qzrltDaBEVCQDGJZ{WTEGczNvN%1ycQxZ)4B^~y=I zlAmlHyY!7OOTH+jT<4fVR{U9yx~ZlD0Sz?+|5!O!8F-R1YJoSYTS+^`Kc-OP3P$0& z@oCk_@seg@if6IRC1j=i1U`MB;w4}eOaFfWmNE{oS}Yl{mpk_XSSEt9iS3cR58qX~ zG&L;mR|BNsQ^gFJY!uP<9fsTDT2E8cT|RXuqD|DjkW6T#UPFFRDwn1H5b^1a&a|E9 zo#2}I3gZFU#T=bK{SXv`nsw4DJTfn1ELyz_$1UKvmr#88S)ebzM(W;Hx`SK6I387& zkJ_?*K7`wvR;PGb1+2WY^jRT=yv9Ss#Sv(axhEX%b4J+BO2U&&`^^xN5_4roFbSi* zvl{UtSP2L ziRECd&ZotQ!d7(^J)u-w>qrm8_V52(zclV3s0mJwy&Q|JFE271B7bju>(P4bvR+=y zU&Dd~qHe^@aEh|eXC9lfp3~n;*&mE@AgZq@v^=!|S`fZWyLjE!>>w88pUHJhSNzP+ z#w}#Y+BxTc=!Y-9*w2ZBb7hdx@!0r#V6Wd`K-KgjCn$8H$fR}OcnNie6;N)ZVoTkY zQm%h`fj+@W_w-?qsyTO2PN0F zw(zVYF)ahWmqI-d!^K$wHwL6<#RS#H&r6-&Ugv!b7Z^Suij}$V)KkQ?EzUo!X7ZG) zdJ7g)_?&&<52gBiWLC$cXhp-wn8JrWY1?%nFW+iGd%FVL%}`Dch*j9Dz7Nl-SRh0C z0J39s0XT#)Obq#c6z{ga^RQhTbN+rEa0&Sp@NbG`9RHX2X)P{(THC0De;ItpWJV3& z9JK%8IUpTV0}BY_~^>X!R(D$ncGYTl~Q#z34eiqMAx1em)N z+zIv=iR1D+pWv~s6w|pDSH&^8s1WKosyK3&{Qx%O_3>6F?ym<=nK2_-Xe>o~5SUx7 zBHu866~6mV*NSxu5K@+6;G`T+I^^ZCneuBP3@D(DYesw^Ud6=XkOpX&0TX1qGPcG` zWE1h!&dX-T?&$<`=+S~m9aOxath-Y+zoeX66~qp;Mm`4qLly-=0CeoIrzNgj2&ZCo ziZqE|$Hf#DM|1M3CRLZ>vjLplM6lk z$CK06zo_!PO+q-zf}jE&qQcr(4n1?kZ{wgd4Db5dhC>JY5n4nq>q256NRXa8;cM9K zwN7x7{TjsB<6V&)e4BxFG#OpxnBF0wg+x>xs1gvEBctiRtP<|iqC}Xz452zv{>3V#oYh-Ef%rPn$3!CuN-7sc z8X@lZZ8=YE$#JeDs!3D=HLg@cXA+g{q_U&9Y=8gQGv^a3*&?#X=gTTw*W+c-*FgpC z%lI8sEjK2nFpi5{_C@2iSWKZl4XlSo%V|IALwhHTKAd?S?8!(`uQeh}A+IXEwJ?XndU z=yd#CXpKXKN zcdtG*Afkhi0|ddr8(s7;OhMm*&|8)C4#^Ou0-^!1nlD!m$5BT zlp+Hd<9V0)mXZ#QPPg9X6(eR90y1iyb>_}K4;}O7XZ2|-aYQ`;G|PHSNe|rgZ>doR zW>r1#PP7pB!RKlj#TaSH`>2MY87|AC6fPlNwebi7CWD)52D9D>*!5*}B`5E~BUs=9 zUs94>jxj9s;C=dSX|?gL!xwRPKk}`5%P@u-5#dpN97vDP_eeo){b+1HaIOwQnu^}w zG)|pzw0`n$&X?sHMM7)4S0jS@QbGEm#Lm!;6$~CHPgV%PG$Fv5&d{JGh0cX&B3K_W ziM$}fnpx>jSU>VJX9P(^5^@q9sV_=IPEMyxh0VX}0)f(Vsam&Pc8O#TxQ0@z3KX{b zPsjtwJnJ+HO}c{qQ~$RR!o3+j84cgRuaSq>zbZ9dg_xvaAF`%`1`QUS?vQs4lCk7G zT?MV@y-5K^tVBB_PdW-AJM(t;P41T&`4;5cU9Qm&c0l1ySz~r(e96g6bogi$rgz#l z2e?042O(>TXd}Rqp;HQ_!FbZD2A#;jw2&s~+ijd>Lo`|fDVp)TNz-wBNvz*)%9^x# zyD<0kOnw8U4$*)D){PvUI~ccnDP{sTb`tB?m>7YEgu^!>f3k0d+MoRmNQ%|G2~hzM z;m7Htv);PO6AV)WFv64*7`gzsw^}#}x^J#n>90Ex8+OR_Ik+gZQK6q0-~Y>}&G%*M zx!4UmZ_E*iPUc=0N^ecmsHYQbFh+b^l4u3;#AB_Y;hOo(o!AvdNhW4h2*Zr_ldsI0 zMuZXpq)E`Z_95~@e@WE}?ZnXyEXR_{WNh$fm~mgVq}${aZm>R5BiDJ^KH%*E@FmmC zEp{(cX3>QFszE>J421NQ-ZOhMVWrqZMD>o78AUpQ7jz2kc2Q?jXeKV88`=GL7Wyr) zTvRV>(K?)eM(XPleoc^HOx zVpI*MK&*3!^FK`EJ|V91*syXc#654+G9_eiB>Jt1F)?Zx=(oCKUF%dCU1eQrv6l9* zio`QP=uWPA1xFciMaz2v*97#e7L7qWS_cYrpo1lvuS1x=La}M#z{R)w;dHU5K!kDM z*j$o%EM}ku&0ozz3wfH-K5YRQ$;)q41qnIJ zDtJjA9Ju)HGO-)Vr(rQBA3iK>{H5l7Rha1yMaPk`-2yYH(WYVz$R-=c$Li>pS^Mo( z)lPosLvm2-3%}WV*WhZ_86e?k`A%JJJjZgO$|^C1BWLdyS$(AtsGzaBqWxSu@;kbF z5->gZfa$i4&sD%~^MBe>9^JHj`>An0EBf9^x$JGV2c7=~Bo?m)mse1OYM%nkt~TJK z*!{~7O+@+iyiVagv~=9gb%g*?U$Gq41B!1uby^tTg~BdPKf_3 z396Po%(tI>*y}>*qjs}1{iM$eTy4H&4@sfq^srdAKM*Ypzh;veZ=jG#s;!ml1+bDO z#g>?s&c3R_=;Sid%sZmj?+a%Ra(4E^fQJ8E4-t)@aAQUgtQNqJMFLEbuSl*98p#V= zx{`nZgi#y|FXB*qJtUsJqlL{E&@ddCs?KCWVl|8N^PzEBNzjq&W5mQpbt@JRI&QeS z^OEg|^dfiEAFvgayVo$Blbs#$FSbv-PMClC)dVsq__rhP=YQ_VTWQ8QdRBh|%sD{O zS9L!Q!Eo{C|Jx8&qGtWWJv2ovQ+j96=l3~hM^%`@9rh_~l^lLPbn$t?6J*PEQj(`D zevlj%C|0o{#;o3hato;x{4NpWN9*Fj@UaI%`SZc0{lRRW(;Q4UpbVUt1QeR7G~Kd# z(kI=|8-oD0VEy}&j0nTWk=YfgjNi*5(LlP62aP~Tz z;VM2MTX28X26vI>7SUm#+Bht^HecScT12B0N#Zb^=8f^G`d~ewR#CJ06!0RiDz_8! zm9HVS_QAGS)Kq>v` zonL6WqDF()uH@U4F8>uTd%hbcN)Wtnw>-u^a6eF&kb`ZaZNeSu4tuU=#{b|s?;ZbD z{1nMbvQxr_dfVi?Ioy;L3Y?nFk{2UU>Ht0O*yhxW5{-Mn4+sZ6k=pC-!Nsx~rTwWv zJ(;yjdb^!{P5Z(gZHe|DhFD65o50$H!0P(F*o5k=N8`+-&qBmJ$@r$9L^+oFkdDr) zv2=W}B?i37wacRqQ0$`aN&;L(&gRYBBRiiB9R{Oi89z42VIr4Ny=z25*q}z;ie^Wr z=N$tmREwpLm?K9i(;tkXh9~)zPy0els9V~%{N6*BGR#>#lbzkv|AJGzPeKq>_Wa-? zm}}4fFIRcdHu9dEjm%k0WanG>u(WF^ACX#>jKBk#$LqK>?z(#(H#=@!>BAJQA1N%w z=AkPWn#3o4TOI&AY{k?`Vn_5AL8d-_H&SyN+K1#wbE~RX!V>SlVve^|$P0o>YE>n zjRSCIA8wNI)cSv#@iqFPnZ&b$7ZdY~Q_-9xeu*$tmtG|jq0)1urUf{6fawZhiVUl8 zGH*u6Av2+?tYZ%c^>|d5`WhUbpicT7Fa(K=6RGoV8Ngig$owqA3j+ZzGXSV)5s=k2 z;AnOgaEv%*qvumI$j=w!2RUHxlFQOsUYgt#pxSuc`TCd~sFi^wRJ;Y&K4MBKx-;o6 z0?)AoT2Xlz*30QB&{R}tktsBo12hqGJZvG`okXT-456l1d`CPW^Dd8s8j=rH2CEOP zE%hl;oiAODApY!j^>Mv#!Qx5Dz0>Ul_7@?=lt8aqv3m}|Ws3~GEGU_nXN9NVLxiO$ zJZ=y#%bWC^)ED2xqwZ_EKbdp_I)GVCzyd_SB5e76)L!Gi_=i0#KfIo55#ICIkJF|dO|G{sFuKPST;?&pQexd1-qK{~E3H_pjRJL2_L_}&YMhWY> z;Yd!<|EK#SbPuv>nS3bjJ#@)Eg(M*}Q~J8~{GEfPJ!Qw}7acg@^320Kt*}T7DOF^Y za-~4evYSaC!Ecx>e~@na@+Eh^Q-PowO>l&6xpr)3a-j6(W-I_(CFMw3KyTtz56g2d zv=ILwsi|_QzGPCOL}FRxevdEriix@gK=9rUH1B_4q2BjY^Lr zL$*=%vYZF@zYwTySu@?gQrwmw2E@waf;HZBDggRznN=iI%KwcacM+%4^Y`!>{~5oN z*gSbc?_qwGc7Q8oevv#^{AJl9x_AHVyrMSjbunoz6=`YnBO zd!tU}g8#esE&#GV_g~VCuc(#V^8Vm zK>mqQwG9B8iF&Wy{B+gY<|+pbR(Z8cqMvC-($$0EMJNzYd;BEyVpB<18M%eG{R16K zkxq~E4QZO4ZNFkJnUCTrp!3GAl)ikKf3_B{u)}fK=-YDG!{Up7FF6t0mNcdR6NrO< zO%FdDicZcG6@1pTGdFK(2jtN_MCkGYJgzR=8&zHGdy*!*cBLQH;YTKaSA6)`0L*>2 zg0swW6`i~wkacEays zR*w-qm@hDrCWPP}9M4qY6N!se;t!0)J&bXD_t?=dJI9dbiPS6`@cv-=@czE|57A4( z&9SNFqzIP{oWf-TL%%a<6H;PXA0Lb{nC*o+*(Q?*e`HPtI?yh@J`iih1@pxjZ%p;a z;z)k%ywEk{OQVeuyxZQnuV0{xfB{*E5~bW&lapY=V;-BA*SosF{m3`zFpQt3LFzTL znbPp3r^YY~4pnPC70a~WO+G0n4!{BFq#CAk3(Hv%_t3?lf{|`~>yp5DL;w%Z>s;+) zg(`Vm5W~G;*)@9F1g5UO(M|P%!6^>i9*fs(dXi6p%)72}KIcR6`>zYI{PIhbkT+7C?viAv|3c*WGn%S9!E`5FWe*r zJPx5C94o$?B$T7kmy9-{O%jTK0WZ`+rP$;+P3YIdxdvsCe5}d#YPL|VLp_*0p_+je zL3LoGf5hopijc0F!E~a~NP7~6lmIA5|Ge($W(W?g0Kkt3JGn(BqkHtvtMeQJUf-yd za(U4l1c_bg!HPOlsD%e7VZYYc4R-XtFTSe`wG3c4uLrsyJHec)OL$1E8M}S`CGdak z_zV2nd+&Y~6D`bt4Z2kk)Tkl)iUZtTUb=toU7Z;s`Xl5)>1Fl92_yhTDZNtJ3?aPw zx^+*mN<8(;FSwTB`89YF8gHH)z9@vV2InKc(2&Q@_oR|UXUyVt_SN~@tgUtSox7Ge z*ewjL0d;FQuC7>^n^M|H)z>wEYVG_JG1OxLqt(+lvhs*DYTC?w2jZH)O!E$GOV9d7 z;W7(`NzizkMSWakw=_vmF>#W0w3tKys`bf*#JMm|=uf2ZEiRi!Sw&7Q-$ZAg3+3P! z72ke$WbgKdG#QrQ|(` z*z4)OSrKM-$LuJ`9b{<-Mw=QKM>Ft2D(3q=k{%C1#w0A`4fBo$^86_`zV>0c=qEVF zyJO_RWSTP_q8^NkDyhGnnRM5N8S~2Bz2$@Wd!hqYEe$!X^RfxV$mD?+nMl0e7df|2 z`e`SZ$W~c{kb%rL+}*^N+0d*}W>6t`;_CpYc;zbXXr&%z1UW6GPLwbRm^m;r5I-y# zcQjH{FWb}ok=kXw&mUA8VR6+-SL3F+pCWDDsUwDF9bwQ~&` z4#mLWZb0;=-H)f5a^so<3_AG z@ftGyXN~Xc5q)3~FbyRV!Yqw;_MpX0FQnZ~>b~^U(mBCDrh91SV;LB=O>M5TJzo4x zWD}NLad=>+UDqj$#3szLBkQ?bbef4HFom>W;fOv@AVi{;S?jZq{`}`2t_Hi(^EhUw zvLSBvy^VtJuYNlCO>nfnsrtKD(QUN?i_zZ0 z`cq-UlCY$5&UI7u1DZZ0i=)MQX1HivA-qvomA$jiBqCW$v1Ybk{5LYqQ7U5!<8;>S z>=3D_i#u$o^4d3C_49Cy*?OJwMprfyos9yyf@@=T#OdIYcY}FCM^t!u8s)ld12D1z zHs~0Ek#gfkRB{F?=%xc>isY3Ei%wx)$ICW1JI`l@=Z3RhvXvavzUatVY^gKrCVg}~ za4&TXJDceRj7SqOs)RrMdD$XOdZ)_DXija1z=qumNvVj&h~au9e;qq}U2fJ^IB7&j z2>X>}f>^KgrM0wAm=r=-FmfL z5SjKw*DS%V~jCFRB|)qLpUFVr}oKCbbMJrmCc|`u*fun{9R3mN{`OW3dd9*x#0*O2G|sw3*OCzSneXq^SVeKl8_zAowL0708rJQ^C03TIzM@y(=+&!;a81g0_BUk!G{iz#%~sL&=lY?IJUFX7}JP)+9{*6w20>o)r#v@$CG_bk*C z0ROoC#_Ytf_A_ejzCDy!p|`8t3#8}lUQ?D+!CAxX>*Ie#c8Q6x=PS%fOkxCGqmQ?2 z?eCey>@51tBGn-@T)=B0tN_%>mvC0gp(oxcH^yETlVm&`8t>z>T%pk(b03Nts|>|u%1y7@xplYWQgYsw|Id^h z>8$WUU7bf*M+Kk^DNuSm>OReEx&MzC27IJgYP{d2^PhkE6k6`?)>lT`{E!SOdGQ`D z)Ze%3rFUiUi}S~k9|t(z>f}n z_4S2{0?L1w!L?N5S!5u=6n-gV*lBj1O3f-u23(y1p;3PI8QP6($40c~8ro#}T&3wye`xW;H9_8_G2NHXhf@%#6<*d^B=nERg|tOc7`ySN3?)pd@7xC+cI`ljC4^AWSt?k zfd6kjf)aWRksYR7rxe+z+9H1iOqTrf5yBK8pb{TbXpqKsT2M$3*)hp38J?dOhEPivQgAW#Z|hTbs;hUOSN;FbaCd z2OTf13M*a%Uauf%J|b0)1Ag!2$OwoLn3}1u*OSfTc7#6JaJpp)8JzV_*B~#R`ko+s zs=aEs!OZMa)`G&kNQgeV+QS621Wf?R4oxH2Jz+6x5~fKsqsO#317{U#<)R}y+R%+e zfd77IU7FoPzwz)XOt+shP~Z9oz(% z$K&1uGJtdZM)J1d;PoV|wV5yt`?p+|)Tp@cFg~fdQd@_zgi+N{7V{eSg85OGuIc9w z=Z;PeC>vHYoZWalIB$=vSQPFogp+Ri&V;W!&brTgL{f)@L|Z|%RJ4N zXp_-VTEK7}8vYV%zhcs)Yd2vHLLdOktLl%J8f&{LyS?BYDmM*huRnU2# z29N93`wdj{ujSn2y#G2>)0h}AXW+iF;{ON~ICQxQVgLj;kR3(Q9hNzppkphhigs?A z@?Tb1Vh$lUF!qG-<*cyE)Kg7;_2t$nf@QA z3_Iq$IDR=Y_;L%WiO^Vr>o8%Y0>h(Gqo2{_#|l}pgc}xaM_+aIzMoEnFEM8N{ALE; z@+OMjNsRVh+$wq_XW)%c(Md5GF4DeF_RRh5?LRihJv{D8j&a0z@$HKz4_|_@Zd69U zn-+cHY~-N&R~hup@M}cC=&Y(TYh)y%AfGoXUZPOVsgk3x*42*dPZkFr11E(DM5^E_ zpI}`*ZD-G9th{Edv_Jifiqr~K1gXm(;i{Lv@tzvpwX*_%t-@*Pm6)*H64P(Rvt|vz z_iug-26r%Xjk(|Mte$u?%vz$;#&< zxKiNWr)Zg_(uwiTcm4%#hC4(kQ)Tu&_sk;ChgjT#1IQhAMV_R9PFmor9#Yx5pCXrO zWJ3KHqb!6jo$c7`cdo|oT}x`L{J_wbXK_qK$3EozR6rx{LxGVSRWsTmWa<~NraCth z%XH#CiDmblecc0wAF00CQZM}3#1{9#82V9gva^~RYteg)A|IG2e)xX~d#k88n{HdU zY1}PHa1HJR3r=vC#+~5q4vmvQkl+r%8+QvHAUJ^_!QCx1u7Br!zkT-DW9dnIFfw-B$)rAdPVfF!JBuHWUlf7ixVc0C6fy?x7AymDn=G9oW zp#W1NiQ4<1rSv`VdcSbX6+}`!_hNn|dvCd4heZKhnq^8&xaqI{8G##$E}fOpYrqE< zO5{H{o8}f>IFLWrCMT;%_8of0>D+DC`pu^C8oqtU@J|;NE#>|`9a%9KksWp&go$G( zP2%p-ZYPnMiDDUiY${L|Psf%>kD|9BYDb02+i(cOyh#&LpiB^04GazG*&Qq`R`U4Y z2aH2n8`gR%nhr)|r1x_Eqm?*s%s+Tkpq`rF(=FGtHrRa%*yZYyJwlxj8tT|cP^whvlt<3kK0s8rSGMbaey;n;?HLW zqRTw8=`~rAxOpFh83czW2bTDPg0*t%6FNo%JFgpJU^4zD8PQu;++FCkp-G=G=G!q& zFy;`nZH(|kxGy}cJP@`XO5{9KiCED_HhabizeBP2FrA~M9Xkv2_{Tl;eLCfxI}bXh z6$eRNK$+OXPL0fJ2}yR)f3;Fb|7$Cih5ncPk6Ege|7ZRTRtKbWAneyd(X>l5?hpme z71YK>#&JAO$!L3~z~^i=shA3$5b7bN-J-Ou5=XHN<6ES|5Mi7%SM;$P@nQ+;$=_>FxZBK0O#Gb=F2#9do%hdE{lzTNJ)98N-=LK#g*FIG z4>m~(e0I)gc@YRU;d9o#y}Q4A=;SzSx@pEf3J5Yj&NuccbGXNGxrTY^q8r3*0Brjc z-%HmMphDTQakf#QT9w?d&AQ+^sxxqRPl8NE%!0VGc6An#ID;RMZR{1p?=VtF&dv=? zn=X=mA^6k$Ro#hnqLt~B={*m&$8TGRQF)1(5pC&|Fyf#pwk@4<1jH3}|x4ATK$i}dmkLN4|E_#Uxe z$f4zS;XeZy>BNTAk86xR4pPu0Y+V;Ii$&oy>ypk9-O%Y!D;i)NYN4P;vw05wlgSw4 z7QqTSRNCsDhY1)y+x=lZ>;nZNJ=8j$-gPIY@TDWiMtoj@4J}cFS-}Fzwpf1=t$6JN zFA#;oq7Dqqf$3r;sGOm$6ZH==?t4&yU1*o*vq92}2*X8+td-w&=jjro%*__jQ8&5t zI|kb@n6kmGi20Srk{9Z*7Gu24kXgU0O#09!sg&53SXeEXb2tCJLcMUa4du1rhc+Z4 z7`tV(5X2(b_f1c>7zGNDu+{yW0N)-Lx2ybD-6f6?r~L*w7lIE9-oMC1D$ebEo`{~T zTzDc@TV?A4EqM`1mOjFWPY@xggo!j#`-hnb3fBGYRxRaZF$=)Yck=?Fr=%Z(J-W

U3Wp@(=H?VD41NZCQ>hjA>+p zfYslOvRi-l3bc9fsdetu!{5|#{wRaV+jyhsT4~p0z~an0KdI*<8>+_P=Oq46MUqIO zSKbO>CDkGp!wc-G66%hf5q7}~jp%=uG`A3tp5wM6+;IeaxnNj1?XH0ydyygBVnf;`$| z(~XqIIKaT~6TvT4*=QrM` zp9Nw=OoaS29{=V_Q3oE;{_)R~3K_lsZ!vkjiDX$^aYWeJr+ZKiPC7C|?jMec{-U?C z|F#%^Yqma=^VxReKX9+ZkUpZJ+Me#u^x3u`Sd)JkO$qN=z$H$jg*md-tu2mtr3-mg zx}RRUpKEztWG-mYOilhy_vW4G>khFD$b+GV?@_eX81qLYGP&nE7`ipaL5R_u9?nt;^LTI{`BS?C!|$l8dBQ6)`I=GoF!}=IEHx@S^l*0ZRnuQ+ z9V8OA|J$hBqUIx~*a(B1BPwhjDnFK}PH^;p7j}49|FvL*(EvZ0q-Ou)yAhGi4)er; z;a~?&cI^KrBLXgQ&gTqooTFj|cCx*`M^AJ}z0VA}@1^G0PjY$z>^WMdtjaI!84{qw z^i9N1OMWXBiCcelwb)^R97GKW36h`@4->*goJ@_^jueJUp?e)h)N*SrgW1T9bQGya zMI3!Xft zeVfy-&IzuyYDSLv6r-}IAE~L_{&|1GhZp6xTZBgIGt#83BfT^J9oHw1okZf3)!6E z?!m8mye%1M(}n3d+zpLC-A8I(6UW-EjXO`aj0(M^MWZ3{PTJ)C4C+^GVvkjd`hZw! z029s-;ARS7AA;m-v8ltS8r6w@HJ=Pn)sIrY!d0^2U>~}6vF=azY>Zgoc3$SZX0Nwj zNZy72+u3c2+NTyu^(=Wi)Kzwi6_QrzGo!<;KAb+}wZvFSJTjW|!fIDZ=mEAAEk}Qu+5iqRnk|_0*w2 zp*Pzbg8Dh#diI0w|5|FuVtt3TdtSHyn~xRauZn7Hp}oedzsd6OpE~b<=xP4m=+8J* z1!yniiiX6Sl9P3AtoP|9T*#}V);~%ll_bKmxN2i5EN=SniiW)t^oEdRbH3m(!iO9y zcWOL!)9KineT(hTIpOP^sao35IMK8Bf`0006d=kw^?J=k+fEI`CGk;l<&6&=%1D6z8va3H~aK@3>X58~Y?;kJg|1(}@YK z0~G;wIx#Fcrp*iqQ*mO|`|-!okuYAm@iPnM${>7LQrDAH|BvCu zQz#0#^gQ8T`>FsPtr{U}PL96=864jF(0}-VzdK}x@cjHj`V?OZ-Sdm;blm{f{n&bi zUpuLZNL6ZTA#hO^)bk#hGMYUwiOe&wKz-mf4sq0CsL7>hKd0vb8P$4@^FLTPOnz3g zxjsqUlqvOktv5nn05>!Rd+oSxQxuL=6x=Jlgz>T{P?}}7ke-jZXRt3htc$n#ngnN} z1?Ep+!z8!}){IkWm&Sojd6IR4pJ*HJUQy~)6GIE3m@&~jL(_}o)|+?Snf?j{f1PNA zxhZ4a^c5vh zH}2`z?w5U-eXlom4*n5_V5yGr4vWwD7ea`r2cIFz6|!u zJbjPVwq#d4!>AR`u(Coy_8(!%{r^pLFBRSMRWlOu_lW`+DTgL#r1chdARvq6vjP9q zj7-2sb^Vc15ngbyY4MN)F~Z3GpX4~i3hvu(FC<2nMjKXanulWF&~ME zUj>n-SQ&+T{++i#j3O#g z1RPEvul=C=w2`Ta0* z+|g=7kjKqq9{?T>9_L%pk77n4cIs-Iu+1hqjqG4klULLOn4SB2@g2CN_IHaU;QF(! z<15kQnXW1++)w@|x&!(b4ZJw>*G78FO-hol%rZ;lUeif`L_o#hTrm>6hKhiGYp%@9 zK*Nl}PIHy=OCj71^$-=1Rj$yU!CIjLge6ernjdc0Vo03%9obR&%pTsMTn)U^TiH&m z`{hkSvphj@ndlyH3KU8Zcp7xshJ9NwiTMsWkKX`qT~I%(iseG5!35Rtfg`wU4f0Uk zLgb%9;`j8hw&LyoP}NG*bGVY%Rl8PlZNxF!P~i`X>^!XNJS1xr9;U&r6)Kwwc*;lt zBkPXCd;o}c>DOTkjBs&|MhX8kHQJ}RZmgJJy>Qsh9{6^Tl>=7^w3x6Vo6qztZeYyX zRGo(*?sIiPI3W!BN#wzWl-r@q7fllswv0^Sw9U~@917V^O6!5Sj*9uu>Z;E3^Wo#+ zG{8zxi#P0JCgEv4;R>d;-GW1#oPgt#cO_q9pA-<@dw<3f8aqTXY}=Aooe{J#@Y5em zC3AHkR1vI#%eMP2p9{#uTMFs-UDrcJEuq3xhy5lv7dS zZBp>&`~rni2{xA1{YiN$7<{s;7-WfeKybK(Um@LX*r6GyfJas1o z950;Q*>`_^Z%DHG)rpJ_T(KIOsCB=*weQ}G^ck40)86O2GN29R}MD>GqM z!W~kT{9Q(^Qg3^w>z+N>MGvOcAz1f*_tmEyLwDe-p)7XcS28yyaR^S&yVrxpvpu+J z`8BwjoX)s&ZA%C^97q*dVi8FpPgK%T1_j@gzwMzbeio*7$X7(ez9necNaJA&(TLX< zgTz4BA%$NW#U>b0ebiJ#t6c&@K8C z9N=@xlP5y>C2KdP@`&~rnwI;TDHYo|(DuWf8M+RIzxliozw@^?``<%M(V;dLzn<6p z{qDw6A4x4#6rqly$~VKX{St?570U~m59)s}rD6oa-98#&(}Ab74Yp)1*>t7_+)E~M zlko!=#m7uxi3L50c+5tOM5|9K_Mza-tgUaUW=W8^HJ#}kOX z-!-!bycEPaj}T=@AgBZ(W%38IkJ0{K1?cSncKQFvu-CCaECEldSBdr<`w;vBoy5Ec z=KtWOW1{~0gWF8kpcCDbHGc%B`x(Z{-d$%kEjKcB{TctPj_K?)of+Vy-0wzt-C=6o+H2DN z*q~pda!r9Dc5O+nO<`aKC=aj0d)u}@Ql2m=YnJ-MEINdNhp>Vwevy-gHT+H9V=Z1DL!Q@@gm*RMkVr*Pqg zHVsGV6zz^uLpsL{fU4vjbr{!7UVXHf zb(P?}4w3TVcB)I)>ejE{1gZH&YjZw=h4J&(QXor!+C_x!jiqSH6R#OjlUcc#eD= zxCMPwjuUwe{(!@c&DX>HGoAm#Sg&ikx=hL@11+{pwpAheGbCiqiDk|F^-AAHEAcMs zs~w_EnIPu$QOfST!0iEAOit=o8P7FoJt6e#zbmj(+Wp?6hbOtp~?R1m}9f$u3cBS>ShDh zxv|&Cerk%y?QTcFUIV-z3il+Q7&bo|s`m0$wZN4w?d8~I@zJ&(=%y)P9Vq790gD+_ zJBT?Jilu~XP9}2YM0=LqN@&};B64UZAbA$*VruT zi%M395TLt9OV}#5{7Wt$t$A8rm1Z4M?OnT5+;{5vihuOLxj)X*i=Uv@yqW(}u(Mlk ze&!+JQgGG$LhD{%IuHFeLH!t1KN_*`9%B%W?tq7u>0Oj7+7c_;@=;W0Vcn1QLRRPx zc85)#P6dKb+?<^V*g3$Qo~nDgb;kG@B(!?8_Hx>aO?r%2>vvPi5&M*`mpeSq&)sW$ zweSNiiczWl2Ye+KHGlW$u>vfRtZr8CS)h35|5@oMsE>~vgD;!g>|?Qlo8~==X%}AX zd-bs*$>{dd3$N!mPt(h zYN$TfkT8X@d9i4OZ&K9>+7-BulJ@RwVEUqg;vWDI(FCvvxPWW+gJ}JGHnPK>nGA({ zq&Ov*58)X6lR=X_f|)0heD2D#S<-}2)e;A&r^~~2_MPjq+61q9coArYit9WD3<8>< zQsJ#T;6&?eaI<1$#z`m(Qkj;FWMvx=z5h7UVOoZek7uI@HIOuxGwvfJoX#I5dtR42HE=X&*h*Zzy=bza#-SfQX0ga=>FzP_*Nq?C&o9`d5u0HQi+^Vchk0YQ} z`yh+`jcRl`8gSHLa7Y?Pz-TkE8_nk%pdBZ6&n7}^Ap)p?td~6n)sTL@N)StdD!Qw0 z?hzL@L#{iyz^_^;{K?kHb73f?WEw-8`8wW})+L}vS4jH*N*fmo{WMVJ{#fNtSnj^^(89m zv5RxHMrs#M3-F0aUXQ|;@!nuKT;$~qEHPcp`$I}$V*5D~F4vz8X2%b>) zleI;l_GCdw<)`Of6<_p~%B7;SKZe%Rvo2GXX!QNiZb{#)I9Ids#BRxj9`~KsOv39r zULx;IIIHlemY?FAk!$a3B-bJMF|-ty_o}^=8|iZE0*5X`+jIA$!se)2z1l*TMS$u- z>PRb|R_k?$WT{j}(sda>lK&9X+)wtH(ZaDlghM0wCKwjvk7*)sQHOdaB@Zwl@ZAq8 zylg`VoS?6n6qF~6+PQW7uqO)ry+zep-Zgkr{ZyF>6$?JdNIVHTKUAb0#WrZYSitW% zZTH<8HyRaHk$Cl zL8Ea*&4u@|JldZ_>6cAzuigpMu;+U2BTa@!DE+?=?w--bY|I`zXpG^ zMR^fgyJZex>f(<5j!i`7L%beQ-5T?%LB+;#+U-X^5Ui!fh|J9BUE1O}K%s-RWGTox zGf}*_fX*MjA;iIj_ThaTa!plOa`p2{FlM9`xc;vo+3t zBS!40yC|G!OOf!$L=67J>nR0R5URT@MuI}65O3GG8i@PkyQ`#8NSRHH@IF3bI`e{; z4F@91*R~Ssq0kg~k^<45l5g@^MqaO?ArhT{(y%&ZSDO2grI6BgK&ievqA7&_9Ch9f zO1NIezNC&CeTdN0-dZLo&IrLzU1wUT_Na=?(rC*jS*(83&{8*b>bWH~-lyjW`z`C?r7d9z?huWC=hXtJ3O78bViUM>5XGY+% zbHVsUQ7IlRC;CMixn5aBJHPO|ni}2W(&u42QbSO^PPtfZ+LdCiGuP*Kp`JCiu{Sui zV}RJ}>>2N}c^rR`3?%@E5)DI@(g+70xRI@DHw;}voF)cD2k(D344Tg|Xfrg~vtz&E z_7;BV#Ovn%gO~99y}!*%i{Y>rBujs=6-b6%DRlo3KXFxmDdxTH^yTan(=+AK&5m9T z(U6>eK_Sq8*|zKT3O+ccp=r3;N|#wEp;~3+>uNz`LQsh}O{ds_)GK&x%zbzDx$!Sf zAE<%IS8Xjn_jVNntHVV389S~hmrw@1- z*LRx6U%TBMEQ{oC*}X;3ao*fgHhCUqqoi^9MsroA+=d|BV>jZQFBZrF2U{Wj^l$^C z#{SQfr?VSRuBCE7&u_vx!njZ3G%j)PG7PpG`_cCUKDr5IB`uL{b0(%!%f-^ULA)E0 z0W8_;qweZK&^H*A0u7T}+RMz{xnY^hq^>3Vf?3C!q;zydrBLgUbT_ivk&!WSwROeD(%AcBdv^f7 zWL@@bG#N7iE7t@BqR-F+g5;_SiD zxlfmaM+bZFR?eY|3iWv5DTvKR+J9n%=)}c7p8KF4(iO&2XaICX=;gso_n3O-He0#! zQRW@L0{p-%Tx#ltH|taCmDSgjN+n6CLB-#Bm^iX?Ev{7Sv_DG386NQi#WZXJ!&P(z zcC-fn(#pg8TmqL}X z$r4tB`s$}CXsnUeQKm(@dorbc&`f*kqSb`D&7SaPKWp<^0K-dV+{}!0M!&;^2eOsd zH+TLgk>6y=#bUrl@1*0KM&;HWMg4mZXIRu5*v8J+u@l$OW6^38Dt<4W~4@-I5(O zi9FBN#5akoEqKVSKWY~Kt~hM}q`nSmj~gDnl`(RRx8skM#I7cxr!w(k{o6kdT{J($ zT*C?h0hNE-M;!eP=K05E*LJ7iswC==yqAHzc`urXL}XctIr7&o+tdAXVU|2Rv=EH4 zjM(CcC*%?wjIvhjy8KK{4wRM)RbCQ3QU+s%-x+s|!Q}9h<zd3v zgMcC;0mVuXPz!hW?0(ZcU?rrSPKn4$=?LT8%?h>5%%U?Djb#@M;=Z)6bbfV<&GWeA z6q-a#%rdskqDfK92)_se6cRw07RUT*Ab7PDK{&A8M|tE*)Z3EplGh)H6)->o*xQn| z*PzvquusDS;@`rvBFNRD=t|l^VN^0nBe-tfHLVaWU@YTt25}k2`+>(KRN{eU+R|)z z)nQcXsb4gf-VIfziRWOT0;*CIN@2XXI@G)hC`f-kEZoBuQie8#PV^HCs)CyB#_?zl zSo9#Zt56ycb+Y#_zh5#SBN6&>*lggv&eqa7#&zn|d?b1={s^?Zmp?Yc{ylJAQRCe` zuOXrtdD_$@1>WGPNANJ)JaFeq zckTo0(#Reucnz`7ZgoU8!gY0{;9dNL7uw*@iI4WZ#Ye~8hkCNePD5JGtLQEuBdnY9 zF5KtsAsRUB%0nUxSXtkSZ8vuM%TvLie;R7puL32ig6eJAY>Y@ zsoKpEU5!2=su}Z#0T7}^SXd|@pEU7CWHe-lExch-YHeoB=8kObjnpB-m}1^Hj$8jG z$DyC7=&#^6tY&b}?q>_y!KRQF%I?&;&}t}x-M ziY5zdfjU1GN`?}l;&n-t+$ zcD38Z|BMpMedOdHCvNe4H?c?@T2CJ<;@$uFiW<1Kb4!#;a+fpSlkFMIWKHfHml;rI zeFxY(uAtEQ+k|cT?F|L4)O7xSTiVmQBm%k49A6jOvI+7kzMHS8)ok0cJDzKY*RE4W zisPkE$=IRT5{`V3FinX#ObVe43mOq6wgY92hZcO2X^e-fkfaiZV8hT8tz>*=W$`$Q zGGd35%jKwW z7s`YxTV6F|BQH}$D**72^Yz2c6!zGQw6sG@7QMd$Trh@c;yF6W6BJC@OIR$f)%PW| z!wl}F^nnQ)m5wigCU@CGUK0s#=K<1g7}}bXF13c^D{ap~L^^aSQJ|%5Bu1wK7XNMG z^$L8%Tf2`XWYI_Wq&XZPa?a6jom;7U_uABv`WAh8^BS@cPQ_+!WwXsLOE9{5uv7R< zYZvid2JmU*3MgKQ#IhHeVuPidbGKJr4?H?QbjGt?Q3tsxg`J~~;aaO$ozj&`YHy6C z@=RoJ3$(h{!zF)+4?P%Hfy*xdMB{=bL}rAALMq$QVF)-dIXaVYS%iF>G~C13B9PND zZ`<_aT5bU9uhMsz#vSPu!mV~{%t8KScmk(A%tK1dI0WLUIvF|_50f%A#A;zbjol-v z8d!TYr$f?j-%c*5y35p6xAj*ok@fLGW2vC8;Aq*GEOMp7;02u9N!+7H-p&(i&hpVR z4YBD~rTZWIWcqusg4LBLkX1C!FA&qaLdJV`>2?L`KD}w2eZ?6rG(FJHXa&@Gov+Z0 z>_G+BD0$J5U7A}Q#a_jg5dcG78yXrOTpa@-2V!PGE}no41f57l+UBHgbO1_oa${$XjT4 z&XbMob0e1in%&We;q%DMmzZ$XnE=?JKv9^@tR;%CR$q5Zx)wqM}H@FE8` zN62i9Dn*cWcWD~Ckrc1(Ph`)t7(*Mv^WVFjZG2C<8|oA875Fkaucg(LIA z%q_puL4#}zvj-$`EiCaS)}B|+dxN|K#T?vX@<3oKuR@R3Ef!kn zRh<4-t>cIFyy7Szx``Sb_lG#=zq|l2(vzvE=uT@rG#t-{ga9$A)3QJf%o2h)9-R+B z?go#FKC4-(?E(!J0fObzP@|bY$3?9a;Q<;E`85TXdI!yN{_b_AN4;(@pF2<|bEX=<7|nEgO&&gFccp)b`m!*7SmB+*w^{ zo`x<3)BSnw!&ifmuG*4r2JVJi6a53L6OR5oWYxq!WKIU|Hf%XfxA0q`?n^xhP2VXN zN+JC1=sn5!bbXIM8ZE7S`?FhZNm&Y>JmZr&dFd=on{-IwOI5$Z(r@Nndmfj#`E%8Y zU}Xe+;3}AC$9TN@H~Ed#BV00*rJ4bUf^vY9l_D?J7S8kh0;G6lEW+!3hPM#94}27_ z9eqc})nW=XWvD7^b}RcKVTEKv0x~}ZxcO--skS#hM=`a_v@;m1) z3#%zUpVZ-N%@k8Ipx63`QSrR}QF~>YjBG<@c=!*{D2uXBnP3P)8SpsK!|XiR@*96xT* zA8xyND5FF=ZqRr4b-~XVrVa!>`iN(1|KGfYop)E_pkLOZ8&5eI$VSR$FW+hCT~c<5 z4;(TT&Fetu#L}W0)}O1iET_pW(ZT&BIcS6LcT!B5^^3_Rfo@%y&KPnLNa^`4sASWh zgnUV#=**4wI95s`uuANczS5d@?sln}GBw3WHT0)M7btX%3m(>UYOBGWHdX&Zhc|M# z@E*Nb&K9a^_L6_OPCQ49yYACojGC-ptm(uJ5S8bKHz@e|EXs4X=WRP^9iPxcwV(jH zF&C;l<@tI5Pr#Jy=ZSp9kT!_gtkL3ea!6hkAjVSvi|1g!Il-y4Yglzp@@xylFs{kC zbop3u#^mr$glp2**|uO(+Q{_s68Y*xBJgx@=^Lp};KavOj!uNQ2!vOd#c_@Q@QWsp zty)@`^Iq4J728n2Ch6KT@$C8T+_Ih|LL9`cte)d#J5@;u#&!6hYSFEW@DP~Yg8s!o zUZ?eds?XY7jSI2y1ypSBo3Us@fNBv-*oyHOg2(oe6pkY{_KlC+=k;tQU&Lk9F)GkD z!kJJWu&%gztkBJZ-0-)su{v7H0?@2;CM+CN0y&WG2pn&>>0MC%Rt^T%oB_=0cxQT;$y_2uTH3t`Zn5;?yMU%7N+j2 zo!WP_01#@f`Am;y6(rSQ;PHBy8XUC9H8SYzYERGZcQmT4Dy zXYC2+G9vyZ44zEO#XHYL2y%LYF5&Vb{%$egj9p>_DXxR+)ob|z@d@a|i8frT+uj)# zDg3gVuOCdw_HLObfFl)QtbNpRCQ@9F(zTGHjpNp}_R9;gur_uqfp=QJ&MLqs1!)q{ zeow=H8MC;zPz3Br4*%Zx{=qDUflmg?6)mXtUI=@W?LtqM-S6uIadmQ3f5u9gd>vRD zLJO}Ezw*h`R-d`h*N$~uVNn^ld*Ep)%J7a`E`l>XNw;MCy*uBky!Fd41mR}jX|$;BwFi5a&}~0DV~WN zXhUqVw}cN>l8?3KRapmWs&m;Th3&J?4xIg{Te#Nr58x^we%Xy2%mjWp<7A6g*9q){ zszT(X=B?bE$-7t~JD6Gn*>H$VV5X2VHK1+9um-ri5#x;2CF;OipZg~IqZTx7q5K1B z!bO=7D)jS?=1`B;7+b+E`MBABfRTr2&k7_@mh+6xQ8^vFnvtY`O!>982d9m88TxOx zpUY|UB~<*v{J714xn~-i+()4uC;!6uhxIHr3bNyvPIK=wXrOJ3)X>Ju>wul;+mFFB zVMy&LZuj%$;n50*$I=E-a+Z}~L7K=p#6+^gS#Ui$^%TCevq!t5#j6%iMCr6s-d|zO zjZKRZ*Y>wC_Vuxz$kh+{5Keci#9h0NX^iOKvh3m)j*CE zR9+@dk6{cm{)$aub+c?!4EuI3s1RX$G_DGUY#uJG`UP+Ni@t9qg01UF;>oP*QiS^8 zVgESZZ5!Clz!57nm`h)4l*X-yd|`>aFMiph!DTS zsXmHpqUJMIy{rAjyuphQ?N)H5i$0+#JIWr{ty&wPNSfkd_VK*M((dR3(w8m@F_*98 zaQR3x^1i+!m)BPH{k^ouv+_D2J00{3BbF=JE{eloeOAN?2(W>y2WF0o=t|41v%3L*hZz ziCTi5YXjD9?T-Xh;}r3hoFs%+_D`9Sf1G*S%#xr-$ea){oKrfV+FEf%Ww5E8us|mz zyEpvGEZgHPytb_&mA>^GD;#*?%zi_W_+Tw1)1)L#tOSZQh${nP71%h^v=N&YRE+KrfwQ`z+P$=E5r zzz&8v5KRnrPX(!X&$M5*=RUsyM@$iZ5FCUs*V=J3GkZu_k$Om#;O=6e!!x?6aVZs8 z{LRzm#u8B$Z_G$&h4P!zo6_-ly=_Ktak@2uknd|Y`78j?XKk(;8wmGPJ+NdtXi#n- zn}>L}ZQS!?2!!bHHDHzozTw`2dPVwlTUai;H=Tx6dz`%6Sw$9*Ym)NaKt zV=-B!)Vt-xvnS?K`ouyg_ntj4<GE0%kuRL7k|$`%2^(n+=%?+%#C zaoTh7&%NOl;n=0n$5xrVeo+-yQbI#ab(!YAoVFN z--O~nYvDgDXyf}un!?mmt>Er{ND>J9i`A*2TkbDwvBe74sl2r0NnzZZdfD$f`jy>p z-Zih(`OJo1O~Xo}Y- zDOvP2fVw!gMnRLz>o3vExA}<4T4-NdvqMTM;AOd79_>&Bf@n zNf_05WeW4n?bDAEZxPnluk0MOb9rt@1!^4k@>wvhc1ak?+>=MuI7-QW2@RCRP17W# z!0|3gFH zE0rpahQZG!t4>RecL%1V3p-dk#ws|%`rFIxO_+Ut>8!{AcX2l4Y1lD`d5-zYgpXU&au+FldmEu%gK>iMbn{bT%ez5=)Qo$wCv_h}nC7e4wg1Mgd^;W0Xp%a|)m{np!f=P^F1l+*l~ zCE}%+n=>>{<1G<)Fp_s?SWF-UFo)(qtsGY#em5w|JA=1GV~h74)r#>XGRTq&)2^t; zyXmCw#dbIs2~rQXm|p*tdfDl=N|o&%Ovv>*UAl^I z(=y;A#mZb$Re0fo5h+s*2F!_FI&1paNzn=omC^G2KLv+_nh$EtkQo}ZVuKW;cZSaF z9|d#?aXRI)l=_{!g7#u^oNlFeiuA6<+rKZpyxYG<4~-KJS%GPSvL7xi=L{ai?ex7VS%Ziq6BRN2WS@0^f+{2&OekIx(Z$Y>yegHVq4B*OuTOsaoKeHS#_W<0){X4NUx++^!8-QHP z9JQKqPHMMiu4e;c@z~q^y7!B;81lW^^n5H95WII-N9DwBBt};7acE^D5zj}AyclhR z8qz-X1B6iN1Fa-?2ju)u_XQRu%>55nA>{(2Y-XVIl8+x1))ORfKm_6-MAaOi0R2=l z8MXo{Rk;cxooDqAETOFVPj7WmDUe+*%ri`k%Br&+Wm6_xDTK7Umudy>VL9Lh&@%RJ z;JR)7>+Ec5HrwnkU|ExXXd{`fLJuqQh+p~$NIVtZqOJTJ^7nSNaGqBz+63XAe+t%_ zl?_2Qed`YAr9#)E^V8NzD%Y%2RA8N%VK)Lk7s{>X-C=yWPpbgVksc8J`p9}1xLTnH z6u+TBjp$ocEQlHr9rc^IYQH8`ho)SzuP^*kJ=+6ueZ+}s>0%M4W_8ZJ=UKY64O7;m zWY_1tX1|t(&3cwTTC7ya+N4{xl`!EOX&D z+1Q)$P{?;u^8+zY7aY2U>%?!(Tk8NDfR%2nAZs{KRcb!-%G(25{ZrIRb1+e?&e?iN zH`glZl%#l>IkTxPm4PFAxr7m$Bc(FM81BB?ue8-U@f!uD8~lf|z83-|kYAb(@$Ync z8_aQh-!BZWeP|Bc2jyoGv(fXVcD&eaT(wIqK7MXQB`vplUoGEYPMK~8SMol@mtbTv z!k5sUMtnWL|FKYzQRc-CNCn3LE5%?$Qca}4mnl<-ECLE)t!0d_;sKp!(<}FKMRQJq zDZ2vjc&h_^?j;{qt(%pcQ3S}T=~F27yUZ^!2py%8uk`Qt3%P?4&!5SkP2u(GVo^F(@?L3{ z_f?>O6Hukp44xihJdic(3MKTZ|Fw6C8|`2E{}J|`K;P=b^wskBHqLk!(8fOLo;EhSxwl!72bgd2wExYyO0}*6+L4eSg-yaL+vV z#m3LTYquXe${XP{>30Bww=0(Dj-oH8A;VB}Eh4{1) zKhjv%UB6Z99k_>^YH2H6#T_E^ya6U&)xbmN-hd&C;YLD;)k*55 zJrQk#5QsaKlo8l|(>$-gMm4C)OEn|FB~U7W)J`P;>!kpmF6dp?Vwx7`*9dc8vPGsB zaLknyHI%8$MKgNRO>nzI(wP#!UhqQM#73j)*tm3(O$*C92nRs1rFXez`^wDwJM>5ijsZFlio^@}eHXS47 z+WYt@Jr=1NK9IEZ_A4c?YrnewG7YdQ=KauzDctvoZ0Ig#o(U)AC2bRszr^Fl8-J2N za`sU`PcEG%{B1<3B!Hks*|HACOl3rI#SkuC+8y;?Vn9xOuyJ=zEOvADRxbLAB_4B} z(;nAg(cN*=;kCI^FZtouNFnUyPv4t+_S{~XbH%A@Q}v97L87uacRUwe$ zxan!UN~5WOs1IV{x?H4wyd)W)YDg;8E+wBP8SQ!DQ&`DSx$BVHT|1>BV;@hGRL1d| zLo_?!sWHmW3wYBq$LT-FJuy`e^eqzDe*Q=yKSO=bS%F2t5&*Zdap(`*?N!_vqt{}@ z(qEYm4C2gx7ro&W4JXBq^Ja>6X?s>MbMns`$m*$%$Ls==;9*nCsx6!MimuKx!`?=V@Vuo)n3w z^jL9&Jgp)>F#P;K)}QV4g=d(rok?G=xi37KQt{i5m6Pm9A02^I{eN#g_}|hDFzWZy z|B>QMGmtB&qpF4hv8aN6SBAp8;WoZFLO%j_VFzakFoh<(Ns`&!GDx*jX|jNdm#tYb zgPX~IqcVosnc~+PpXOI8b+^EE8P3^trX30TkUa_^uIB1#2u||YsW>n#<&9mS z@|O_D>2k!Kdz_~Yc-1t%Gj~h-0BFCz5aZMV7y12=4VegWd6G4U-oL3fRt6LUp3Iv` zyx*ndKVhX@?itFjLQSRgQY~ zED**4D~dFgTlTZ>{j|C4k<(~oJ1}fRn1IrHvS2QUPxoD;2&b9#au=mV-q%&)_a-us z&-T|s3ILTi@0w#DmWkHJM}6>ESE{Hpl;6%5NI;R?_4yP9ESJGP1-@MrX6BkVyp!QO zXI5I*g;yBcAlN?H`0Y?cywEqFr+kSVRjrl3^(jfV`h14oF3WE8UDevo4?1_^gsr?( zd?GZFsq=(}Dx3nz%ODhYmkvA|b6B&9g?;*KB_cQJx?aXVzJdrr`DAK482rK#kfKZcCnc?-Y$7wbcxcnh|PWv%Zv$;uC zzw$Hjm%{VHRFs`I2V21%9W&4;8dmlPHC)}=#`EYqz0#kg-(`qB66>cAhlCs2$jbti zWW2)AoTyvkp%&fUHzL*j!nN(i^`@McTtoa@@}eShx63tn+Tl1g=Uhea0cG`-+|=nG zJxPrARo?d^HAPaBQvW^IzBB1e`TX%O;tpaw8LHgVa|BV4UHFAE#hBXuo0MP4mv32; zgEOx1RSN!l4(k(QBfNUj@vXH+5mDc2yaEAoty!L(I8U|tdNBgZjyE) z!tEq>^uBr_PIyBxi6f+GysX-m*mg`LjEQ#*@r_%(B4)%s*%s6IzQb}E)qj*LMLv4~ zpg+xz+P_a5CejuGCQ!Od;OpAV)taq<|}LeaMVSf)kfWpseEwp zybqVJyL_2Gu<`w=oL80~f1E|cH|2SiPXlF))B%?&@3sj<|0+p`C+w_I&Y!uE(NL9E zTxrBcysf)4EAU`U>Zzxl)?;*ZRfrAV2Lqt_c`s4cpYI`#F+bc7&Mz!{ll5@+l5&@irHP%|pVZi0sv%4X8qr+pHND^w z3{Ri1l`kpfliw`Kb&doBpHsnBHcn#4wr-iU1tEsBZ~JOj66zJ1w6kS54!=Y5_$g&i z+J!GZz7l@CTs^U-ATri0Z%1={2$2j>f7Y3&1;*VN)r`FPpx+eXm6nuMYk3|oe-QFK zIAtk$?&2lycbSwgKC{FC-bVy_ve(Z}CdA98i{j|)&h%oK-m#wGk5R$3?bKJ_0i^$-;4fgO_DLEv?e@lhbo1|Ha(b#Zs_MJj@reBN>sVizu`pWiBid;usk zQ?8nG!-#Z!6#hG!7M^g<$5^(Vlt2WPL8(||>YI@~e2fwCkGo=-V50|r*Cvj`vjOKG zPlCoC%zOdseI8=LZwEec>x%ZSZS(XGH`}i3?iiTn9vl(*@I#@xXAz?8+ox*E+E82O z^VcK_-lq!h1HgT_+Qech>&@`3hh6T+Z=bv_MVacOxXaqjXx-mZAt3RPcf)7muh&-Xe{r9`T=@s zMj~&`cf7jSpXIkWe_nDe`&yoPS&Gb>fOx?s<1JKnj=@mmkg}s@M{uBeb||p;CD8B37AiDR^xX+jw)D>sqKx)>%SXV zxV{pLd5%Nf?OAE0RzbhmY306dwAK65oPo0;aLTPDRf6SB`bw-OJ*a7FY03bT8B*x) zC(=aHWK@M%_i_;faD)Tq9h{)c-qc>qa+TK8w`eWrPxt=&M-zz%g@scoGR$Ij4w z79t%XACLk{6m|OoznU;-t`DW`1;U1H@eB&uSDhN~Kd9_4yH(_&SrfO&eWmj=+gp|R z`fBTEJki&rPVI8|;{+^D(QEd+K0!_`v@2ZHp+je)|94wC@6$V+PG6*Udj2v6&;HNJ zzje%izqKGORgwiVh)MHCgrM0Vz9)C96NO6&Y z<1`nIA9bl|fYR}*7058!+0jv71_n{n6Y_^7PV5-y%x9IXpw*@xjL(PR;xY+1c}OKB z+K^cST+X)TWi%Z2a+woBFGQHLWDG<*>)xZXt3-j&uixiV^U1{ZaOKvoMadn0gwi+F zGa#1e%5!tPSxz;6cZ*Ckkr~Xqp)tXqrk}?p*SpK^*P1N%eBRD;@p&(=nptbysgt83 zG9hmK!qkaMm$!^P>LF@a)#R{6bDIn4N+oxmxt|I?fDtssyPH*z)5rE{KN}Ks8W)6J z1b#fH=Bizu@Q^it>O!cMl;D{GvQUcNXT;pmZzs>#PjdH<<;+82zPQJeR-L_}FMNhf! zGQv>wF096}Ls2s#{o??rWOFx0`rAc#%1nRI*b_+xSj8) z@{jjMtW}Tc4d@jt-%m~;ODZev2TbwA3V}oTuOP939+FOgx1LQlcW0ze3RdqeJi-57 zAz`dF*A8Z}$7k24&9>uxI(Zr^;uBJ6zvZV=rFfhH)`OU8L#n?_6DN(UB0_mdZ!Ic8 zLO>yl=V}4IZw0U9boRux8K9lRJ);GX(jy)pZ(wt2K(K|%yq$;-ofiHl>GD&|0hR8$q4asHuhj`?}tT>KVc zN%c3XA*uK73uRvO!fXAWc0raOU;Qm6Qznh8|4uj2HhAgtUDP{ZRQ}n-?&#y4q=Mo2 z9_ue~Ua!@Klj(1@P5Z3v*)#@2Z$k6)B=u5TVhTu`0%AVnqG~r9eCJ&{WI{UtCM?~T z!wzLX&3WqAm|4DP_;Kyzx&1Zk4FW*H6&g==#aMsFZ9Rcx$%uaryhqo?LqbC;yv>9T z!3ub@VI3#&CO^lOqT)`;pAe?D5v|dkf{IcuFX3DiU=R3jm9B=B*c&dOciohT+UTn9Gs-)g-DK^ZhSptmOt*9e?}*#Gbc3ru5(eEDoX$|UwdUAPF~Kfm+!iWuuD0O z_*r`-7E2M6XP{g~>#N`A1`47j}G<<2`Z!(2CQ%pEPZ`qBsDMqR(Vv$d& zsMlxYCp(tmgdluU9)%wC4l$ShRHI*pmntE(R#NDuS4MSh`m_?SOq^{^%+f=a{b_Vy z2#Z2)E&0W~D^tFfZELNeHFpnQjl)t-NzUl|&*Eih{F66}G+9#V%q3FkVD*sNR1wD_ zv#825uqtnWfOblc0KuqW5#8}GA)s@=RH?xy1{MRXP%T897so&`p-h@d6SRfilb^Lc zt!1lY_ak92ib5H|QFz~m00Dl0oeKbjUS>)_GSlv^I}B?wpow3%D=ac8p5E2drfM>E zlks{9P^vffhTTmhM zc5^XC$Nv#@=aZjHTsbP;{%ys@9IZbKC1&F!9-`R(rx5)6Va3yMH<3dWZ~kweU&2R8 z_WS6C5OgDJ*RQes80pVz>X%_Zo(Z;&!aE*%Q%uPJ?jEiE_OKCIgVHXFPX^BbN-e`5 zF3BJ2nXx=@+@5TA6RxHr>?u>@PC^+p-jz2kF7%=AAAp0v>3Z<@#s)#Gg^oHP zpS2g<>sF;U6D@stNn}4n$fMChkyIA_zf%M!RH0!fZU8}GIny_CB2sz^gg`AF5wa(H z$>&2)kex2XZ9{j|41kXPLS=4)r=bQ)Ls9$nEHbQ-eM{{zp?6>)pfYeT_v;$(b1b4l zl9D5vkTO#*a~_Lb^CM-UYx-13o~Zscd*+MM7#+2N$U!4|x%fqZ`V2*Y0M;ZZ@2cc4 z1gjbj;EeU_Cr&GkdL(fADWtTrIi$5&rd@Cd`ApR5?*ox^vpy{R^|BRLBcy%5tTJF@bb8@FNl|75~KA}LPl_Q zIwC;Ox1RJMA(vbhLBuEE_U(+vAo)Q*`BghKK^u-?n}i8V(K+2kzDGhyZnT`r+e^A@ zW9@K9dBn`t4H$_ew%m-aO0UYUHo;;8{&ac-&=%G3@$)L}?$6wV_hGBEzwaLB^TTFa z9_$x1^4BQNhk8 zNg}X`@FFeyOpO&tS*dW>w#`*b8zuF?;xWFsS zCq^^7bB5qlRdNELy!CV&#JFgxN?ywvfZ$F6n#2K6i$aVQ=(I>dhar{n)%$dBz$RLQ z=8hbO6{dr_5QozJy->ghx+XH|>Or+FRg;**_a?QQ$aCBE1>q|BqF;m#5@ewu>U_Elc1_h}!LKBWT`=dJ}K?t#jX=DvUfIl-t8(@z z;2t3BG6aZ_Kc0DGiQMI?{Xu-{myg zBiaTbSX~*!YUfq)ocas>DW7(z0M@lkT=h)|Y+4Yd{XSk?Vd!m{547j$RO0><^Y;!x zPo)dsZ`256cGH)O`W_2m`GAimz_BJO5Y7j~q1nA20v%<_X3SZ$qSHDy{luFOx#NcO+t23Y5iqS@#=5W> zfvd>4V1;uE>!Jb*kYw1miuT^0v&~!85&wXW0-6fRuPOgVfNNvOx>Y^T*Y3q@{dk*7 z6p5+k1!~Wc%7VAOW8fL{p0wtMh+v*SOS40h)BNo3DtZ%IFkFLdoTU$F#)K5G6(Tgi zPa}DptI9+oOyC6_gU;UB2m|#+^#TLDtK|`p3HT}jT$&K0Z3FlL_?~w9)Q!Mt+rx>$ zM>kon5Mbi}EYKhm$<_x)JqMz4RGRv(Gm$SJ4rXsNWTPfxM#5n@p)l5y)J@~tm%lMtoQj{DSyCKn8tacII@&m>(~H7Lb+9uGNI_fB1+cp9cfIA4 z9+3u5!)8QzqG8?Z!ei(5z%Rv)$mzmz^>R5?um3I`C!Jn7+?W(FkWS+tbxO?wXV6*Hh ze~Lar37mC^-DTk0)yH!)4vE*&uG%8 zcyp@bffZvBT$6OCpIMTYRSgFwHUK?emh`ViA_UmqO>nh@!D+(>yP&%dP_UWsuMw=3 zm3y_QSWD;|yFm6YaVFbLs)Wd00SPXJPq_sfKX)}}_x)}R%QwP_@)TXuy&xSzKjUu+ z@yxxg>0*2zGd+BvuIBiN>R~=_3clIQy<%@Dr2}~y{dooW8%Z23 zHd7M}NN>W^<>i9T6P;*a)7V)68k7eX)IP?^SQ2y>SWN#xp(6~$I~j4vxT}trYWd#y z^V(&3K`Z}`{67(o_~(A7A#AB+vA<~Te@VN}|42I(kQLxxr$1-}f)9|)4gRiKsfVXn zmrYOF=C7f)N}ZRGZUY?2vP?l|eP+pzIbu?U)ZK=OMK5SMycXU~F-Z`_rT3zpxZiCS zz*>;Br{jnAJQJ(&`FH1qlFK@ek=f&wQslP3cH|6cD&k-DNQ&uxdHQ#r-&eE;(w9R) z;rnoYgA%WLYB$+QFB$J@%CxUGWzM$RX3X&`#gvE_>43ZYFDx!;<9Mq<{A(;g!ptp( zwKjx=zwBqENNUQBW@?K7MgRbxA+phEd76+DxAZ2@6Xf!BU-MF8MV0wif5P-~Vha7B z{1U1N^_a6GtKC2}Jhf5ctM#WBxE{p6l+G>lmFo>vZ`NZ6uHhc>r!Je96s=u0b4?|S zLz?l8MM`Gt9EV+kybu6`f1m@F#ZOP+@g~>741s%P6|ketFqYMCLNksg71+ix%zHupvtUdy(QRj#Ih8!MK-nIbP@5;UkPzy~T z1o!vG(cz8L`2)<0A88o8q{9l@)kL@8n)fO(+Ycv>NvYcKp1XQ|A8XCE4rtc2N3~64 zF8332qP13fm>D;agAmF;7C95s<;GEHxC}C!*Hn>o8}OKBP3+z6#N%ek^99L$H0PsB zR-K2(R5ouCF6@WdX8+Y~7v4VJJ6O$13>r0cd=RR&X0`7Sns441Mt53qCNaZ~UkRcr zhB=`5kMS0Bwl%lkvsX3(~jLBI_n&lS{&qC!5LpI~sI#Xv;}jKm@SqyH+kFfKsE z^hv)FZw20@;@M#>?RtYth~5Uw2JjM^(&wr<@mjF9*FISEpag zm?lh|%8#r$98);S>%?xg=T8=W-`GE|zFgD;wC%ecn@?}RfQTc4N9C@^t)%4OcZ4qq zR{uu+y|~$#r0e=vZ$^4ro+|rd@3X#47mtFh8;m;90$44ez`fl0Rr%h0qCvzJ%jc-7 zv(0CR1~IlDZ4Iwy5>u!hLuuH$Ia4_{3mnkw+7qjMqqWy?j?<3d6Tbf7y|yBEiAnLKPfp6PZA}9R6xQ zMPcw@fqia`}5O@A%k%=wV#i%>~6nHm@Ppx>2wz0uL8&Igw8t=k)2sZB>W zF1_w-^;8Vx*%8B{S5Bk)e$Iy85XOxc4+fOdJn+3eB95c#s2Jb5U2lxMu{-C@$WbvG zh?u4=d{Hyte(hDI7=<+9WPbLLqrjzr#KZI9Z&O@M4;c*Q#Eq{gcepFYey8oxa75o1lu%A(WuYRZT9QfUlg>bFaUr590Xkc=bFpcEmZQ_w7U7aU(CD}J>R^Kbrv zFGi2;D2(J(H`wX8-r*SnTo7uQ#NIwqfIfAJ5{|ZuBxuNQGg<}=(&E5NKiqL#bZjSm z61_HS8;5idB`iOUnTK0{BW&`}>;`{jbeh2p(vjYdnMg{$1=&gg7gj*;A1Z46R@nxV z9$7@=Xn9ConnIQUaNBZ7h48HkzIrMXDg=o_rN#>dh`oMY-s+GNmusC->d)xYBHN5E zdR=WYyX;DDzdGtGnBE{HU5Kid3<_cC$R{HhMJ>f}RVoLQe~xeXvGHt4)z4jrpN8r| z(2nhK`%8;?u4n3GMFOmZOqNHwAs~kWsrT$rvX9Hvx~E2|5Q0FqxCd7)2*InSoT|Ho zDU=|~R)6865~h>M)%VNHLy2GSC~a-(?f!()vnk0{Ip10a6r!+mQZXEc8wM)HLzodE*7eIz zlO|jcO(*KN&p-c+`PIL?&!LpF?DhX5kTCkhyZsAR?+?gm@AaD&<^D6~!V9u$WX5S) z7!M&nf7WY1ZM}a+6{4zhf}woqcxOWx%K2?^4KpYs__&~}n!aS%lI#If;HJGkqGlg2 zmhf(sjc$L9%Q{f-SMP|xQpAkT+7R!W`d7Brp=a`l^u&Cg|>Y`%o@xexiRR9aAe zC&YgRVw99*nQl{RN&Y-gjke z(TROkFI$np>ks&gfLSO(hc0kHWqSxuR@})9dTl4W;NC#f6#q0Z_<$?iG(7WhJ}A^X zG9KFP6WsoE|8PGnjI#Wl&cuS;Pyhtq@@8TudI!QuxY3NO)J&D8@L#|tS6T-7gl|Q_ zoRN!26-rMLr*$lBH^9( zI6}G>lrIWV6T;-RS+gR7P4jsT%lG!VuqqB8UMbhgR(&8;7y~=z;DbtHQGSOUom_6%0|fKEfCoHHkB`KtQY7?Jc$DINa`6=RL{Yy-L-LnVZ0VWl2aJaT?fR_3I(LwT`-a+$dcyy0W4;-~dHSLv2F!-vVE<tmKjc}p2$2P+crUj&X$euY*=`eR`Ul$?|lzcNi*AQQnNK#wY_kN3vO1J%;Oey)az5 zJ@4Z}{`Vsx57`qBj-BDYlxvRzvj|c?JWdYGM0*h1B1d=e6Qk30pwrI6E6E=f(=BSC z{SHF*U(7QhUm@&)PW_VozdG`Y>x1c58_CE!t^k%tw}k42~aT-Wm8>oXITErY|^;ptwdm}{h$|XM6f>ZWT2?P zuc3SsQFP5aLH0U7Kzq--8$b;zRr6OKIoNXf=Ic|ixp(r;?cNXAUpNWH+s~Zj9O>z| z)9F9et}YyDl%K4(BP+jmQ{f%8yUcqd1%sY=d;=>=V!e2=5IRC{EL+TnN{%Gb23W0~9^2KC zY%J26pAQh4?OiYEk(*lp(dn$eJ+j4{2AH88bm=0A4e|8S-1YQ&v z!VI@F<{qrx;`FH{;WeN6acGBvD5LIJpT?JRBT$0X$)(em8I4xM%`>Ah%!EtI`GIksO(A^f7MBPf_Ghg$mTA~_Bx*tOil zS)L4j&P^w?5E90#$3iO127tm)_ZeXjpA-JB99=Qyg#mPfOOVc~LYtW2*cw(KRVcxLmTyT8N zww_%D8Uw4!R{P$3r2BUB=2G`H-aZ-S?i(h)Jw4997T-09-g?L3Ra!Ml&b8F!HXh0! z+lS1_jZ>bsvjyVwL@#_7ToaJrz#5A|5A61&bbA}XEs4(d#8@G$i4SW62O$-N9}kfU zClVK@Njxf|ZJKUJqM$6^EImF{G#ApKmRfph3WH)CBl0_f6ftw33GD$I!xHcG0^Xih z^+I{Qsw`Knicq6HGSMeKkUL8UnervhzU@ObPhaAN{Hpd*K|6UabZXDMjyewq)@ToU z_|J8!!hYqqT`EoVul2Ly&ujmQMqJwoe@-EbZD7HE`+2Vz?!jx*uAUHGMU4~;vrSw% zv%xSI3`j_+WXJn)+O;IUJwg9mN?ImyvU*~v6mVeU=!kWPx3H4XpYl`{EtjsD-+wUFJsY{| zpm=4CJij;uflL3W0kBDJ2jsVfg>wJq^Z$x0AO$6fQttKTjB3VIB{wG4So&puwC7#R z+~o2n2eU9=4`l}v|MR0yUPk?|3zdbPiy%pw`7t|ayQPjlanT3iZ_$3; z*UQ7%soNK417A8B6C_N3WmofhW4SA#|M(f_Wo`Z`51Za%4|UjlW=9u3}ZTeA(^p3}@{<0*yCbbXYf&s@1o`E8!cH~cX0r}62n zOWtEi^*^dHr9ApsVPL4Cg%}U6JGPHA9>=jON!g@Dh-!fz2unFoV<9NTQ^YU47-?R znFoPyj)k|)R}V?~=LbDzVwbhw6QnMa_zUA1G?scr_k=O|M>Y9UfGi)%*bc`PiOYc5 zJS+ziq?61^XQRnbkfb&=dR)1Rwa1a zfUA&{cud)Y7L*6Ee&C<<2{YcN&s(LxXMCN$9hBS^w$$AMHMMWfKWbbi43T}1teZ78 z9HcMeCj~5Tpa_8$5qI)EyYI-tlC8#nE}|~T^Mz68p@Mc%Mvi7CV?VxmgBq|b_J)<9 zQXBCBBu~jrZ$JsC&!^F4Z4pDV+ybrq0WR;?tHNH$W9L!IzzS`yU{xmmU~5L~g)mi{ z$n|PL5vnIGvIGFh7F-J^KCb(ltk+50T_J-$9iKd4X$j!3=sGM`Y_b*XT+v_;{Rd!p3I)&5`1DPEXocn1aj953b^LYpVD$83$&Nu{ zdyNXG6w0S5?7S@_s?g;755LB7TQ$yu2Nc4ud)LwgrMP~01?Pf?bt2Sv29q$Oz2i#u-VA0$WH!W+{P}`hC^zz+O@;{QG7J|Hwy)V{vmH~ znzNK42@LX%vi{Om@ddr!^9~OxRtyUG3=++V!GAV$KhMjugK*g(84?S2qc(ePfcZlJ zh2(q6t>pP`lQ1p}T6wjG%=Cvic*rGCx;UvTJ7wLS3}q1b6klD_ynq%vM15$qLTz^1 z2JHHZQP{NkBvX99y!nNet#PwhRo>osHiz&fQWIBB0a@(jkgJijc}4%A*28?hALd4? z${Sy2?oO@eWy9Z&TZ6Y)C#7U`2_&7&T(q9#eBCQO-iOfOjQ@-jixOk8-*q?ze8786 z2cL>UZz68#ZHgxHTpUQq?`6Gycp=be>OXmbR}CuUdjxu1!Wq1cB5qx#4PpVoIxos% zLiA>dzsJ{VJ9V&Pv>ga-s%Q*)!z=K{iJu_N!ZMg2la=__(z!xlUa}!33h|*;18|Y zRmf&Vs{+g!|Nf{ZJ<_d!*D{--cKt8(tUn)_M3h!IoZ_DzbxmQzBz#y_VKMdQAM|Cw zKGTq2byou4`6yj|Uj~>|)ma*Lf&I1sFxS~VjhVDjYeoOlk))Rkt?&wm8Q;S~v%-dq z<2bDxz%vjWe=>39qk^YdST*+===;)j3Hhd7b!6{@%eD#bY~UB@GtZaab%gB2%7LLcIH@a+Kw2h3$t2 z$)Ec}+4Fl)CJ0`hlGbWO941QS(OQx4#6;iiPn!Aie@}!mzW+<~4E-Iino$%O(Ksjm z?}7y^(`@8aNX1yEugmbC*z@XN3BCIEGwy#zM@d0or_j>qso?GMIJ)p}_r6{2Ydl;9 zErg`*Q;^?t`RT=Q2vIY!(X5MO4mb<9>Ir*&{6Hiw7T*MZzYEj2DuX*97jCV= zECYPldVKn{K11-IUeiOOFyZl>%u$I%}0xfODKCEHR^HY5>#(T@UFxYCxt?d`|Hqdf?*B$&-VI!NAj z0Kaz~t1tZ;YCd0Nl zWafo863y{v$L#nz@OJ+TOZd7@I$o2jbnk5n2S|xJQI5>gri7UfS!G5(uV#AU`UPy> zNJN%{70txDl|C7drH)-Eaih!6T!D%q9@>_!9gr6n!r!&jlFujpeI^8BVf(V@vNEsP zDtHz0>O7B8l7LntP{xbCG63W#AwzYN9y1Rw4pjPv?^t!GD|F@kvPih~;mOCbVPE)W z=|L%=>dOz$j;BP=xB&Z0pcU9OHhkrvw&MH#T)8G_gK(wr&zMvB_WsL*7hQqw*N2eT zTmL17;qWH2e`#T!4wh>I?*iNoLkvi|ex04X5$^I~2{e1_dF%Mq{pz9v&+#d4?vs8N z_{I9=|8$tjyj)4oSIo}u{9%-?hwnX$OYyv+>{&dGZ2DAxI>QG6g~+3D&)jh7?w^-R z^S=c-4B`kr3hHC;F-Kkg92@ZyI>0ao*&kh#$>P4S50RYsSaFE^qjsY}z#+1>E-lI=y zA-2=C_0G|(jMS}T< zj>GnivA$KKO>W)%UUiMcq?rXEuf`Wv&jm4YJj%uTfRE!)BZFF_Qo!M&9s`+ zx|qDYxmLiSOP}^GkdwtL+G9XmDN~DrIEX6TUDhiz&2T`Z56cfp3Nib9R{YCxdf4P{ zcyckbwtz6?)vXXvG2_Pc5Ssk0KQ0dxQWZAbz<&3n=qHB7AM{Z-=er&$1y!=J0hDGpa}|zq64g zgAmvWxNgcHJcu|fRX6kUopYeQyEVK7FXW`=D=mBVfBnMUivG^;=&39eRQ@I{qfz)gQJ=OCTjqH!I1> z_t@7{M>xsrm?p=P7{~WC{%=~BMn#F*|2M5`p+z*|tYYzx=>OgW`ooxA6Ri7Rp7Ytj zpV5DL&c`&Wpb*cd?A;PX?Q0|9?Gl{L;V*i9KMOn1{P$0ZVyOK0b7HHWL|e|GVGQ^S zNsL}u%{FeLV*BN5^Kmddk(R0S^vXF9*(ng)x$hU(sm;jE7Yo=}qMw~>uF<~f#d$B` z+K6lG^*t;$rkFtDSxV6z9M5B%@zcL)(YhXN_6x| zI3wXmBVGX1T3#}MH71jk^xn1rVKq%C1k{4!We~*HVN6QLTl$VEFxrK#;Ie&Cb;!#f zt@=WP3li&nAI;7QDsd_t$t-HZ5%_WPGI7r_tycR|W>yg~&Vj!YhNgHptoSDPeA9pnltH)RLk+W z+391A4k(?4=D4Wstsh1JYpIY^{V57fQWVNuW?IOxpjdB(9R20k^EED;iYfZGNyyZB zavW*m9uGEQQk7G)A)~Jl7=OEqUy~{CgTNxZ1{uKsyTF=JrFImw+xDG9?(nGkcZi+uQZywF0 zBJ>E6UJLZfG0fW*BLlaM;5tI)1Xymwt%$-ZU!7%Xx5{4x^|rCD-xRDSt$X5A`9dZ( zu)x9;-PM$QpdFro_15xfTa>^3;(UQ>t9Ha0O4WAHAGZ95@8F{j{s-Ubz|+RL5Sfv5 z|1*ho;HAd)YuhrMzOq6>MMGFtUBV9W($II@zouiHpX( z9)HnKgru*gFD!$C9@DcN7g4mwkbIH3h_8p2l8p#qEFwGg@lu_w98Fp)n7!?@5Vg4yGG? z;#NayG}cMN-rgntQp!qhRYC|O*Z%0qnfcwHN{jO@ThSstMq8&m(v(9X3kmVCMBWe8 znPinXPifRvC0cp&xE4x%G76yihOyZ^Bqf~ZW-D5$qNfonEoE30qUOOMD(`S==DB%@ z82Vw|L{j@@NMfF5_yeFXyiJM$U^U={I26Yw5@1L$ak#*&%5E+My$wJSx8R+N4E?-u z2ntr|F3lt{&s2Wy4ygkXH>^pLIvOODT!a_sw`$Q60l=zn* z@Fg8lpYiKIGr2H;#e>@#G4m*u5Q@WZob7cA*@Yp_-{bW7ao#9@A_8FPx>bx@KTi>K zo7&oYZ(jdX)BbeD94 zG{Z172oe&KN=PFm&CnnnGjvLaf`l|99U_PfrKFUc(dU1j_q^vi*Tn}u^5fq3-fOS5 z_PtpOMAUivO_G-2K*kZBhpd2^sfLohL}D{4twyZzMxYy&M&*W6!;(u)SoEZP*~}*~ zLxgQB!H_j_=PtREKl`9F??DZDNXr@hS#Q8u4}lli*xis(EKWWHsq+pKNyhI9m&^yJ zU(6YIk6#n`Y5H}$|M~Ca8pPnglWX4HgQeo6HIPf!7BWXC8n;imVN-!&NHb0peIEF) zidVA^4Ym4DD|VCeK(HROZc(q{vxY^7x17{zN^xt%rIO?V&;@sM!Zt-k;$X?W75BI>Xp>} zeNbKj5l?;&d6SW+2-C556ZHfiex{jt&+iXloBKIbK)X3)$@w%`*z}QVY~0V*UpnOc zzmNm+nr{__&HNHO;Kje>_I*8_qEbgGzp< zk5=!e*1P1iymu~N2Ov;KdwU(?fzKWYW|D?8#${vU1+XkqQz=`#eG$d%5Cg#wDa#^K zI4a#~D5u&xba|m=#lU+{^p?nnU*%1vdC!uG7{`^Pd$tlLmUnUSTgI7444i}aLsTc?iV zWrRZUT|v7W=^IfBOmg0r4#cwQ18yRN6cHP&s2CF5Sy zwUs3aZ2s?IcQ%D5ld~Yy92^~%yfXlk+*fyMES*jILQ)d zz^H*J2X53xu0pP4_#T>?PCwZ)vBrUoYutK6W0)?QBy9XapeRwjpQw-u|K&E$8pSf_ zM^H7S`bj@s3*$>s^S~>((KqUvKlUB2U!}-H({(Vn#c^W z%$0x(c3!7nQOAFN-qQ1-=FG?qovvy_SIz!p{_-W0;`WBkBPjX4C~P~ITXF7r@l+I9 z=+VR$)sL8@M)ZDeH!hJ`d`K#;{iDSx*s{&asbx2r(9Glml(OJhgNo0yxn8nDk3HABLksSUm;@OF$@m9R(Rqj10^ zN<>xOFt}VMVfeSCu%KjIYoB&06n1@j`tOS*6YnY@1oH<@l&=YcW$#mD?II)!@i5yz zsPHARH7ujfEErUvGGLo3)KF9GKxIsda4}lX9V&%+rZ`7mg>>|rC_js7+YUEt7-Kl&V9s~b;=~b zt25d@5l3){%|J0f_Ex~JCBlrV*o@P&(H85bz9W&)uwW3s!KemKhUiRj?i0~0aqLfD z-e&uxQgc$~V2u1G)-|S4mSE>6?OuQPe(^Z$K_drXwaXw#vSpG#DNqY%pXwVBQu~|) z0%m3vS8m&SeqS$(;m*uaR^rS`U>^06l3E2%ERhuZjT>?WV+mg8wxiN^!p?_d*k->< zz+3@9sq+dJvHriDO_E{yYx87nIFloeMt#fT_fzhr|L>S+#u98#WY zUxoQnX$St$~|M@Osd!P^QWp+^Doy~JMW4U11&uuH$IkR z*&&R-P8KF;zg6`SU3%);^k`oO?+m|6=&u+7KdkE!AwNO?LMUTNA7CWWeR@4;uR%U* ztasYHHdUEl?&t)}*95$^^+6z6FP)qV%^2hPC=SIi=~(EV6b$EESjfpQxW+ z^eYk8Ov=3|vLC5_O4;Y%8)4UMW0S>!vxig?fOFfV^-CD}{Go8c2gT*dfPB*5e-IBd z$7zmC1T7e2&~d5-v<&sS2ui&ZG1{TQhSELY2$d`JKuYJr>Ev$e<1GXP`!xfbpix0sX#xbF{dL zTIRe5ytGyr9MdJUg4q8NS`USkFUy%C+yY~P3{m1ml=6vDA+aP^u?6N17m+Nj$7gM7 zAsg;@HE=~~%b)i#C1Lg#VJXA9rp@3IT7$}q`y*VHXj;CwHCs!j;4i5^tybMU4E=IfP`L-bi-ps$7zLiGKTq2RQ{ zAeC%l6Qx@K3B-?SeFe!J?E6(#EN0#IQW9sf2cecUwHpD9pfVKjoVzDy_D`)<_8dqy z@A$1N%VohSY%^nu(pdt9SyHL;+gyY_UMTI#2j8RkKlp=$;MY3GME<_{FG8$F0Kcv`3JPId+a7K%x zwq~T0U6FjJAH8R*$`$oDGZ4UQ2d7AupkVpW@|h^Da%(jqjn|R*TOI^ONEuqczv@_y9_A?ywI`wG^u(d`+V6LNyhOwy&0jgs2UaN9D7Df$1A z<%Ot&R6-nk*p0#PNTUX$tHdAS`-_p(mWoCq*OR#pzf} zBvl8#1+7LM9EvGJj`PLt!@N8E!D~N3$0GUqOm|mcz6hX945H7D2U$U>20xvzt~xeW z^ETpI88PLnKb5O7D)%Vg8TT!evk#1%iXp9UsnZs0KJ0M~cj;@k+GjDXPP&kIhw4Ur z_Uw|#JPaZNYH%~=i*HhNjk5dR`18b|_qtmc)yd$y2>OS0dMuOIc^Dx(R%J$vVXm-D zD=M8_k!f1)-}!;gS}!W0Pr7mfX&i9RoLCGyFtm*Fje4GJgK*dztwc$iR0Exum&E6+ zpIPO}K;vCuH4j?#{KH}sO9EhyqO6fHGlj_S;9P%E5+f=Qgf`u}G9fGHu!5bsrX%tpM$r0=f&r7vEAvAk z`c;qdH~j*nFOCfk3+SMNMvjq}jrWs@Y$Y1&AKHCGddAr-SifHsSXt_aLNpVO;fECG z_*uQu8VzH&8Y#7df~@8ROSDTM)P!;!J`2Qgz;fKj*jX<)(q}QvEtXq^VF|y4YL3H! zk8m5br_ZXf(|(ev@peLqa^}0v?V`#t=-5txZs1SJ$6~%>x1zVP^A%6!?Lq_&Duf-e zBb}P0f!QE-@-ZNcs3|JbwYq(&Z>@yzRg<&NBbx9EoJYB8n|L8E!>h}%g<%9FQQFrg zZ#Wfk0sEC5NycDF-`vrtTUAEa0T=e=|Cas+4X^Z;DOzuY0fZPTww^%l7n^i5h|dgB z@h|~n4&KyP)_;(mt~0Y_%8Ew4P2P`VIiB~wyLUZP8H5lvIc8fgc?l}Rl;!a89~PV% z)v%S(fB*O*B8tpcHD38_^r_6Auu^|pSghf)b?oe97JXESMGj^U{>er6N(W{ajIv|h z2lJW0udPcgryWop4dzp_qGXj)lgh#;Id-TNCg(Tw$$77M3s)oDoCE~lh--gHcc05P zcCzUytw(K_wof<;4#GaOQ<_&g8QgU0$-&lY9`Wf{GHK(TeOGGoewyD*{Y9+Fmr5%$ z>Af#k#6x=JBwc*vngbX7=egVk<+n+ybtvV!T%u;3MNF=YjkxCY_lN3_RerqKd4K)m zZJ2OkrAGkzo}i+5;R@^^MYrOldBN%Hf_NBA2hZG*vr13xIFsGv{1DLUnmFan&rHoY z{(5a9BlL%xM&p=Y3z+d}c?;#jtG}Ysk-+r1x$xZtGbW~rG?{S^F;B4ez*rh93lbjm zPEzOD2G3YfBbLcJ+|$Lp-Uq39SngW&$_Q@PfuT9y?9BcU4#sYb56sp)GU>@Tr=q2X zgT->mu=s(bY&~5vG?qZrQ{Q=3ZuX%Z)ttG53IS>Z$w&@XSw(0_HF4Hlo!)Z66Toi* z!)$*E!=hqqF2`Ka6t0jrA`?q25!umlEP!~gkD%nhJl<0cGUu3#(?Rvou^UX>n-y-% z^T1R+v9uTBniou_lGk|)%yr7@p@`GU-HlkI`^2@BTI>5EpJVj?@Bo3WIbzGuIWP2ke1B(T=7aJo;qirj1GOKrJ!93nnZ!MnZl#+G%JF8Q< z2)y8g(W!#5QKI;K8ggTWl-&MP*A82Z{3p12edXyFs0E~xhY)AG4Fn?KE$K>nX6?& zE@B}5l>oGP~Mz@H%k#~0aJHx{41azrKlx!{t$y;zQGhs;}Q*wAkIGlrwK zO8YbT6WuJ!i;BGSZVSX&hAEmnCTO-++i^0eiuSwHl`#iiRuhep3;BlsPEnfeLBlMg zx`-Dr&heu@hK$-nPg%FY}^p0=iHdX!O62bZ8)tq_}E5{>sf2mkoPX9IoKVk5mH0vE##=%R=?fH0hq)fRwY z9B`1O{nsXgNDU$v#J3@)^Z6m~uOpcYv$V(QbvJr4(!`qV{lD<81J@1(pU&wx_b>Ez^$=6WXrk*e|x1{}8C~BoHbomvF76r*TJ( z&y9aus4@VL6yF?wn_=c z`>N5~xB>+*HGEDV1$NxEDYh=Z)5ga3I~wQ&9XNAWi3^HjuX5U&kGiTmN^OJuR(}~j zT4!hz^=N5c@SDqK>82zL{H0E99iQ(Fe#^fpv;()>hRbl(VTx|cqizl?MWMq&ii(R{_CDIMVwWeGX!IBx8=aCx2N$B8VIKc*v{Z z8cfsSimo~1gpPRJ&-*{<-LQ5e>cI8hD@Yx&nUW<3BOX>ZqVwUzPrxU3U>=WFcPDTdd&-gqu^%6PX#sEt$mM(!}k`b1Vh*65lPq2 z*&uJZW?}T-_%ZX?AXe`$#zt0L;w{x)ySr~|E&ZI_CrBumd@py=<{r{^bG4q-Qc4&; z7xNdZY>2kYO8mnr9GI^2onaWF0BqvVAcKcxsYK5=u2$5W$Z29Sh$Ah}Pi^8txOsWl zM(UngXNG-@)v-|IF)dcqvt87uG26w~A|0hm$rdVu_v@rtOFo%u=^{l8p0Q#Sya-{J zy-~|3=vp4@l!IbTZXbtN%6#vMxAbei&t)c`^*2}9bOB5jHa2X{RWZC1*F``W0p#FH z8|2qdEe7TWyYIg9o`Y6JS%%@CD@Fuj!7t1U>@j64q=)HSy2HYK$Sv&6eva=Y$G!HbwEzeht_Je{NUF$Rtz;ZhJ zEv5_JJ9+F7X=N&U9dD{&yiur}Bi;|-p;&e~6P*uBFt3R!K@(W5&U@Qm!&h+RC=w2a zr^h+(C}qY5Ym(2OsD|lFutJnda-ViYGHyaOP3KW3)>Bi0v%Q&gi9m`1?B%7)LQ;gs zv8K7^3eyu}sO`Gk#eX~n+*wK`2gslRTb!o8iiA4{!en9-&+ap2t! z^Kput?4%#J{vNEgu%z;*zP0s`UBV{=Rl14WptJOo zr8I8B#L%s+*Bz7BG?9L_7qaWY_JY#%Rdd;RUx{xh=okG2nr=3>Ltc=f&+w+Hv`e4Wy3a(N=H8L#nR+O|Zmf|iRV8G)I+|Hiv*lp9CjVCW4meuw% zp)#l+o3XT&R%XNy2WT{{rQd1#{Wc#h8Ez$E$Rz8e={Gq7Kq9d)8A^P%i zehDYTVpOX`%~Z&)s9n`PyZqzHBPc+T-eqW9PiG>G>@kWTJ$Ma?)oLY(XR`b7e1oKU^gpR)OarW3{9uJ z>0c)`Xh`ty(-Gjluz75LGjMG6Dq&>(_1vbx+=!5uqM2&DhGPx;+q8U#ozoMTN|gcY(6Nwkc&+ICE`c z=#d@0Yu}0%9Kuw?jwtcX97Hqpc->v2Zn?r5EdNua>Ur`8=^_=TE_UpbVmnCk*O9~3 zvOuI8@9(NTdedE-f+uRp1sPQF#C?Mm>TB+kW8lcveH0jAO9M*uMGt^us6xk1#2it( zF3T0)1a{MHiKJAV3Nvt?;(Jm88*5lelMg?0nNs+Cw&`9RW749+Ac$LETA5^1*j6mlsVQ% zdqg4enY66s9Xq7@8k|xy9Zr8Cjk3dCRBJ_bqnz1HM^OpelrP_!adPv~V#*@u4M(BXtycM+NOZ%c z^6St-%*!Cchoj8U0fD;+^vINmU>`9xOlB1-lUV{48N@vwGyN%j_aH$jlp2OedqFi> zWV>_TU594seD2cr+$g1T5FKdJ?_K^^F_7dlXZkVf27@}#2-vUd@||&4(zk`*Qshfv z@W%**M8Pxu0wF-GD4(R4js!~=o=Iiit`qZ<4Yn6)f#?k^cnSSl07PP!LV>`m?aZB} zu5^*a2sbjNsOkz1g15HfU{(I+^rXLq{hFffM)Y4{zXUpSWTtodm)3d&hMdiQXip0b z`a@KL<9Q(8)x+qu|L!(Pc3RUk=!54YjZq3KDM`*XSri@h_s9*3j{&>Al(@2v0=5aF-ABwSpFnU4l1QlIRw=1Lr;@G{>@svOv!rG0i-E zSn&E+A_+!8zWEIfgPY||!T&OaZwxdDOUcfP2@hXW;Z^&@JGujdK43)&G77e=cP zy&g~*Ll%lfLIg?^XQCE=%Qt47+ZSQ92 z57~ypZ>@KXP>xc&rrrg#Lp67QfxL=x|7()*<1&F_g|2i#=TN5`dFI(~D|apvbT2Nm zm&m$Q6vv8m(-vNOiyPINcTQi=So2}P6x?wj0JuE3{BRpyv#0?8U4X50&U|lss)V&h z<1ZS&yfPEYf`fBslhtps!qv6x@VI%OA^RLIHL#@B47s#EL|q7v*CaGJ>t$yxKk1*Z zHg)%F&YXYe)OZ=8SuT@9siqd_DX19nsR{b=a*TL7b)zTemNOQ7?A4fLHe|x_3Zfq= zR2*cvQ`)&4PmN;){nB-ZBe7J%`huM7#c|-(de>+}R^NKto&{d!#-s42=S@f*Zj0JS ze;(z<-9P>mx;%PIJ!Z;gA@}3^A5j9*Ixz_xdZpCb-*9%0egf%y%8 z_|%Vf(o^HX-CqNoSWFGyY#TKFm~)cSZ7LL97S5t;bi*jO8-*HiA<+d>S%rH^}o9Z|^&8NUI88Gksvp+%w69etY z)eM)_l5Y7*ch$|YK$=5A2Wxyx{DN)RFYvh!j>(kzP3V$FA{Re6in)-y7h?Y7K$6vh zP=OOGGMXI!E;SQ4aOD=niugg9dm9s^TRlom&zpfPTF;Brgs3LO?>@ICo z6tpGCQ^Y(L_1#qS{SN*T_HH)ya*!|lRSiaZfC(9ca9e|4?aO6DY+-L0Qwpcb<|~%z z`lnPxao@aV{9#>6w%noilC*Ev7HTj)4fEv*c_AnRfEHaXS%Yt30hFuL#!!f(PP@;*neWPuAc@wGqBxGfWYdYX0ZX=qA{V z%H^2V5Fb-(Odg$9CAhpGH>s(iqH?(GbcLZD2*i=_F=B%YYrXI{;W|uXCHt2CV2zb$ z8&52P95st=>QMIrKic?dF)Il%F8^e0U`x_oL}%kKp_s-@*Uy7pO7~w{pLEMCx$e7u zf7xEhWg!>gMir8Clkkv5-=3Cx5UlF?by)qVeH^#E+5{2!#5h#WCM=aR0ycj%tE03Y zSq+B9c|3v;nPKGyB6)(&{krDrMXmo_3l=CdBrotk7kWvtHiQUhSSo4-psE&HYEHE2!K-3v~{FDiNs2-)#4nX)07#1Ga zF1J0^WGG>Gv=MfDykVOcDeE)rQG7&%SiV-eDP2X65nj0`(fe?57Ljd9-~%0VtMCo* zq!o7Xc>kBe#%3&iSS(j2+XUm=wac5|vmyX)3q3jZKEFz%-DC$Hxi1?a~V9jknl z`jsCm;p1nN8<`dDPK}pYN82QJ-iVMOK?i^WdsvJ(%e5(#636$%ud&ol$3=-92JqXy zqx~Y9usEs7_|*|Qnc?`dJa%*JI8V(_bpew3^}{u1VP~*~-Cr~k#wuh5AXe|uMn6-h z$Br4JTen>A3M*XF8v(Ay;+@N-rrJQDmmTVpT zD4lV$*|Fz|e^^#^-sn17YMW{6t?IWMdI+eEECA~Hj z<(R33L20KK%4T4rM!@TPAy83=rDB;BVUC+mFmp({O&g)N7@@!*REbPy$p{-2FYzr_Em zaT#>yn*VmOe%0oA`)Z93^CfZCp$qT~DNls|FAo1I0Hhe=zE+W)gt=Gi3CZi{o;I%% zSYaEqs~vle&4jB8gE5YHW>#0{S{50*N>eZ|N1SKbYJ&vHWufUM?bi5C%EUQkMoO7Z z;DG@hzN0|~f~D-+%uZRYR0I$GQbIg+nu#0w(G**3ZrT1vnPEa3w80tLYoKygetJc^ zY2o_QD*p)!nH=LL)I^FsCXS}sGTBj=S~-K z>~YP=5TsqFy@O#n>nt7k_Rsuk>sG5o9LzHxwu(yxrTs_6M?hZYUOG~T jN-uByR z{e=SSu8m50gN=8~>e|1UtbaQx?;9Qge|ym6{-&Q4@MORh$lW%Ej!ASt6)A?{)XSJ8 z_zN~2H{l1b1pPJ*zKQ6xxQ~e(*5crOU3b4M*^?D|o`?JT`swT559kk9?CPYv`w(xr z!$%5aV0Yz&bN}zR0K#S;K^(1f)tj(54Bp0qm2u_jus!1WZBsthvg{3X%1~UUF=1>G zjerkN^IUG#wj%};P7mEYeB|der4t)o2UXL|NKQ-J04m@KP1URo1b|#4cd4LZ<%F(R zgc`03T!$Do|J*zqZb&k-g31FxS!BbH2PWLi6V;ZP;wIQ8Ar}fGoQkCpV&(3>m~F~3 zm@|fo@hPn#5|M#a*7Z5boVxT#fXwB|68GoZjzqVlNja!ik@^o4^7&Ky|*Hm5BVmT#f2mUb?j?Nrp0N(1DVNncP z1LmiSkjna71IBv}~kyyLH%kSpzGkCfB(%#GVl>KYXTbHZbpi4EIO%DKy` zXcS=SVv|~zair2c>+2%myW4Yv_Ys6M;?|_+`jV;%MuEMr%{P{9S@Evu28XNvT5MOD3Q zE~b44imLNk4z13&WA2v?1cY-E7+b3wPnbX1QRkC1>#DC7F2$bkt1pAHjs2$Uy?J?e z4u8nCIRF{d&`crgnZqb)6)Ou>^s;?vvtKLdYUYe$fNlw^Y0MAU$x{%?CuL7{^SxLT zoTLZSjt)-@p>8~|qsSK4WrY;jE%PD~-Flmp6b@?atx?)NW9&4ckx7q_F}BSF6bmrV zPb?NxX|4NYYMUJXQ$$}V0>#us>TUjj-wN;*$OF*n>>nL?)9OHN*8|tz!_&TM=BVvD z7vBmrtc)$D|UI|Wtu_#)HEJ>_m%TAYuJND`jI!A;~r)ph1-1%%!o zC1m@la8_#x#o*&=h|+@I`nol*W}usK8{Gvqf<$#Mz1CGC!mkOrtf4R9{~Wjk@0mHz z>gGP~j#}Fc-0JH;Vzc6jdsV*IA>^Ia`2I%#RW)eoXSYnF_6_)#)ItSsy%X&Gd>nCs7sc5+b&ek#cdRworvktTnOU@ zMZ+n&;YXVb&&UkEttS>4m6|19Qqhftjt8N&la<|jP*1HV!!Un#F7nkp$qI2pXD6Dm zWVY^TDX`?+0bwUmaZGA^Q=v>r;S6ZYg(4q@v1Juo2BMoIXNXP+_^qsq%2p}52V)yV zX)sHw6Y0aW4gbe~9Nyw=gAy0#fGey@30wadJ^1Chr zW=9;EqXLmt1obk&n~SjJR2b@U8L1!hJG}u?STqN$fG7%7gP~SuulrpzL$8eYwF7&0 z#TuU=646-;YReI z?JJ&-OFZwFp{sQIBQ%O$wRkx*8~r7jIL%TGy`6dOM%hR$6iuG}E@H}cbGZD* zA}eduifTh71hkYd1l)<4xo}HweZP2_ew~wsoWW-}KtGs7iHUZi$c%1g)#+K&skfy? z+rm5)3oW~Be(UoKDm0RH4-hl2pWcqIpWk|Q_^5TymW-I(XYNfEoA~9pl$(~jYqOcU z$TZzf4~sxo>Xf(2^eb&%cHjCL*UO+h6R+8FH6b3iu*M|`wYtt1hp^OL_jt?bozvx! z^ro)`3lFJfPrM7fbKf0?$nWs+%c^XCWJgPE6!)=*I7EL@dp16%D_<8)?(=SyCZp*? z&)%~ikdVuZt57ZD>w3K%zMzqhv8F3ZvY?M3E7%A)^hm+^P82)Uk#&ebRZ{NtPr#di zg0^%-6QPcxf}UZ`i3*@5Skn|rj%%0Qz>4Q3F7-}bi{u==p(+ZH&`#rbL(6!Ubxg&0 zmfut#VniYWec*H>Yp{mL zo$-7${-QOYy%U(NSTguqJc16Pr}Ss&HqRv*`UI-mQTT`(icMc7uK20s?J)u}8n5n( zhnl94Ha*8L4?pQLa|!+T&FCwG9bSP%e5U(pz_efd?yO-HykL`mw;rOjn22V+z$sIV z9oqa)HLA!PwLzvW3Ds^L65cEt3bDp@wS00;BVgZ_U!ODvv`=dYu$+jp&kEZB!Gw*j zA%z%PEimaMQR*+BU1dp(YXi#yX+~4os}j~0Fe72tt|gKa#0Dv7c0LJ+d31xqY^Egd zWl?|3^b`yeMqGK{y`9nYDr~iZPO&?|7@#z{f~xt#^$3 zm}>W(>acbG-uEq1XYC%>G_}nS$?KkgaWCrdymzZE+xhju`zz1CiL4)dF1pbHdyVbU zD_Bb=^z%G*T`CCv>##g|*SJ>v@6H%2DH44R`NkOVThfIe;q_05DX zjc;USfwNymEI5#fAO&H4tG&TaRU>}e=b_!|4qJBT3Pf_DF!lh9UdvlHJ(DNH(xH6hG%y@6bquGg1Yrj&K* z_jS>(RCgg~7|Jar2t9uZT4q1ez$&w7ZZl4!`oYK3JdoB~H{OdA(L=p2DlRFr@(Q94 zpOHnsu7^}XAPX5X*l)Cbl_`umgW1jCX9aIF&*L@jfcaGUsXgO zQ~(X7*}f)T6lxvn+x;rh@u67u-u=+beNPT5{H4t54MiIgP`j~04@LY`5FKf39m^?M1i zrmwdaN4>wRV@6Zhp6jJamjvMhhI#ZB`WtUk5p*gGBJ%&2nWQ2itgJfWjjX8Hv>X&$ zx#=WAW8LEH(;wzJxwUm~S$9=Lsd~+rF0>TGGs#=UaQTK48?#D^?eKLlC(CEn4Q z>@I?PZrz$yc8v5DK8B29<~wugXsQZZacoi&o`l3tw9t{(=Ar zI((#~n2e$26p`qb-f5Pnq_qYeF)a`Z1$4OXq1?WxTtUwTmqZanwKCb?jI&*lqxUCE zjD-#WN<{5!A-bfzH)@BAq@}IdlBdG8sjA`2o~^3}kNm+m-mjBLP5D>O9N4E#Zr%RE zP^NRq-DZ*sPckX39_?(g|AJ8CP=^0{h)#K8*wguhB9I)L?p>@x0CYko!Wc6BXTKY&W*QDC-1=~K zXn(;S$>IKUkbz*sj>BUrwNGdW*yJF_u)rA?t+$0x#yKc=qD%ZAnHn z3W8yQFqz9LS@ifu(uT~+%)gV4R4HLF} zkZ=K2xHKQ8xCoDyY?7H@T{Neyks0O%P0IXmNkCl-pnKlM5f5Ih*NX2AL)VR@>jk<} z4QCYy0B>z7rOnrI*U=q?3SXHgmAVEV#JQEPfb6@zc{Z>agM+G94a}9)LZS!Vg-|t#8w)R1r2;YwWfU%|eyg%z0b_?-qIkYT`^aU?$+N z{NBg}g@x-4%Lh6AjR}kt3|v1yb6a8#=R7Vm8>UY22;3S>sqyaos-nInW6`hN0m=@y zMDCMGsdJUHL9nqqT+>;8%?`U=kZ;3Vg||UHl}Nv(dD2Be%op10*%t(k0K}at#%t(`5E1K4Q&NZ$GaU9#joVDAe$4&= zW2FiA;LR1^7@k!q;~y$Ih}=|{2>o^e_B=u&;nbuc$lS=b_t~-ef$e`G0bclbB!YNv z_@E^wtlusbVa|VCT24^Q%t+Uv{qZc#GCIBKGmO=2hKYJk8_b8pK9ELU9WUWyhgCJ8 zVT7<3BX@UldteAh8-;_cmWMxAdc{V7v!048$I-1QNAYjYWropZoMC5%*?47!wX;HC zaI`#o-CT*G&8AaxVrc2)xY^TiarS_v%^Ap`0Pt69gL~9BQPgB==(fY?CEu+lh^SU? z^zu<39`7k`R=>-5y>YQqpBISiRdSQUI){p-)~(Wbk}aZpz`<6C`nsO+HV75`oLztH z%}Z+=vjgAZ!^U>^ze5(6{$-4gzI4X%1BD|TI+QGUFdb48xnsitc8iAl3aL7Px)q3Qr0g^0g9jJHU z8gB=QB829EtD-50Kqt>#CU(y(JmRj)q=Xm8b^*eH3W$9)&mlwvcvYNPnVP)gB|_?~ z+@n3hkRewz{}BGj5K`m4ui6%&s76w_qE!Yud^gTYb8YgYQo20oCZx~a=?ym+Fd^Rc(#+%c3Iw@X9R^-10IxY!!h%kuOw@<0+ zX0&5IUL_~H1ImDR7n(t|mNRp~W@)wEOo@BwC!8DbsT&?Vk!WH%+%e1y6ExfnOnzRM z@lLnZmCiG2#}3FK3*btth0DS}eS!OR<#Z}~)%JAcJ$3&JOr&~&V&&H2qijFFY8*>q zE=wi!Pv{?sc~3%m&ktL@Xg&R(P2bl(uTEMC+Fj}W_06vtxGHVgR>?Z8fp&?$=5`zY zoA1yxsU)y!X7%{4d-TVryu!;1{;AMYYP6^0VmAbuVk~J7${8 zggppWtC~F*LrNL0±t)^$>@mtZknrN8~-E0~DK@L?cQV@LEPGfX(#zD2hI;2<87hFz$jfD?#*5)w*jFYFs5=v}?!*ZL93DN53ygbd<$- zT3B89tc zkZA?orJbg#6*c2V@eSeEV9dbDN8`_hGs9==PD+?!q5{I?Sf12G*tYKM?y@+&o;~kg zx{Z^w^^2p@YBT>b-%IN!=JubWAE{Qv0jU)WFhyXA#~sa#-877Vo2&|{y{xw3fsik~ zBjA=~adNamL?oYvUIE&H&lMP z!CZ@BQ%uin{3E;ldTCO`obfb?L~V9kt4vgGK3?RhfDvbyx_u=D}}{^Z5K%Te2G3>Cld#rlWfS}UKtd@(KMjU*=Cm=oa6uRGl9 zf6b)&dVU0a^KqJ3q35q3_)I>i%jX}Yy+{b7+c~Ss#7i5`L+ERohW-(h3*LWpzc9lI z1!vW|II{l;==)|muHB%mn_36jvrMg$zbQ(YkrRl53G91hmGp!^y+f=Ea(;M z#;6$lx8WkW#(LNF4t@m^;(&|D2L+(<8|jew#%E+w_&IYOA#GF)+yn4+!}mm9)%^41 z*j?<8e?`9QB32tQDdfZCO``&V$+pdDptSI4C?A@5x=zh{xA!}si)veLV@lA-N@_Q; z5w|S8-hcTz*u8b)A1`kU6ieeCxQ-Bc)_D5=Vc0bKCb$#!e2_v~tvy z+D58|{}0!}T-0B-aBi+G2(7!LZ}c_rF)MUt%(^EIOdl@G1LYSsI+04PK}|nqwRAZ= zK9h`d6Cb;Lz37~)G-!fcq_Nyk7!b}yn{_^+Ti`4Hz}0e#tqjCk*G(idADv3duQbvH{aXh7I^98td%GjSmBTn)MGbY=+_LuJ|hl81!AT z8b>^nA-B?k&tV+P!{kBFOwx>NP!(umOQJM`DUh}H2TR_$#N1D#jPozAmq%)A zE?GvVq*`}#EYEAuIh!uur1c#l=IP-Ne7wK1&jX-aLN#nB65}b50fhNs>o6uz$^WDn z$Y5%tfgTJlmNc{I2)NX9-`jkK#$b=%H!tPgbw0h5t zvqG;pN5|5?rlRlYHEzN&m)A+3{@9NyPsmd{-^i~oldgC453eUDW9|>icP;K)jB1Uh zX?PV6h~vYAy*BNcV4%)JDlt9Gh#@FfP=AWuXLPy6gM*`D;+Q@yGDdNBz_B_|GsbOU z>?7cA&(=Kn)Pm_s4Qqn}V6(7W_%Lf6XnGUG74zB`{kKP7{d^P?WJ6wD@&r8~_vm!! z=X(PVK6(7651roB;babok2)--b~;#&5*~K!u@T6e!grSjZG(c;pXe3Pk6Z_2$$xPw zhc|o;sGCa;&Wu@Y!2T5zW{>#y!BRqiTu8l4gGD0+BM9?RxFr~Oy84w7CtP#S?6!Im z;HeGpfvab8J~z+ha$3B#!vc1U5vc|$*BJ$MfZ5Sj=52VGzbIDLENOsKYPD(&e)-P> z^h^NShZM6usaV$5htUzg6D`z58ng7yZ_siZUY=%Ij-j>gV}@Br)DAfFkncb=O^P2r zn@p!z%>SQ?&!cL4?Kf(ORiMxMUgIK@H!I1-X7m4N^tC`6eWPXe{y(Cvz$@s$edX5e z^0bH1MdNSdgu`R;Uvc~*K~X_ySJQNx>qdfyNXzTTP~Y$7mo|6HkY@&>*GD1x4xg9> zFsI}wUSAH?x2<3Fyq$SZ`_-DqBP-l>)}vj*N- zwdxr7SB}JRn4utEm)-38-}$7xJxkJGF6B*9Kjtxq^m!o25ilU;=J9VHjrNJkD=?>X`ePWo9WIhPMmn&G>pz?k%p1r*B9^I>J&~e^C@qldmz(uUMhk zuaXwJ&g0SpwxTk{NcmC@5~r}B=n{`b&8vADl;lg#61@YteP3bH_S{H!xBtNPGdZ{J z$I_gH#IW?h)RIC`k))Wrs{v5b|2|x8Y=R@iazEE=8LbinT z>_56H6nouW25hpn)@C9bGu*e?W|)r4tCu~)zlc7UF@1z=qzfM$nImU~p$AJn|8%`S zA7-2sGaEO1hXXc{VorX#n|*TP9!Rbj4Gc%W7yCoAk895Vs$)rR)Un^}<~IHj%qIlt z^zQ5Az;ayZheLGh&o9h-E0ebPDAGK4FuvvVfYgtomzh=P1@1mppF*O)OXQoA9>GD} zCv1ptFuQ329)zrz333*LI5pRy^OPm!=kl7?s#KM)C0G!3PV_IE3;)HC=-*`c(WPGF zjshx&UP^7boD)*&G28{gJ-6L5$Bk~C5VWZI)|N5ZUkh-{>Lrsa>@Rk9%MVI>O4qgD zl&5-Vmfsmv*^qAzFU9+=QF3PxL{Ueei>kKU?`G1GGEVM4J)3zcxns5*lDB0PVn^>U zcT$t^hdI8Bsilxvd0YRJO&-=Kof*qpF%g?Iq# zh}$xnq5i=oC6^P&%e0=Ze7g>DKf-8<2jgBrJR9>qgawwXjr9#EE`4vNxVs?Ek%sG` z`mRsD3*A*-L@W+-?xZzEy_n~HTuOXl@wn^7+Zh)|7re5Kx$^b(x#jWzbe}9gI zqA^|f`Zp|o@3Z=(%U{n){VJ$}t|$7PtE5QpWAI?id7gDl)D-jK>^!eh!T6=x19|gt z4GGO<(e%6*2lm}uRwxK>|JdrtpJ)T&Hvi7Mh^S5C5uhrwDMPDOH}-*9odyi-@>yV1 z*^Uk?{6;MapWtW{>C0iCt2|8kSL7b`bI7&^hbr>|Z-K&MaIV@H6HC{P*Hi7PIiM$G z1-l+c0wsGRWd&xr_EfUPHrcwuUlcdWyLty)!aaGp4L%X?m+;8jwk*xG*1yu}AxQjl zbOB)mHYVkogODe)W=VPd%MW;rQH=oPe^Lj+$4w#uxK#;wwLju|*CC#lY3kVrWFut& zNZP3fcpkFl`?;n|9oC#@2>Tf5B7&MafCYBM1nWyzT)GFRdi$X~C{Ew?{?wB6_6H;Z z+7XS)ApyYwCS)9A*%h8}x@pn)ZQZ!NGwxq$1d}t3mP(?DdcP+YUu#lK=%#DQ&sUIM zIfmC!>AV1FPQw@4*lH}=cGH(@BRj&q7494sv^W#La(vtx=)bi9fCFLjmIEGuKLkRE ziBkR*ig9U7iy!?KU|2aEDpMn=BxrcN?04ERxiP@kcOGz}cOjj<^n(p^fUnB;Ah#s= zT0YS~fu0Fu&Z(Un#+U_W?*REYa>oFf8AZCjqXz(sWcU?P3n;UIc}>0!6xq)9KzsV= zacs?`(#l7gI>Zz=FUd6LG{^C6NQ?)QjBUWr2aJtNb^r{Le?LH_9s{0GuY&9#dj#1F zkiq$>!qirDR6ma8_VJa8Y-&6GkLP+DXJ^$J{7;Kb2M_EUTGDHvs>t#t{nr$Kt4)BM zjLbTOa;W!3`eYGCiu{SyqR(_U1b{$&EMwmBbxJGp@pnUh6wq}OY%k5j;gp654P_=%abNAx&q*y2G*$;OLGB-uy!zIC`n?Y*X6r%B|YfNwf|zYd_jV zo~==S>>Tj+Mt6BLKZriK$t2nHP=BF=tyX{W01x`19~e9Oa|~!F(k;(M@S#!q1r~sD zn&k!n2xq9CLHv2JxIwLKaa6kWXRMfkerr8L4z%6tndv>-{lhkAjLqkSTSg7=ZpEse zF{>$UH-9b!exS>@i0y@ih@h0>PtGmw~P`*9T6wEB}$TQIAj;3 zJOW^zx1YDd*|q`fl?B%QYo2|UFm^Y`!!1Lj7J~amN=e_Z#mya7UBg@|fnC7Kzq1|M z?VS?%#XQnB`W1PBIB7^)ZZoes%!F5L36!W%8_2H5h6f{;M^=N2GRMPTCAb4BR-c~q z>*nc83vtY`p0hUPiL=MGlZpQ*A6f+_BkM=rg)klQ3A|KnIF;*Fqap4P#oI?XNpw{R z0HH9wI}+#ive+~-POCKTW}b}{RZ%D=sGL8#`xy)X)_46!A|97l2*c5xq=9b$jP?OM zfd@m1Qa1N^6D+HNq8+s#^|@Q8!U+&Of1ZRD17H>6`ePMJ_Ori`_;RT?3@aI;GN{{w zeKXY7T0{_z8Q$}p&Fg{$&=f^Ua^0Wx^!ZEX#r~e^=g~Lywu5tw`3Z< z2s^fXk#6JNMxP_iQ;cv($`}#;u2w?SYvq}NqYw{MX z>w-MC5*)Ta7GmmttDn1~TF`+cRT&b0EkXE)Px}Lf`ri<@|A&EQdg%WFVoArpsNY{W zo}T$nhlcxU%MQ<04(LWx^QAcdDuFeN`t3skE<$Rhvy{g(BYMq2&tq71X7>4{Ytwy} zlCtL-{Ynn>fc9}T$E13865h1ABg-P-_w730=)z_rqt^dny8ow;H6th)@=eqa_ET;s zWd0#}1Bi6!p0tu?^!1Z$d%^0Tjz4EVReM75%$=Ge3WJ6ooQPx!o3*|#fN%sy6`>z@0=`NKkfGj6x#j3@DG`=2!v zeco9ivzMB{{-)YnK0)ybkz8|oxpJ`P_us=!5U|EgJW>Ln46Z_Iv%kMBd~Z0ZQyRlf zO4qaT+1WQ06*fK3grf!ERL?s58OYdIrnz9Ug;}zx7s7Ce%+$wf=j01cX+gWeXZ==! zD*;BFPyj+xZbGOPKb=FoEY{`hhD1aJ4LLXUZORtalWuCF;H%P7oXW@uJA2#*ps`fmtQnsr`K+$s6!XoggysRtzOwI* zBS{vz4!Py>A-2|Q)@$4A3TUzoqOr&xaI^H~l17=# z7KPzdIfYw25F+_NmvrWJKyO-FHpFAn%my3u3Vw4`2}g6n(R3;UFM%K@02PI#=o}S@ z*C7Vr?nxWlHddFtc4gB4-p1h8c=F5-o}N8A^_%}6OzL^|;HT%XeLUoB_&`TEX=Lbcj_%^#q; zT8=hNPt!{LP?z^%d{tL5OpcMvffu5t5QeWBl$2~B{hw^z(#|f+f(86%M_Y9i_ zX2IM`zF@mhd;d7jkn(}Xh>8~(Vp1>qgFRVp^jjz>;;ntjR(eHxO?pGRpM}HxEFmi_ z>dmX_pT^{wmZqMQmXhAzGTagqGyV=iH|Z-tFp+%g2+|Gz18RiV23p2_sj1X#xg#&b?2b)0h{Fpg~}g?ul`ut^*m1TM>!^Cv(Ft z4;Y;{e2Yb#jp}T$>kxDH#lpNjI=~Jy3;AzI5>5Zx+wK3~di$fB-rmDdJ5&7UgNB5* zs>dZfGZcsS@$z>*0!YTS<-O%MZdXBqg(F2*!(+_4XR~wCUpJmOu6kEsy$>jv2!ysRyy$y-F2}~hG-8Rr(U+ivBxc9@a zATW}8i1WQTDo5%-)VhX|gI)=mz@C|oOfR|l_jWpu9gKNrGL?0?Q_WK|6~~mWDL+p$n|uz4+sN!n1`OQP&RqGS0NBLE~Dw=n?8%iYwM>pI&$>0afyk_992 za7c!duuO3v54%l51I2!PST}}$k4sX=ByEBGYP)oz>ki&hS6)%m|qy8~0ryzN?ikA)dxU zJZ9^+~4y~fwyrwuZRbg5&rKR(B6hm{f-66_+v5V0f)SA})%JxG8l>;K<8&@`3jEr{W7i z*8GISY{1n9g5wOQ8ubb9CUo)wYSKUJcHk~UIlj}ZOig@>>e7+5H{b#Hnv-)Om0T-H zZ-;rxp35T*<_Z8-P!%I!r^U-;!te{%Eo+@rJnU5)mzF6mP2zQUPQ#!>*#oW}``|)1 zytCHUs{dqrIpkJqO)8`Qu#CZyfZPwE{#}rhCndg@DJ@@u1lw9_8}EXi3ZQ}AA40OA zV)u$1dFZrdKG{w-*c}{PkrJZ6WEr;X!;5PjmmdU&talWS5f3pDebBgBWvpd}GNf{5 z8pr$NURpoSpsHI}Zt%pc>zVA=@{!{a{PzfFm|CNM?AssL_`JORe>U-3{|_$Md%93a zpX+pb!_;vUV@Zm_{5DfX6LCO3+4x}VV*dMbs{eU{PsqlfrE5g7Cb9G-nsU2=Cj74pQFXY^COsH>RAJf|H5zH_SP3k zl3Z{usfa(h4MexZ8jMO3r^-mIz#;N(9@tl%+^A|FcTjK|=i9vX_7mr}OSU`$!i<1O z`LUwFNWQ)4!`~n@LiV}LiGaryRu6=l>4~eu+FGp%@M>HQ)Zs^Ts-+r8D4DxWZKWM| zLlQ)vK77W3k6hw_%?0=%bNh1CixI(-QbybGh3;Yz%fCI;5f4wO#IS10=#$*7Hnlnq z>>JnlFW8jyMQse5a5%(-I8-5X6C+JH^BE_VJa-O?08kA= z3X^*73M2>vsZBo$Q+a_eCU-LKsw2+UY+u{_O`l^lXdqT zwI@!58?(ki<{pu`vXY~!`DT0g9eh)H?Y_U7F`QCfzHaZWm}}ZYkBX6_3Mj8q)(j`Q zyHRY=%O+TH)m4AR0JdLDVh(8Hmg#Gz=Q4QUH+Nnz>*AO^HW73C3PN!g-MUP-y~PZ* zAVZd7$`6A`PTzf(kZ6T+hw18dC)RxG_aXn&I9|VX{;5Ctv(6xxEq|q8Ma&AH6K)Vx zJB!;ygD(HCm6K2h*v|{uZ&0-wyFl#`oEH_N}1rW%jVIfLy19Ae{JnW|h zlD_h_AulGB8a_P`0zrXZ)PW$v;f_b#4WN`Mwvl)WkuB~qoxoZLwoD$NQWe0e8VzZ) z$!3nTgp>nq6RVvq2(5Y`iod98BK|Ogtz_d^m@WeRJ3mXqLU;@S-B=hC{XR*CkAM%L zC|HNoFa0k&v2JDQdAyyYn$3j##;Z2z+Bx+{Nb-KLoSryf5bw}zA=|x-_rDiO#tg53 zX0${cg3F7wvp~7(v*^ibxVU4`ur(l6yPe_3+>?Fu^;&1PH_3>k<)Sp<_Wcd(-TZ65 z3lTZRYc(^Nm+B+x!u3ytSJ#a$T<5HKRyr|1Aun6>yztf*hsx%G$VdGRO|DmHdLRHh>8m@A`6#0>5xcI3G*G?~ zG&!jpC~Pd>Aa;)lcJwQXr;*Y^v-lAla@qYSZ{&|_Y2Gu>&{c9aFz+VH)CWNQ|F0<1 zb!_A_z7Zg%MeXK&6b~YuvHvr@1+WIo49e+|qx9W*ZJ>`M-owO=4W3?h9Td;(8*Y1@ zg`|vKx+0RU?FILYJ`$SXbe!>DYF_D~jt+OO=g3g<94h6TEz+hZ6XQ#vU{`!iv~$ED z<89`5h>_m4iw9(?GI#gE+p=YMG!*q5C$#6lhfw0Vrd*P77 ze7T$+@d+X@is36lq*v@K0+quAm9hyDGe>Tkj`V2qcS-9-?w7owU25{K(O~2TcQxV~ zlGODSen15VhPrSI3wbQ|9UYGJn^gq2ny?yE2ZUT=Z~g}Dt}(e-6nhcME*9>?He%ts z5gBG)6cL^o+X~XZ$gVv0zmg|!a-Q=&4M|?^nIgP1;3wJJOh3@j^@t+LaDK<^$}N+Irq{bi508r`(RKFp4J*ErVPpuC12m6h@dNr;Y6Ka)9_AY~F5u^>Z#;YUP zdI>J_QchXwEP&#eGr`f*xtufHb%>%29r|@VAH_}0zZg*;9@FXRw@SZwzP~{x60X;y zhiIhB0L(tJ2G14<3Hdq6&C2WmWSKNM{%@8EC{x6~)4)CjkP|oe%GtUD4fk%Kzv=w{ zE>qe5G76ROG0hsHL-pF2^T8-w!|Kr*;?$j~+vx|dhWN6jCAzk*r;YIZE~;pn=6shd zb~Gt*t~nzEulG2orz=Zja}dzQ7QZ9SUH6u&s!>08*RrE)vLCLael8hiJFv;?y28Y> zQU(%;-~9)EMTFjBQ2*u%oOmkZk+)t!IL7a^jOi`}n>+%}1fqT!7t`11ZRH%WL;Za} z6719s0p81~1p4{x6$eZHxsyjan8y&LM=;WV9^9iMKuNbOxaD?Q`MLVrJ?ewci5)&^N%!d%ZKi8+$#r(n{B z(YHEAH|Og*OV_WvY)@V7p8GUcPrmxM)*Zi<0rD*}=)N(&UWY8~i52USChNQ2q(r=+ z;9eIjMu|o0Lr`XEj(IZ8_A2aI$ptC*CTu*Kw^Hl5cvoYz%(XLDk z9?E1Z^e&f-!#KYuGkX2JD-v)}sEX^|=pFm31kt^Fw_`7qldo$+eDfO`{h_OV1+6>3 z>UMEq#qxgddNV3GWl~fE*{s>N$i7X-p%TLmvcMhU(k~xvrj=sJ=Mq#Q4C$5)b9~M` zG?5_6q8upNrdz!<#T6g3#u5);w?@~>iz_izTMiFG{T&<7Jq)lP~zy%0~2oEx?@0 zjQ|db@Z^&527t?_q{6Fefc!d#o_A@bLeOVv#bWY@gR|X{h>L6As4uXM{ik@+ke$CM zG1TU(`>xf*Rsq<@Q#&72aF;~EN^I7j0XdAx;vCplReAdiN&KLqXT`L*F?NDCQF8#g zkCC`#^KY_b!UL;7Uy`GqH0q>YH``*Jb(vaG4zB-tlPI;?A?qzNAo8oT&FDAI&P->@ z!L@jJdFHV4_29L4$wlwX*NY{d?(0W`!`F*4+qN_u)(W$XuTP%S=v^O9Jiof2y|-~* zLr&I~AJ$eD#49WB2n#T(&k5UkiZG=^(LB&m1+MHU2E7kdbqm0m0QKu)N3OAkz}Y?I z-gAd)R~SBiJ|U8!I3d1#+l2C!7^_|OsMqW)!6O|EzRLE`UlB=DF*CaXai$OpTu$`E zG0)~edSg`0H0GBE4}9A(IR*bG9}9a^Go-XG9?`AOOms)nVrE?OG`V>`eD~yyw(?A* zj%@i;1VP{Jz7L0wxot7gILh!A&fld>nn9y)SDSWy_DEa|qJJ~tY)#Lti-piL2Hf_0 z@2gx%ez_u6eGs?OH^a(k?PkIlL0sC>z7w?nKg*RQxxf)(;cWVW!O@mPB_}DTRLw{- z0-nH^LL2#Jr-ps~u~4=~K3oV6yLLoC`tOw56LaQ*&>mK12OPxa3`#FJszACtqxmo zim?V4z;&C_>YD87d)#)&i^)+4TRTrF#Y>ehovghWwtl*@&vJD$5v~pCd@l@D6J*x` zkYH;}8_zVG8(;##*fMjIa&{L}krqR-GHa}AK*{kKUgY6iRlDsZkFB>~v3?7RoO4~2m-kXNZkJ_1m z6cC(vWcTnZh`L}`IHe*PSyWx;+3M*PZPH` z(hB}bqh(91O~O~Yo*c&qVwA{!PZSS+wqzUW`rfjtAvw}~YW3L^pA##4nM!n*R7uMB z_%K}Sv{kCRH}rUbS%kkW=3|DJk9(Z1O@5CtYrkl6cH&@?dvq)}Hle-A8zz@}4Ub#U z)Np&JcL}~v6dCexfJ?g!)k*qtJcYP4>FCRgrHXm!eG#8RT^}Ha zLg8+s)Q)%yPEBLg1znit+y3=@z`tL9SkJH74E$(j@W@!JqZ!r=&cJ(|pk(CPEuM(c0dc(%n>n^KAj>Ji?{nir+4Q zNQL+0n2n{qovZLIP2zwxrc5IiSCK?6nLd?$(yXQMBc5d{gQ%DyCWv)Z5ylxBVE);V zV!jd!)GZk^l6V?1L?2087M;0H^ePIkyI+}5oWiXs04MNO%I6PBE%(u{{<_qS@qiN$ zeptv2ALZ{DL9Sb+Mta5~NqT^F5<$~4d8I+orQzdLs8*b%V<6Om77qsJRzHqQBT|IR z#f)oLwlG`O$*OL~6cRHGU2Ti$wPu#yjd~gMN%u+i5-T+lBT~1u+gM9FIoTV8Dw}u8Uud~ZlRVn8P3yx!MB0` z3w#p=z&DMM|4oq37qpLVvimca*064I-pr-){vz4ze-R;I6~zsr#s$8Q;{( z8U_xH*WCDVZgk(P%%TN2!_^)#51n!_O8cVxhPe7Ue{6`Cu?My*e#lCpjU~fuB>F(L z6h2tdpYA`;%b@D3T-+Y+-()Jy7MzkxR7~t08M8IGF~dqdeIQ_#{B!sVn`3{px4|}+ zF!?q|q{MX6_REurt;%VijVpSRd1n201GhylC3e@3Cv|XtzsugGEY6rYdGwQ~G0l`} zd_&CRaT(TP_5^6H?qQ3aK~-N~aH}R1Y?2to6r8y^QdElnu5=R@Q09YKzwn6!aB54H zR5UMoGCrx@ZgKBB$Tek8kwH;A>1wh23Y{Nx&;sWEKWk*E9(InQf@gi>)Ph_6HQLhl z`?6Y@xY`B1JDjR&t64bnwKG<89$>Ogx#)zWkExr&Ww1cdNC(ZSsj9YnJ&VbUk!~KG5Q|*vfhgPqxex{U3s&lH)nW1Y44Tg z@+YUMllrZ{7P+QC%g%e4Sa{k(=C@*48*4MvZm)~~-El<8@xW}z<}8T9>OF9!uJ8QP zdTrCsnI3eQ_gR4W>k{v+oDD^jUw*&XeRFN7n90zyT@{W$AND{s=RC4((Oc}=*x^@h zAsdkYvH*mdM#3AVHT=h@HrLA^)MeU}A3=7LJovLMG_s#kupMRdSiFccA>>>F%BrTT zC3n|-oB|zo<>@{$N_;KzdQn7*s|Lbm;8cgBgZ1HFnTmXr4M_r^IPdRsFrUcFu;hf~ zZfM`?KNZvXCD|`Wi3t3yGjXfgUl-kUujrNr(u?8Or1N~;Gg9IlL`{e`M)B$cn=tQ+ zxpZ0vk(=kqfg4a^d920D81=hsReLVmwCQ~xx!3nN*mGh6+o9={8>xvQ9{AbyLk0IBrIv0GPw_2e>)*W zg4i=hHF~?XUmgL;D(np6!v;l|a!VH4+6)tqK5`jg` z@97__9niTP>B#?F29aU~Bs0%Dy}5tv*Y{U{eJw~dU_w`Cb4q?;6k+>&^2&L_heUoa zQoqW3d=pR3m<_X1ikBV(d&G18iGpeEfPil;Ir{jzDg`P=SprK-a9*Ir2%Z^oe%lne;$i*j^?XbfEkD z-uXqMZ#`lxR&r;xsp5%)62M<+1rH7aoG!XK6?({+-ebPMdO zKJ}hvf|s(v9pe+CPjh=RHoit%GH|5iYC=vTy0|o>e= z&voSeA1|@CC-C6otvx7vgO9gby)L2FlfQBe zx_j92<;iXwi?UPLB!PhJ6Bas*Q#kja51Aj!phC4*0IjRz0eP{f1tx^T%JI!{>**|t zhY5Z|X}>z%9;1nae8gF4&>20T{>=_968~N56AqdGMFyM*oZ_y17zdvpxJT^54g>GT z`V7o3Bmwz|jRuJ~-)GN>g^mypZ_$5w!2+X?7oPu<)XWtc9E+?}d*>9UiEN5mVRZ8c zthu_lSDp8SKk=v*<}ev_yf1FzJwklt(m7jGeb$@OA;X$(<(+-~Dfo?5UpC9k7wVYf zTAEztJLU%rTf6ojtNKGGa@DvmOq?hzW8>Rgt2MYsM3 z;X1C9-Rx_7T`A>wsV*l#UfY8-=Y#Du4J$$!`rd z_bLw7ZP0mqeD)RuV+GH=rBFo$5;@Jw-EX$MF1!ydBVCTLIa7d;!GP>QeU_%As*!wj9`GP9r49K*mZX5AD^BOiYeR@StTQaG(2zc`PpSS- zZhbczz}V3GWfSpn3#Sd`s241bz(~82s5|%i*AuI!G;b4aK0R;isjA8~%=Ig7GAMY~ z%MjCp^DE5;{=9Pdxs$Bg+jt909PM$8r}r&O{bF*gG5HCfLV$dHK{d0gFA~hd+cy`? z-R@49UGdqeJJ!cHE~~ipn5`<0Nr|O)^joi~sp4_}7+5*s?y{q4~Qg=Z^RT zTt~e*^SFP1BjnzlI^bZ6oj+J<{VWi#tA z*HC>%`#J*T7&rV|)n6g=w&z9-7lKK?m`ZZFTA^^iGajNBU&|M3)=MUU)e_6N$*z}Z zs~f^DOD69Lc&lmQkZ0d)Q;TTskCV%r=&Nas>L$cTX+o|v^~IDRq|6W>W`(wmivFha zEM#Jm=OZOZL{Cl-7(GIfodc5c)_QDhFr3LyDxMkq1yD`}x8lng+rP>YmTa?3dNEIE znGFq$&aNp`QD{@zqWILq%_-vGLUPKgi3rr{fZEwal)fDW)bxff3>L(9LzKbl24(L} z6NH%Cd1UJdzVU;g8oI$tw~uQ)$)C75Q-Pq6QSSEDH(=N8<@6C%z$5LFZq9zE5?u7g z$EU7YdWM>2v?@hngv0iq;?RQybE^>6P5Wa6nU426R+0(uv41otR=SU?a%7&LCp8=| zZb}>qPtX->zEsC402h{^%pY(;O$Ibe#$GUcfZ6d?e{`-nd+W8P=8^EG1!8kxn)vqP zZakcd?Sk-VbwO8+X)bM1k?9e=!#ZGJM}EY9h0~FU%HZ`+OBg8M_wmbO{k*Msq|1(w zOHtbJo3QFX{3RetJ;YPD=4Cifsl1;mifIS9lg-IJcG7lU{?-A{1Jb%9(vli@UsB8Q z(UKERiMzyr;!D*Kx|8m`&gT6YNCh4pX)L`kwp-8*zKAnc%|ZJ1bc(?u9}`==dL-3x zY0yh9Yv{Aa(8#`BNYuCEVbcol+M!M3I{gx4QJYPZ*=~}IS0!r;`<+D6qUe~C zT8MPi9m+~GZ5jH_0!GK-?Q`e9Ek6M3X(qFtotI*CbMLoUAZ)|u_(2bJn|0>0@R-3w zAQ2@yzubgytbRICjyvL~!Q`7>e*SSQQU^FJ?g#~cs1?2vyKQVCUW@uCla@B~m3sL}kD4H41KFLg0 zWVP)PN5vsiuulMGfvjPbLmt0}7;Ar(E$I?B`Bd_} z4g|7PLnUze>|NQY`}~MPR9Fr_%4fWya2`WEK4UK8yYO{KA2ujwHki# zKXpr;br%IwYR;e0h!-uP-iT1vi`x=_oeC{1{j|*)*m#QUJ(2E zCP(Y|7A@o6!)1fKy^;^M)WLU~`CJe*K)Uwxv3VNuP>K;*WoJ?#eK)c^zx z`CU1*c~9pKXGuQQNxShDp|(+>+?;CBMLF9`Dao|A=^j-I!q3}?uj_h`EQ?DYKKp5t z*MLc?Q)&sB#WV>S;u-Nf zGsMVG2U-@_*=kjvBXu4;dbM#P#-?Y{A8w4qm>ttupD9-U$YZ@!ekGZFn@auiOtHbP z?@w8!h?3D03Ao5Lt(r+%!H0;;SYV@*lEXe(WskD!D^v-omuB(B*-*XVX^%|T&Rs`# zf#fNfVNo|hDtZbU_gKpDtUa3bdq>;XR@yc`M#z$^3()x_hn;W>l*itY}Hr z1Ty4g`MJ_u?2A9cneE$K{M%!bUdlk)sE9w~o&j*G#sR-bfUg8R;rQ%{NRw5e%aHps zvk$@H<)4@|2h&aP>w1t76Vd`f?GpkZT~@vl^ihrn^RwAfY;QPS6)1F-=fd|>JBW07 zfu(n>=PnxID-C+g6n#?ml)4TY%)A5J)uZbCj8$~ zG7iL3@8nY}tu@ggO#05^i1&@zpzi2xw;ncd_W(QRWGUV>5Ygs#a2e)D$XVcPng7&m ze>LGmv>SimH2_B5QRnU@y9L0KQkMfVZ_i%R^)cRFbsb93{cIDjUZossL(YQk69+S~ zKwhNRFBb1>j56w26ciDjFHGZBc<#nQe^<^HMd~y!MmKfN>G`ZY5j4`1J4^e$HsFdt z67m%&YjdlmT`}*i16N4*9({`Xm4LaIEcw$Wq8%cq`D3rGz`wKS`rh`CM#^6%R{)zf zE5Y8_8^Ud9oK7d9Qd!UVJNMEf>I}p9>WAzR8}h-IL|LP48$cxo&UGcme$6$;PF@r) z394AD+J7>uI;<7T0pl$q?t)lCgR$z%uMdWSNXqqf@wmyjA5?2G+C9ZnIZgWm`-%_!w2TNrj+ z0*?!|f*aJQ$QQSVLQe@g<;ProlMhN1K=B@sgL_!;RpY$X!Kx10Y>RI&8ujPrWAm$t z;s8r5DKB}wZmqwRAq;*+gu?e4q#|a#N;IqrCzUIJTR=Jel&k%*Udtp_hL;lXz-h&R z^S)I;^R%9zmo4z6;XFqu970``-~*U@M+Z`sdoI|-d47EKeerQ+EpN_}=?N1G^ByBA~yHv8Zg$>%XV z+XMY)tlJCr_i9g4z4wOuFI~b@4{xiW8&Z9}#P_jC9?!kC=&12|uL`1ansW6~&Pb0! zo=iXA7EDvbE<(CXpb4>mwa6GV3?)$Pt-C%dPSlx8babs_pr~!%({SbX``qTyHvbK` zL3HqM#U&vL*y@Hh79`%yIT{U^P8mt_y_jU(XjY!Fztd2o*t>u1Pr&%)g2foMTZP$8 zhFa8Ym5}I~c&;!0l|lVe86UR<(fsicA0@I@f9p2-AVdJ3;7JC zvSZCKgp2ST?R?-3lfE6cCHfh!lG%MWF?SmOGMuEtY@m;5Hko(2Pqt7y%!=wvJT!#1 zh=xZ4ai@XcIyIS4RVL2Ef->u99#W(d>D*q>I#eC5$&!SF*2c+3IBtISMe64R4|r2T zyUj^G_v_=Ac4Y)EjHLdJs}PzrO7^CN{slJffx~_Gam-_A&fkaGx`3zD)RVXHw$)b` ztJzT`P+V%FkQo-nR&1^JT4oYNT-Kw9X$t?&)?ww^fL;v1Q_~7se(|CdB%ir2&s_`g z41H!fn1lg`n)88H9V#||(%wz9rrU^wQ@2CH!(Dd3+EgiB5Y{lY>A8*X>k-ip3Uuf0 z%ltRHiN@gxnD5fMuF2=qG+V7hC~zu++U-kJ4VzSIJh#z6DzjZIQqAbiSr}&(5SNMV zdlp8s)Oz12U3v;7ce0>1byPM{I2-ML$$W9M8AmF#K2xd=vGmrYS`i62(MSNf#tV0E zJH%SZ6~sMC*!)ckLA<6(P5ESS;yyb+!#GI=+~i7RiOlm`vHo2&tJAcNC~)CJgqd+>8c3 zGaU8pqhh!WpirLMYv5Nq(&_w241FbGr7C7iD zUfFu4s(Sh4{HYLD1G#gM7aABYnW^FwwaL4v9W#Lp^#oEiq+=-LIU<1X^}?VzRzq}Y zsjyt8>^#z8M(^sF+lKdry{6 z_+I9=3EOM>mzNg8?|2~gNK*x2szKtmS&{dPOUXNoQrz?R601>|1!hx4sNG-97tuFDm#TfVk9)Ym^&kM$*a-TIoX zv{_(?CW&|6D(eHl`YhMj!~R*wK1yN|(ekJ@gSq!gS%EnetwtSiGJEFB{R2SG^{;om zxX0d{y*4spfbDwK9K2anT>X|-bzx?7wvDjD6uFT`})t;>chO-!4 zr#ok;1g`U`YR8r_EGh_hYDpTm%6^ttf}t^UkP=u!2_wZhE(rrh1w*g-Vw&a;+f{yy z6BIwK)}42Le=GDhsgP$m+JZ)Z@O()zfHrf)Tpme#lU#fkGXxi-$u@+I`}$SOro(oX zPo2VTzc902!zn!)$4ej~fw#`L>7CEgqvMLSIIf6DDs!Gk7M{@=S8PS2h=Lv?Q%+Bx zjwa9@9-z&_Xq!VeR`#O5RM^lq#%&P)4|(E^P3hR|)EL5$+@bLEkZcxr0mJo?wBYc$X#}q^R@oX$MFyy9^Kw zt)H;lLhLe!#&t4~PnJfJ-UkXFQ{5~g<4Po8+*kHL%Cz<6W!>)u&1=M=yP6Ppv}XF+ z*DIC@t8#(YF;yGe!-v26X!n0kBhYUZihtE?Z$&8%YW<4CgVBAs?Sk_;g`t82*1a`O zfs9YVE(~|WuSDMQmRVh3(UGkUlm~PMw-OMq?h|t-d738JVlLcTLflh=V!TLW0$H|K z!Kq%ix(FAP^Ih%&yX2?nh4m{-7y@RLa~b$5DHi#xg<0=kGJ+@{UghUgKHKjsyS2Zc zx}DIiGz%Fv*|_IQT}}Wu#k@2E%!)+%rHu_QRhKuJhyYW?Jx&y*oG}HUS48~upae4p zSb|aOgsjJYdI<>gi?IT*k0T$cA$+5DZz%u|(!IuV`a9m)z4U_O)B3p=N?xIhg)A17 z%&Lnq$Q39P4j@Oi&=#j17cMEJQC{MUe@hl8lpA>8>92$6P`7;^^h)=qi zvmJlntr`fu>AV`1mb%ovUy@(9W?W#Li+haK_S*!Uq_3+ekj#$tHCgA#`8%28v&wpc z3Is(gyPg}elm@coayh%5rRwqi^WwJfdf33DOm97CX%&_1!K}sb%j15#n9`e*a2Ipn zy-W&l4?@G+asoOQxfwPKkvEUhU=y6Bmmqc?4Gk*ShEVTMeHaRVKBoy zdDZAkP#CU!m#@z!4pGlTp}%5={bS*fuWrD;2L#lHf_IT{as<2(S{CmG`9_3;RQ-S# zfUND>7oL8f4R2;lxX%eolS)YxTSQ#Z5u~cdd@;yo^9kA-ygD^5H5nFM6}+ize~mwi zI2h}{Iz+Zze+~^)X%r=h$zt1I8t((QGy_VQLyBjFJvZJEMHFYsQ z=`-Mg!Y@d$z#}3PUBgC}eMPX##Ob0pco<=bddlhgK|)^@5M=yHITyr^c{OZ}QdekY z`l#nG6I3;a%c-S+^$7jO?@5IfU+Uf?}~#+MJN zneL2w^~6y}hJx`l&wRyxhIZz0*-BPAr_tcWWy^m#kH3h0zZ4C*FDX7|2F^re z6{=wKw6j1mjGNwi2=lWS)u9m{cfpKgM6pAVGIcO%SAUO2?S6Hj6JsE>Caxahwea+a zwRn$$uu2%6f^8&UtvOm}Ue?n3&_PQ%;)!2g_D1`D36DRBlt4NQz=>a+G7b$z* zB~dSy_wJtE7h(>Z82I|!W}`C}_9Mt=+vyP^(T~f=p#Kz>={Nz^0O0&mXPK|sj!G$s zS*EXzTiW9hRe;jj3K-aF^}t1;YQdk^8L8O=RMFgcdSR4BGj!RbaMAH{=|F8rCJA{S zBw;0tLx)R1_gNaK@SNN=<@v%m3d!-cqJO4D8A`*XWAIuDO2e#E`0}Ajd3$@3{6`Z% z;8wbm&tB+tQJK;ksjnIXCLnMU4s*7cyM ziOYv?L5TuLmzd*_WD&AFb3*NT=3CYTSrC*yI}6~g)-lX+T1_Z29YH=Sy2ZMjwc34I zi??6!Jy5s^g*+%k2T@=qhP%=L&32F1=NRek1m3IE#HpTkRZ zBOqg@4Drh4jc83{s3#6YtBvWmc>3gP>58evT-kEt=zRu$Qosp&)hxZM@V##I6LAB5 zE~wZt`?2?ZnJ!4Wwre-e0gb_Kdmv%iHKE!}Em^<7s-IW!{aL>qlf%sMP_;V45#sZ~ zqa*uN*9#}<-e0~+b zdeWKb?dK=sCZm_@&92v8B-fIXN$D>K3Rlq18)rM6q^v&y`9!qi9m96wdkM6Ph5io- zjec(KF`SVI^|ThvhyFz?=$>PTvbJn3Drf$j8&qzyA;uF_1fEjKJ6)xF3StYbM0Z&6 z?|e25+QiM#_NX=gp|)hAV~nBa^3-ADJe}X6!}7a|torb(KOfD=>(PI}RRBKgkLvQy zR?zORcMjg2zY2=il{J*n>hjzWy##X$<0d&C&20(cGQ{^U_2>I&?RQxaDyoO*yX4B{ zhvqbQmv{&h*|Ow6A!-YVvhRAgdhn^PB@}7Yp=SBpU*?tXVs>+DmTP7_g>FVfxVA^^ zWIMsuUq&9Y7cpx1{k=^?4TK3CJx#z5E_w3bESV=fKe|HIjbPSS9x|GmrHx_7f6jj8 zr%gEw-xCS2X%D40q(I3d_H(!$D71f|NDeTo5Cs`y1W?T~coR!(IF-*%swXuZtquaE1~klrCurnV~yG8Wibn zkQx{Sq??(ch6X7SrF)PRl#oVRdR=(m&-;Auw$|FV@7Mf=ne)2NH z1FRcb{i20AzPTL0@<@YPfD$HR7@MwR`uT!*+Au@OJdmU5z6|FgEBwWe&qkpx!?9j) z;gZsE^=c(nFhgX*|%@{o#LCF@W#Q0<)&WqdZfJkW@mMmhN2`GZ6>i*}R) z=eEldRhx!9qz082YMm{q(AJwDZsqR8v>}7WG6RnF*)%IohZ6iU=8={7&%HoFWRP5h zbeRcu&@s&$D%QXxUusXTTF(sJ(m-}_KZ77NALzxEvzIo22!S$I0wbXgH(3F(ey!}C z@H;1_dr%WB*wS4I0}lpOC!@}(!cZKm9Ze+SRdx)A$jeOwuMd~UuezaAEhW8L`#S+T zNhp(h+`2+h&voBevS$T;)VbMq82(eQR_|O1N`w%^@B}F^O_Gh&jDpZQ6?*JQ2 z2o~52&yrfa`x&_~ZonToWRA01dBeAY6h5ZFawer+8G?=p?REh5o8hD`y*3wBpTF1F z<}^!Q{DesX|85hsvO{oFoxu9j%hk4T=KC4`t>lFfI{~RzKB6uZ?<2!yj)yM9daiCl zz}-s4Dhg3xWMEiQit-mhm#~ni<>CxC3q)^$r9S<`_CR;mk$3M#B9d_eN#|pe9iP#G z6+8S+Tvi`Jvxm$#Sm_-J?+H`)f2|40ga+>`Vs9Z2OAHI=at>%hp3s|4-pq4?ri`VA zoAFNWlp_?LC1krXaag%lCC8c_ndlrAH$k0`3fCWr{9Feg1?^k8H`4Ygd#-!dEMXdJcx$U)S`71L$ z+LxK{pS+gl<5L|+kxUdfzgl@a&Vnblp^L7B5+V&B$jMe}`vEvf>w5xNkL?eA`lI(>HJ2&#reR*v{A{ zavw+m;y&AwLhwKkXQGnlVcgKokeEU;tnqNr6uTgzSJpStTfMw~Rd+;A(-eOv6a0nW zILzix-ple|XOGv@3wASRy1&iVJe4rko#23o)ZEhfh2w7Y>$^GAnj*KpjZT%;ao@%7 ztHq&C$$#-yn`;O5!!ub!uZ#=L3U61|ljgEn@vw=l=%#~kqryV`((%)1pkA3xhb$5r zE5zc6n+|Rz?$q{%mG1@)ccMRXY{e+GHRqMtV z(_6!gV1ARozByIXuk3RA>E#98%UZRWCE?@Ur;8tS_2TMO$63s^_doxa9#OicN2cld z0iPcGMfK$VK=#?F=OI#p_G9NwfFF7LXElrbCZFE-{fO)zAu@DyjqCKks{zsaB84ar zRhjPfI5?O&^rcUf7o=*Pfb7eB%hKpR1x*TXJ2|sr$`Lcf-DMKjHV9VX0dTLH8yGRY{HK3@l`-E7`5Zp{rkY}KKwMIa0=%@yLG?qBD zzV4yhslSTbF7K+~?oIM}6a!w_17{7O4!(q~uLF8BT3w__$IBp|ijf;plx6A8lh**Z zO9D~*TLWM8UTQnpfcj16c$!6b8PA#14s(S(dg|K>WwKS#;U3B$9WW_bRNdy+yxA`q zV_~5nT>|`XZiKS-ba@vtN9YZmiq>wi!|hejI??`$MBTq`8-Z8&@)=`1RK_31dYk9w z8xE3Qlo1Fhlu(_#myFRt3t-KO4^o5Ih&a2<%ss=n1D*DdkmP z{Q`VnfSXyY6bJlMTW?mzn1}j?iC`M!?=xQ!ZIT2nKChgnW)85kz^;3{{A%$RMZ#)- zME~A=4gKxn$H!CD=kiaxqDE{!5?^hEnmjo8^Igzi#Z*0U(@Urdhzdqo@*?ysR|&Jd=# zNr4ltK%$t+2rvafIBP2nR4WwP?>%_qp^WVt64uTmKg5@f5BtTzcDIFOY7?YH?m=7p zW4rb+%(Lzl)lbre44G@&{%!?yxQ=X<%wQOYnvp|{RUzS3QgEsp&H(_Y>B}5K$wT-E zQJ;lCt)WT;5A}RXi)Dds*Ru1c+O6aE8K0@(E?wR#-||h9&}@=&PZiZM-m69dbuf%u zd5ZPstjG2pCD@P*#_iXiBgxkGYcZkI~OSatm_;?i|Z*cMp=7d(DdEL{Kx zv9vg*WU_>A7@r>Ab%O#CX1e9;Da0EJOes%lcn4nZQ-Vj$q5q7l6BB{uKm|G8(^|RI zy0wtA3tr!{|Ih%yGr{vOr&IU;!RgQoi~WbwsR?d(<{KY{@lyQ=^W^#rY{M z5Bc7Sv!vVN_b6h|lwSE1C>#bWANW7x4}EO|L=h^%P-7@%_x+&P#U`J!Sd2@ZdfqSW zzWM->^oMW#+YaU_jDMus1=tikDvkrHV$F(_Uz!VQBbG;xx6i%s-{S*N?G@X2!29$pYo_T3Y=x`_9y3eMso^Ms# z3XG=d*x5iU#GHA}+TGk28kluyOx~D4o)sq?OZDr_f8N?|w4(q7_QdXX`VBlZ{JWgE z=up}kwx|@Ce~6RBJu@d0MTC}d>k|7>wqfs&6i8MVl+-wkqijK*6&4>a?GC{PT-u>U zlx(92=1vDRj!^5A$CIVfX$FCpHIK;haB-m>4Ys=p4~GO+5z2RI($K`e1y&>Mm%*Qr zC(v1(O2Yo3JflOc;%U)pK@Q!E+s8*w_dzzba_Ldw5w&6r0uKLiK=4rHkoW@{t(m_X zt$np^q}ttkmWiHGgbx_{ z$fY!08<{dR+2hWBU3s5XnmZj|=RYal2`UQtvlVcBU_GKflXaf+o23)>!wyYCjXCNK z&ZQ>s`YOmxS}4M8dBXeS{b$KiW2j8T=2V1U)!X3pyIR|P7EI3mb*+oJLNYVtX3Kxu zv&@LZ!`s7#IJe7~kCw&OL6&`$b+FQbya=5rYG}9C9}rp z=*AB7tFyiQGi+etKkx{!X12S^1x9=AHb^8%3=I<16q?i~DJ-;jw6^UJZ2fo#^bTz> zo6O5ai3KbzSNQpP&FXK=-CpBm2>Y;k0IXTPH@dFpRa=~(MZhK~kKUUza9~Th;iyyF zdk3Y7e(Ii2`}x;I$mDVdmjX>370MrA5_I*4HwdmktU>IGNQPsvAmYQ9%uYH1uwNe! zJGV`2-CZ1rx&b5Y*J?B^-zzM08|3|EPDR5Nr%)d zG&sifmOim;4HY!j5%Z}xZ{}9abFIOI3k2CFUNsOqJ9Z0DA`Laz;goKTNJllReYBzA zrhkXgOR$=K@AwsXLoR5t3W(hMDB&$9Ni)z#Dnm|?hcHL6yaCC$9d>q27YCW6P3|P) zg>dlsSl6e=>fcX5v6Sc0=*_V-|48Q1O`}OPzS|OB_U}pClmaB*#?5?|+i#2Olwi?H zk`GE6v0#cra22v(a21LMgM}#3BzD4&ktKk(1eb8@PF3_r(sc>9W21NXoZ{S{-0bA# zfJDGS%`ewVX-?2FyH7k&sO^>-(v{*>`+`BgKUyY-Hv8*g&F_0r9=f=vo|4Lo?&~s# z7rt;pKbc6_hfQxCWjFq49|J#PqQAu5h&+&6>7q(A)8qr>_1>Dg;H`IbLu3{4xtfzWu^aXRu4~w3ja3m81D3R}1v>X1vn}+8>33 z8tVz(P0==*B+8Y`8ouBKWwWFoegaE@;2WQ*+<-nYhIMGola$Qn7J(Ea>#cVysx|)* z4&m)p(hG(a872<-jE*!xUw{6p+K3Q6E(BUToB4h}kOIKCq;Bv?0rYBu$}UGLyu#T4hb{i?W{fZzVF)H^MV(m?d3W zZ1el`{8PH-ed@ipB6gbRzf%3qe$$NIxhLX=_-fggeHgJEuvck#ISqJL&q6#d)vuh- zXACd+W#B5rK?R`-M5gchIh-i@l<7Uq0(6ElBI-b{acpIy++{!~JIrg^lCM*Itw|KX zHuDZkpELbh$ap#CDwQ2|Hpi3Y=5@fbqj<`+{o8ML#ivo|Oq+Y=nV;$D_mAsz^ZCJ? zrzRK2VxDmwkqQ^a&R)VZ|MWr95|Pfc%SW~~#tcaumi{u0{C*_Yk)NHwn^PwwjiwV) zuF-4hbtQ&_id3{rqFu0IYPE@tVBTOikf1{&rB3l^)B4!~kA96>g{%scu&k)C&%fPE zlXBt=sH?kH)Z<)g`M(6x=ub9lBWUcW=nL}$o>_t#m^B1h^e=uER4B}GproZrx7h^N zI@1Ou>9qX~-6(aex@VB@k0KyyH24$(b|2o3ppw)Li|l4_IiR|cVgR`JKC4vmT4Pze30``R>t z^GUfpLxjPu?Ywyu*Kfaw!~kytYgmgE`s^ZPn)zR4m&fJFVseC+TiftOB_Y7^7!o-v z9yp`=J9ib~;de499;n~<*3&%%eOG|ujA0nCjwk4GTdqLUg4$LA6kh$`!&l_%pgm)>S z{9?pymJamZwbfv_%e>WScf&7AanI^5(UU$>jD4kX zXr_Gv%KylSso#<}QGokuA>Y4d&;N3|S+_61|FC7 z+aVI;dS8mQb+?XNr?dU>dx|QRkWcSU1Rn|T#7Ntqjrw@Hqn^FK0Z^L0cgFGV@aPfi z>pn6bN5P?+IGx~TTIk6~A{fNUYp>ps^Is4d@Tp$wfNGgv7;-@7d|o)m!qJ~IwKI#Z zN*d)+$oHRxC8|_x#05|#zw**`fXZ+;sM38xB+TFMP7jiM5f=`9P>z5|jM&9y#IfW$aMih$A0XWD`jHIJfq8sfxs_`HCU} zS-DP>UX&i_3kHu=vzR2hwHs!L#(Av-!w|{`F~^ClLtkj6saukX#;{%4X&wW;Kz@XW z$k;~^1Uy~7+>Yu)`D2kY+bL3B@o7u@e>XMYqmmbZ^=o-~BAi|t0!y)kEh9cEv%JxN z|MYorjavq3P}Ci?L8BJry5lLFXmhCK>bKh+W6CN!QI>6D5j;0642yh0;VwU!Xon54 zU|%P^ST$`K3)qU6?7s4v;19_DeH6tBl~`l+YS6l@m1>>7dMVYEcy-9gOj|*;3^h7vup!9i7 z50hp{7l!aal`xEbilOqegtCcBmJw$jC=@x$N}!!<0=1T@o%lt@7y;d-fP?T_e^nB_ z`Ihkl$lBD=uhA8ruOj>H^rf@zFYrDof`9Z#X^yZ5_`?l5;BZ6{YftpFDhK3&o{$v; z_k&E}3PK4sZV~xNdqBZFIoE^}4ni=Jz~Yn9lOXnclo1cUvgosNQVz+Wqm3D?``Pr} zje>mnRA9>vH9;7xPBD>~GNu2@8_Gy4C~BenZ0Q{e!Z(|qlj{S!GOyXm@k4T?9cIK! zpyFo!Aw$jOq>lm}=aNcwYr{4#Q<|^Qa|685ak?XOg*CpLcF@UFtg;Fnt~VBZI)J%+ z5rdceathzu0-D(&Zx2W-?#8T{(wYG7Z-_6R#1ABQHRirr<@X*w)_XwA z*Voq-aX=tYjFJ24&pwUN5@6*({?E$c3t2*5YtFtJe~Y_@Z>w2dbgnrXGA^=Y{;JwC zJLs=qW*6tp8@lChuO`mERo`qeIT{CgTtdMi z93D8kvFQ$o7&W09O7wA$tU$%5=PjnuFrg6u!I53I3$;b8=mp7N6yS~6*+|#G9dmH+ ze-};&?0FRlGsL3h^Tv+LcKjY|3jQ(H3tU>dwA$(+3o>RGFvvC|JDV=QyL4oF}!?kOGL8I25K z1Wg3iqjrro2e#4%Gk?dWA2McJxiuROPahB|b=XOTVA=G`KF(akD*uGQu_e)nUZ+i` zAJ4r&5WCDEI7oGhAX;yk7gx?Ts>L`*33k994jeswGG&+^p)d#J4f>~OY$Du@tXX=1 zzz(Ke2iRcNEomn!Z1TgbBxnC{#D8bxcqu+2_t6K5ZS0*0T(i3#e!2}h0H#AOCB}|nYoIfX$CbAHe{lwypv6TwIJc`$_jy( zdq0)^j*`x2|7+%`$36T1vU0?WwSN1n4b#Wp28o|#u>Phd8kUl98@tm3y*1hWgm9$r z?PHy?(-KCz_6xCqhRgczr&`P!oX4ijja-uWBBUeh?}D)4EN{flryCGkX%bu!1RtS|O+;yte&@Y2r4 zfF-lq6E6>Y*Us|vl0gXK2i#zX5*ug2a4hQ|*GXPF+^_gSGL?t5!c4!;We+;+wG4d_`oiY&6s%DwbNGVdeF6qwXX^r%=t09x+R*9^vQ94PL%NoE;Lv=#;)im>s7eis%9vIVJwD9T4g zHE!|X-5N)0VuEcs``QQF1uGv}&~po+4tyQyMUghxgL~FD(n53qJX zJK(*hilyN>%^9o9kV--wH@I73N#?~S7Vt`T%5LC{ZySgtCB4bxGN>vb-vXh;N5H_2 zo`ke`4~o~vTbVJ$JCa)Wv~P!26NwDrF>eM&c% znCmv*%}_VHNuZ1ao;N;;PZj(v)AUZ<>uliUsyFHHmHNrb-96^yzm# z&6FU|_db2?rDwI)W`dQZ$=E-tFMG|RIT-b0i|uKH*P7q^O1Zj@JUP5)@=O2Y-a}RB zd65mlKbs(>3=nP{*G=orQV+y~)y(S%WT#`z{;{zW=D>>5a96qI%P zeU{mB*$u4)l??v8;Kn545qKCjoaLs>d+d87*4O553 zA2fVv3TBKA&oTXmQmG$-0U5Y?u7r7TJ=pZh=6nO-!zUQC^MJo_e@26XnMpVh2ukcJ z=0qgUjvGR``;y0+S&|Q;-!Wr=o2B1e zBy`L7iT%bhf0V;v{>D^=zbBVZ7eo(Kiv%uy?Yg~n*bIS7Dd+7u6%AY3th6>v=YZfZSyreW2Ju<#!cpozsk&+LLSh@Rp)d z`+6Q0Wf>laSXkwi><$QLy%O`0HdfSHxxYb^jW>=GfLXNbtwXDZnm89>`t@lJZPr80_8ml=U$*UTA71hc z&kSK4lwj)g`6{rVny;j+z{&~>tuF}h#g<1xo>_*)koQr-K@2c0dKpacb51x^La~Ym zuk_DPA~=XJ_^v_^i4#J%dJh)FVPQs_+W6R8I-tXla1fgkWXx$}MxR=w;Cbvs;}3vy zw_%GagP!no=vaW>=)hS0<2*#*Kc^dVtu@hWOY3}XstAl~GF7ZEqLdUqe9~Z@ElvNz z+lBnXK(P$=9wQ)&H9A|BdYz(MNtLMb8O&?14Ft%R=yl=Qw=1cxO-iN`eoeM1}jo$b9W7jF!OG%wpX_`6+<`2LXPVKi_L>uQKEQ%+DSa z)QynfQk0oo{dw!UtSEwMyKezP$WS57gGZpfp3m3_P+O5OSLvT#3O~J|1vo0OP;>ZW z2vvPzA%={41et`jgLw(xQys9Q5kfuJ4$bB=bGriKK{2D*qk@`k**~01KrCp@>pMFd z7l7fXMf-%%{&$+U2oSg&>ipDCG{BFD zrU~B>6LSUvbb*1y>K5InPd~2=O2IT$NbXY(J_R0YP^5xu&{O{h7T8XXf+r)ExBDId zw~_s_(U(?i4lJrDyJfd3IZqc^HW8SAvDIOQD>w09&E{9eAmQm;e&3OC)a|p1mA?|C zgZ}?5LGl75NYgq0LDc_W<1M?(Br!}53zk^tZd5tC?VRQM0~@B85_mOqynP{KLUYAf zsv596m0LyiHY6|-=6q3kB^KbFfAx*-$8Y-8SU)pP#tzCns3Wsp5eN*^t`3%K-(}m9 zs{v~PW2~8iAiX&6ig+4$Pjtv$2<0ylqgzGlfO~&NpD@~&i|V_ri0s`bFvA85~pb#F-Z4aaPWmRB{ zO29y`++!yOhYDz7;Q8dsEN+|xkP;|GF%tPsuEHR!k-U!^S$Xr_By-orN0uNwA&3$z ztQ)3G&qd4u;lYc^aOc4jD~~KuxWZ*uUOE0v&h-kDW|MO_ zqMqm-_SkJ%m`wbi_2&h=M#w2l400bL|vp3fBW+76ldQ!WSrg zccfz4TEOW@q%}($p1%CLM=p!<9hQf1Y^(3qTS_?~a@&Q#UhBK=yu$|T$J`F3AEc*X z%x1N-Ld|G9@OFU+ItYCwJzWob0EfyLI6c+-{wvl%+}Dcy4}CV&ko*++2g3f%3`^)E zvS@Ay^t#9d){W*i(l#evC`xlc^a;0*ldZRZ{&#vY@=3NT+|!Kv9S!@=_Zv6`c{w^k zwZ=K3U)c1u?#b3f=iLY~(kIa7{h5q%M}DHn1HFD{ZN?0RaFlV_)b@(`??hi0j@c;JCQxQBsVw#riyoX&2U%IJeIFE5fPEGP~P|EVGCEG|a-L z7JJD)wRb}oZ(uCL+e0aGn0>g5i&=DyNnn=)uPZKoC`w(@76@L5MC?6yzN%g(!E?E8 z=zM(jBfgxrP5X8v?8!usb`A2b9Sp!(rmO%Q530#YUw{I_zIo|yDWA3&45+BUKq|0% zxLmJp_mFU+z6kyvdl$`ZCKz+^b)1FWFWG8Yt!F*-5_hW#$hns_G0jAJ`_bo@Fv!w=5X4=f{RK4B|0LL_@-5FeC*??+Sk>j z_(@Qo5Vq6T_l8F4?j6i^@=hxPP>UHUuDpBeqm|!Tw*kL)mYEogx#@S~lC%;GcXwrL zQzQ#3nBd+2Em!?wf+FfG9Gw8CXaqnEFh9^66~p?XqXdqpS@8KC+>cE^cPf&>3BQc) zgY;H}9!P;(UrPv9Vo1h|2dqQo&>&z=D=_I%v4gd?NoUL0!7%jB<7lRYj9uc3a50Zl z^L!iktuJAfYf_KpQD1~Bl_v%AuvMqOUXlYPom%)v*eOK2LeQt3{WOVgwr(1ssNbTp zgshss5qn#R+6YiET5*n`p@n+Tp-7zv`caMpwQ#7(l1UfvWw*#Gt97+A z{2uhy95~M33-QrHSd!^+JuCyXK#SnRYR1A<-X684<=tb6qukA$e^zMVu)LwjDK~Lg zbfW#g7=ABzM2q@wOn&CY9dQ#1#(r*7hYVAA-VN$h0;sg|(ZD)pl|a z^dmY|b-F~~{1zORwu!sHWS-37W$^h$=CojC_RwcWYbHlv45Z*mmcGC3kf=TN?T>SOFCxu8qVgMt=))T)?d+rri-J~7?TiwvZ>(tWsgoJ!ys{G7E zaUqp$4mrbhCZ8lmz3VWwo_K2Ff+9T^4rx_?R@O2*{YSV=?e9W)Kc-yerA>n4ljND8ne z4UH0Z*cIEC&wp$kdr)@wCEETcJ__bEPI8?t`jtY|0U!0X8+r+}ut)Jc;?Oldt1mJ* zoYeIa^{{F@o7%ZhKCYi{F*%3h-pAJgtVVarS@fX_fau8KO{sxQIaLps%}fw!g9uHm zC*`oBYL$W%f`5`Mg`p8u<+JEx?> zm%RwNJ&rQ5-za5-#+#p5&4OoOueW=khh_r&ZJ!MB_XH2{y6_C!Eh_I$0+D8vwej3`u&s~1 zOc?N_6++#9#rP_kWcz~cdS*ZBERApaF_hKUa`=%-sJy&E^WwSU)oYaF0@cu>GAv|J zejjh*e)TEMM4PR2)iq72hL2;CQ=wjv<4E$P+uQcTOqFr7y!1q%BT_fY=rAc;^=ggJ zd81ULIJUA-xGYMC>_@cZ{{2I+KJHU3iflFb%meC5)yf; zn&jmRv2LT#Gv>eS&Gt`dd4A5(i$F-GvTHPG4;S1w=|Fs<5>S(`iMD=F;y5vKUjAsD zI7_-J+L}J~(R27q{aG$EOh*(+3b&>O2kPP!w)Hf;edL)LP*YKmz4FjUni84Xl{>mn zsaqc1D`-X0AR8@*_lc5p>}h9R-A%O^Cf1dKqd*LwfKR(}fFUf4wK9MRsFJ%h_nWME zEOO`R1O0nT2WUH(hAJlbtvs1t658d2c=SvUfq8~(D)oRA6|1-I~1GVN;Z`5)VJJa=o! zbn>1xy=A2sSyu-}VuJ@GFCinn#$HAPD)Wmh+72A3kaMk>TCyO^@EbdbE@39+qd?(% zj`-6Yx?8k@WNApPicxn?ZD>8ixW_XW170EqVuX!GIWh_E`_-EqbU~{QhnEFK;WX$5C_xH!YTq4&9cr@h3uDkWKPr{*C7IWH*Wp;1Fc`X7|%d*I| zuo`&vwQC7M5>ZyndWcDa2^foVD#-xjR>YzDw0ZdHNszk#N?h;D8Zy?;sajtWHFohR~)IE_ge->Y>Q|CCiP$}rd zJVoFAJut0fUWs%pNMhmX4{I{q_-TOVah}}7nfdb_ttMv%!8P^U_`Wycmoy0=xg46} zOxAa4;MnRCeF?Ij-T*^9IMC(@G}3dENlcM^ccqt>nCO2B67mac%Fgk}yBD5o&GHlG z`baCO*NVF^tj&t0Y{S2+u=}2Q!arP;+(#I8?wn2{=REx#@`-Lm47xU$Jq*f{JNm^ zAN>2X@4ujQYY*`Ov#g_WlJL5lp>;e&ga{9)I>YM*Hl9233z`(@$3zktbOlEn>~(<& zs7ubdUu{C-zf#LeWUAaHZQ@0J@9Gm$wVxabjiadr5*>56^mDBgQVGta9q?IUPzA)2 zSE|YYxFR}PR5e|u&h3Kj&<>VRHhdH}#u57n_r6zFu8q(=4a(YiSz!vsof533(^NSwC-rGv|b4S2`rUaHvzP8 z=opvNL2bSai=Hhs7+44BsZTl=vofZg0A2G}x z&Xj9b3mYUmodEnY@TUjy*@bqYj+=Ocm#);mkMb1(HhJLiMa(vNh_iJ0sp=i? z?|!^R<`#V~fMq`AONg(tVj(@WC)R0aqf=tBtH9=P(4E%k++nYcde{z6mxjAB!FwgL zGc?rD%#_l}JRvlZn%D${lM=M*|Jq5jK~Qdjvp?}|X>f!VRj8KK>Otm)t0}(9Y`CEZ zzO2zNQnA5C?Vv3M`CMqjC+Qwe-IMtlYv$wMLF9Q%=Xd{79*fcKwto3dEA<9C z6%1XPcwcijNo89sKdLGAG{@qYr!j@;;n9RaWS{r-U}BTQVjyPSSte7NY}y}X&YzGW zGjuJP5{);WUK-anNPB(ko2A-c9)V0>vtf7N;!iNaoSarF=L_fN{n=RHScue^>$mck z+bp0#c?=u_kB8I^kA!z~` z-?+@D?$Oe8^j@h#4v}a<5I9n4)kZx+B|K~74WBmj4x*ab;u$RnB9BFl386)Nw}pFo z)Yu4gPk6i-luWeNj)KkVESgEYXNTTAVYKQoXxvo>^$&zwziC6lU#RTlq+fDZ0K$=X zOz- zM82sXxWIIP%Dz22wHf9}>eP+Wt=it$8xT2}lyH`GF#H#s0X}{|CVK2TTKweO;1f~& z+u-e4U?%3G{^o2lzm^P()3p|I4TeHQ?82_Y`s#=TCVXA)w2b{I8LV4E_Fu#}p3dF$ z=oB;3j}1|6JKwlOzhu>xSQ7-lu9@XOAI*QnxQEqZGcGHm4LF%5`z3jm_J=v&sF0F| z`LTIrSPscUT3L2{fSSNJNX8JBM(X+k`f3Y0m`0=c3m~AKs&x;V zjHKXauM2+=)DHb+g~y4;@fk)m0h5I=yUUN)JwZw6H;MpES8qsg3u+^lh_-E%gO8>~ z!}s3vVq*o+6h&rQk(Pzeq>^4jUy;csBX!}ttIP?h9m4%QRF8t+F~C~z#BV;KOb4BL zLuoMb47B>UGVw-8(q`GR;o}MhG+yKw(1A^_y03n9heY&z$wctfb7=Wb$atG!%88Di zwMQ?qElW2$Yk>H(NlrR5tHDS`&Qm6;GLs0a+JdvzbE`a%$QrXcdEtwHmLD@IN{U?F zYIQ~0pv4jsh3qs}iKb8I#nTzEb{3hatlCJ}cpB-2GB)}hCq7Dr6Wqg8$!)Gj>r$lZ z#_x)s67OPSrH6Ze67OiubWE1JxF`=pi`ITyVDk)!ruaVf)a%vaJ4U4LgP=7MT1HM4 z5ya8RMb~8JzbZ{STpzhWr&If+?HZMLpNcwT11$y|)qa^=oK$rIX&C>NVp~H0100`E z@wsj&Yh2q`HJa2QL%?uWr?>#r0_*g5O4`YlYai`=a-{WK25Xd0Ulx(~eXk>iHhT?* z`(hKy%P;fWcBK3#iFE-!=wj{Wpx@=|UB+{c?)CdtpOjScZV+|Ebw_BcM8O{V%L))$ zApC0>WJ-}n1`GaeA%HZZG=^2bk(UFt_Tbkx%R5h)??Ur?l|=O24hnzgBCFaD%xRhS zM0XwW!`%m|hDK7n5iCtP$5Vl3g*---XuU|)E)zn}AF+0tso}zS9i+E?`~ZMJQDL|z zMiLAgld0eQX-EAgWc{{<-&khOQ^jz{us#tTtU z?I0#V1DnqxJDcU5jbs!A6(Di7LGF@%_KD@Iwp@>Isfq5IQJL)LtLCR>kT0 zNx{be5wv88H#TgjTxa|T$YBlG2N*7%l=yvmtnL0y?636l9}JO-A_vzBX z9ndd}A^Lbccq}sC9a;x;GXX=qpRDebQGcK5+3DVj#E1Hc=FDIDxkqO!mP43BKxj=x zTz}259^nOru<4PVmUQ!(mOJ7wx?vTMH3E{h4@8a@e;m8ThU*9+Icnd#O<(rY`mcSv z;Cx1`3%UBeQ|~t=f#8Q@n?;ZdL%#RD@^5E^%4|EzP+;;vifW<#Z{Ojw>S4oK!nD6> zUWQvCX!%xjGhQwgD0(H(ma5+5K#k82wDY;pazG@D=DBqL8b;%YaY@=wOC1l9CsV6r z1c{8s3}h5%n}!NLATb-8@%L9fb%;npRg)+F&v@{JAG{@GGkHiM-u3Ze3v5uG)0*E0 zfTGvUZg?{Kw68pj@A+mEZ%%agHa<%X&;E|bt5FYjID+5_SM2)kgPqC*bm*#yBc4Ld zbexq!BIYdQODKp1Lp_dYJuK_>HltTMEV+TQK%@UW;@sh$k#>a>+QAMe!42Id`Uvgn zyrICW`1NsFOyXv=0Ly3Pj2y`5hWSvJIh-&xUhZSwVBxJFj(E^tD+=}UfY`)NAoZuW zhxel&k2@xxH`2@ojn&>8K}O_VCu0anM7ZxT0aM>wO;^Sdm4D=ea`@9VH^qlETd!Yc z!dMjib4#`1a&HTZvceYEA!6g`#^IH8i%Pz0{yJ1=N!t$=B0cAjoPB~H3f98BQs~yA z1F4=4an5a0_(;Fc^YCXAAj$j@u&Qk0_1{z^Mlwncd0jolq^CQ); zJ(51HL$GjJJlS7v|C-py5XWyl>|N^;M#@wZiF>tiO6)fBU)MX`LR_mg%6mj7{;mhK zjUNBA9z0uMA%b;E*Ef}_9+)WhKo2t79{zagyYe%Rq||OUB-*90v5HFE!3u?~-IKcf zQTu(!sA>3QNAv2wXYg+ka`}&+U!wY?ZUvC<&}dWA$XswnvXd1Hh!XXXnA666<%@O` zwueIla;$0-`gQB8q2IXm%kr!aKhbeQVzXQv15R91rypK#2Mk2-;v)H6`v92bV(Hh< z`9k>ES?4d7KQfk1fwbvP`Mq8pT^!gws-%+ep{7`RqWf=SHlM)`VwBEQ>w0TN(RmSf zyDJ4&Eyz>9VhQQ%`rcih;+PqKHhtAF%yhl2Ekv?Z9#saLxyw6EIPFVQQ1qy%0NCCO zk9{+rDl&j(Z;Hc3M#tH;XjRLW!N9th<9$RLb$Q7hkP>GBSO~)W+`6=T9oRT*W70v{ zk=>0HBP3c%9Bc|r+Qf%!5ne0h{etZiY%x9f}D(*}Q9tK#X^xeO_eP296v55YaEF(tYu zOPox8$woq}cJ((S5f1-QAHXDRfnQQjJJZn8$!9e3nO;BxfiwO4?1)3|~RAm)i+ z_9FAy2l=DkylvGla$WVZumY1DT+ZYz8=J$1tPnS(S zVgk+r%r1Sl8+6XSZ&Lz%{MW1EvHe$)JTIg?q^jM@jzxNu*`Ksn%zjp|LU#(6Qw zl^f74;^NkWqpR6ID<@d(vwZs48Pm#5$RWjLX*Smo%A64RGzs=tHO>1 zd!s6^hk?U}H1$2hRIQuPX!r`17nu^w8jO|zT+6o9rPm?=j{;0P`wWmScgpmD3{oYO)s$O~7&h0ugB+G@=sf(twj2P{{P za}FohnIvuol5X2g&Fz`mxDX$T`~HIvNm)?73FsK_l#F%efPMV^YM27mBg!Q8@t_jS zEPo6~Y}DWvW#}`%Q%d5C$gIBnQa9o0oc36aRo&s@)!B+O5P#+ayhGlfuf1FVFNHYZ zsVI^B^5Nj&_)sAAJstted#h(Hxa}hHwn-(%g7{XIdA%9>XUt-Chek8ku@Q?=Kx~A& zc=kZn`V@t2l8c1Yoy$vql&+AiQcE;R>-LaJs@pIXMOw6sj0RHsD(&g)aT=-w7W%9y zdTSPsK7W9Syt0MnuzBmg&LwJzG`y-X6nM3&|LaT-wOjcwiYbq^t7l!j=t4#k1 z54+YDOI*X}KH*NLQj^#4Ir+%{hR-ud=qdg_{@m=96>Ih(9X}Z=Is!4>yjPFv0wa_g zRSNRISnO~IG>XlgO}rE*MZ(zhTP}db_Psx5f8IVc?DLyc;?YZ+UVM7cm^$YbK!B z*4KB~RIX^%NvQMoly%BgZh?Yb;FR1m>dfIqi^|TsucRUrL&NO?Uq1r)httTSw0Q_E zPylKa&CwCz4b_XZ>GIZh&q-UEq!XZ+Jqt~B20DHqMae^0y^akAg+7!YD23iJ(=jQ^Eq_ z8l0suxJ4kcfF<-Fm$YG2WH9w%j5taqncs?8sm&5*(u#dn`^?;$Qvr38S<8j1WU#>6 z3#Tvl{Qsfry`$j_zqjA$6Nw-(Q4_rl36YTKy+@51M(^_ozt_ zMDLvlVbrKW@ILbWmG`W3&Og?&mWAP-``P>2*Zy1?Zx|cI{@k8GN_a=*qO*qxKb(0| zVBaF=*h?eUy$E%uT57&FWm)szq~bV|e?}(s-z>d$V@g|GVoSh+gqI7b2H=zkHL5rT zVb{lT(6RHE?-1yL5JRk))BqnQ!#-1mmJ*X@EAOwqYQpp?vV;u>%}_>YYkGdxZ$UY} z+``i;HxK1+r{llHq+7Xl;s9OO7%dO@bG^%HfrA>fkzb$dYtfCLP=S&Oax!CqK3Tu% zs~5N&C0QS?F|Bxf4^Nn>_DQC%N3*=C!OxOWYLkE;D>Z=2C)4iqe>k4oF`^dE-PVN& z$%R?Y?N>l^TR}~@{%L1W&Pt3y;Gy+qLeIIuq&nXNVmI{Es5T~{Y83Wk_>#%pMR)kX zUV;||GwvY;lm;#pb5=98>Z)TxYbLYciT^9g@n z2op&ivhB|^6f9TW?moaQVvFJ`xFrtMX&)WOO23NSuOvkYjBz!BUguduG4tk-2z&73 zW~xjkk6UV_ogQ_mtTGkaZATYG#fE|xZ@FSH%~JU#`IsPE=zVtOU8{p+1k-oORrj!2{ zxpO`C|AXB5|3={WO5G^V_D_9F@R{y~@s9+$mjg_?l!(i&*13-NQ4x9`hP*er#nxiN z>C97;b+3jOy-12}SCwJAhpm^@lIY6^fARfMq7#gSh7qL|YExg!xpBc!$zL`tiY!<6 zKX;U9%YE#9qu`KGGxO~%rUIq(*;DF(il23wjjkR2Ce5(NiSU5*Qa7u_^TmzT z4kit?JBPn^LAjaN5W;X`40a{QAq~U{HBM; zNl5f;KE$vK0fe%A;M9#Yg{mdmiK3ITgLyM1u;c(W*-IP%#H`nd4AJT~!oe!#& zz2Bg*SHyJknC{rE#fR~wmxqYU6m;cdK^{JQf?85fe0$hqnI~d0&*yAD-eX~Q$lP-O z%*FddVtQR%U>NYqQ;^3dZm1_woG{6Yr!`5^vt#6^8cjm4YZ|Nf8UVcEdm_A53f35q zeU>RSy1rXKZKHZ!0?vWjZX!DE>Iw_<+=N0LJEdHhKQWgZ*Zv?urr(~mzTy2RuD{didg&j_ zv^QSA^XOpL$!8=iJ$O8yQk-^`TPfi3^rAP>2l4Xy4m_=~hd~;(sz#dDb+oJO$4dPU z0&}e8v1Oo54ndwHa<`*fdWc#BiASpjN0ehGOw0rwQT%TaVAW+*-r+sXieTA-fDo$k zeB(|#mlBHBktd;i2s3F$mXG8gQWprTkCTev{=Eemr4sVXV7CH$cJy4?EU;MQ|L)eN zkBcwB_DeG&F9%S6C5x^e9bq`%Xo$YDTKz_N(KbtPgSFY{OI(Vr^-N9Mg4Ok1w6uO* zYAIfcX8gTZqg=A!d{ojsM9T%2&fjOlV1XyFXUX>#1ei6%y~qI&R|-yVSn?b{QEluU z%*-*AZ-j8(tKNAf39O?)4yh}M0SH5y?{A@&5HgG*z0Ut|5)vqL3|gnU4h$IT$aFXj z{udP^F8#l#82BF7yRMCE0%y?ibmqYH_V4D~?@VhAhPM1{SGcFfaw2 zj5*xSx$3iNJ`oRWt|T!0I~(XcXq6F4j%Jq7xB#$lGS(W_Bp`po zjkf^-8R7HlF@bzTC_hEUs)JP|>08I!`|v8q0`CqsbQH%I&!6zWmGMpn*Y*$AP$2yx zW+=tu+v}r>gi^xM(YruAN5O&Ore=I85E+mx!jTI;;-=!cwBYVknz`7r-Zr~(2pBm|6{Uzm$YR}FZ-Evre_?I$Cp<2402U8>*r0U^_9 zfnO9@6yavO*t@4^)IE-r#m<%YhVAeB?)#iEkPWK{t)@r7QtbQO7Rh}OxnRwpo*Rm0 zcK@y@qw9e}p6`|CuWcq5!*L9$NS_B*LbF_(Oyi0cPt~<(NnJSzAm!H;7>BW&$`N5N zpN-is#!et8Z5HS%3z*df;Q&|(d+i$EssDet#*NYA|Cejrg3>nKRB-E~(0#Su z@BJz#fzd6P`{XEIHgj}kLH5I+AsqTnEb#b7<4><6EemoY6FW%qO{_l3R>^b#?sg*O zj9@m)50kX(9&0`_6OH)HEqFZ;wJ4Xgpm2e{(b|W|J8!);D@S&9KxE2DLBlVL2cvm5 zHrEFufR&$t(#k3_iZKZRR*>y2!rqkx{Z(pFVeMjvz}Q2+{vo*E_zhY>91?p<*(sbDQ^Jy7iskHla@Xz;BbC^eB?$p-(F}HGgttx%8liNg@#Iu541`FUE@h$U;aPf^7pY*m! zX}sbkO1=124rr!AnDrdK(AZg?*NKLVKtmpa;--^ll7`Lc9X+d0;po!u(Xn2EgY=Ye zcxOpzdGy&|5?XLPWxTVTSSsyq_tmN7plEg1659FiPH9xlzIAka$ZWN4`3Spl%ai$k z*<78Q&f5{1fKC0Y6wFf$SWG&^b^@f#md0h9(E*2Z;DwkTgKBV(h7a}V-O#+``+?r4 zdf%YtN>PSZUrRy)BjsvF+|T*0wwA&e_A&)~Ab+;`*d=3B-ijyXsm~xHa2W5)V1oUk_DtRSVD!pG!%~ z3g~j_5BGz(ZS+2hTX{9O#DxzVj_{Be$}Ynmu%KUmxQG7z>b>!s0#&VIzdsdn1p;2> zM{c0;$%ggEUTP#i5n)557x6s$RCkK)!z5$Gq2+Zv$|oO7|X#VO~+v)QRva#{ld zkg=43=|U@jO~9L{kODxkZ5eCN-*ladFyo-MhQ^kAe7+2gw$cEuy@6cLV%mq>*?jW5Le=a3jJAB~F*9u}l*EEEQM3=V1e<6DO^0 z0@$U?9MzJ1{r2k2)`Myw!YkuZJ}iy7f(JkPY5h~Jo3R#ceHotZ@My7 z1;L&FL?$1i zeaMa=XbYymi3-xt*Q?uNBbLm#<31($sF%TN2AZIS!sx~VH?3&e5H(&*p+HGV$( zuYlkrM*ep8h&nTLp#C#k&MEntv^@9|x)V8uapS8G-y_B|{3!$)yN@EbpwuhYWiQh8 zagZX80Ow@(t+*v&(b=U63tdaHCVShaCj zd>JrV?sUC$zPj}0lJ<3DzY){b4&VlYMq)hlBiT$h(h3sWi-l`9q%@8XucN@ z*5HMnW;VC+1BBg zaXvAgjB41E2dwwa)|Lyd+L~ggXiq`kzkH zr-K#k3U~sW3_h&q4$iT}C07A%CW3`ydi+OBL%r*wmD|66hRg+C?2l+AV!q#X`Ab<- zH$pKPGa|s>7hJw@d9)rFc+i)5^{V3@F}#E;N{ry$9*LH!x^_2{{k>R*G+MKBL$1R*DCtxx2?-RA?Yw zlw&b_G*E|wjtu7rXHtJqbtU?t{54T=Egz)&L7Qli^lGFCsA&Zu5xZ48+hIWaX6o{n z%awKzyS)1K0_M`~uadlNvP}N7+A)Ru-1<$T&e zqD=MfJf&U@4Lb-c%INz|>BsF(U3AWf9<+4axv#BfphqH+%a>sGz`Hj4(_$lOtiuzK zh*w#d#0$R%ZPZ#y>PSLQU)JUamK4&Y zOqUC#P-%fcMwuVCB2(f$SqXRgREu=ujcMm@1|Lm2e^O|H07(6nGnHAN|5D)wo?LIj zib(ve16G(noeP0rpE9chcXwWg*HmSHZB4y~vuOTx)Zb+*6}PE+a07~`fugV~?XxBgv+}ADWk1$i4=_^P`jnXVY)r3`?u2l; z>z?W31Til2Vm%Ld%r~y?tuM14j*! zGC#hI2{Zn?q5#jkdDfewgoECuOrf-%A)R{L16zf(02Wov(Iu^Cyv&|398{0?w!=|L zZ48wh5SS+pkr+C`ntlQ7>LyFM8c}?nzHE6`^H`HRgwKY zG;{UmiMS_+smukItObXAF>pI0BRs;=b>a$V`dKOM$3qPzjszYq1=e6Fq`|lcbxGh) zQuXY;m2iv-R4X?MN%t7PA+AKbu7CmH#^=Y~BD!UT=|{oddF0TWalT*Oc;h`bxbke< zTr0~9l^-QqL4}R$L@`>kzJRuPORRU!S83M-=Yt0at%h0GUfWRh$Nyozb)eyUvA(K5 z3;}kvCXgK=y(5PHU`lKk3P#aC{?L9#j_jko4i~EbbJ|@pF2EF`egof{+#6pa^q%KU z8+Z$fPD!vzw{*W&sO6U(4$1=ER&2rH;h-uRa~K5Fi1{qw4ASKz^weIzB*xVsBlmdAMsPyH-ZKvZ+&Qd zcnHRx^%`38=wRYLnI^NEMub9P)3gAv1u@P*dcR753#Ntj4n{yVgowT7$i^`F>cbf} z=8f}pk@I!eyV_37d*uO_mSMylD)g)LJtbs1*q6odOyZ~_;V{IZ^ zHDgb|F4{0ueCW(MD`ju5Y|?2)t2|==Swzzxw8msTK^F=Up-7bDai-*=mo6nJq{QCc z>UotQBJy3n#44OE7G;Z;thkATUR;*$st;~+N~tL{(@s~oSA4ddwi6W`i;*u={5 z)JFO(TB%~B`BKO0e_x~%Yy6XMelrazP^fyj(Pdq9hy3Z0+=T0t-^Gm$swnO!{`G4a z`DvMua5hk++f%Pg3e`?Zy5Hi64`^Thc1{js7?d^+%G?{LB|K<&EL;Bq)yH#Cq&r-A zjDw~K0e=%x|EOxd#ksNW1#MN>q|N`z{0FET;FS@dmXia&#p|t9Ysx=ALriM$&!0uL zUv_&L2>&uzJiqL%2|f?m`Bk_rx!$__*KwQ|pPV$SWsu?pA%4K$Q;OZivx(nK+XB7- zYHBvuZXX|rOp5FpE{n1}nh(6n4$ezD|GTAiNkW>zgjs(z!2@+AQVM>7bPOAL(X^x` za|gk2&UCZfNWKOF@|N=0Q-RqH$#;}QNRsrFXlUk*a{KYi!jPG@4id74Y#3LVH`$|i z{l8BxY=`TruZ?0M?H?DH`*B`r^L8myJVzyKe~nvuZI)jHe=75c96p@^U{(Zd=+DF6 z=qNR+9L$6w#$|~^_$F}B&1|;U7urj(7d<9VH16Qf$`(p5B)@H+md$ah4XzGIB;t8V03PgI&`_pT z83ts2PS?B7yf@A%aFUvnBQ`;L;3pV9W~(`1JV+E<1&zLEnzW-(^TX+oH=%}Ef7z>s zbr>>Om0ulZE7-kxivpW9&)7K_Y{o0kLy3LJdz*z@Ur-KnQcf7JTZ4EbJFL>fywckD z=vB?g3GhG$65pf(t&86|j8-px91WM6-R|HEEIxSdl>3?KsmVrv;@yNX=i@oZ1dlnI z)4&O1ylNf+i!1)2-(sJj1NJ(Az;C`&h7}qZ1wOq60?|5X6#X+?G-o))YCh=5>u47MX?N9nc=>=G+_#5 zA`GF&uVn$tA{$A-uPA4^mqlqw+~=Z!Z+<^u)olTdz!$54_!=QSpOe5lGv;C%KR7Y% zvt9~Uv*~T~r)^aHWNZsXsHKqsZG;BdxxLd?B0{>cjTWr*^YA$n@57fo~{`e$y4I}B}Q}-IU z8MayZAKp7^Qyut)#mhYZBBap*1c>B+`v@1q<9i9V~t`J4hSm_U`j#;7cC)TdpEbw9mC0J_%2h$uEy!ag-E{Id4 znyONT1oA}&U1>r-95*I(bUTR#FTHm+$X=c#$UglJp=y#dZ6fKiyal(MFhjqnM`^ym zi%=CHe?<9t-fiW@9E;`gr|Skx?(&7K98MuDhIiU{VqW-C4bMt-32^S~>O%P0m-AbX@V#D@-(~sV+CMdr<~H4hd%X-TZPWJ~V=pkeyzSo|Gy>B??P=CbD(j$F!>l{%aP+>KGOvE>1M9UgV_v z#}JsJYr5wh;O*#5iB!ushQ3fVBl`mkpUG*%9F?<oV2Da$n&6lMlTteR=aaf9O}ugFYk{%;mY5v$Luo2fC&t> z)Z3)0ldu`3crCeT8l6L?Ixc->=I09v!>BBhWTd5w=>yOrik3vGU)a#W8lZJFygK8410S;p z_~SyQJdDFf!L%2k=HJfidE|MD^^33E!Wy+|?kcnA>JKbu?L_j4NQxb7N*d+6>s7U%p?%k6txxXiRAdT3FXp?iHpc(vo)mNrOwohYY z>%hQIdjTU%Tl|^4`sbvt6aMK^Pm)=*@frj_`dj=3X-D*SoNb6{?Ys|1{n74xrk{`H z=8Gpr5><)Qk=I(J2jTkFfF<3DnxH0+gOrND4F~(Kc0`0&SI`1czQxp|&$C*BSrYy_ z2#gfsOhO6&QwNC(6-^{4O^1zV;$~c&b&MWbwMHV;1?-XGHjo(l^B}B&z6TOlp*Y4o zN6FW;M~RI3X3DNd*PYqrF=fa1{> zcean}T%D3dq>;|glDoNJmFM4mkbmVV!y7qu!K*{FwBLb?v3H`^T+nU$-WUc^r2A;j z6$57^s>G;5zaQ(z-VVk2StoTOV=0?tSn*boE|ss5(WEAqzANdi7RAgxJ*O-REKWqgSe33t2^M{i zkuckP!;nn%nzTR$Z_FE3{S8^b?EE4XG}K^Xzta2tVFj7aEZ>`LAK&?w747o7_S)sa zEnCh57+tN@62?bI5So5t=7h9V(qwm~Z#uVsO4sE=s5gKbi8#8Xfjl4ym&&Hc#l#djQ>qzRtlXEn2OxjCKZSe5Fq^J$%St4+1g1lio*N?osVPNl#tgR?8{}fSAT%&%(*%1@7KPldra}1IV_yLFF2*}3sD7bKPnEn;>f}`9@~4}usETh1Zv{){?fysa@eEYVB*%bK7(9(qBvF^p$~w^N zxL*XSAWVQ#knNa%Gv)6_-)brHdnjE)C$|%o&9<|#nRK30bwK%{nwD^(VR?=8pBmd8 z88l7!kn_xQ2hNhi^!Ibv&@6G*{a_Qdnlo{tsDdu(FdRnt+bhu-;$YfZK|yEv%3S>N z#ZJ+df}CA0srMX9o10$7tX``G?QBk_%tf!m#NJVKqtK(v%5Z_R1EiS`-SGGl_}sh* z%8<=$QSdT`M#BxkY}c}Q&=BFVbn&=e{nsku2n#BFZ+dL@h$Nl0T&fI@>QH3!3Hyd}k9%e1?iAL>8^y8qj{tp2&+PD6IOo?Yl zJq?VHusw#~*Mgsj;`%{H5I(YZRQYG$`7tgATA8XB?=9KShOAUBUi21ykhM1~)|Gcy z<%IuS%ll>&MlT+RfCg4ypVRvH{{G!nWutncCy6?*J+oh|PhdAH1gCb4-#@sPE5oIw z2CFhZu38-#dCrKwp4`0QFuQwuv~lqZYv9IQh|3*3TfpQ2gb8sn{>=r~6v7FQC0l2E zg%{5jPINzl=NR2-{T-rf3LiAK`7fSMmjGl(askm9iubhm)l7Qt(qC&IHSUaKickhw z(T3v!?N1?>ML-D1*=jw&X$$mOeKX8w_DcS%7>%=5TX2$l!VfAJZi1@mDcZZU5k9Gj zPqJjHdTC1{3HH00;q15Gthziy{YuE)$}<<#$eqjMdczk_^c>f=ald@=PRhf{%D47G zA5N<_y@pB$LE=1Jz?h;=+VoXsbtwT5uU2%rBh)?Mm{^L}_80NvCd{7HA+DjPX*Ha@ zLuFYHpvp$*Xk*h%+D;=u{f#78Tkh|g$>HbtD;68U0)PZ;CcNNM@TrH#DHw(2GK)6HD0n5?M*;r+*Q zm!gHc#_1jNP8tCb??frFz-EYt^;38D`!(tzPE1<`2!Hxq^yk{%uABo+fYgrzR+?zy z?}bJubv)tFuY7ldUy*`nAPeZWstccs+~PL0+P|ZEqwkPP zTPofVerG2sRjP8T>zxbL^9(i<{%)8=ZI#M*{b?&zlX~lYiQw6S{5Ya#)f|y9-Sb9O zgfL8b((so$1>#fwW*qi?wC$={g(LuAXK80$d*HCdxT)xpt6nD3fZgC4!#rD?m_vTk z0T;tBtW9oV@Yz}-RM4oz%PLN|Ya1{wA`kKG4a{#7u1a+`tKghiu8Z|PJvRT1ZlG--%cx!}SQW3ED2pGXPy}NZXsU_YI^UDn6tQKgK#Ua_W}09J)E~Jp%+Ue2kL`9nXbl*TEH;-f%|$%` zOy!WF84gk~A@3GG)LENTJB9e>=3zBu5hr+G{;1^F&5QLN{YDf5st#BzzweKKy|@p^ z3rGEzU{wHwte&werm^xn3Cm>bCqJ9)?8NcPiCx{PvzptcaJxU?pZpU|F@^kI^lScS zO=QG)&1St;gZ69|V_U#g`pn-%tLd{-=^Ist+Voz&t7`vf@bKi*jm3T-)9N{}8F8Iy z^@*}{V&}p-8Gu~c)?8jUDy@yBW`%Hd+g7>``{W>f_}g{~$``yHDrEeU++S9C9`Qu1 z%PEwV>`ABvjz{R9JPy3B~DH8V55{(J{w9*k(Gmtpj8+JeM zR+twNCx9(pd}Y0xAxhIH-A_7BJJS8hios^pMKD%$BY4w2t1sb;$GOj+^je2yp2=Nr zFb~-?M_h$oc>eg{B>_9q;=!YAQQxz(A1Oe)Zt%*ZeZ@#b`z9Gv#=c&W7M zL*8i4MAut@wbQZ_uavdFKJ-m$v{Cj|q&C3_M?jJ^rjys*fBeooB_jiL{j$-*zJK*d#qfJRnO3ls_7L zuz`-NZ!Wq}UzicJHh3(@Q>-`ANn=1V!p-5rAcHB1fKg^4v;4xo#5)qe%Oju^*e)ED zL1JGhxKj65#H-ToGU+hq5ujOgX%@0zN2i5iie%h2V?ntwuBe}=pI}C z7uZVXaCK^YTx3V^ER_2Z+Hxktrob3(FJ#-e(b*Mqvg6e3Y=QX~mS{}T@SEr=WA1VY zZU3fakB>&>FmYx-And)oY!jV9@V!qf&vy%Z*ibk!YkECqi$I%q04TGcZrHQep^wF- z9^(XbGqb|yCp$ZyN<8O1!-}o{fk>-@7*M9r=k0#f(->VfyYqhhXRNyY_?;`CFIr*!dR*Il&Au(SoOy&zcoF9^r7I;D8;z4 zZ%Tak1lA_*nbTbeB%r`^5iXbZ4+kU_4&w3N?h0kzzo4Lk7T@nCsK^|rc(DT7JEtSZ zG5Z){ug+GzXcC;zhU|Lu4((6AiW!+|aud{teuXEN^s;=Pe&88~Fzn zt)hdWhPD~rK?t;L3(k(y^T2XuZBIlh*n@@k6?Y_f;Om<+GIH z_Z&ntfPVB%-BKNfq;SAw3JI++Ts$l_*dJ-O<49ZR7_fahOe$%fppsj~tZ$?=*&EK^ zAU2G1m9J^SABGf1!mg-(E>lZ3tywc;r`Ns0vcUNBPWYi+SwVW&N0wkql z7$%+50ASm6ttaHWWomk)A_H2x92bh&!3*)ny-G)n(`-BG7GX?iU{f)A>;h2PB7HOe z$kUch=?3pebmRu{=I+zSA`9*Y4$d(qLOciks7FBGXu1e;JbU+UaXZivYLV%rTC{Y2 zeC{pT1`pVMuC!oiHVb}`H`PfvU7}2NG8mMOJ9m7E=pZ*}+g6)>9#c_U9djdd(aZW` zxow~9Rk_H;B ziWj##?B`NNAV($TS6){uygSpqFE|JR2|;=1*yIw9(cOz>Tn*g2wu1bg$J@O_Ban6P zMUtg?g3GU2kI+Y{s9Y2m{ zmr3y=NDiGA+E&?X(vJ&04|v3&9@hv=wr9XFwHHZ{`PbKd(hj)B4XdV{%c@!dnJwTSVv&H8zc zqmup*hW_e;ZG+nV>kj<}sK)AEk?IpLhQN>MI60LSuz9LyJLLpX46F2+EIzPXj)e7= z4xI%=z;yQcKR|--yV1?+lR#=^Q7|fT4t+ZUe~Khb+$ON9*hICwtdc4F6GN5@#y#+{ zqJKXMzsZveRtPd?PxOloR=-7Q^SfQM6jGJ@68^&>Lj5DBzK7)l(6Y*%mn{1IK&zKy zD>zsTj0KqW<7Iwa{tj3!Yhg%vB8PFPdeuZn2=wtx`u(ysq-BfAiKY~O$eiyg;3T^- zUOzzNTnvi=bcPQUe>!HVQ*esIA9|;E`pKSne)6x|wq(gUHZ~x{blw_Xtn;6wb8r}t zeiSj5Thz;0W1;H& za~&gUbjhzQGN5A|pCV&d599=;1=MmuS0k14NhTzlH;Cta(OPdb0#=D$+1 z($;qdPEgX-d;Ex~!yGHd9^CzRh3topjnn$#tug6v;f9Zr%Me z2+y5)1=OAGxv)=tx;_>BAcps*g44ZCmx)n*(ZY;tBA))`5yC>50fcf1iG|Jv;Cc!K zFdih6#g(s?b=cy643cB1*o>eZ`)whFPr5t9MMQMh8z;@BU~aZ5PL@aT&u z_31{V*PXz(nW_|x!20`so%C%g0AM4j&r5{5@q-F$kzd^KGC7Rp15uQiVr|K2f8}L7 zVjT3DV+7`77=v(F1(bWgjqw_W7zof9UtGLbsi7q-A< z!C6q%>eeI$!EZXH*6&KuUKE7t7^j?_TIPTH=;Esixc$R(fMM1AwHTIe$uq2i4nn~yr*bFkF`8uw zufabt8Kd`1)kWg4Qz#AX#u$H7eM&5fAs2eHqXfXw8#U;{HwcEv!>Mz9@Q1?;Jjy+c z8c9b>fF&e%S+B_YNZX@fnTL0L8yzBGyOJFIZ!&qZne+vxchdzH)rN6QlAOMS!ir4x zANrY;OfLQ+!%qfIInC<>^$6)k_HV7)-iX0_UXyu|aFOLLFFE;SNui}cA01fqTwCxo z@JWx#SZIKFPS)mdY%&7MNuSI*whmac?jDtNHZ}mhdW8!zH>6{w99>$*fW>0kWzR=D zv8>M%*slJks2}pqNJeFXBJm9af z#%<@=SW2+PD6M=Bmhc9!hn{Eh-strv?)g>cW5t?kU%JMU%%?cB4Pl%UxXq%`jEb(& zUeVn#oZ00Whyg*U@UEi#FUiU4^b`kxxtHsEXtjA@2y|=MtUq636&A~&rBrK771BTP z6Tr<7Y&;jW+?2O`^e_H0f2}dUelrYb2!szjTks$4vG*0^XoK9hafIw&H@pU4o_cMD%@pprVT+x5_ z{r2YigF^M=F~RsOU~-T%`8ukRtCyDG@!f}XmQ(j-OYvx7kEvOB$uwjB#F?(4Yrk^$ zd0G((iYW*`12y}-8NKnA%zC~0TGLzxwHN*2CNC}!dOWIAIwLjSD|Q>dFx9r;JgbKM zZ*Pi7pc+ZLhR)A`nJ5n?aFagKf}5tB4$f|57%#*~{)8trVSnuUWTL`Vu9`a5Z0~r(CD10|q zB<`^skDdX$942>*BH}UeDwh`}K(aJis0zvY_G)q^tx_t~c09xu>?L!uCBccb9Z&!A zdc$nvA@5IhIUXYG;G8I;A>v_`Z7@9Fx!128j7_#nJ>`G-&SF03$!0qpxdA|Brv$iu z{oWb7x+ju-26#ElRQ1~4Mn$+<>?&rlRAl@ldwQ{y2djl<-u-=H+w-ENEU}KYz!Vv! z4wmD&c-1CSnDvFDf>-1_5USKtvK9XNVNB&1rV2=uayRsshmQ@LTu%Jd`}uJNf@cf;?0hf?sxlB!=c_p&60{aSceW8vyPw6y=->)}znpy;r# z%A2S|p!Ei<$-IL!jtSf3$uI?yXF!zhRZ}e{2?b^-Y3sTz%S@K)LvdnyBRq&W;px|B z-{MD?)OS>r<2X_AN}Z%{!HSA(e%<1KbY+txi9Oeh8jee&MeN`b4AHqM2VeakG#Zg9 ziRA>2Sh`I<&zB}vH~Y7efsZ5;_Pt00Epg{?Zk+ilN|nv@)5UcdKV%OT^vS1?>SuPy zHD=F+mhIUZQ*iY6)RBE=_CzWV!(XC0ss+{7`po-`O@beYH1NK*k2<&P+HXFRlgvGzWG)VQWBHc1sR(rOl&XD_^7iO^msk6)1DvZn2@ZB4M~heqGCL z(6OLV@{6~utES47n!rMVBNu|7z5ktnIzdt?=^4MHjlpd+srG$_*T>alx^HZC4o$VG z(e9>7x41>Q0;q%?ofjJEhamwtLjm0;_h8$Cmw)`|bD@A>t32Er0jvH4htvK7xCvu; zgqhO>S*?6=5r>8T_B&FcB{L8C<23Uu8O1xp+1TxQ5QnY4 z#oYk`MH{R4n&rikSXQ~a)|QV8Ni$?C?P~hWMaB!uH1X81tX#ure zfTiE6y2J! z1hYH_ph5E$d@TR9D%7)b=6}8qpRojO(c=Ngx#yPM`)`FZ-+r9e1j57YevP(XwtV{Z zV5B#|y^pV6yW(G7LN}@%34-;5G0Wq?#@h(v#q64UotP{r!jhdGGaGnngPkAGj8zES zGik~C8;X3mn#5!D*qniwl#pfh_8+BA)Rh+g2NXylgTE{UOWou4!?3_A1_fn^KX_oN z)od)6{b}!R!cPY+bw&0_tx8053kr>R$QR9D+477Z5kXx%A&@0tg%1qlYJzq{@cVy9zZTZEb;{4lY>R z4@~W@c4sagSd%x1DscXe0Qxrf@~SW5IA7QW=V$QZn{iKLRb!7F-ST}lo|5{{HjdVgd}VMYts+>H=|kHT3&1YW^XB|{~N z$+)lVnXMZJ4w)NbN7(1@3LM#Wxs~1RxIqh_INiq6Ka}2}HwUXEo9&v*mynuzxKn4Q zv3*d|Nok^ZVHq{&H`^ zSSH@{UBNe9SKQo6xAqYB8%!-dK)c{rywPoM?r<>lff5X2I_ITjCX{JMSX8%|>S1-- z`}}1KE!V?5yWoGsugbeJJb z*zaXXB~mFwf=~t$3|=N{`6>7`o-|=;Y873YZ_fTa#;2xNafoxDu=|}y_zd`_N2uyE ziptjzn9a#8doNE%kRxtrUMH}Xh3SH5BItI{rP(a?@u_N0?DI?u;2bLiFl#;m@j6It z-Jqo#pcNZ<;|3tXlfo&m3ab@*QZ2ni%9UC2t1QUD6ej086~up=fD+<+JKd0SLi(XT z^&CUC$34Oo?0;m9`*>CB&Xmvf?N{*CInptF-r>qUA_bVOnqE%N9r;lQ;@81wMa&*Dl18uZnX1qfdP1n-JY zkDIFFuNLT1&K}8-r47^qDJFW0AIIPq-b+oaRM?!e+oQmu2;2;tOSIQ3<2C1X$x|jD zsapR&B~{Q>`IkLiNQcp`N*My|Hg5lruQo=WLwO7CU(2UvAn%^zzx&tmI_fiuoQxvtSV)pW-fhfn)sMy!{{ z@Ag`V@!iI7StT92b>c%lxh{R(A75FUC>#$gG5Q6^w^G(`i4BDOg^=_ZPylL30(wrE z?vs>wTYC7fuO$nXqqGV^gNTjWi&^tCcYr{Nl=*&^y&jUbsb$oidY?3T)2o$F;XHof zUTxPc;Z{YT^A34acI-vba82HE+pegjfnZeo#TV;cV0MCPZv}>6dZ&@-z3fJ>^ptAj zG{L1(O7YayNV)p`ok4?74Jw@_2lt|%N9YZS?@5yp-=sN1dWSjXjMOa~npHA`dAfYx z-`m13Fh<|ydsK02tLw`P9%`*hT;UYqK)i64R26PjVr5goW!8?VTjLDLU!%!~dslb8 zJJiCGQ&J%8DaI5LeL>Ew8-G3{(g?)Pl^e%g8BZ>aV@!>EN2H#p5(!voGdW0A-D?I+ zGL+6_5$_3&#itJBY-Mh*u1wjtMT4J!Q5PaCEM0HC#OU zv*v;K57u5R-40ym{fbu9Gc5RtY&@B33>4U;50nrCS+i9jzvoa;v%{TEoi(~yx`rj= z4N(7nf@mYx%*Od+zJnd;fN%7Kh2Pae4_jKf~e! z3HDU$xiUJvl#M_Mr8EM!pO-7%Q=%dqgB^B4)!x(=6#ESRu_$RwBW%?H>PRfg^Wo0m zjiY{O4|&Ql9-xefqIk?eIi}x>otsFecdv(Alm+7~gcTdp%NyigcoioT^B=Tgv^ctI z2G~Ef6wdhrO~#TZe|6&5w+X-eqdgYVhKsGr^M0Kb!Bm<{C_eOC!Xm{+;Sdt+JM>yc zh7Mv{#UKNO-{zL*-aN|g_sVINub@|Z&S$6E@b;-js*#>0hM$f`De#FcA$3m`#Kqtf zs=!wR+~{fxAg7HqmmuR6+-tBg=oB>XDK975f{Z{Zeo&N<(?swip{o_Dn~Bg+nXp_2 zkuu1eXh$FI3@(a7q1qUrhjUIU29k!Z_f$n|1&JuU|ASqYlK&PCZ<@7Q`RmnFQs_qI zzx}p9p8l5TLtX#WRuIUK-E!;dTx%aIN%4E|m0`$@lGa_64X#w>r2NcvRbb&nCTsTc|eQ8;9JlwW*CNa>}7@8`pnxxDfFO zrTQS!K{@0@|AOKDbrJmm9Q4#R; zewm#QTw7D5O$3Mp^~D>tSSmJ$vE;YGj7_e|$cB7*0A=c?~d)&zl3(|mqQ z8H=3(kDg8L4U|KPU-WbUxnZ4eF2@Jy+i`hEcc0``N+f$=x%td9W{({neHDz-X0G;6 z`Y!tZTWKtxjzx|`YrR1~JJ&uX2ULQj4~tadS>-V~8M%~Pt(b;1YFU{h?x6=)jDJ+| z-J0B_e`Nr`yL$P@eUtU)dExfyWcyT2Q`@Z*5l`737bKE%Tm0rp23Xz|eiSs%bB`GX z-xstiG2ZCkzTWFe&ICXA7U5%+V^THOge{`+>u(kwC~H#P(EBt5ckN+y?O^r0MD_CZ*Xt+C z;hXj@9i&=)O2bYl@Z^Qh^9gkS9xeUiR(NZ1_4o_X=DTU$LoARHx`W+@-jQ8AlHqft z(Yz*A908p|0@ZTX7sIkIMk=Gu$%?W1cRw7-;nNn#aB#8T%r~*Ui)5;idC$sLvG<@! zLQeeJYL24rFNrchF@kmMUHEGzQlq!rIC!_~_q_^7e+)6y($me^)Ut9rY0#x5j%5+_ zaod5J>X#T&)$xg8j_@?Mu^$`uLnaqIg+GBvlH3%Tjb7zRz`sV z;2%k5z%s0paveAbl^Fi$J1?p(WM{Jho(1|*!062z`A2=lNv1ZgJe zp1F(|ThyF}cRrN<9(*XS`i8&Qps9p8DihZ}kk6pl9bCK)EY00He~(IWLlo}iDdg(B ziBq-{%x1H;ZHc+2@?j;m<8@pnN;(rwngI>lkDppJM=sJ*8=pn)dv0rWmcHlP2o`7k z(0wSdz8f*=cG3v^n%4JHtRkkG*Q7)((v+E+I&heIR2s8CvX(39%d2R6 z{~Gv42Dom5pU6GucA>A3S6pev;N76t;$$+%c9$@W8qOVt8WoQjfy2P&R1{2F)z5K0 zmU-?s!IsiP(&L1b2#VOcCnFyTw?|k~0|uFf6KZ?aiAtr*Kk1vr@UsdpujUmyz4Uv= zxS@XIR{MecUxl$^{L@cdq&Tt;s2>B*eQ7xm8qnN=6oZeSo6vVuhal>wtZbH#2#wu& z8SloLEuR<^@0F1_rdQ(`G?+af%d=)PtvisKB-Vggk);lbXM}&ttA3Ve@1DUftOm4C z4(Yx5j%Zd93TWT?zZoh}-h0el_kT#|I?h_5DCjKw_sI*uqUV!hmJ2vHsA_o|P3o3Z z0iIR%i-j?8`1!kwYZN8=N&>bCF+$%xXY*0?VJgKFOtf+SG%R51@+tm8(Sw%nBSi`Q zGgH#|%?P03c~^)n|GGo_vpyFp=py`9r>r@RJKm&OQanZUV)Y+QqM5D7%kj!^ zNvpc;_LRe6?R+JW1m=QAkwG5!IWz+Ak~b|q|pf( zW}ti#9Xr9ts#v2HRrbB(lTe(#ftA+GabKmOT=%;FUN)QYGi8Q5usDwgcPwzGZeEE? zxc`$D@URF8TPPb%T}uy}zBv{JeeTOjTM*ZEa%8}Mpt6Cc#Ndv~e_{?wpR-DUNm=ES z>X1#3&0_90m0=WLL_60@vEdq-dsEnVS~d8obkTp1sZJ% z1dIDwhek*x)DamC5t7jwEQ1z|Hl&2?jU?F36sNVc92Ca5`a+3T6!<{6eO?FE>(^|}8irbM5fT@UHvFP3&xVPbqK17X8dbVXc zOLpU*l^jR%tZ)!-Y7Xr-nvB(jRAe@J(T+X8h%GElx!rVrb5y6PN#*(-7-SDGAU(@WlvEM8)I-8KY7mVf#{6Hhi z{DCrBi?-yxb_>?2*xjpAI=m0;kef&<9aUtM_k65&c?!91yaqI#S6S;NMP|bYhqn>X z;!|I$xsvCig40)HPWv`;0LYn;5Rxv=RAoN9?$Qmz4IMlx8z&ydXO`IfrIOiqB<;sN zshQ5FmF*u|Svo>Ml`SCM`1P}8u?%gj+mn#b0x$Kq^F=Mf7*vu{(dpqV;m^egt#-Oc z8|sN&o!`{!*D(|>!X{*}jh23nxAi>?EXlZS0suUKHNI^RCGplA7hq1eJ-@~c`4rkPVRbMyjkeR{>A2RgvyjXgf)c6HhT>_Heev*xU#PP3d8jy#()CglLFA;4KgBr2oq|$aFY2bb!!EH{zvrH^J`Nb>{ zqU;J=4%90F>ky|^oldH=hN)wfAoz@4y_x|$O?F17=HI)FR0LQb7mJ8$HgX{A6P;r- z0ZWiw0j`!R5J-b+m*VYEW>&kFAg~vSTn_fV5@5uxPdLqFzlv_ZXTf8Qa8^~j0(+Vg z7Z5LpW(Q2wM|514_}i?qm4s~by0Jnf7uOAw?{u%0i+lD0tt1LftD-B71n$4aP%}^p zv2W8wW~EkJdlG_3qwYjWZPwhc9UMu?>VrrQ+J+hxt7!of#a4s1Qs5s4 z#qwF^Q5z%TWYPQsVDWTq21qWnlEYsiL7E@uxP)9ftCbsrS>7HU=q6Vs@D=aKI=wRI zF{bYhHHka%UBtd`_vcsTOTFeqDtw7r#$PL^hER$@=`~&`P3{ zedy>7`Fmis$7gB59EeaH7p5{{C7wj+n8xg-*z!M!*nV=4^Ow{zSfkcb*1YudHL4GX zAQJU~KJG`8%MLMgHsA4ag=X6B6=vCKsM&7g+iLxO{y_Gp&^vvvd+*xlGD{Wd!bSP| zC4yxNgXAvkn=4H&zm4k1zQ1gGkiz=##^#T27Ujl?rPSkJ_P^>SaF-MMcgO#SDG}1% z6UHL;AFM=JR&z`Lf9$8sORRqKtSM)c{~!CQGhXC1CGgPU10FifK4N!W0n;CQ$nkrR zS%^2RGn@(T zYdc{r9yC__4>j34f>hi0CO(qhNf9u%jQQs--32)t4r*;AIQSntW^io+#=?BUhbjX1 zz|L|XR8ZDe!H4sgx~C{$vBS;;{@wF13}m9W8#;BkSm3;sXe>63T#R&%?Eo)?pV|@n z4~uY|;zdzYVx$sJd3+xIex9d*3-~3+3{q=(3v?3O6vd{s98irTa8bum(05-?l(Roz z*Yv&)aHKV&#$@W3VAYlHFS`nFJM3{9<=Vui*HnExJw_j@s<~jj_>of7f zL5&I%G(+(0oMwSTd9_$56ZWlb1i`(t)CE=Yb6E3f>5|zHpMYyfB=KXcR^)8?eFmk* zC`mTEpLUeldw|1Xy#~~g5j;<I8>NJ?I`k)Uh#Zkt&2+cEN!Po+Fo`vlRqRl14uhkRnE<}z34-GbaFS6h z9Q=M@=koP7ACZ!Q&W;103wC|k>22)^6FyFODOVOg7recwyJxcfr%Ju{H}CC5#lbP_ znCnNhgpK9(t~8O~zU-a!=~!KF`<)?a*Mv*SW=Eh8qy|X_r})f*Y`=osZP-Rfpz09~ z`F)g&B-kN5CStO);s;ndovuQxOG~n9ut*PGKh8|%AMv=Giq!=~%Ce+%!ab#ROoNZ? zUXj%4^pl5w?^RtI8H9c+71Y=#{Q8KNoFM5rfgHtblIjJ%s$9HC-Sg{!cPi&hLO}u# zz9DJDGk|D3X`~L}%TcJDz%GniIc#ndVv4hiyeje*oBK|;yM@I*4PhazgxOM>2w%KE zRL!x^&*&c-C+W+vjUjlF`mNN~;^9KXo(azOh-bz^>mbA7`OqEVZ*C({F%BKDd3R$h zuxfn+K&oMNqNtwPtAkp%b$EmZ3=lC9j=g7s0yVeU*5u*V3)@KZ#2Nn)*--WvI&aCud5@Eqvu7Z%vFgAz zNg;GQbiOoLdn4mHY0ui6ck8=>t5U|Fp`bmKsnxcH z1d0|qX#P_&PL&yLB^PHH@|Jodg*S{Rw^6U%A!Fd~ToW_z3-5-&`T?iBdt{L(i9Hs) zSKXixw|grtOqE|FQJMpgf;?~?64uoB08y#-gOkxDVm4&sQs9r*QUuU7_aVt2rm?Xx z;z>EI{F{Z#Z;FiVkqjF}{|Aa7mam~K){uS)z3z1Q{73fc-8Z=bwM3_3sk?-KLSU2| zsOGt$&PL{MZeMdJ&0qUa=bkm@0W{r@ajF}hxTvO(ZXka;^X}eX+Q!Rji-Ltu%Lmsh z2fQcigs>GX`xU^Yp-|*m&D{UGu8Nhq4eQhzg(qT#ZKIA$(JK^rl=Yq#l1CboYVV7qE)acNu3@mFh z9P0Md1t}Sp4Kpwv1;Lx=yJv(Y(8%Xt6_n(}>#rTdZ(?IuqjO?b!s}9P$#Q_`UQW&y zZ=!j(-wGz2QnKDHzuqmam65{iaD%Rf^$Tnt18{P^`zTZ4(7bHF+5QT+ai3tu{xBHw6n*U`1 zfEsz~U+O$j@gX3i!r%H{*yGG0UNS(zK=F*U8?N*2c7BCp5L2vMm$F6z%HmV}0B;Pa zduu^eN)ctI3V*4=aF=|R`#d%WJ5FRJ=LDOYZM#x~n&-FTXz)`hVlu_bI-TRP4k{`H=v)v0p3}?uDV$sN#_Ib<1FxC~arNqDXL!3zA z1YhHT<(wd{nfvQ9kKe$pq6>rkW==iN)f=~5R%YX#;${SdzfQGIQ(>k>kv%EElb$9m z>rZ{TUGn^UsLWT#oa$%Ibda~~U=+a~0uJ@QN4Peeuk7L1G?~Y^w#?$yyz)=60KDSL z-!}LE8Ed88|Btbjezx_KRxe)JP2WrVK z%EJ14&n&m%W}j6i!Z^$mouQOdbe~b5HH|xHM&Wm2qnM^sx1_`)kMR0HerlVh)yX3= z%xP~!@$h-x9-Sw80Mvvx#<)Y95UiqgE#+$Bxx~q++h3ItE+f#fw`Vl?f<#hUSS3Y1 zXs5gAL&8irbfTXnPf#ix+MLD~&zk>srj7*f$6O z4eT6S&yg!oOHh_OBC38b?QTVVgoT)VH!LfE3cNbC+Q0Tgc|t6m!A^G6HDJ#H+z(Uv zC&Q0ZVHLD=C%7n2y8f8r`G{_ArDAW+o0kLrCQVP%xZ=`2J3)Iy901v?3&o~3I~VzZ zN*r+is*tS_Y7f{lL;EC;UQ!w-I~TyZ?HH8PhZ0@*s*j$a9;Hl5JS9RP{Iw{HeuqC@ z$o@ShvFu;<`s}P=`1n6Ur}uwRA^)7ONA~VX7O~>P|J6AciCdBXFO`#t8QXqq+NHuh zxCZdVeep53WAvbMd%K!|2nm4$d)A)`19LeYKS(Tpm9awcb2Wmz@9w`~b0?c;#PZR^wOUXa(hs zyBxKO)iV@dVdbzx%T>ZnB&%H5r98{LY-a%XrF}k%-I%{2OW%h7d-kITNGRS9JW3Rs z;oukjQ5xYf!#BY`g4oYU$3e3b>e1|?WSXD*%IMA(#x6jBLiBbkR{x)V@6Xjddh_Lc zq}oy3O`+H`gWX#WxN5;vuOEA?Vxs&=XqC1*w;wn9bHl3Hg+p-v(jPc%&9Qcttk1To z+*DEPZfG1vF_hUS9L-BO+g;7a*;dMq%$5Es{!47)P-RX#Rvn;EbPBl`UxJVls3(Y2 z8|+u(=ky`obW!*jtqscBovMWro7{~pWGnXUyWbdiN({1IdNArk$*9;tSm(wuZ#m$j zQo9)UMb`G&De4{5{g~%uPLZasRn0LtVA2$~y`Iby)oS(6#pZ!-4@I>3_>tkkVZ}9yA@MpS=UJE39wFCMP>c~V&K;DAlVevC_BP;+>aol4Wa!cZo-zJnmFoRl)S;{yg7% zE>GR8E}V8|hJkC944lA&yVF`KdCFJ@u3_z{_`cWI3Jh%k`< zcek4L04yTy$X()jkhb|QLH7si?{(KM-y~kRx7hRkEp-sVuIbL2a>g+I8DgZUn~|x_ zEf<0-jyk0`u?Bp`ItY5=9P7MQ*dTJ?tKB7A3RfYn!bXx!_9JMdV?Tp_z}YR(e8%ss z66vdzL&8RSwki2n-tWBMik^0rv@ZNkmNl6NWE(8ldp)-e=F8z{=X+e(aGLvpiM+VekRW*58hWl>pF^0*40ezo0wHsew9 zRMUWn=Zp%KeT=D&nGFe-Nj_i~ryznYq@Z;_4FHDA4s6En5CiBrzkKYHFn{T6IMiAh-*8?QfCL0_YQ->`-CVlHOn3Sl5ZUUxvXS9 zBwGV7_W4huizfe#`mQbxGm$6Z8LieEM=RnUg)&16WRYK0HVC||EnL4|0ri~t$cl#X zhq0c1Dzgie(-&gkS3k|&GG8e^#2BUsqe_U0u8g0(s4~;EQiZ@oHX$!4RRm0Nf#o<~ z{EVKT@1y#eJ)mD;995hlb?Rot+_ty&)yg;ru(RF^+vim3)3pZ#$8TUPcimZWr*uZ= zY~u&WOQOWj;V@}S*M>(Pe#bN&9QU#eiAbT#XTcxpf*XH}ccc5J`=a!q$s>6-uU#gz z&Zw34ar$fblzfceyq5@@n(Dde-MdFo3(|Y}Ru#EBpfe_h%tYJ`vsL#A0R#QD`q%ub>K%2?`8sCkI zEDp#u@St)fPfY2&&Y{#(Mw9V0xntLa-@hsEzV|(L>@vovm>jna1|EeYQXP^BH5ovZ zo%h((gZ30&QMkzVs3?m^`pVY#v9{JK@8-K>dy&%bv$GvWgwhl~J9x5r`^iUX-E*M4m#nha=DLT`2kc6*#*m8dla_Xb zh~Y3QHUr;XBiH)`AtgO#8|6CEQ;B|;(5_lQV}c`kM_bLKHKDElF#u6<^L4Z*TXa}( z-Dnv_GpCq*zxt=VzgGKuc1`hI#lfft*JAZ~IFXvFsO^v!%MnWzCE$A*aP1g$bvN|# zo)}Q8JWXteNLNWHNeC#9XzrUgn{i{$h>@WBmZedXr8hVlR3RC)3h(#6^U|lvG|{2J zp0RK9!G<#ZTVKIgD#DZ`tu2U9hSAbdvWBj~NS`JgrYRw4^Aw>!oi+w_{HYQrGgIi; zMTzhY5*jBW!nDuVepKZqX|H(t%knwMAovrzsGq$0G~}5<<8LxJtUI>T$2(u(7W*2B z&;op`YLT@WCD7H;nJ`zRHB1O7YV&?Wc(`XNHF2`i3b2PfszfHrr_(@ap6OEv{8cb` z(vmFa=x|GZp5lyhxL!vq5uF4YZ%Z&D?bCmi!1)FtvTu2c^lpgKQfR*;el?P*SRJ;`-hAs1!xJGe!DB8W7`y~d5OKu9ue8a)9y=;*d zh|)U=$^@?tU&>6w{lsVok19{aBTv~2WPLwhebfJBE!1}d$k>$o@9M6hOA8O+ch+A6 zd(DHfJSWg1eszQ3$!vFCCot;)8|fQy0C2}?f?56ZL9qD5QPJmm$1^s4DZo@&vf+*1 z9pR||J@`e zh)Jg`b41_s=;fz#Jk=&#K-9~rDNFHRWXb;&Qh7r2xbT2IujdzQH~oE`aGA4rTAewc z?Dyh&S`{LGhC}1MTZVP2sWv!_U3LoZgxh3e5e@lX`U}8_f{5J|jF5z?G%EP?-3lMz z*7BtwKu9jNRxJ{ljAEB2;@E!ni1suo$Oyea;1X0Zfkub{zb?G~8^&~tf6F}%IKA&e zV1YlJ3LNHdRu}1kE+cgXAl727#cshJE_Wlad<&jaLCIgox0ibDtr8a+v1cODZ(o#D zCZ@cZyG9(`#mp6gntcS{&#Ho1^LPwlHtnk2 zhz)5H(3wX5o>*MZNV*eEig{W3T6Safdc38*UL35RGR9J?k2>!CGin3c3pWpoSlDsu zUj*b5eev(mYWTHu+cd`!M|&SulO>x-7{TKfKhy8OO>nd72{NO=yUl-{?<+0x|V*cObq zS?OL zlqk7Jw$U86Jk7oJ{d}p}3$gC=%CNvRav~yj+bA6ymF9SN>=)5*euUs+Gw`q86OTM+ zWWezf0ASVmBT(XWJax1McZYR*`c++4iD5E^gsy#yH|3tq##36Y6fAQEE)?-Tmh}Az z2N2cgfPx%*rRzlJl{*cGaa!I?9n8zI_`||jC`h3;`!b;S9 zeUDB4e#oRYXd~p^!&z+paU+4JublsEX$IB3aye2uZSlx#&lY9uL%6bG{^0WDbQlXh`RHgcq{I=o;rGs1H0Hf3Pd%A_17fH_BdC{MwWA|l`9#Q4M{pJys%vQTfUBLj+ zn*X+irtK^5wPYd@^ZvHkLFD>>Xx7dok%J_k*H6$(Y|0;;Wh-Rm9-Ww+!PeP5PI(kd zI1JLbio+Uz(f2#BQ9VB7!HISR_j~IT@VI?Xr@TS7;B$?DCa|w`rz z2^S`2k0;vQC+UHkXj!_U>y>er?Jl%o$Zugw_ML19kof*O&p3Mz`@cq2$|lw7sP}F4 zUp1GGO14)Ad}Avuv&QvTsP`TJ=M6VZ6C^z@D|g;t(kqSr-^S7O{}`S+As`wx zR$H*2@KQbe+&3)%&F(D)+IZRsnlspV+>Lk5qf(vD(5t_UO{`>!(-r*a=8s6LRzE5E zGz9;JuM@wq2(iTo4W-Kg_EMY0gWFCYaFRvaa*!}2J?$E!96*hQs==TfLeX#@qN#o&kWCM8abN$(MP*6N`_ku1)vGs zziL-~`2xt>Ta~6F5`ZqE+UR3i(DzPh9{De3=(sAg1MuME`$-b-VpJN@%0R6kbBSa%7?1XU3$iUHC$Kbx|V2hS`ld0K`H`fW=7MSVb$rD8*B`=L4)FI zpDD!!+kNYaV~oz;v?j^2X=<}RW4HcPRoHpCBiA*ty`1uN=i%wU;Kdt7+-fV$7xRs! zPg}Y4p5)2EsW(r>+&cLS-%4{qWIAMDlbo3@pM-aMethiP9WxBMkNQFjx8&1*LmvA= zG`=Zno-l4t?=41Ki)Q$>vT@7u%(67`Sw51NQrX-OHz}k@?qbeqMSgIAto4=zbwH0M4Pkz_0D&G z8xORD?9Ii!4Ru}GbwSu3Ve)bz7FS(yQFeBel*lVeJ+X(t!W~U=88)Uw;QbLEn`ROkdDvHia>uS~N-ZCxl1zc3*|Hzj&U!P~`~$SV5y4fdW_SXq z$>C>(LS64u_qyp>=Z@hc(0Y7#;12;kms0H_=O;?D z$ClG?&&i^fK<^nA{PtGx1+egvsA-ZLE(5S1yV|t>Z?<7p112wkS6rFGQYtXe9l@V#4#dTjo;#HHgy{W|#oe&s3|l z{4DYv=x<(#;xHS0mi|x$rS?HfML+^#@TBn=C>1nAjOQi7qU&iVpHO1jTtJ%BuIbKa z{BT&HTpLYQA$LjFwnrlDw0-XU%~d_}2>Pq{zr)sSOiv%)A4uWbNjCt2W!Y0&OK^9= zz6puxF^qgZgCKX-fX-<%n4ZW*8r?UNEO6KwD|xA32WN-rv;Al+er)>DB2(FgLrS8D z!`m_|5*W1J7YzIm+s9I0`OB4rUvu#Mhb!q!hBZiH_GX^NYa61_n~^>w)s(<%S}e%u zXcnJ^#C7+Uf!3vN6@Ot}TgtWlDzy_wGW%}X=M^84YU{}&4=A1A^LS?>tPxq&Z?_4ifOqBo#wfJxW3er#;fZj$@0SM( z-#h#Wxm8{#Z667;D%CUCB!HeK*}g3M5xvZ#w>8G&Ee#epKoMLt0m`@c0;S~ACUW=V z2OA+}GQejg2SBX{kfIdqP#4QxXi%aLn8atq=jI!|-xyQVfh5aTzVLmuO5+cB*LDS4 zQkzI=hSSb<)lPV!#3r)NQBkKFDZhEHwlKNeGV%jJYn2i*y@J}XG&(B#s>#!|V;&^* z1hJb{fxiBEXkUW(vI56Y{#a-RSho}dMv4r~dU!xqZ7lO!SO}+?B0B0-ft~i3L|C>6(f1$uQsI#ify+rfg=k(nTWx-J>TMl-XhR(J`5%DgNSS#wfao*Q=d(`&g=#RE2Po-^qSdQ)63o<=jKi(B$6M-bsoh^rB&fKXgVOR3lh2l z%zJl#v;5=g4=3E2{B$eqGAuFcFShnNC?NUidJ$_zrAk3mjn+7a6SOW#O=4by%k0V(!$nZJiBORK-1Ji5(+d86z+FTHFr$3H_Qm$* zJ*BW}DmnWt$Tjw*h{Pu3dR#1?5?N#nF29Siczt9zDaB{(pU$UTt~C^4)d`tU3!+ie zFykKNGZ$zWeAnT}e^1EzM`q1w2r#1qy({7ifSK}!`n+oCFO|9{jhzjp0J%mqKXC@+ zWls04LK@r&scEgk=-)yi9A=8|j^Qx#1lYsy(6=4-t-K{*I5Z8|ot8kExJ^Oj0I#oE zpRPP{4$Wo@GEtu=?$t|xT@ctwn3QQCxPUuM4B+$$*)N(rli)MiKI(VR9A}qp`vHFr zZ1c@j06^~l?2;;hKA;b~*DH!#Vkny=kD^*rr*l`BxtZ+(0kxEP(<=pYl_X>N!p5*2 z*bOPjxj`oY08NwPr4$x%zz_vc?(_QyB5-`XR+#%)lUpmeV1%6NP8~1c?e9ta_28SM z+Jarw4=wbPfU%=vt{Qi!*%E%+JeS`|rAqeol%>V1Pl!t(7s4%yK^w7-i-#T1hug&q z5O-ve8%K4n0Jc(HVQh{4qwj9^PHYC}Yr!vv&@M1Rr6Z!W>bba&!K1BURBAfHxY~}{ zTtWUF|0jnBKX{YX>*S|8{Bs$P&|5+6W67!`Xxcjae3|XeQCfgsyHEW$e#dJw?DyOp zfd8!O{u@S?{%06@VkzPq1r?xm{bz10rYyQ`IeF-5i)fhaa7&VGI9Xj}QMP%2Eyq?& z>J^T+8{@n{p~NNNZ!(gB5>+_7pZy}JNSjhh60`{sZ7S983~g;DGJ5MDTt}EOFLU)_ zxv5saj;eQ{@N!`y3w*Wk(tS&@UcLf^74~BIEEeX8|GC~X=4Y1?-rWl+a-VrO<8=9I z{15>jFzu)}t}@k2C|_&Rku3n1@*)@!@FJs&rTb_@k`Tc>0QnBR8tt=X$3rD&qBt^9 za@lu*x1fBeh2xy~Usr)RPM3}vMuhLlSBd3np!Am_^7iK2lUv7-a%1kty~qy{Rx2O9 zwivM+)HVbuG2vl>r+MJ1Yoh)$SnyGsQ-v=w=?9F8JM%IOo6s5F>B24QM8J zPkv5nfFm^%BSm!0mx+d#v77l@ z3KftDlt7p7ahx^$KO<3-Xczlbk*8 z`qC9`2^(X3^pVDzIj%mrp_s1b1Y@fvc)67{sCs}j0Y0OM^jXA)TR`hoCXr(vnZZzq zTTku2AHHK`6f{!SCattc*}JBHTkI@ zel|L^rVxOp78;&h8TDumZ>B6715~8I)4y3c*M*#kwvPYcQ{4WKhy*GD@F_Ae|BFw7 z{hu}|K2k@h4`@(Ia^K8o~YLD;cyoVJMa= z0m|QCTo+}u1x42qy{s2?b;q7Pi{)cz)q}t&Sd&YF1O*>13fUzR@^IviH1|;)E6W`% z&6&$No;l=*jjWHGC)Z{J-80HXHxn@0w?ZO{f?m@;<3QaNMR@3z&(*J+z6*BXCR9__ zt0qR$yD7L69V?YV?xIucqAb8l@8FOFr1nBYDF^-FrEo%$?V`{jiJ)rrZ>EI^;{^cU z*T)P@$R5UrP8*)j`r<60k#^DDV&wm_Neoy+vbT;aHY3SH8Zf};u~jIJtNUQu^(^Qz zBo{btechT=E^m-(P(A@{w|nUx?hbJYbo?2IS&*1O8pg&Hk%|T~a1Xm@n9>nI?}|=i zU(qA_dz*E@m$yx{bJIlLH`b?Vyk^tif@_5nfS=UzcKK~>d_G06GIQ%{(^&o&<+e;{ zwo`J-)Ufkc`IOyw#?B^+7^te$yb^3hJbP~495Fs8FIrhbTx1t%#GMWuuQ#Z-BdS7b z0MS^Za+?>dVqig>)vHhongR<|D(Zy@)@K>k11x%@+7i1r0U6_)>f_=HQ>9li+6_;1 z<=7>hvTX_9gy2?N=*n#r$8N3(K3!fTZ!SRemUq{XrC5plel$X^FhUdTzkDW%inv5y)br z0vZTIk9&p8u$=yrZNt1d!Tbz+;b_)-t@!I?_MFvk9l)Fb#wr55$+f41jWNKSERgH> zv+;<)i!%1m&HE?IuoTVG`nO-%74LYcq8E6uqU&EkS&F~+gUD_egFnC6y!CA&c5?XG-(@<` zx=a*V+CV&xQ)T+Ea3c?t!)2B)-j0c;#LRuGih6+S9KdFl7y~s;@2GSJ_AbwJjRK-l zD^%+HYH(@!_@Kg-MbO?VBkdmWQk6M?@%@8A+f-GP>k9e}R)j+eUxb86nm|;Ad=RcOYTlfRQz27{Dk8-e zCB)`B60u$l>pBiu^VOYv-K+iU&iC2X2PZ5WrnK(le zEGhce+Nr4Y@l6p|g0_!rhM#Hnoc0#68Sqe9c}85BED%dpM@JusOtDg_rSoL{Uybt4e?E?X;zF$l1O{4nMw`{8ac_=(nA} z>@_?8_6Qa9i3f{wSs#mn!(cRyaj#M*QfCQZ#1W+Q^jDhfm`ljLs)hG*xpnKuZ8%4r3?Te4vKkVhGw7=jsTUX}2gIS2N(It~!Bo zD6yl1of$k(;^o>8*_jSp7*%YT0@7X4-)WE=(aRFV5BZYJiDA!kzxsNXVP5hEJ5qmF ztUhjUrNyWJ+R>s^(Jgg_fUb6#IvA~&l-}+zy9-RX>o+Y&x|h?WRM$+eJ4B~Uu>{|; z>vd*qPn@n5Jk{+Lz^i-HklY>-)zkhWXtQu?{x@^OY>wRSCS*44aRs(jA%8x#X&;{t zcun&3xWouRvB)nil6B-*+KKkqg?CGSvZ0{+V}g5LOK>_{fX2$7Z;Wa zqI^vOjMjve<1AIvUZD6Q>s}7aJK0yNHYpsb98QmtAA5KKmKx4COcL%~djRXefd9=xfx+J2U~2 z&_$d_aFG-26i_ah69T0-ywI<$jej)11Sf5-WGT0aSmf@S^tR*yR!fN?sJhWgQ=6xlXdd$6#CU|yWRyktnc;Dsg>pT| z$|&&8aOMUJq@e(UK(13b?_|3{?eR<54Qf6U&CXq1dmoJe=;H}gA!NSfebQ@K)AzSg z^X7$-md%7FxI-yDpCA4Fllkn`6>4{P-pKz_Zd!{QwR?!34L&gJ{?phjXazp|6<%0& zo)G~pkPK3JjoNx+H7WKMCBQ_IKF_@EIEwTzKC4@;-FdlkVDs@n758}sP3}8S#-AS8 z^V2LR3n}VeAW7{`t-z@m)eS%h8hiCiorFdG{qg=QTY_?XhfSJD+6RC?^ciA6oqIQv>QA$mPo4` zvsp92btRsLaIy=HzMpovG^pF%N?dPOkH6(rh4}-)5!T)|4ym9qsM=Zi*Qc^eu#TX) zqb|6Is^z?Ei-Q15rpFr7yB2?>=R4=H>}(@myj787wRargu27g64#Q^%s{!~6RHQlLswiV|r|;ddH-DNWO^ z^Zq+Pf;;rmCr#7b{}lGRaVy1#T05*`|0VFXWCN;XKRinc6w-4NT!4v*x@p2*^|(^B zQGgb`0Fr<8+{6xnb52c^5|pjNjAqnTx&#cCw`Z>0r{*_H!+o+sFwA=r3y9*dND3-U zWh^`H8%8{!J=PZcCNHT><@e_OsYaFDLmVe%}@^=*##7$>9sMv z2GMcB?pP0-$$JSffDGN#`QuaKt90*G@k6Z-;(GqjB@?eEGQY}M&k8P5hvv>EiWhqy z)(yhD5`9hZP&|4b+c`Rr1l?%aqNq!RY!T2Rxa09~!zEszvx>`j%PMz}XK_;MVYy2w z;})dUYJRWCVz7-|@x>Aj;P^|2qmk9eCXf&X*E!-}lNHoV&=Y&PdA^f`1OUZ9Nm)vDWa&&tGjPJbZffsj}u~XYz&ZTzum&qX>3O zw%Xs;FPWp=b>`>gI%%lnH{n6UC*rEr$4p+&fIQddGa+Wz7z?qFKf)h=?+C~P{_BM4 zahp+jV&U96>5dHD&qJcZqI;HugWJ};i|K9KVoyu;gcZ32j<+}L`MxM*;HD!uDEQN` z@IWu&P}TarAr7a`QMB%tqA{drhHa-6r`c=tZo0ESO05|wh+BTsfN=M>N-)m9QEeEO z!TBJHd}Y7#2p{xknhSfY9ri{^i1lZgX@o_&0`zbAY|HD`BP-b$SfL{)3tg|X>Uj7@ zp?;v#QRvl6c9ch`ZS!_w@cm+%)&griA zHPKipHH{5xSBR_mbc00o>22=}tPX)yHyhCHF*1L)>wzRi5g#a9S zd~ERaE@HCjyv*OG+?l(Pbx2Atl5uulUs&Cv__**c22z4R6 z*W#7UsaL_f+2h=~%@t;-_?JlV0Fp7Lw2Z3QZeFc-*UE(=C-Xc)mcJ@>&S4+RO;ZQI z_5=5kQB!;Cs&C4v-a)n?Hce!e`i9}~RUon~R{zPKL`~bfECFs*aC*#wVNDI$V0x7w ziizemLVtOlsB{iR%q_5BxD6wQN%b&o{a_#a)Cf&QRl9Nb=eP80Cyf{qc*`qMrlP@#Jnmn6J1?*+^|R~wX;c_GI4 zyh!AIX8FOikji1iB}at9dfIruN;StHhA`jy+MPq=w@r2ReD?&+c2862pGK{Xy9+0U zsith10wm^)f*0#Mzkkj8-!|0OM!5S=wtidB4e8B@Et-1)sRwVqA~3?d`S?yZ;ojNA zi!+Fb&yuS0enc0?d2^n!&1}2ln(SKtPepXP&pgZrtn;gT{ZqO7i_L=(s6kC%se-(h za`l%7ib?ks2Z-k(Lb{F=3I7jOZypb2A9n#~U$d`u6JsAl%1-t@yO}|@Y%#VFLiQzF zLdKSTHw$Ah_E0HCvW_)NA!V)XTfLY2exCPv-~W94V?N_|{jPJ)_nh<1g21g%v|hcv zyFwa_Ye>F(9XAP(0Xv=z_EqF3gG`j|Nq*D5rxV@GNO^Z@iQ}zN*71^tsoA=JFJP|t z%M%@R6^CpTL}{45^#wXClA;Ho;lTSn3!%GU&(4P2nQV@$mGG(YaKn9@g5nQ~UB2JY`iHT40T>;d_t_>{8j(X?9lJr=JCX5H>rE0j2`B<`Hx> zqgr^)ABL5v8AdIAsA!oqBEkvi9Laj5pr=e{0eLsmC-;c59I-X$c73MfSrFfLJwfhg z6>IXD0zSqGuh{Gu<59yAbB(lXm##%eIb0n$&%BT9HXuY7%NUbczX2XlC8?oI>QZtl5xBi&h~UJ2Cw(D!6K zraz3{4Ip{Xo8d{^iNth>R!>-Sk$u$MX4ysq-xb!BiCSwt0FujlV8!GElV4&`(t=CM zdpNk&6PyQO$!2cf@+oVr^MB8z4tkeLR8vjPP-fj6+bQGHZx_Em)fqC&t;#lwLw;T` zk;SSM6>Z%nFEFIU#$G$y(wSY`Eb*6EK~*lX&_} z2=<)KJX|`p_d63cB`+c9)78(u^0wsfq=nSJpv@2)T02aG%{y$WeF@qss1-ALqYWq;nO?*tnSy>73_f>Y&I+pXYa|p zeU^I^#{1^(kc?o($l&nq%Nop&f4IY>v~7w%iJbVXhIIx?t&TV_{x%A+J$dt^s*s-L zNFhJ`+qc{lhUz`hb*^%D@y!jzE!We6W6?u-N8AmL7(K1huo5H?4h|}&$V8=8t;MdW zbgw=!iaJ?fRMGOQ=Cl5-+zDYk^h*# z?~RauY{)=CrAuYp9~m3+T1o)G3xKXNrVPmP?(2Z~-8(K~Ry}#(Ws&hDp36&XLBOoe zHo4OJB0c!;_TYAVSI&jruKi{{T`ko&F_L@t%W!OB^V)$<`}erX#g#c+?|TMr(QWa(BF)s00(DakQoIzy zrXIzmeTDNCIQUM4>kl5O9;J1K5zD&`ev0jlR9#rNwQLEFzKyZJzuoBI7>$}${`sL9 zR%-ngn-$S#!QW+}hdC`?q>L#>-qsh}k1yC8!BuVM5mrHuA>9$VvPfO=<~}YniWiDc zk_;iM`}(&I24SNHd@Y*TlwZ^;^n*yPS;_lTF3McC#FeXs@co3(Ci3uU`IIZW^O~$8 zz%yoCsx;@ac>3$RBF#KPb7p$!dH1 zBJT>4x`lTi=PZJ(P@lX2pgG9$lV#5+YHzh&!QiiPOr#sRX02%^BHmdu!DdyRrjR?NL~J;)u9LVQH6$Jk^!a{?Q! zq@Y~h>8p>VgSIiuh#Gx+W+6i&NT4+#k)!_8(*#OR{)%2YKKp)$TGdBowEXrb58}@u zum?hOzU}iXrQ1(G8Jii{jN~pnv<(fuUOiP7oRc7ke`?2q>1b}50v3k9xM-??@_)Il zu>We_W)j$l=VRK@%e|qo8dZ~9#X=d?9w9I94agCmB%D~PIP?6_-{TRp9s`YpZQKy}fi=#(pS#^W zvbV|eAyaj^ZapV}uxdvb6Ycbahv!Z!LDr;ZB(kSv#|P_7p9!cIs{)(ldjuZ+^6^yM z?$$U6AY{g@+4lV7o+v?H!BHEg!c8@awYS47Ch4Q*s(-^=QSraa=GKxqF&k$~EPlaD^jTSCSPB;dcE(GtIoDVqySw6 zk+57<)l{4$`DuIjFlPOLP0HJkXCUW&{*H)nNvU`5kyMp9xXILi8D)P zK3m=S8#$)MY+JMDsyo-{G+N4*@BJFS(!s`mr1G=z!8$rYhtJnSRoA9zrOkxvE*(<~ zEg7Y)Q-ZPbRE!vO+raaBp0;Pkdx?p~@tL9Za|Ry_hh$@rH@Vw&g^BcT&UE*kI^w&9 zLAjr(`?Z#Sosn0`_SRPt(vx=@!a*iA)A~|xHrwN36aZ1mw5j^<; zbK}L=NJ|5eaP_Niw@41A)n}8i4Qa9L1TiM;WGa;W@VTlJbcPH((36sq(pYEvuw&!f zjR%(OOAFXK&He=&SM(N1{x&03)o4r4*&l({+Z-|W{L=6>JWAppKP*e{n}_dMEO6*SVwKwp; zUELp2j-3QTF#PHLJf?)i>7K{6%mGBZP9x)4hD*uo4oLkhbTmVXeh%IWP&g%0ugxDn zj_Qy8tjvdukAOYYUuqAtz9A*-){;^_La_;!HY6p}P`lDz8IWemV!zPk=ZJk!_>lSJ zsrksS3ZRhSnV2!>`}hL`t4xfoat6j?0NM?Nmj7v{xq@CvD7(^o@pyrAY~%3h2i92v z6DB=Z8`YHK(K5Z%C)OF6i8_soHCVTiArT&cfDV(F<~McVq<1!MuKbe}%g%;+$Jq!ze0gTV$gfy(1D`@hz(%n=tXJSott(=`8909OVAqDI7 zT}wp)>2&Z z*EczQB)rK~iT=hiL%}}iX*rQ5>T(>*6SJ=uYxf_BtmryLPQ7+sd4Fy`GR+nm!EzXV zpX^j7lBh&+*xxA=4^CzRC;3rBaDQA9XhYd(BfOn1B*qxBH$*$0`QlByR5aJRb--40 zVzKAk8hb3wY<&Rt?Yg(|YQlBoN{-FEW3*P@FbK)$$VU~oa#ItYJzIPlACp)plVsBj0#R2*IK@}>S1Du8M?F_3LLTsV*qYf%{;2;9{QLWD$%WGMV?R?;u}qXngwm6u8&$YQJ{WUi z++eL^W#lveD!Vmxpl2E8X`6)-bX;>ah-KGEd&(||a?q^3nGB6>epz9cK>g;+OY=F& zLC8^YXVxIc=;Iye6gj6{q}pNYon-mm;t3G8hdBenN6You)i-6idiRjfGLw0cboJ|< z=gq2rBK4daY^kPv$PUsbJ;vBr5S~(^k17Cx%UEcAC|+OjY!qtIrgx^l16vYu)!Nbv zA6*^`{#eGw7yG2YHyHd7y{3$OoiEM2EpnKpLR%eEa0F>xiNmw;3wqcT5KKl#x_N({ zy%A=bS7%tK@|_!gI%3oqAN;ehd;!s{*vk1}Y%!roN9*}syj_fZubdgjb{~!J7pCdR zJ#^rYPgzOOldp347-5nxT|1icwxs<$ehw1&*^#(}D-EZLdvCda`ONo>2~a*}Lm5@OQWU*@_-^F<749i!_-(k?H57@IIIM_6423LI!+7l>w za`NS6WMtmc#1g0LqN(avEf})1P&|hsq&<6t%!J5IxA;~MNm4!PZ@fyA$x;l^2KWY) zG80vxBoe`zYRnTp>i*zlR0_IwfYh&jn+y=e$F z8-?r2x~Sod8o%H{--!g8xc5=y{80&qh7UZygpxcJeQ7Rvov!1R8B(*)aPwmqXF@@_ zM-_7?PV~ij(qraB7w;c{AwjG}2;* z3SVA4T`z76nTnZry)RB}7FEzZ)ZFE1AJJK)_K4aW1j|372!K@CLR!hwpa3LM^$%bt z4dfd0&Op_!j;k#TThu%Il-UUdK=2y%&MztvRsKbG>`8WOIjcyKxaPC?68ml~Dk%b_ zMxuSA_kwBb>V5KR3X)9pu|T&%{U-Au0c`V}&L_P1BE8qavU%>4oY{!JtJrf3)VTFk zkgaF74-IL? zjZa`(LdJ~Ln<{_A_!V6d>PjL+T}vV(jQtHPCXRrRApgZKI8ud$=Y~2h8fnc^y0_6( zi`ov#NzeAlnZcn5pHff91qrq|$U0^c&A}m>VeXIdCL5un-yqC4H#Ya<8{^-Q@OV6w zN;{o0y(@cIIAFf?CPvA@@-TV@TU&0u&c3LUWCfu(X8%;fw<}(2HfK)YtoV-8J_fML z4)ctZB(l<%XoRf+4U`b5Il_BuDL6R}ko4ibZ)eD=hnTLw0edL>7JGOv$lBii;CnT+ z-S@HeoB$6!d4b1>Ny=y2*`@-Knm4%`tk0{vAo-}@d4_mXO}toOOS)mpRGmuoNOohc zUUsMCuokW$#%WDLIbAH=oT2MiO!}6>$Ve-%=-(Q9MQmHZ830;;wnlz=(9`tl!lx@l zUeq|ac`jl?%j)3!`+gs?TeQCt-oQDC#JwGk~Z5% zN25O0gJKrU=*lhnmFSLx2;zYC6wiaM_W`g`*@WpAg6XFI44j!>|7$5RJUNCqDmt|DRxzS7DaYKo7p{GCu2QD z%>jt7j|rpY)6vJgCpHxhRU16j)O%70f=JT?!cHECOmRJ__+V1Niove-EO&YG6uyw0Ey4e(JQtr zcr(xBQg4ZrBX~#OGGXpfk+I*o5K!0Fe|US0))VX^Ao>Pio!4}_SjK1xh^$}vda8gX zRRF!fbGTyElO>;Ay|$_TnxZwOIXnq^_pFaV+hM>Y1GA-c*KoToT22hc$84Skh#%ze z+b*=Th9i8O9)bKTUfOHnI+|CX;I5)u8{(5UKF(*NYX#@za;_*jX;})Ul2NV{zg`fI zJ!9uv!PRD|r%(c0s40(!m4auee{vp13!fxwK7GqOy5P#lRqDj<%qW@)o+bNcFYniS ziU6b=o##YsJRP*f2QVN=K9EWJK^=NCKwJ=| z+`?*y8J@=2c%c9}(}|B1+70JZv2606tXJ>sZWN}5zWvl4wlasczcyqkH1yPv`W@zgyov{OBnWw9)Pq-Us%m zYt@4Xpvk#-r+;4F_}ao!a*G%+F5`+Sz(KSRTbg+FHbc2ET6lt4i|r4=2OBG5T>C)A zIj)@?BAx`@mP~=R0vp(ADCy*Pf2^b*eKY@{Y=Uo*Sk%r_YnxP?_PZUo zIJ<;-`q+Rf;4&?jBMokzZk)&WKzVf1^1NER>>sWAIx0#z&h|@pR=- zHLt@r?;ENa=LDm%>&?I%h=KlJ=*+x1A@yP}v8uz#0J@_6xp@lWJtpHLd<4dH;LyaU32)dnp&A*KePL-hb~x0-_Li0)Q-OYN&!Hi;)wby%NdPOP$ZnQ%$N`PoRdENN zgAl6C(D}}Yr{o17*mEsaY{wb03zM@saVm%Cxu%1($d_mV!-*7iNGnGAN?)O`>~4OE zZcg7q0v--$K0ycsY_iG*Y&Jb3{qsb_7a=RIE{kM*Wjec>Q4Eo(0o`$s?TO z{}9198}CqYlBBOdFdoH=-{~)qVgZQtC#lNC*;gB)ewYcYBHM$HDm2zx<}U@`L68#- z^kx52al{QiY)8(;TShgL$W6>pw_4}9{I96utY>Y9IXd~4|IT$n}*)YYAI{xqj^OABRU9q<$ z#{hoOuz3#dS;oOuag}r{F2tX+)hP*5O|;vwgJD>>8|rP@izFmS!aYwpI~9TjI+N@U z0!tVi^r(~re~B=)r*P(Ymf5)tK*XTYuZ;>NQ-`exkh?12>YxCK)Q(WzP^phu2I_8F zH=y~0U_mAv1W2M?i&X|{vQd2$%?RTb+LV$Hczam-uMS?S4S0tBsPxF0s7nmO3q9ro zc?8K1ft#~)e*(vVeu>^kCo*HSYqEltrOVm#iz1vMB_91sU~AoB+JKyrQm7jnrqVs( zM(O(T&)Tluqbg@a0stO=BSc7{hOwz^35eCNW$qBLU66rYjC=o;IzvH}BtAwAAH)AY zR-_MawH8lBB>c>qEao@NiRkFzEHuLaV|Q{iLda$dllxF=p3L`IGrwg%OOb;_F!nHd zzN}I+a=#CLHowH{)j0*ivXUAz+kL3wZCi-owqU17qviJDEJI2!-<0b4_@NXx z578joF&A(hOshk)4>@1v%P?qY2o0Pp43ao8R#o>UCi+m`&s?C)R_#jRbb7$(__}*v zPrFp?0QrC;CPb>|APJ^&>=h?L3R7#7J05z`cCh9b>E%BPg8@@UN}s4nAIFnETnqJ6 ztDjp(5~MEby+FP76`X2+pN5m0>Y1KenDQUXI@@lTzoi z%n}f{=s0xSfsg%(lZU&5!>FExHu!QF{{DIIwA5m^`bm?*r#;&&qQV${UCD!bOXyPg zIzYY%w`Pu^dU52;rxHiGnFih|-c&06C!Lspbp-(e(j zqx@O0_ZDXLQQDbU#&`5W8hvo8`QEp#EoKksorSRP{6{s$z_jbCvy@Xrw1YxH`}Qjy zOZjxD3HQAE1b$9q7+!Jck!9WN;bKX({OnVp1J{9%ZB~CRv(L^GEVy1&tAYCrvoM1 z;Z{y_E5Ir}O((rr;%>y3fj6}yo`09quZ_h8b#?Fm(I+BJ@NYiSZ>p1C0ZPoHy&*Hy z*GQq!1ol<%ZN{;b{Vi;bo9--?ZcsU-)vUQdXUy=BPEP^qn-+T6x7~U$xA3>H*$S-<~cr;yEaR-f?txTIbU zJ|KkyN0lSmth6iWHhX0lOHzuz>hf0pxjP1~k>MRu}U5aJ8Ne?hN^B^0B_hctFyNO z8ludmtc1Hp?!2+YlSR55#m` zW#d#gBUDHk8#+$(xJ(v~OQZKsPNv8riq^dbtCf5jYc4)-QjaFudVJ+gFS*cUeb(~f zAAZu-NR_bEULfn;aV{)D`ivP3_3cvATusq(3q5>~jT;xc6_BdVM)Cg|9^U{og?K5B zs=owcr3bLx$FHMKY)`MI-F}lDd7?B+$Su-woQ1#D97PZ5)Mu4LQWi>Y1Z`-e#+9T& zdtvSKboT+nS!a_*gZ%Yo8V0mhbH@vx5{$ch$&=4lVxMear4EzD62gxpV=Lo0mI4 z_PHuQ(-lfh0y-!~J9xj_dKEl8(E>cH1Q};?`b0YfMdOmVH;GPt%4gzZ%207JSpN*E z1OI*0!)z370C9W8#32*)$x$?tdF@3h!wP=d8+LgLIbwvFv4SY!v6c^b>U#_%%ovlq z?0NnHu*S~YFnmlWq#21O-P9IOh-N~`ltY^DuWzk!1vt_YPJUW=!|^0f zHgc|7D>&5NhRV0XooLaCv}yup$EJoowimPZD1L|*Y4otS$#32kk7*o9tR{tmZ){=Z znLI#k4a#bF)_ZQ#kXtQRzvsg9SsGvn)Yi8ke2#+R6sN4Z6nm_OzpXcA*v5wVC29^6!NfzDxvLKvKuDdq{?|d-gKWJ3lWf zFy*LY_%F2Uw?JJDsx-Z z`$eG>aUfyt-;tDnPIOq|zZ#}L-<7cq7zzn`Dh9Ik?Yr%U+yJAs`$+v= zQUAmx!5so;C6+w>aMPF&Ej}z}ik~1$rET9PmkOEilf%BY~K-Wf{4G$w@=%%c! z_-tcT8c(Nr6IhaLsnVnyyR2{@5exZSpMd(fRjqUiW!lDF+FwMNJ-}^1K}Ij_W7aO% zesRp9Jm#0P!(sGXwfQ@7(S}u3W-KhD#Od{M6`(~xOj=6|e zrZCa>%;wt72{I`8k!f?ciXxkt#7)J9NW$PI$sh2Sy0+Lb1Kax98djtFxf;f)MW_q) z9mt>bm&>+F*t^l@=HGnLa&CCE56S_5X;xa%tqEv(Z*n9<+9kKH8*yP8wTOg0zcyihWA||U zI}!ExxrhS~pf9ksZw&m%4T-N63t%^}|Nj#Q&Tb85Yv9J1NQ)M-Hr|_PoW@Hr^NIcv zkGgodbHhHg=O`#miuA=juS`tF@ztVAJ}3kUeZ8JwP!bLFLkw$+YjT>)o0bQ?&v7P? zyoR^6c6i5x`I_&fJl{J%L%9g}#O4i#k;TAtkE6oJa=S==j0tsbsIAL=PlbOZzu%%| zW4I!{)yxu;g{YJuq4B<$tMrAeBAZG)FTT-aqb45=QPhB7%r+I`9GVDSc`Y7d0@*jA zM^${l!3EbCEOQ62qml1kTpru95iViCtBa5X%43SP+mV5|qU}Vxe%Z;gLbf}HUH>tJ z7wRc{R#6Y`-Nw;%1xWHv%x-xCFKy14Zrx^(|7z!bB6@#c8#i8iQUv75giguwQz8IE z*c97e?Pd^JbAyu*I|@iR=(up|s|>CSrdhN&r|3B`JV@E43@s3NT<}Vc;~}l$HICSn z6PKP(GC9x~zgh@MaZCo%l~eSuy2x*FS=1wjEOs9xkjk>5@NmRbcoREv!$x=w>@(3R z!9--p=Be23xW$7HzLI1ItBi9QB{vRujAlPEgp-|`b2PmCi5JP3}})#q;TXBRK7@O-8_`+IYn z$5-e7SVzl&g)UA7So&}f=Ov){VJfcceArLY*B<8Ru-#mYlzIXGBd%r*Aj=BU3EmH& zQ6^GgxXXk+#B#(uphlw~_MA`qg*Zm-KJ0FAo3Rt6&Isqt>E`W={s(xlGf=>P4JbH} zaj66Praai41DTxzGjB`k>O+g)3wpO^ssGE;f~Nr5#d z6Xta7gc(v6e%5xwoao2kO73wh@=wl@f)sTrxpu2aD_0;p;!4++V8`C60oYYS=-T0V zlBMda2}|TPQ~&BuSn7M%V*Kp7I`!L+*Zduy$;3&$UI;26F!3H;%0|rF3^)>huiRK0OD~m~vGi&GC)?j_`1&T4Bir6A!x7jd ze2=2ea46KES%w@o6*ALmFVMEwy{Vv=(|vyYac9d0wKw-;QExfKK@m6l&fXeu!>KNpP`{2Mbz9Y(ER()Qfm-e9K)gqUi9+L~?Ir zL{tu}$??ya@`BpaBinOXr3-VM^Sklb(inal2p~$r5`lOhRKEw}u|*zeg~=wP{$a>H zYfe7~Hqor%ObnH$1q=;ZW=@o{Ck8KPBaMGMWBAT@7FIe#Lzw*uH`?Ju2E z#5B&zpf#{jeZlb=TWhGdS8xw*|7h;Wn&b+)W>Tkvfu?U9}3jA-%?lX;ad>a zi@ZSp3jEpP3=I?+g(R_`3z(yXyM*f2bBGt`rLt&L+U}E;ANY6oP z=fj*+RYn47B7cMSJc*^AN)Zs zT^D%7G9 zB<3j17FthI2@=bVm-+lw+~U0w6{&bZt6K+2A?**YefWM!cMeOLPwmxypiAw7qS&3; zNqvDyOAbCd$4-_zEI2xZ1t`Lr_Di{JryTqRf=V6!TQ z7{-xr<&L2%s34U{j*3QVdTZcZG7fH-Y-_{3_JU%)d|HZHsOhzPk+7@r58-kqhS+kN zSfaEU%+KyR-bi4c`6HKNA2O4!g{lQm&2-bR1S$T9A>#o@*>3mJ*)u}h`*VJNI~xE zhg3O1L~N6j z03DG=fS7bSF3#+5reekfd^Oe>mcusKLFOp4upoQ!LK6P_2K}@(YBp3~1*m_4vX zoP+OJkvZzdOp20yQ&yd`kkB6nKL(Xt6B9&D?v}X|wT9uq%jLk7&%VqZNH}D{@J;kq z-GIzqadXHj=ul}eMF*nkb7rW%)fmIM|G8FbmPJDMxkuoK`!@4Y!+I3^vbFn|Z{}41 zSa1fMz6em8l%mD}GUQoBj+maE`XW{waQvFi?h$IEmFAc5OAW!?Wfh3S_}=8P3L9m2<4TCtOR z^Yz5p#Sjz5rND$uHrHKRBJA8OiN$!Elpr`bFF_RUDLBL+h)pOQB1uqFf`_+i7Tc5Bwax9ngS z?f_hRrMJk1jd59SPL@3L&tu#f0iw4;y~SQmBt?x+s1akTePr-2(hNTxn$J6EkQ=C58$(b zfcrX@_R?}(A*v_c%2gLQ=?zq?V5+&m+~;2fNV4wZAK%Z0?GHZbanu3RwruP7IVt1Q zqLpk_z6NOL`&I5$(XUPQvf1j^dXQ=AV%7UiGH93LPME^iz)+qUd?dXk%~4bk|c*9=yC+&`zbsaINn^gZyy1KzIXnMpt$b zDsNl|yV^wE6@G%zy~5)Pbf{_E-hfC&?cGm}^L1~T2!oFv>Ma~1Zb;lu?bV|gi5vk* z>yqPztY-};xk{o{5jsv+NI|xe7k6I&Y`q~ARYQ1}6XaAJR;HKHYii!=OmCrf50Hpw zffrJx`}Hh#vDq?jvR0%G=5f{|;PDnLrpXRfT^D-2CCSEAD?#R07G_03hD_Vsv-47A zQm3%*Pmq+MRC&wn%~D55yKrfIqw;K;R{TSXHKdiS%K{1IIc;0F*K0w}zv_Y`>X$N5 z#dLq87h}@)VYdO7u_2^CNvJdlayK3vP)|Qd8w(saPs06Ig1mpRu@Xz>R1JD9pDi=3;2fkH;Y{VH2}1Y1JUa)U(#ve^jY#gFn!UGvEAmB z%Q*Kve@lAW&o2EPx3c#T&MEk}CC-8#j!0__x2k5hZqiflU9o|Nlks9;;`ecPHI&nt zy)UA!M${s^v~XOUX!$9db^=#}WUH3H>D;k?#qU8!!>B_MpfddwrA|Z9FaDU*4v=*P zZ}T0f6Q-S^El&EFYLy%}zBPXv%G7-@4wYj7lca(9F7DKwyV3K*G3IyG;$xRhrXNo4 zs**>pBLo0&+tNbqO-lTs_U(UMVhsPe#1O`e3v4PNH(h!0l^#z6 zq3tfC?t!}ctoXz57`?wcTo#aIzrWwv8#mInt{e3uQ>LO4=9&q)(8!3Iivh=Mm0;LVG6`E=XXl@txAn-q(wC^5|R*$ zxK7!a`~KMe)L{CTWhQFMynaKPy&XB9DmMU3vGuKS_-NLrTZfaE^puM!PjJw%^`Ru$ zho2eSViKhDeAtpnbS+(nMbjM8qXpvwjJ;%B#lIRW&r$;CYV^uvqBGkpA&(-fHkQ4` z9Jhh831vc=HLzSvnzd&YmAayVt1zj6(T0)|M<-ab_J?DrcjXJel$}X8s=3aU z`}ob+$L{Akt!!?Ht^m)$TQm|uL%kRuaaH&E*K^<8)YMVvl;OoEGsc-i^JVfJu?Vx9 zJT-WVE?ilbWa%O6IxQZ~tKpImT_7}Lc~ai&%$hgGufy*(iHha>q&Z%b^k-dCg7m&m zn)T4~pG-HYhAon%wI_Z#5C1qQVoa;s@=P`fP&ORYeehQjC}SvV^wpXxIxbDYb!C*|n~(wTgi4^M)Imh+ z+I|KdA^%fy<=6ef&)e9iYQ_u1P^)N@VnX37$q$O%t9f>Y!s{dB8J+5uJDOvE8**ON zKQrKFoG}_OS!m&q{34kvV(a4S78FBW`aQdnVx+CUgTS;YO$PdHy zdDru(RD0pO@iBWZUG!q7)q+&4V+|4l>G&n}Av71w1@1ScUabpB2zeG)hlLM*(Cnto zf)GxlZy97M=mHKcw8Lj32CBP`BJbDgt2gr~KP9;)z=;y8$kn%eJx8ArTg0KlRB!8` z8L$Ay&)<7tM%;wt2uC^vK$`>PYDOl&XDxemM{v3%rI8V2s+{pS zs=kP8I$TsN-sAU})10eh&Szo9!pO3F{{KkS2%<*evHK&7)DIem16L&6Fq;RHN#)zx z9bElEJ!nK77_7k>A2TFhek(LZHX{__VA`Vp8TlQyQsMTk}VynsgBGz3@)md0?bBX zw+kHDs$7GwlgNSdtzoR|kvA+S;j_d1?F71LTCuad4rg=9_3Rc&!gd>!a>P>CQYgrs zz-sSydS?Yy^3^&dP3y3@hcQ$ot?kivP=-45qIAe|Xb1 z4RuI!Bg0GZ%!ja-hpx~0oR)Y#B(ttpK>!kEZxqFCfhfsvv$wYQ*w89?{~Po&!AUqs z{Jn%AoBC1D4JaI1#u>l$9zU5zcx^KHI`Q>*EB(_%D=iz!Pl%itb59?BnHMtkRfVp+ zzZ1aZqq`?kQ}Vwl|4E9wt6ndY0$F}qJP*54ZGjnYdU7r3mfKr#@jr4Y5FF?vB&w+c zyAM4YsY$jw_+5ACR~T)lgX*!Wx9ETX3xTsGXtb-}fTR-w*W7WG+I?V2HuUE!z?PNd zh0%v80T7N4-)RJ&P*mFk3mCCjKx=Ari5r9v=WSnCR-`}sCy8@%vGQ_uyg+>~E{oXT z+BqfjsKb#pJm`$;a!eVaRA{)_s}O9m)W-97BYZlnNBTQ+k_=?Ln%Ar;%N`hh^J1^M zsnpp1CSK6BWpKQC8Tp zjk*h(kbS#n!*#%J6Z=2Ti%c7;OtV*=uRs}&t<}Hz0E3(OZNQ07`y16c)b}NaJ?>q9+Ht3|&># zz$Q$Sp01Tpz*3g^w51K`M*n%5%OI;+U*HvT>v7AaTNTU6G?VN^wAMN-MKNtB6M(06Wl5etHW$BV@cM$Gm*%~ zzr#gI5S*7CP>1o5xkzP=H^w?BDf;=qquoE=2g==oXVc`*DW`+a9(o4r=0rc_hmCz7 z(eKp6q?ly~p!qD>U2+ueYLXP^2szu&kg>>ZCS)$9-d^kGz(+@J0hJ|jo@>m5v@cUG zQEh%UO1u8lpw6zP3(~$p+TeBqwGDtYjeO=^07TI|71Ec&HZ&w&+zKa0y$*eSI_es8 znq%NRL#9f1*)-QGx`vl%u#R|Bv2Bq)6qqH?SAO#nzS(rHwTAS&uK!IpCd{MbWIf$GWu~8Iku!3%Ns8MAhJgM?1Ch+5SHVKlj zxxsfpg#gI=tGv6EQ#tiN1qXi~|HG-C4?A3Ddj}RHJEZ!iQ!9hY1D8teuz{j$QYqnM64bXT2&6& zv1ITAwIl5~zR=$91xgMw(C&F9bSbzHOYV>U_?$lU2IGVS<&o6nr!88NNHMz3xv`z* z1||=~TIW2S<%JNkFAM&)8_+L2j_2NGys-05lLrw|4Nf4NA&Bvy;kf34C z#*GcCP0K3VsxyTXtDmn*P97gqSzrJ&NIGzo4Kmt9A#MW}{8CQ(5R!N#P%`zM=>@H2 zhcg|}juCGPSlbR)-uWDS_5Cga7$UofkFTyFwH)$zgKdAvtRh>8?OUFgeh~`}iQR|3 zmp_4@{jS=Qf^BY30!JDP1mS>B8asNrX0Vv64~5nDn^?~FzwD9^_f73}PzM%b!0t{L z7y|8?=$o$>PIfRaEIOG@`pl0y$DNJ~f~9Ro8kbV>;-2uO!Y zHw+*x&CJjx-O?a2q=-@~AxNs!eeiw%_udb8t^19|Hx@I`bI#d&|8|FgIeV=h(i$J6vUP+H(kGDj`AA^Vw1aa!|Uo-67!Ddwq=Q~gi42{J49d- zPSW#qy2{M-8A~!Its5*pA?Cq@Fbr!1d?R8!c-5bkX)_8}D_Kj~0bP__)Ftw3GC> zGL53zg0IFblFn*wV+{UUNz_M{T3qz9wORyJ+^i7y09oob-nU_LZ%~gvXRdM!%~vf1 z4#bQ64~ZzuMgwl_z}ccTK9Xed*5~~jpWKaB^>{}4v~@8xrVEcf`ZGHwi?%>nS^Th4 z8eELp7pgC3KelMCb95eF-w*{2W^!x(=~4417XMq162)PPM~=o-qvT_M^^!wEFQXw9Nj|lN!Ey zVrGo)?fx??nC})6gyt!5ivnmj9PoFJ$RWZSgR+se*M2$p&R1noh*1)e_y8)jjU|t~ zF}i&=vCkt3pLGa!R!eY<`#~fgSmbh^IO(fYnE^M~`C(W!aR6YQxFT>b_7NUkPUp^T7^ph zv>!w26JKb(Xwy!)MRLou2~k84Z0xBM6aS!Ma;3BOJNE%b0`B_E2or5BIMCibk+W)c z7dU5?`hRua`rNsAUvd?-E-RJO>1b0z)86P_62Mpf6i}Xe0SIPLOm}R40&mJK(_|FC zZJZ~}@`2ipI6N$f;4Elk3iL4pU^V36-W_EVv5%voPLqe-GDNR$9dU+-^PB!i2CQ+w zQ+5DPJlb%xUe4OBt$t+J+9M04Y1Fg*2Vvf>kA2m|Aa+6|ZacF?(NRl(C|e_2Ig+%#UEI1JYJU zAInW2SQ#S-V?sfYqpWVltv9x>2h>vNxY11ExbC?4XHQQ9lQBATCnEy$DHyixH!u0^ zkV9^uTCLZT)aWILHj{poukhK#<-yADYaOF+>SCpJ+u~o3@SBV z7q=0t0WXIsHH7PIDs2iz!^il>`nqE7H^mteX*@PE4VG8%O;F&YyDz7ja5V@@etnlRW)g z2B$rv_YUMS9Q(BR?-cv3rKL3m`4dS}8Q)796;0W~HmMQma_*jRymh+RlSQ0DI5jL`kn$(D3{5>J zL*G6#YHAgsO9%h+EAKMy^g|F5@P3qsufH++g+18h(ec;A=v*?3!kah{rx)-@T?|xK(qIq%?7ZR!N*-wD0}kGySWUKFc%f| zD%d-MG@uN+FQ<%MdWxVW?ohp{R^ zj2jTv{Y6T=QtivUuu5(dFTL~Hr&M1iPg)FD$7b@Iw_!yeUd#@V}zVuiAcpI~(!Wq)UEVWodoS3qQ-oa&qFg`rmnnH~dVQY~B*4 z7SOY_^|()ZKP%9=R>k_Pz9BESTMj~c{_ZN|xVK%4ZJS)rG$;qJ7j4KYbkJWzt;a_O zMGd$G)ddjLGDNBv@Z>V!4jPu9FswVgNV?a+AA2bh?tmQBWK0gyy~Pn_9&K2^%~Rjg zn(@0*L;^7nHIC7PFg}|rzl%sg`U$F*qCV(WYpP)Y^p;Eo3JkxS1`jm{p-bPI@@RMK z%Y!aHOa{qZ;Q56rgB72splr2#&~Wy z?UGT|Jv(ywyA7Xn-jfoE6RYVGcsbc| zoFQ=h_b4mbPK#+lsu)6zBHA5gHjoTUOMASn(wgEAV^mfvY0MwaBH2uPk{jHZC4t4V zK9dc}J9#MD1`rCW0 zOD^&f&gVrQ=Kz5a=CFK|pIqP~+SRC8;+lUdd~}QB*^jeqt_@9!j_*>q8WL7Ia4V&{ zLZrXm3GsYo+5;x)4&mY;AnIN7;y^>Z|zxN9M%{o0n9y+<@XY(E`1R1&`*e>faN zZ=%!yH7MXVBY-we%K=ORQXWBzyBubfCnDr&0|P8dVSb|3l(ck1G#?gB=Dg2KHQ<)o zbL;?3rmJEKzos+Wfx3{! z+=4?j0C$@r z7e@*xq{=E=oCtWjPV0B)t%40WMB;(=v|6%R%P5DE+N#HnbF}IHrV@e(LTH$3n${j4 zKP&Nv<-%v1Pmfd$Jw|D{WaQT? z9qGuC(hEi;zo%i-Tp8y^gZ&(no5CAOg+60l{ua}^zl+|f<&NbwV7wd(?U0HtfWUTJv8100Qa_z}rS zOGo=`MWwfX@p2ipWN9k|M-iq-KAvHS=8ZXHgc2FXT53zhWB4(~$kZeWGt~Z2$)G^5 zu2rJhRYMs}#>&Bi)~awVIx+7g@`eJc0v3eMsVAu2rs+qS&+9XRMC=KO5PLvqDb}>a zX4G6^FDPIMa|?YI$fn&b2KZQ&0r>ERq(S9cBwjN_mq?qZYQHD{kOKiq56w$M1FwJn z*d(+;B}vdMn!{wEgZuj*Kn0s&bA{;>xDAh`DlGv+s^$=El&8h@@;kIM@a-xe3Z^my z!EX?74nESIK_;Y=iGksZ|NBq%E~{oK2(W z`epK}Te{OzGFg62I9D@~j_4!}CIaVaGr+ST3sb^J7xp4$RY=nwwUm{%+puq%?fRD9 z3FCzjU|H`}SkjBb6sFbJ-zbzE^Cq98M@_IHonnX+FBVv*X!>sIzZEJg;cUU^N65`g6J1<5yMr z(H4y)EvQ#KUuO7CfItAj5G-jn?ry=W+-`dF?%Oh?O;%n1Nc(_$pEb;d3t<`9M!JZK z`_6hCX}`Y;7U9G%)g1@s4L}O(`YZB`Do)fD{x=~Cjs;8!9#~BI_tXZh@)(vJh^E>G zq%nUs0XPjbRBU_%7lK$&-KpO}?m+CnaIZgw6VX%I3#O_rQw~#3PRD}#P7C!|1_rmkz3z@^hHA>45e1L4SKSrr(Cwia;oFsoppuZa2b z#;}rR7&eMiRO?p42X}lLJGIb2C}c#q^V`>zPm_Y+@(MarZRTE-;s!7SJ*aB7VQoq- zhvzz4pNenHe&YWlTK%~Ff1CZ7fgsb0Lqti)$#Uw0TKSzkm4axko}0TWEIo=@kQs;$ znh77jo57=Kff&A0m)GRwnj99L7vmQ6|2|NuTr#K~c>L0-h!K=0{qVfyi*@_q+wX6? zeXo~YaAfpyZme3D)=z^|a-I)=pm&d37DX(O;J++!EWpM2Z4Fqllxel9exl8WMMV;g zW-&vZ5e_BYp&Ia!b}OerM-3&YrOfej%IS}t#&yuq24*RFGim_Zh7!{j_Zl@UMrl=| z>Tm)JALIRAEdM5t<8kCMNrk=FkuP)+l2I2TUkNN-xOuN{IFtK>-dISXJ~4}G)aivo z@d2+VjPPAd#g-#6W2;r-yhk4m6&4b?8qa>5;=$kvW8^giuCw#{P6jousptL+`I?C; zK+ml8FyNQjOR+e~%IhVV8Meub{ENqUo=fn!FY(#-ls+M5J^Vk48tm~W z>}!)RsMJFojO$1!!WZyjo&iQf_HH?w*ffj*o}{W^W6`wDo2Il{Ylf*2&OweLxn#_o zg674h^D77MZ0i`^ofm{!t!di%#WX1lgnQ14Frr>aG|fkb1tnwX7WB(W%)PC2mIWb@QDLCT|XT z8%m8XL+fCMS}U2D?_7~mxWE)kvpe}o{8`d-JH!TRVXyRlzhJzrln6vlI4|$=TYiws zPw78n*M(DeUPirIZAS*4_a&y*gAqVpO>L`jxRZX0gdXG(1<)bDCM|h}b!$z1p`5QA zkbco)LcQXecmxEH6qqp~{wXyuy%@W+`q>P?p|BFTqXvu${aJ4Nr8yS$^TF9gs~PsP z_*?a>uO-Vr9#EtGJ9f|8gRXnFQgsJua>+wpZVjAd9YlfaYbyiQ*6qMuy;Jh8iHkd4*{0^gj`R?b zPIeYHe)(5*r*GvMDdmxlY8{HU615N94GPy1MPmRy<2n~Wr&*Cog(Gvxig z6t#lewK2sY4fsP%p$O}NN5Awf1T*2UJ8}6Ahs6P_gEQ$1rlMz`-6|hj0iDpmeU>=| zt9f%USP|E$R9g_2BM2ZbQtsw2k%oXsk#sa=TFOX=3!vS*=`FV^$wS&P-&E1})s6&z z|4=N`eVQcR{k8(sRgvuSxeF@*e8xzqo&fGSFx)Pk6t!kNXDN#YtTd+a>%)s)yL~jLjOrrE_C0gnSq?LBz$_xXeJh zDsDe9Lpa?8qe5OVoL*Zl$$pAaq*VsV4E~3Mx8yItgdSsz$Kv;+LL-6LRJlUz{0&}PEZhK4H)*I`oT}XbU`TAYVlNSUD2)} z!b{C{?Ora1~xVzz~FRZZuc9DOMA%WdR&!PsxV2d9~o_4kd#UAiU}^Wm*P1O?p5W z_-l^}7Tx~(5pz~n^l-}~!9<{S{tRgrLx_y15CE;n*;-DSz-cOUp5@9|97bO z<2ab$Gyr&bw4@nPsCncy4ei;#ucC~&N%c8zv0d<|ma=^kQFD7z;~&Eo(Lop6O$6vy zPIen8|0l}L#UoDMZjeZcp`unRidf?k$5w0d9*NYL0*$I!vt`EzIyK;~fcfVMLIHn* zYh5az)y{k+5=Tbe3YzsGygk`!wYM+Hau3uS8;bW^P z+v>8eUI{8_2UCB@^lz&nIi7{vzj#Ij*!M_t#LM;-q`1*Je^@+`LU_msa;b9tIxSzo zxOeYaGq>QqKxmMQ%+p+h#;KUfkN7}la=Z626#3+qAQU-^qSurCdBg72fHiMfiS(l+ z|0!P9%{aTZmA~uH4!smYO|g3pA)=KruyC7tS6Gu~V+=d4?(c9N9?oq#QP^5r36ISj zZEcKo0w=t21#rmpq=!c8tN#GV5_S0PEw;qvyYMIUx)Erv&%@10=wm6=Z`l&k|DwVG zn5ii4jC$R4ylEkpOF{2kEw+2;=F^HENl@q>)rD$P;Nz!Tx9Y9g=XLWIP z$Rz`ZA0jE`;1m$oG+ex>_=%c24ys$@);aM0QAH2!zmxjtIo+9pCGqchr2(gKZP7X~ zs2lVvE$x5?tQjZpVLMTlZ*o`e;sT4AdGT&=;;UP^T|uOZDtwjfak$g?e*`{8eapTa zd$@f^1sn5I1h_n_NStUY8{>jmH4AfTK9=FfN;m?H2fQBFDwXRcRCdVgDb2unU_T)} z{2Yj0^)NY1i#Tf7k8OE%{y@PkcIVak@m{V139c@cCE$eg2}h1B=+;KpvA~}w_o>rJ zI||+>?GlQt_{s!@b<2>r8J(&_B9Sx&`^e3MpvG=^4-X>Q>E(B3nC|1o4uia_91o-c=v8f$oq1 zfy;HEBsD%9S)5bsfK<&D0#1BuPGFPqDAPCL*f^mHj#$iwp}%z0nCC)5irD1sSnJ=) zzq4OxTpmS-ndQ9)E;ZTi7-5cHIxRQ}ex^+*LsCGzBgX~=a}Q7y=m|@_3pC)@3IY2J ztb^XGesilo32_b*`IFMP9>l1QHt|2u`u>uSDKT0nMgIJYj^u=)p*3WwQM96_5C8fWo#`24gCfxB@G6Bw$ zYoBS*XuBb;Q=S6F7QL`A)`F3eCNwiuYY02$o5(4xaP0b`P?S_X{_dm7CqSL93oySwuNyGQu}BXEn^+pRbl3#vm*QvNcEJmI z_2$1rw>N-zhi$!AFn7xH-YAJUcH1wWnD)eOSUjqdq`bl8@DW8hd;fj)u72JjX(a{o zqI#CYKOP9Npr(E9;{Cd#+A%^zn)!z#xKL1=1)mw7$yilrsuj}>l<|P!wf9m$u^+%m zRJ0ObZWRJLn$MOmNWbK+IvCI{=^@Rz<}<-H1jE}1LLUH@aDqQ5TekScV#Vpx&Z_as znU}%QOn?S$w32RV5GuO;(I#Rgd8r@e7&O=rZ&+LLYPBJ`g3DSm^hsE6dzePnZDj~I1%C~Li;=^GuN~7D&Z#{>qVDfTC zbA4DpD>Qv(bwy-k(=zZhg5Pw4ReoGlIKSxx^Ns{M!PIl>15QTDRhSp`G1jvG5@K&} zXe9AmHFv{dexS+OAc;BN8V^`2wEQfFx>7qtaMf}4C1Zv>n~SY!t+ zqBdgoCzF00E-5-d_ilz(E!bz*)LCdkmHt6+JJKb1AzySvF48tT>eEzV!3~WXa2>bo`KEIbsfT)g=r-Zp*_|vP1|pnpV0#5CGnW>0xhNk;@5d z9LK2-C9~49RJES2Ow)WUha=NNn*sQcB1V}N2)tu>zGM};n*7s5x=pEm@8`F*90=#P z4XjRGme>fUU^fM8InCk;hyrMCGLn+`v(!Q<*ouHx%ct>Pz|%j&YA?uE>Kom#){MV| z;AUfqBgb^*n;7InL)>(KPZIQTx^*bD(R^v}fbcW@2PJQh(J@OT!0Txr}!uIX@aN{;p+rxxBn(O8|-cVDu`*MF+Y*l_X!F9x(*|$WR;O z$J`%b0eXCi$!%S{%bZyFqvT2HzLTq*|6W9fwfdFZI~xQ%sE=r zoO)^B+vh2^zy#f)i7g}ogE1)6)Qv? z1U29>K+|u^kJhzvia`@@!HC)~++AKNNWzdfFyig<$M~g&r_?p{w&$l)n3ft|nbW?H zi?p6FZ+mf~>JM`zuaT}DY0S{qg4Af8Z=HlJ>A8Q+CdsEp#>a`|95CkmW#wv$BQU18 z83cfw%-&As4p1C7NqF!F_h^L8U>Rln;SYAZGskQB+jeJ7$Ca!ZK!&&w!qw%fX=pcd zlnAPNzF|Gvyp=nfqjyqO5%ZSeX#v5M1XjJnbovpsv8u`3tAD#Fw(sGI7$8(aQ2xK~ zcK%iO6NAZHR8_szRhDnx3f~(^Cpyr6yLx!5?MU$-o}E9oJG)wLl_<{~f$y{0K0Wk+ z{rZy;sk!5)y~h0&*K3cnpsnoKIqvuoh+)-=Q=Q8fW z>wxVtpd0?39H^_RDO`w4r_n61|8pxc#zyrtG&r4yNOm*f)5pN2g>|y<O6zoulg(Sb7F&z<}nUfKK^ zcQ$=TJ3o$gKh>}GSbs`odA=CpG0$BA&;3C0vAogn1J5H+>mlyOeU&C~ij(JzVs>C- zvp-d(!<|gt7YaOfHhxwqHn%f*7amVEfZeq&ji0o0SUzw2(ZNXGzfZ7uNUhK|NL;Tn zND3|#!pWulxX}7V8YmrY;g5p7a{53x(yPbbG5jI;-+dDBKK=wQl-v6G(yiUAE!XeQ zEwcVRw!M?dm=Za}CW!!E?!dp81Mx>4QANDgfSe!|Nw_iXffqJ;7N_G`gbs6`H=Y4R?_Em+NqACpPJY^oWVjfJ8%Qds9 z-v1*(SUr{t&USNRaehoeQZtci>h_f0{1;ng%}A&A9Xj9Zp4GlTyfzK=wrwH2>lcq{ zPb~P3KPcBgcpQFI1+i)1aH~(^C{Egx5pFSZ*hi}?C*v>mT z9(~zk$VzV5ZOAp5HFaPfCKivG^2LZ-4w%3D{7a1}(kmm2!)#6JTx$y~)Nr#y>?n+7P?~FnviiUa{SuIcnz9 z!@vmnGw6Ou!1>B?@(PEdhBF~DYT4LOLf3?02VW(^45%;u(Q*KFy<#E|T*TBTQ%my( zhiBTX@zH80gNy}@F@aMQ0FPPcwelnh4_bMgWJmD>xcdds1|5G}cO0?%7ml8L2B$k* zd$>$QqTcIqvRcEaz~ni%MqS^~TLZRN6rCPjGCf4%N>wy0WG_VVZVlq-s(~#-s$z6X zT7G%%YyJaFi>B2?ZLUOM>Rg}o^ zo4+o*qo^x((^I0m%A!MXJqT@Of+Uk|CVE1NGZ5rm*1_FM@l#FJWZUL3Z$)(>Dtnyc znaYb2kH>A>>xw6yE;CDqsxQ2c*jCg0kx^CqDlg>YvI@4pqMxR}^$@8)IUp=x3Dwo@ zsGi8J1zZIkFU6|aQI&U*JeU!L9NoA~BR_fLR2g{5Nnbh07D!N3^zcM;@d7 z@?HHp#!DarI!(;nA5c<=<{M3%}w~FVRB*T=YLZF{*%%zgO}X`(>!SQ#!Nl1%aX0^77fk4WtM6xwBg0cx;pu0kF(i zy-cDj_KPNy9r7NmOZR_9K$}GBq%3SkzVrwTL(TYU9c?Bbg#A)A0YMYi#l%_yeW`e1 zT545)SrgWCb|#;n_+Jn-T@FXDUsciB(*34BBAdlT0Mp(im2C$#*Gp@Vcel#y zn*`y?qv=t?T8jR9aIm6FD2Guv{=>n#v7JUMFMt}KR%uYJJWRrjJbs1HdT#4v{)JCZ z`O`-KvO%4)o}Wkb6Be{XW2cS37BTGUToYQxPIj$ z`b|?H)2r&p`%{1B;g`!Z+-9@$#ohfb9}V{iz`7IP{ny|3w_U7EW-H(epCOtQ1tnVX zRpjWdXcK6!zo=blkB<3Hl8n-2t$bID$o7t^662ZjM#=TKjp>Wz4#;=6gsr)>tV{r@ zX*GZc3+)4V^H0VyMFR1c$FxJCE4@FA|`!s?NV<>AiYNvT4Hhaws?a^`N73oykQ&SwZ) z*A_qJKEv>C+7B-y#KTu|`a))&HBc|ibJ7aD^VrPEjdb?L-sehGJN(1PPTG0=vu%n! znh-9h(lBkypu?{E>&?5#uMyIVHpXFJ)g2&C zY@-SX$jw!MWtii68Rq_j7pwl{J6%N;=Ttrs`%tRSgItFOF4yyWN?fr{E1G<;A69)I zgMN+uI4ER7lK3TsS~g_}Y>qGB9O47M3rj`OyZ4JB%v#T-!*tYAQ9FM$a|S9_BVJ(7 z9(>T}(h>V1w-$ZwLe{eVJJ}^ z#ipZM_%OVxM#J@EW%}Nye_^97JBf;p5(DXLk zOdNf_mjb+VKC+H~0~&z|uex?%zeyL?@Pv`F@$wkIXTP;{C%N3{C3) z3C;=^210D+o9slDBfbaITb0>8LJL(+(w+g%RxTG>BvYKk<%@v838vTyxXe z+k8ym<8(dm$MKd0O_}dCpU$=tO@Fkj?Fs%#gp)-vLEI;k%40ek{n*P-;+;)nMw2mA z9jMp?4(N4v)%hhX2;a^cTa}v_o*7=hr3hIXk4Xqx@&!70zr9Tea(zGq*Lyf_fw!G7 zqdr#_zk64_x(!2nD9db7;#23BK3`VnZ`QR~We&`%^kEW2;)awv_>>_vCnpko%%zac4%y;(+yS@VGh)bR z0k_JK?y`79I$}#-MJm$nqxsnCw_)cU#p0V(-iNz$|FQl)Jp5gnB&Ajb7PVeDF5dmU z#P>LntIi-G>*w)mg}hPryPMZNJ${o~4YV6ACA)|}{cd{b4FZH#x5<64Q({$8_fZwK zivvr?Qrx5Tq8}fUleov(^(ykg-ZG|UWdUz`Ew`6-B*#>hPQVOw8v6ajq4Gxw=GrQ0 z5X>MS&3_BO}_ov75i^K520L$2z-|Ru6g(q9dlxB=p>_I_D!}{ga zPW3&{`|Bq4%xZNh{>UP|TYdu5De37&wU+K@MuZZCehPYVb}*La97jISKz|$=?Ig_S zR}MTcD%72=`$*Y%&~ENCOdLFd5}iP`nlW`*Oyk^I%^OjpUSJF>paGn(QS9nt_N94S1Qig(JAzu;w(8F5YlrIm zYx>GWh01+hGnoh96oOXeuSg-I_y-XNLq^(a8D4mFXc}ROew3WVHcZYz0~S{j`N%m9 zWqQVG@+P>uHG={T=kz966)DJezP}TIL;DE$#RD`7Cg&_>=PKfVqWmj>I?oXg{pV7L zx%1TjAJMjZCA`}yzwM-c)A{P?{8ye@!?Z&*{QTFC1vi=RL_?wnOR^Pz$bSxW%qi7P zdmkk{(I7Nq<3+ew8VlT~;I=AxN5UKy{#SSd(yMdVE!C7b3B&!M;9&68n*vTjvel{9 z9Tf~p13uj*`U}gSg4Fw*`#hZkae~Nk^k+~BTjv{|PQomNDj(oJ$F|#{pq!af?&Kn( z(HT#YJ9pcBa0U||;9l6+b3yqo=5j8M`wgq+cnL}{!H8~8f2;#k2&nT z$z9i(JZ1!%DNjuX_nQ4^f#u^-BhW5Ez3>oMSAf{#nq*tP4h%E2%o68lMR{z>$Hy4% zd#GIsVF6h*fwBiQowofY=|pu7lVtv*UPRR-;)Bc#Kk+u}eRnSw)p;pX_;L$RD|6<4 z4CQ&Jzlq$|vPT_IP+&+OCS~n+)#O%T-vD*S4}r7X(cNktv|I@idORo7iReL`M8#UCSL_vvIB|!QPKTs%ToN98BYjr4C97^!0w5)O+x^v&emW zmQgUG<0RpHdN|;cL6Apq5f3FBsz_Pz-4k#DayW6M;L~*AAJZ2&fQHs^T`5lK-8JAr z8gN6kx6C((HcU&Y3W~gQ zX8B2ASZf|_1KjEE3PxZw;AZoqUTSrs$;6MSZ4PC~%VIv_rhq!knv*dFjxMk9xCJ{B zyl|5-aszr~hF@}9qpIhUG3^wRryY_(4%_#<#+>e?AZdjk#P-X&X~N4M6xP@>3~S-} zix((K?(T}570Q{`a0}wpQl>m{cVAk)*=(4VPTR-JG-hVb>pb~YKW!$HB5in+2Lj{Xzb(;qTt z@?HAE<5uawpZ)AM(QZjf!9DX8s`hu$BCz%+sGb#}$EJOT`0HJkBJJq;f|h0aEQm{D^rfWxWqqG=HpN1nm|OqbgX!_09U&VWnZ1KdsYGDQ_s zGK{>WG%c37)gs#+k4PLwNHO`++N3C%*vffKGJciBj;WYYxYcb?l zDDcjF*X?VL;scWU^8PEs9`er~n19gQ+Z}EZCdj*9eUhSEl)c)XCwDR@`=bc|Ic|85 zIL%dxpBe=N`C?Vh*C+%%)y(}fWp?H)fCw~cW5UzDNx=EKl!JVRIwsJHg$Ig6@%>30JWd2fE{ zesPdK!JZzAgP`H>wVf5O)zm@DWYq!Fsd=I+a2+<7IFBP;`YH9eLU>eZf zT%yJns&Z(%s_2L8);Om_FkL&?|3zWarnn6Y3NJC{xaIvkoK0}1qq|{!|N>>f7M}S2^UY;ZU%K{APb(7 zg|*^I#(WNzQk!%+(*%s)ceYbCw+VqTWT|%C!BE`D*K*N1hpr)*ouNw=o`c8`OJ4Zj z?x>uS>#KH7`H9oZnwwigW%!JK507 zRJ}6rAyCqHe{jpP7qsUvvcc`{cs5yhJfZP^d1)kO<`XtQ-NMgAAvoZu4dA~_U%gMm z_d7E;pFa$Lps`u^){xKSq#9*^0{bn#*?TOdDrCffz7F>``|%c>wZ3AJ0AtA%Gm{Gm znR-qU>o5@djM@@8>trZY8C=EH`nf6OM59eylgMChEBpQD7BQgEhuC6g>}Zdn)5yt( zx7u(UCR0uv+ii%CE$qzEOr2R7_!50_WXmV8WMTEh9U ze-$RRqgd4|M{Wlqe*`v}dN%vxH==cS>yhF2=%;p+zu`6-+@z%FCZE2rm|$6*oCzH) zEVJdo&tAxs7MG@~DmABCf12}_=T$1-KH~RVz69Nts`HTQKZlI)&n?LpqT}{2h(*E} z<#8+`oMdjtP~ge{A0<&%obL}y{1{u3(1FQS->DQwNDcrJp=uq;(JbJ^>6^LIcpyO#@i0+Actrexo8r5$4X2tTo6aigg>kiZmy9tK}C}Q*xnNP7|j68b!fM zC3vQYLJ`3oMQa01_F>%+eEES14<0B#Zh44^J=n$&K-cbdwIs_09-I-0yUg|Ka%R^|b2BzZukuM{pgQSMi=q$_IK!>1fn z!>a*r?FLp>W@Bgx%|Ogc%|2f`*>|?E6wC&xDH$}(6!_`8s<}#?r*;Zbz2>D*bqWTT zz#KmxYyoVyk}qu)heo(lVDB^irTS(`0pPIdJMCeUrX%QO?l-d%C{ShZx7EjuKHO{z z1UZ%I+5@Gp!3_3Yh5@}nZ@MFzj2-t%f>6V=wZhAaC@B@p$VxFc;$03n8{%FjrT~1~ z4m-0))3u~fR$y|~KovovZ6lO0Aie|T;zqr~rQ*(he?C~wHt7*{@ldByF{)QGp8BP@ z3g|BJ4i>?Vk~>L{fc}c@ z{b+mgulQ@j0CY*=WH=xM$2wZ!GpM`m0-7Vu?)TO6`V4TxlsNDJTz}W#oFW|^zFV`9 zezf=!aMNbAQC=AtS1uq^(SXb2cPG`8m}v?+7dh}ijb;_-8V;FMLTRYAip#mEiXoc< z)McoOrP4ntROMsOOGL5fC8DrTvAD?0O#GT6yNrz6V1Rqb(c@Da0VhSAwYi`>M zm}*v{pqO4V9@oYan>U`IejudSPG<))>e??dZN$T(w6oE0VxeI6zpR))cXuB|o#Osi zXSFdQ(gEHXfVluxzl1evvaTcArN9wDPN3^i-MS6)6)oKf^Rpgy-oqdO7so1F(bmm8&X!*ulu z*UW*jg68uEgAEx4Th5a6UrvP@ulOwMt+xIdy<6bmn)rVzVmnw>UljG^g<&EjK)cI7 zt-xDA{2M%~{xw*Ys0e{uNj?ArWCeXR!f`1D(UsALVzB!^2Ty~rBh*#DLBARQC~~hc zs>{bAdJsF@)cks{#~RWGch%S%)xLpi7XKevL0c|>V`A6fb@tQq4mcpKjZZ?9O5FG~ zh*vGy-5;*knBG%^9H?kt2~w|in}n%Ae^@LGmWO;Yf0JvTx(=Ur__Y__V(3rx_<}bgS*~@hRCX~U>Zh#>88V=h+-A&9oN;6+D2V<=(Z(Um zr#cydPt=et%kYbVN2AhpIw|U&GJl*vQg7n9q=Xz2{{D z9}El*eC&Z@TS&Xs)boB#d?LmdTv}krq7L=bPNP30(U^ExxY#@qaiw3nORx#~&|=!4 z_6g&XH)&``=F*N%FMdN5e)2^H}0@X7B*g zAU4J!j;l9pkVyxc4XG4svjv|kH@wLypYRV-v`_2PPnXRHFDHY9hcL_sySa9KFBlCF zMNS6+8=>kK(D?^?haKX5Y#V=7O^Xc>u}2m4yJO0_Uq>jeEdPb-X77O%vSA|5(kFIp|lz+6!3m$=55^`j(M9Rwy(RQHNK>2I@MoWZMBdhE@M;} zN03VS@vr#$k;*nXi6U>qKtju&Voh6(IedvZ(a~zJmv8vqP|^Ju{_c@;o*=5#vS(AF zA2{3O{k13gVNa@dQHt{i1~3>Kt%8+>-=JJJU$ln>ZSe4>Ns`65_ph85KlXql)cO}UiluO-Y z{I-O7?n`O?FI}I>VjdP*p%EIEuDyir;E&FO&Pn?ct7x$pH8S&=(nKOwudS4E1xw6? z2AO_ua*qAl7H{jop7Gr7!zHt9h&*T^YtpX02?*_W`a1Q>j+lxK z-Em}5_2;j;H9-f`vg>T-MfTwAGfH$6w)3u{jE}fZ6_coL0TT$O!rq%B>`JWXA!;D# z-@T}N^_IpK?*$X`fy1~KqhSejWe$^7KS(&fRJ%Dk|AL^Z4u^}TVkWVF(KYFd`)^A~ zs)Jjz&K!1Z0h51ytw_%P_+CPd>D8a8U%`Ig)gZroCe0}Z)C!+Gpko-YC2olC2YdM0 zh1mel?$c`C-)%tbw;JRXvoH|yVa|={99I7VnZr+k(DWSiltEQZ)j;}s+5B*QcI~C8 zW>{$`3Hf4a&AC9VfjrzmB(Rw*tY1T4wNMLIqy;-msqStvYs(7SofsPU_2OLZdY0Ej zZN=zN-24ZlHd1ZcQI#X@)#IjM7S7I8qt2hF4q zStectNc;`cJVvt~qG-X;JhoWZ=-P$1|1)74y1;rh_Q;8J?@Ik~9Fz-;5F=uK8+BM!ZOMdUTPfj0@uHrt z^l0{z8v6T2E|2k>f+=m}PD@@E6`9^TM!<0st2s{p@T|iAurmd zEgVihk_|YH{9!USUH4~#QyozNX$?k@1k*uaKo)SGsATtYg?LXDvdds0b4HUweuJr( zbWWWa-=^q5nsI4q>61)N8b<^&c*<5t8ezF?XedL%BNwg4a1Rfq2*OK^*3YU#x-jWF zh`qFwQYao|)Z-vr@0F`V8CTuPpOp_8PH5kx8>V*Lvc7|d<>hAnmj%%4dPz=Jb!%sn z{rFM2nl`ux&L%bCca)4{9L#7%{!xQ8fZBS{3;lpw$s1o?Mw6o<#DR7YpXjQCc@o9- z;3k?;M;A!_Q=r$h#M$oCrDZYOt*XJ;=YzEW!s5VY-cjQnetf0xKxj4X;j@r*t8lA7 zPeu)BK6PKkJCfUe28HfQ8JT}Y1bVyv_f;VcRjnZ)yVRSmQ2@U1$G{iPd*9C6u71|1 z{fK9R0nF0c$OqJ}W>b*?O6LqFEH!W;X3onVVHRj+A?G6#z+d(n*KnIn9z`-gh-nN1 zTWA5&WYD8mRHG@r6_DMf&&2+p7hAdpdbm{}qaijd6dTHX$F_wI*D;CMmA2Nkvw+We3&ip-2YQK2}f10XvnfW%Ngw4###f@jJLi zEKsaUdJ@`5m4V3<`ghzG@Ca9u8;Y_EsH{Ru#c12)x)rvJI7CdwZHr+{MD|~H-$xvP=Kdc z!5ASIP-g(<24{Hu^}TbHLPyW~0}>wonh&1AuLyME(x6%Ca9m=c4r@ z_{OI&hHY)C{IrsuJ~rMV1Qg3f#sFV=2ZzUw>8kuik(f(A7*g*@qL$RA^snG02uL3q zQR@f1E`NujB-fUrLZVr;1BypNLg_@5zY`k)19d-^%U+4W;%g5@6dYL7!;qf+d;^1J zFI{9%OoTB#k5nLT)x#F#m*W9`6U;gmGl4jv795c0B^90KUy)Llg6cVqnHfI%(nhLn z%M}j6F=Ybd6z3a=_RzNE#5CEn1J?g3=3KD7~iEKaY@?8kvnNV1K( z(2z_|on|eOYJ-8=*EG0M>%kimQ1+L@%Let1HG;LH+wXAE6vuG!Y6Qx&2J!qeuPQTYFO_FkJA267f6^#5?lF2bOr8GA36r}+ zM&TexGn^F`?;S9E_OSjL;OR;oGXm0lrM_MI3(<1$l}{m}O_)u3R}|k*3DPQnAbQon zVlb0UsTbt-Q_!tVQ1c4<zm+_p0JgNTPonAi5G6_ERJDTDfz)zGZ5*;(-m7B)n(?G%Z(ckogA7CJ@*<2z3 zX|&7-wU&AFj@ah`VTr9>uTDqZYBgVdR1l?-Pp;M$$y$tmSuM%*D zgM@(8-Bpxc94rIS{Ut;5Z(Mi z*wEe{-t8pm>83nYGmY9cnGEjAG&qOnz52x12{gf>Gm1gnFwB=zy_qXR$L+gTyxQHp z*D5>BmOL3j5B+9o?YfYeiE)Uy-l5%Z&89idnC$$3a5g{gc^~M^uX>z;Jo_CjdU?ZpQBSChBy0Y^80Ar42S7GK=S+PrR-0Z ziqF$L(O`XO63?&B$G?l*A~B5=w?K^=my%PL+20(TQU4i84k&e!RIpxM%p)3cr{V!( z!ly%m-BItM>nv7iX6mmn8y4cs2LrKaF&P~Fv@1Q3r>6Ew1Nci+1dhaGj$)eOm|#SY zEPx&~*xK|V1;>$OgofET+~o|)5h>tZo^0@12T@XP`aLV#hS ze+Jw{j?Hz+^E&B|tlYh(sv1l97e73_n(C_Z8WG}4_Qr??_cnuQKU(#QyAXTs1GAs^f;P%a*VKmsS+hU^Xiow8aD|PvADdn*Vf=Rcuvqncj4!{&ngR?TUh` zj<0J8vwch@P9^)2w>%m0Vo`X30Ox6F=@dco zgoTBjQG8$qVYmS`h8tDD4Y{p}GwS@Pn;}YVpf4)irk-!g4VUg9NrXeW=45;ugw`X! z)aGsGShv^K5Ik6PRr-ssHkhfkdXO^g2=s0yOXs( zTxW2t`WGo|g!wflgvIzi>YN8~l_{ZJ?O~{*_}xjqy#|J&LaUZTUh=33N;M|(&Al=s z*<7ZEK`QF5&5&HZ1k<(2v&h=k${M#N5O^8>I z)V+QFI6N4u-{F>&!0e~OD{iZ4N#)k6fK`Rxt^1sK6&$heMQroF=scU#s=>yzjpg2e z+EScAwYKdFowIFWYZ*t``f1^UKd40Zj_Ddc1#Rj#0NUA?s%fYla^5#~-AEjQJ*tf> zp}wK3J`8!L_c=pY$wVcBP|F~%1OdIxJR9$12OSD1Wa3w)5$Zmc^nM5cXjPs9p-X`m zF(3VeMEmHIFAf5oekG)M6|iy~Mw9&`NYEifpZ=xVQRU9U0d2c+y*!FplNz7OWicBx z{h4o4ALgrS7*MRmw>%1~?K7*Ptb5fw&0MQt1M=PY5;Vm5E)6f`1gn8W<4;!At>BDl z!XRzY1CKvB3_f2lalF~=+#*KHUv~v2EtAiNTxx8OuGdz$=Jk~OtVOQXQN19xAm48F zTJSxY+xR}!(3AWXlyvT&jD-96w|#p<&}&;A(D;ec{Q`Vqg5o#`l7eA(K|LY6j%7kK;>LeSA3&Z3Y{lA3o-3Wy(QHAZkESYXm;9J zViK=tbSlP_ZNRMn&MDUPVmq3-A(MxG5bKbfB+{~;_M+*_O|dyWYD~B#)-(?;kk8Tr zdX`J2qapDzHR&660n?8cDkRE#!Y-ttu|{4iZ9jbQP_+urRPZ|5WK7wJK4qZygY{m1 z#9%2Y6z~dYh%-{;N+2lkvzTjCL+;A;sO3g`m=Yl_ldxU__YN__wLU=H=31i%1Uryg z?-Cnqy!S`B9Vrqc`U9R>O1mPG|IYB`t@QwAQX z_bYAdNNU>l`G~m#ico@ke%6x^6rqTaEwSh>BpZMdirbyM;$8u2^ETA_q|g{{=4n6$!5w0a`yDQ4g4v62dy;*gI>SxX=W?*svPT1Z!x~bd^|r z!RtAFW{tItI>g?8SR^GJqOV2bTTXv2FAc@3Mz`7hQ(ekneX7%tb@b(=dQ$63mvfKX z3$=0R>Ae>^H2-ln7w|E&>nqj3Y2873E5Gg8yqfvSheb>eXTx62{%7I1D#OI*&=Bf} z%M|wFTrd9uGO~qp!7}~G8K820y~1A90>Bu?L8{sbC}~va^-_0 zzf1mGH5F1%I^^X-P&X`mdqu)qU`c66wZcXEjsN_nJ<--8($sb%h5tgJ8lN_ao`+)A*u=rNNuJYZ%}PIwbB1-_l}j_B zmfofOgjr91r+8UUxnp6hEWltM6IC+NFqj6iY^y8TRWL9uE7h1B2tH=bYz_CA{*b?7 z*pR7tNm@FXmw)14J^PwC!xJ+rwrZDQ`WMBbY>@kXKe?1^=?>}fTlsY8bfm>usZ z%)Ygd;75I(n~++V*4V(6D^HPA>~Z9gI)kCZVFB~svgpc(uSNn)OiWu3B3|34o94)1 zvhx5<*gk0@^c3Fwyh1`64=R|fm3OnAHZQDLDSp|%mYvrKbYZT%|F;Xv7qR=d3sYXz z!i9vVKll2K(=KtHH-pQgC*`isE{}{K$X2bHFXiRl#;Tx|)nODbJ}b z!((A!HLe=@P=wCx0YURE*-72E2I3k>y?Kh7!BC?Tl+8Xb*6l@o56A^Hv|2#xnEl_Sf(3CI#sl>_ zIEG?&l%jnf)`qyCKE%M^wp14zx1&t;iZs|K4)a+B(7+-+~ls;=iRnF z4DaJVzpT%jH6|Gtbl=*l$?iD(tXDH}qt4j$rK^40P-X67DY{d(revZvb$5vYHZ)xt zrc-iOD@UhD@$tR5K_{>D#0wdY2ys2hTi>DXwAiMjFL6^uNYG1oo#~t&!Dy}Pj~53J z`RvH_C}H}2MRnwhxv|;hTty2q^vwfpoE`2Yz{<62G2OS3UAT;|DILkEKb~HAQz|u6 zqer38rFL3X9$wfe2+-r7WC=e5oNtV1E9(gXEMlb^B?q!!c9H{RL>EP=RG;^_%5sn3T1e@nA|Qpux6#Ejn)(yu)gp)c47z ze)H%4U?i@ABbW&sxE(r^uL?v9(HsxF&V&p}0pMO~5p-+s!-gS1wrSq(% zvVuWyyzU}J97XO6uYao60&GmyT>I&%vOSuDdMNM|N}~JAqWn(Prpi)a21rl}c@fw) z9V&|o>tcu zI^#wP1hM1`2kKt6bz&qZKSh^R&bZZH+29d9juqNyykxBH!P z9KKWA=X}L z?QR0>u$nvTE`Xmd)q|gh66#>5Y}}_EJaWq2kkIA@cOtLDN90!rmY81^D}i|@bwnSZ zITWK=fjc3LD1a2$u^7W%8l$Uq4fc{Gg}={&vH8oSk%$q@{sHE;%9Y9{g_3@g|avQ*9*OH>EBn+^$Khj3Ss#e&Ie937&K zyn4h^lxBG-9i7=BE=ov#TX>I#mt<;~`;(+!x?-qL{|Z8VIDuy9kKr~`a1;eA76ufD zLf`QKgKeX5e($>v?)z+;dSGICdEhgtyo(+-XHfOcX&;>v@Va9X45*1iJPXU8i~x=V zd$J0(6j-!D?>gW4djN66(ewbD-MCO2hNXuZlI<~xVU}!5ttVZ*4X`KSd_k^vJOP!P3dh96Bg&_dGFe>m)Zo4)!`X57%1y# ziAmY0DF!T%ZR5#N?wHC}hR6L?ktX0ueM-t6 zxS(|OpNz6E z#jmLCqbD|?6pBby$XqKh8r_lxN6lTlB_b-{os#ULzy7N%ard)kvg_Bf|9Ah#EcHe+ z!H;wjW0Ihi9{^AXXbO(M0S~ihc__oCREYnu(+a<+;fcb@#~v-3kH#$c7DQJLME?2} zf42=J&eW~IwpgF3YjzLzv|h!4*Ek-Yas8(OciX~^wmF^Oa=k%DLwxcU|9X@~L(;2y z((Hw#<_2+GK z`1Z-ybzSx>1S6_*!b4z00ub6%?UNf}=)u@?nQOIrt*|*zhX{<0?`*vvIyLc&$a61g zo!g(O!-Eq2;$zuzGu`JxFQb zt)nIsGokqkxMM$#&bRs5PBF&skJYw%hk>6_Ittt$;4{onai9u&n7%MjWkuePtr^b> zbh~8B#{Be@O=^^j4r08M>+*;-QHeOG|c?sss85E-l(P?F=cOa*+7wI0MqT(0H_=CZzGdm=+NJKVtDz0r-Q zgE%V0kV87kBOP(vtLT|!saQFGs7?5HJbdWH;ZoDSAK?IeIB#2FOD4NEFe_LzIPN_w zjJ#m*sG8gYT*?{1zKH)#Gs(=Q_RW9BIe;FH1wGaN345y``b!(cy}F}l+Kht1|1d_u zB8M3v>ix9o#PQcdFZ6nW6j-6CObN-jS`UcHWCquZAT*Zhfh@)(rCl_HMD+KL;}B5= z6WJzJxgi!VeLqNtl$leykCpEjlF@90by;hKX+>%T#cHU=pSl?paIqvWlmBO&?j0jT zC0@0Egz&kpw=5yEeMObck~z3PiQ1X6_4=>xg$721P2kw+J*O%VTFiqP;_8GXp!Rs1&m@~z}}{aaE4I*ps5 zxV+LLSMOtr(>I*2o~VT}HOtkEO}dRwjSVEFtbPQ1h-4+!-wyqC9uAoXnO~Pn8H!;3 zV?y+@Huedc<(;lFD{*?^>+KZR2}lf;>3swA?^8$^pR`r)sLubX*B%xI5+4C0;{0x_ zcRnyC;1k0o5?S^IN&L`MqP(DuiiK&hrodEQ2p>peo+t;IKoU^c0o&nSD_LLZm3;eY zl76&_vf4bljL@;|H*lb7c9p}ZXEYojAyQ3N~2X55})Hw@LXquhawo;n&8R z;Vk_BMu%ktp{gv#cxXVqUqLnZ{*Va}SJ0%_XQ@D{%wlMCQShfwrN;Sc(}MZR8R~`> zv&v=Rg@A{9GLa7fK1?+Dq}%s*uH$zFW>CMq&z&*py-{ zqnQsvV@wn!EkP76>6)v+f!-inm}O36`vQg-W5znyTk-#}2FwXQj1RtPl4<|2P`lLq zu!2~czJB5Ic$)wC$421@j=#G}aK~fe*t0``traID>nZfXUmpjeV3>>$W^RT*=838N zS_)39ATk9^s1BLGG_hQ@g!YH!JT=veoJyGgI^#_%DHvW+=!LzJc|}%sf#Jb2Ve13XJOlj|a~L7P^uXgAdCiDjxqjTR3;E4{p4Y0t)?@j)i-O z7cCm*)H_3UxY_8GE-yLftXF;|%NlWJbdNQ_&6lA$c9l$w;Jw|P8+GglKict9gr|g& z$AV!j3|O>4f+Ycio-H37#jNR2@Rfp3RWe^}9b}xH>QGALF|9d;_-TWuF2kFx*tRRW zE6vtC31x9N!(t_8ECHQcx6F^TEqt&!bs}6g{w=q)rEF)JIQJNx9C#($mh}@QX>Su$ zz$fo|n1EPpH8le!@VLc*9|aavw!n(n_g#lMSuv1vXW)R}R`+@e&ctCvt_Id0me3*@ zz^*o2K&!;GoQN{2|8AyrZ6W61^`&?Va|fu|qH?_fMz~tyi?{5OKWSAtt-vKl8y|W= zc9J!YWkgQNJC)AAzmEzu_WhT^b1M(5@6J{PY7A|N?eRkas8OF1%GhL7>gKFv=b6`X zT=R4!K@srO|I~^xDIu0yNO%PvufR@@MH>rWvM4g0$8ih`6k>o3fOLxCQp-Q2x|dKy za^%Hu4&L*}&spq83pV#Z8b`#ELqeCRU-JwunHpow;0>5&rn4CD-}Nm6R)5|@<9dAl zLqxerrErX0iM1eQW^7H2*zmBde~5uWaSlne$%Z1!0HA*mKNkQ@rc0|W4%xrjcSNgd zOjro^;c3(Td{7Q)^Yhb%mB6Lwh3m)S=$cdAwNb6F_pw~Tk{a~A1!luD#HOb+pZw;A zfzw%oJ0;B99>1OSk$G<8RjIs4hi@uYbmbf1i)5yXXKg;jdUTdU!0_!uNd+KTTeZ}b zN&Bz)t@>Z{o5z35Z^17AHNUYw3SGd3d_P}h`NPE7ajPyOJkiDX+do>Y?x*m6}W zxQH1n={6C3*7JwR3Kk^}mW9K~%4bsH#VWR}(T1UvC(od0a%ExXb|2vQPdeo)9PzEd z^qex~VCp9_Dqz%u6J3O3eOfAD4LiBj;uNE;!|s7U6%H9ndhq&`S01bS5iba1Gf!5@vD-Di z>EXk`r8CQF~awntAYS14Tu%6X-A@m(-~t`#t}spMTeViX6@FA&y~%$EItaIcevXaIh$?@ z{EQL&F#yP=_0+1*PgZEqE0NQbxBa=P)T^e(;b{FVb2P|6-xx3Vnh3FAAV}@86y--N zFCOL*yUA8l+8iuwagp-o45Qblf*Y1p^N&NB;9j5X{f4gPV$@C2d3W)LMwuthDc`BH zqNmU)UtdYTLCiKh{l#=Q`8X^nK0cb=!FwJi4{UF&^<^nf*>b;Omw{!7^A{0+9q@(+^ z1YV-KD@8frZ348G7z0ekDd4~{4-v5gtft+#HtOM4e!%rb^H>)H=@GCa4LmlWIzjIF; zU<3~c>lAY)!>8yClMvPTTrm(lNGSU@zv}0vg8@)A8=|uSKtJ2P@ex3D{*0yF`l%T4 zND+h))#X=z!!Vk?U55e*Uv(4GQZ&fjV^tzXZ|7$LAbFXDbC`wDZq5?IP>F7$|Mta@6PuBcN1?DjPykzu z5tYF#dQBgR?@lMbMxsk@ZGH9fx$HigHow?Ps@S#rov`i> z9Pn+OOBo9n$PVCV<(jR=uvv z|K04~1@Bcn7}^qK|8;fJM4$p8-_f;s!a)N~nV!+s00IA7UVcm8)l#D5#AQXI)c4DZ zt4-rb1C|odAe8J;DjbLyHek8xh&2F&vGtJ#LORtTF0^OF<){pjKEYzhG-3BT&^o3O ztj11iM!BFVUhna9TbYU(azJE}eka2v&KW?A7f5oCGPC$|>6WxcrCv8{>fo;+-tL#u z>@BlEJAc2?;zEDI;Fc5(nO&w9?0W6?$s34FIaH8gCg*_0KZ~(B-G!BTvv8XFE0PMN zg&>_OT_SB1b%wIVO&w{pdiZ-l=6gn1j+!={-hR-Z)GVAzPcFFA-5Z#omN*#$prj@w zI_W$H;7Dq4B>${ugcTWA-k1zty?%iq^f|4{%;Wpn&X8%gus?Vhy;=dq0@?Nv4Kox0Kg5j;qyNR-_wwtu8a?6ylN#ZXF!Eu~WbRE;+Ee zRm%KTJ4`vap_(Hki_B4=-4}X^c}5aivLOow@7eCNjDt@BE!MGj-|*C=tt4^;U^lj& zsbRjf-9(c6R*4R4rpXOll7}y0^O-=k?z-iyNe*mYe1q?bOU5+DhS!Sz2 zH|F1sWrl8d+|e6=b(N>W-x`M(!1&0dq(%H#XK#g_U6s+WDG7;>g^+8Cl_LNWyxQ>P zU{i{n&0fpXuXK=^CCeRnF6LGWhD_ZAtJIdJnZ!&4cU2CBlIWyw>@ulqM`Oeqv zJ|^v64Wf#l0N2ev+F;`|fU&ZCw+&`~*hy}!L=7PEdX^1153MoYIyX}`RzC=sG*x@K z#D`b&z@)j|5#ueh*oUX?3B_#9Rkn=^$YwJlgw798PUjKSsRgnfT^$1;=Z~(7Gh`t5R5(3#{{%0pFka@9A;#1AR^F z{c;Vq_I?OV2qyj)TFN3{4veNr%k}eB(*QRdM@!7%6TOhxnsC|JM}H?SmvXpjj$DH- z&z8B-6;0J=v8Ek#>1iUbEugRcdIig(e3J!0Va{WUbtx$@q%ojw_8uAZGg1qyodRA{ z@M-mLig9o`JStA;%9n_EHFS)==L&URCxot9ZWxckZM_&gerZ0(bN-*sfalH9rPIHi zLBjU`>I`~)e6CEdB=4`iJ4PZwH&wq(fp{0C=NssCVUr^HTJ6E>v=XC2ZtVoMttYU% zZs*_=!(i*~8}GH;XABK>ZmzqDm#+*(B@S+K!wAd3f*k59lrLo#@*NVjNZ)yl*QiOF zFMyNE1DFvuI2q5yv{Eq344YJcHV2Vd5^Og;aFLT;h)bv1d}7n5HQ>$N{}HfzDNbw> zm6Ij4_8V}ZX&S5Z&t|`Q7QRxfM4zubA3febij(tF+UIm8Ie08L$Wfq`aMznlBcD>Z zFQF;ed-v^Mx<|rP>!z6XO#8i$O9tEp@(<&v7dT(#3EnN4F@^64|3vDS%nAmCbX(XSj;#A#TS({J$fs>a z=+1?&!ky~f+a!A5IQ|_SR{GDy{5vWYj(JW#o7KwG*nk+irN7w2&ulR=#rLoH@;$fs zULsSC`#l06o|i1%0(|gO{PGMNv%S<5px4!!dG(5lpd~=$S%KCbfQup`n!Q!Z>Ri7Z z`AexQ^*2@gm|`H7N{AA0crt8T0SPvwGJIMpt|9Npfw;nDG4I$@nF2XlrVgGk(#Wvi zfy@=r@ed6%t#aExo6yFFP>B&Yh5cq`ex+q2i?J3PZc83L_r}SA)qJq!VoK8Nska`W zdv)D2Sn;aNb>~y@q$r3mVRIF1Sc`qeOZGehJ&ylt8EHx&%<$se%G@-yEn`5TfYi@l zshY$Ju6}}U6VwKGE_?eyFKZGV%*KVFuyu3Rdgh)W*f{^R0HTP%y(;&s#_?nTSNy(- zrSki{xB2?0ZItZNj|DGoy57M>@%aN6l18CV$hLe_Bs6TAN2b=i2|;WRLfQwIEMs@vMGU$-U-hqamXQUhWl^yPn43orpq|>F13Ddv5TCl*EdBBY3*VI z*7Ox6X-Dc_BkacFw*3lQv3LJ=rI_B=N9v20^uUxpXWOb~FHZX6kM`kZi_cLnpda%| z`2|^FX*HX34A#HG{mtD3=WugrrwEk~bPyZ{4CKZu~*=i+AH>YT!WR-FI)Ya~mA0 zOFX3y?jd^9YM)b(jc6ukUpzVJ^$>n;xyIAtPhA~}7lB=b%_8JVSp z!biUBRhtEH>A&mauyr}016WCoZw1O1ub=sSZc(7qHVrUx4Ewlid9};vNoZ0e zZPy-?o%O%=9X}+zUfCVd05hTREMRP-SZ&;V>348GNK|U4WUF>(4rpOiH-Y2AzIp&` zM&2=YuKav8vAPr=5tkze2`${ZHW9E}d|^l&(tgQ#_uUxV_3?PZd2}6wgfSeyp=H*|qLnE5< z8*)3T>PISTu?6vE1@Q^`IXAor2T$p~_n8I4oahBTz|5Q+^fwDTFCW$o2Rl?%A_k6Q zb-c>fZsyyF;CX*lwzkGzOx%v7o;T#R-4&l##HzrUu?p3^gP9D*XTNDPK(U}R%AT<< zTvvd;1^5-jyeOSqt^g^C_$M5f zfc`v)3yly5Q4H^0t3WnR4WzY9&hOlAIB(3AT#xT*l^%Uw#{LZ~-yKogbFKe?#{gPC z+A|#^oaLX>_WUsrKS8FKKvnU8r}%8f`yC+STGPQzj-5{6an8nxvjKB_LEgohpfT#f zm=UNU0!J9GAqszZT@+&+zt>dyT^1OM75S|UH={R|FEuTY_aypFfTrx!A05<@oI+)5 z`FBaF+?IR0SBGAi(amq0pIS?zYf@J#mgGjJ=~GFF4Ous+OZ9Ow?)#5939Y#PooTol z9(_^-t>O6_E_7E0g5fSKHEJ^7XKHvkL6cpH9wJxCZ32MUHX1B0^)Lrm{~EOy5)`M> z@pc45W(?AEYU$Z4EdyT~yRYBga!twXb||SMk@Wi6Yh8*3<>GI>7lM6h%gUctM(aPx zef>7yY-*~75leQuPhARGi*Hf_%tx5L#?0D6)>ZB zro0b~H!VIp`|SSE%)n<6vkN}xU+W9Vu|~XwnkC1M7VsePX4ND2jKn!oOqyp#?jk7& z5mlSV+%ND3@CSB2#Z|1`kTI&j)rAk9K7gXfn$#Nx_l&*xerX=>F-_DosMvHL-u8Z8 zQ@8qy79(C@zrh@OsY6yBx^$X7jNY1X`yf)kT!mrc%Gf}PnWanqu*Q_|R=1b{^$Fo{1t^MtGXJ@;I`}@F*hGY9MQ&RA z#l!^`FIE-+@HfSpo0? zMIk<&;KEbuO(uhi6u8MS8<4}5eWv65Y)c~Qd zRgZp%okkcIeN}~YgCOL8)F7c>*B@?CUE8eV+ed?Wia`z%aW=NBWXMN9fc$W3MMTK0 zc+S{-TEWf}d+Iouc}z!qh~DcaiM8ibJ#D_6r^|aiW-lS3Wk#AyZr?91>Ol}7ePS@~ zOA#%fnE2ul>YkDzE(~DdZ1dNdZKYQ;@!DSrO3yfQg*PnfS(73SPF( zyJ-6k_0CaDn{<0FE!P)>-V_V&YjT##jfk*nzXiNm_%!`nv*tc@duc{coLJ$+7A&9z zdzB?n;~QjK#b>bOG=-s<&IXM43kUUAP@1V95Q09B2FO+QlHod4KUeisle~6+K0b9@ z_Br0^9C|I*ILuxCG)P{3syp%No1-6oe~ZM5{${-Md%Nv%@Dul5m8C?BT3h;OX(NH{ zGHkAM=S;e@KCkVX>7u_5|Ck8uOMJ8c36B5&TiN*H3RaXqS3i+>jW5DQ#QzpM(3qLa)Pk9eaf<7g=B&4pVlpJY-@!`#bIjwS2( zoWgH3Nm~qS9#f9=hN0>04Fisd#{*d9!-x&}ka)WsvAZC>LqfS}mcwV#-aU5@RrGgo zHot7~RgP`Dbk^m$V!Qv+c(mt;WQ}TuCN)9V2PMMiAYEKt4n)=5XK(VGX1WOxN`|qUiHiS-;9<7^3cKQgD+{RDD@k_}o;fh&pq#=k$2K zGM|oZZlv`3diN^f7^8ASO1eL_gh85ray0jlWorhb_t|qb@T}M(fjr#fQF-iioJJ|v z(eF_wU-li}l+RN4JKg^$At$(3O|(5U5BkY|9B`Aruw)~4_RHWb6s7hAXWV9A#9rH; zm?|_xyjnY$&y&ZTn699y4iHk%AUe~tcgsi%2R(ZV8N=H{<1ujYYaGsNK*vK;n>|*lO zfnP^#l8Q%7k4>DskC)_iBWC43J|ewWm&wHSsa)#i;gjwM?dA>VclZn&vnqOnLnXhn zIWf{}E4wdsdPe>@L#12RJOIm4M!4S0QqnNliyjIu);Jpu1YiF|$}xc1d4jX2BZAXa zgUC&!O9Mi{aQi3x7&A3ev+l>ZSQMW7FR#avCWgsVM9=MHJz&4*y=L;vryrr+v@uAV zQvzuWZ0O8&;OJdCp1F^t{#50ZqxQA1+zL?uK?vHO*bhoB<|GMO@dxkY*O{n>)bpd_ zKWn#D+oio?l%G$%tG5!n;J3Z?oyUHf%Ds3stow9Ah|UO;Q|Scf;&4CLDMDJOm-2t4 zBZ&P=EGX8iGw{n39ljN|8azD;wA48CnHGlrY>?^l6LhBDoMI~Fm*9$~!8 zeq*)W6}`;s4jPsZeA+u%XO(ayM_uH~1_9jRlQ?&)8kzSic z)lxV=Vbb<Ie$yV2LCp!IhyR ze~q&reUcvTt*S?3{h;7E!tHoxB!%Bmz&k31fI!m&iIHRX0b68+5!wu%NDTz|mE+yq z9!|9IEc~W}@9JT}cWGPo>&aNv>ers0N12o%REb95!ttL&8&T=WNfcc}?4-Jz(>^Z{ z@C9%S_~r%lI@6KO#OeD$EUV!U$d!w73+)p0r$PFII^o+7v+{VvD zIX7!e+O_smwSc+l6&ykWSH95KggG9rQkF$+`G?bYr?V>)r~XoN{ws!Y6(Y-vu}xa? zI@ev+MI>s^oP=yeJO#O+o=kspz0V=;P0DDTB}jXGKHv`z_Jv`)`xrNfwX|zDN->_o`f%TO)cyt+6fQdw=3;$nxXWs=pf~y z+N6SwTcL46NZ8}7RS~;qS&#pd1@Q6m2m&d83Wo$&nVj50sr8{X$ua^?SV{{c^~Nd2 z6T|XmeB#AD{%}KG-a^^j?>ILF78Kfp>g#8YmQsY5a!7jtCX>za+06q3!)y`E zU9dp|c`(@pXKZoLXg*b?$*bg)eYZsO4Wz_-1{L7Oc3^q^O^!Y+ii<^8g4MdQjfoEr zcjm}{4CjGP%IXZBF-+0~N>1~-*vyiu^6?>M$isF~TG}WiSw_h1~sQu$hX? zi>o~{zG#2>L%ngfJv}!Nd)OK>^*w5zJWKOa9?y^t*Q1sI3E$0=CXh5J-1oD9(}YC; z^nC6=)lEf)`B`C7)ivFcOs9c5C2}-dTMXUj`zsOM!AHgmmjci=4+i^tsJ<|=C6*LY zPP_BTN1e-X7`L0zJsAoYx9H!|jdsB@#h(p@B)5Iqr;X((XMWC!y~PN#A{^Lm4@hNW z@|h^F%ia~|KPMW{ewyTL{rIIrjhkpkM<@Ll*c7Sff3))W$CbxiE+H7nwBM*>o2^b! zr}N0X%R)ZpF7a9ZwJ2EU74?srl9FV1_`vZF<(kR?L}M5V1;}6H5iXJuF8&cNLLZuT zC;j{<#iKGBpK_)KV#FieM^@Shj&+(&u~`EtkI;gnD)HK1eZ)FA zokt(uE$T|Q$-jE@{kIM#eX&W&`CJn7AI6Z`fpLa-Qc)W8~o8cR_>#y zp#<<}198ebhS8Xwk37t6kD8SOnd+uqV_-=l?gaFp2CQa-cHAkz+OskO*|bA3anSE~ zSG7^FzPs+$^(5Tmcflr73aVL}EXpv;FSg`iNc}CRt`yH`(&QRZ9pXyS7gAzaNvdHW zTVMT*&Eo89A~5t44TGGjmiVgn{(#RgLaRSCy(9X5cG@lV%$3i%F===;6jDN-S}Bd3 zx?NJ&%{pDKKw9Yjd!!$JOT-Tsmu}!6S1?#+1ZlKL1!lxb_`c((iuj^o|%$i5P#fPZnplSdJeJGyINz zy>cN$4jnkO2(&!T`x2o2@vO93zuvwdcq{QS4dT}3abfnd{awjkp#B2MzxHL&N&jz? zYeu0NDlWaK=MCgTAG3f!=Zp6j?FBR+UM3=E)|FqE@NSZS7b|x#3rq^WlNiPH#BZS< z`fbc}Qp%1SCgD;DrAAkZls4W#hPQmfI;HPV`@Q5;#q6HeNMT%!!(#H*N`wC!N!2nA zbrHjHIJW|$zZ1e@mbJB+s0Nn*%FQeHm)Hg^V0y@YRp^ekC%lKhQ*oC%U?ZuM$;NgXp;po+VK|~M}Ukh&P zs%L{~&yy7QSDHQS0;4)Mhj*fR&0Q58XgE32^4ul8I;!m{dh+E4H!BapJ6%%mXF)#` zLMG?(d|UrQlTgQkL4B?TCcw~=$*<K35TlD{BC8HaOVYYyLHd zY6ty4CS)qFpD>CAgI?d~oM>|J49GRlF+*NYiA3}M*>(%8j~^vcn?V?0rI9DeBudwf z=wSI2Uy==Lq$D_|f$y>GX7bN&I-F&fUoT+3(#>6ke<&`zhs{K>fbZ?Vi(PM~?i-56 zn-?q>ZTnR3=4@ZB6OA|giYw1rKCw=o{^?ja6PU|N3HVMP8U+dE@b2!ym;8LLjkJk+N0DkLNo{SGdRdg#^N?z=Q zddogi=bcw8JSTTX4@F}6bi>3V8kq9z?SDQg9=vHjzx&!%xy}m5N6_oSfG&X2s!%|7v~4;+YCyXlMY7x`)MvIxU`Kov;OYmnpSbCOKf;k@bkTmq)3d;SFb9UYt3^T%+ot^TmZ>%UuKfB(?Sm(5{+yEb zj#r+W-AmILE`7}R?FdN}VRv->lZWP|qS#b*iWO)4R|Rh$*zl*mL{vaAL&|HJo^=NY zgO)D!ah%1wjQ7kAA1oE`bA9fMKAXHcBuW`@+2b6|h%Ch|k1_H*|D|?gq5-^J-H*SY zOK{P2UJr*G3_(3Ac<_uXfFGs1p2?LJABJU%(LmesIQso*D(DZikRjCf2j$hb+pa;h z<&@91uQcG$%{$7LUGxo5jnj}{5Q9h>lE-e9QJ3olW9&0iLF!UHD5oyP2IxG|9l9K( z73v37XLa^#MH@!au;tz>Qh(O=ImDYnKhL(ibxw7X@OVLnwk5QYp^M);YCoCSdpQ)B zQ|qQ9vMqX~lX;TYc99);w@(=zalazUd$ly@+jaLVe5=b#4wsr%E6H5HpmO;%alcOh z92b)b5oAk{8PE$lHrQb0zHnq!*4??n{gt?!cuU-9oGH&Vfj=o(n(Pe!1zBX-Tovbf z2!MmqVyEhruS$RCTaqEHnE59lQ}+m!k6^@z7UG!CNKD?L5aPL|4}NUXEz0Fr7}l6K zY*?|a^7>)qG?4i)x*qmK-pFb;6^RL__zxU6t0m?nuV|OLx?JG)9=#yXqrHg%{I@An zI0n1jHC@KvyXBY@xf>M@^`$nLp4?`Eu{tkOHZ15cye^f_)hWspU!R#$K}0-ii<3db zX*Od`k1Qynrxqd(2)xj1DfqdA6SOgF;Ptqc771F)%4RSojODaKiiQpthyoQ+TUTTm zezjgzWI^GAweU7~KF~J{ZA9#|h3A?aza@@N?KAYRnU2})Ar!o=zcalj1u)LPmF09P zcsPM{bT%>WFZqk%A?|6{PZ7GWzD2nFV$kN8+NRyw}>u5aNcMYLDJxWYk z;YT>h&4y8`e`I~I6aByf$X%WvnzRF9^1!NJS=Tkm_*fiLfUe>>&h zUALCt`q|Fx<3w~{vbd&Va)0XSdUzczi;2O(MfA7((I8H%)G zCXe#rW8kYNfHu9CAnP}H1d!~YZaun`q9VG{f#S$FV3DhZfin8&4;&X8ml5PkL zi`>h1tMLgOL%oafEggB7 zeEJfm-U-EgCIKj+`Lq;|{vEjM- zBPr(UH>@ajBp011GfCA9L&eMCjKUcRzWfQUw9Z9O-TMoh;<$#;mhR9i9qc|gX05@CMYj^W7b=yNE?@7Kox|j0`YIDXPYkH+1BT|P zdH$^(o3p>4yDdbG=6G#Ph_Ga?4gBU70~0@|1zgh=R~TSo_+Fx_+nb$BBc@kt+hM-0~w$)9BdE(R3~FR2Oh1X$y$C#E@g2 zUAIAnuvs3|^f(d4b0k*@S`_@CfpW_EWu@@C4cwUlKD}N(xd_O;p4)9dypW2LdO=FD z#jLn-p4C^LGWnV@ZvD{=Z2J+UBZBPKX~!}G7kBXc3`cdc@6qQ<79%bNOF&mqB|~hR z$3T+CVR^?#cHny_>^OO@2aS~DBOGy-+4?}6_Lsx#2v*%}7mnGPKnHOt)w^xuQHpQH zAzP`)?uzXDi7y47icWa0ZqWu-?vtC(^Xj)NqThyXdJ2n*-SE1^ zVktHzWLPevIhKFK&XkBI*>4V2I?h=T7|TL^P@z@&1L?Oa>kt#+$D$+%^sL1=K|K@Y zJ29_onVI-I2HwH@=OEZ)%D)<`=YBIo#y8|58BO2rDtL5}PLrwH12@mg-kO=}8b$W5 z1Smg=X{vJ8;V5_;RnGMQ2W#84;p%2Hf2VjeZy?pgz^VJt^+3omn>k|hYIzfCeC#vWLfi+E!eSPBM zBtdYE%vu>m`EvNWTc~Hg*iYKOk0NtYcqwFh+Aj72eUA1)zPD{PG6-8`l}6GcL+|KU z3>OIrZcyZ+ze~OmytB5Q+(ai8djh8qnTqLZ3yC%>ISopzFjAGSB{zfj#p`Wi@v9IL zKgYk}$C47*LGwyR86?`>GU`_P5d0eB8iICQ?PcM-9$5G0w(?l<&=XXh*gibZP4qNB zDu1a|`>O+3uX*Z!6f@NG;Au>y+`oGx6(4xgQd-nitUFJ6lZqA!LL2qIr*ewAj63Ro z=5GC4-&;Sl{i&h6KDN2=nRTkBfbz$(%&WhOix%Qc1k8PvewL{!H*LBj_FU{@m~c<040s3*h7XTQEA(ki=65upu*#c2v!$h|NdTX zt&F4$z@QLNNH1#N=AyT9KcV3btFB}+ZPG~pGs}hq zZE9SG%Tv}hQ0i~ZG~#pMkcU3}Z0`>_pMTm3q-)~bJ_Ts#*w8&MT!L-HGn~lZEmq z;P3duQMu%+!FbD(#yy6*TZVe6DWunq={>hFL!@8`+b2~*(aOMgbW}utZS*&ggS`%V zPhD-nTv;)CY^DMp`($pGeo2T_YGQXNbzgLT9}{zn2|3)5VnRrE2BnOns|}p0yDI_G zi=ljgEB^#@18JzdwPi=>?oOc?Nggq>wc8^8Q$bIJO7*ztV!30keb1(=@8(j>)4Z-T zDIgmXyejda_h-w0n^Tb}=WSvfkEo#-;-avjrbuuz&YAdZ+J^O$N|+|m>&=3_w?WC& ziKjvQ6S9H9r$NQh#YuM-F|0TP~G};zGsl3axBU5Fe5W zlB6l|cvvv>j|+J((iNCkQ~iuda{t0$?F^`nv!Av18D zePJ=%n}9B{WFNmb+)D1uV#D*yfBM!3g~SCKGig1)?EVh*Bve8$H`COiAyiFm$eT2z zP{BmXM~7`19txtmDwX@c(W5%p{a@a^yiT%uJ~}9YGYlUK%G=P&0a9n!Lc%vE(iwci z&sIwQs2x={p5ZA3DQ#_PKNNwO@VM9Gf(HElY1*V-omK;K%vTdXyVaBL3YGZnW=^Xq zwLek$Mtb|4XJ=8PMyOHUx_{b^OhwH2>5y*QiCOmzLV4Ja6oYtWm|7x@Gf z0&pXkYn$z-s;4KyDD9yPE)N}%)4>0v+FT~ceAJeZAJf^|PGy{OLAgsnZzh%GNAbgS z2+E2np9XK3zhH!gKQA()kgLTPObC$-eA`7b;G2Y!FY`{Hak-=0Kssd2L={Wdixn2} zi74L1Wt;9}-cdYZ!`0P)e^laiVSn54vP$nxDs2;kuwG-6 zm2{Z#1>^5W$6^L2G1Rb`S4vzXS}7)`o4z4>(aLZ~txrmM#KrU@gRolc-DaZTcpyaK4+JDw*P`y5fWVW z$CF5(x>#GN!k2UoLNtl4ITr{5yQm{X6k7hish(AT*j5KA{XzR{kXlUwt7W)2#m+-@ zHIuFiE80!iBA+q|A|!<(QC&hg6jL_sWfD4pt7QfC{4W(o3X}=%FlZkO4FvniLxHlT zo2G5Z7NFC6_TkrlVc{o%5RlO1;a0+(51r*>H6oGorjB&%M(Um#l5+Q1By$!gZ^<=g z66$?MCVt*kd-c@d)l`rx#boHI+X6PhzHcJ3(e;~%X&mkIemCdN(Ck@BxBF??=I5Vk zT7Yv+ogRHq#U08p_9ud+7CFJHhO_LIZWK61N#m-jn%YAo`D0$!P^&cLIFFRI?RDKv87@eq`Bvy-yO{Nh0AkD!?_un?mf$*ec@(94be>-EmJ^D(w!I7;G~;7_?Kl^?c)n0oWeD$%lpT97gvpK*yw0kJ zR|4!ic(&@tM6L?U`_;}KZ3*qzWSaz4-4ykF7HC*2D6Fl{Am2U=6yZoyhPI6LFmE7_ zjC`E8E~1M1`x5(;N-^c(w1zziI$(5*0OiZ4{m4|opfZ>hafJXbbnGL&9a_UO!MYBo zZofHiG^2r4amW&^(w?t7mQ_u9zo8xrCfbT|EO{7t7)JjFvmCbhgOK^TO;q#1 zqcnG@O68)c616n0*D(omQO;{{8f2XFh7?)vrptv*ynnS29#cX6AC8NI4vobi_mrc% zGSG;axJN<9n9KHEu2P8w)B;4K66*04H<$@v(!?3DFlf|h+Bm{hHwoB2JiTQoq>9^u zrmFxeS<=|~NNeybY-1g35fc{qrDaA*Pya|#pv<`c7HRBvb;L6F<*Ic?!e^-iKKet- zEt#|YA#wAK|Mw%-`~MR|K;WTRMegG4dfxe0b1{^Zj$jjF(k z(N2AU)MFq>i|~(^cC~7j#`My=nRViYbD^cLFAzRgIO^a{O#a(knbwU>1w3}F;L*M(wvV<$fK^D>xpj$#uJ$Q?&&WS$w zrkd0dLm*O8gj*Gckvx0RRdZ>uobXVE*Z<+0BgyCYDW9=vI;0j&a*o-K^8?RXfcA(- zN7fq0^h%=z-$b31$!t@+9q=R4kc|err!kN@KAr?cuM4tT!ay*7B=i{VVqH-6mt+fu zLgjAa({DtD8~61=E$(SaDEiBv)a@BxIYTYaM!K4t&TUM^PN*;THgP0JKf30kGp0Dp!8M} zqGti7HLhE*OfUL3V_w)0)b}%fUwkj0H4BT|Z-%gE7#?4j*)5|%Qs#~cQEUvT5v8Zu zxfHXkc~76hSe#1T=67#)Oh7QNHWk=T^hn5Be|17=$4_=<58@TM@H}N? z8jS5ygd6XoKtn#3>5`SxCHTS0-N4~R1qK>R&-dWnFLWMLH)dxDkh`iwL6HGODUC%V zDXZrU#l(tC^$Fn9s&Z%LKHyN`R+Oiuq3?xi?S!%nL3cwZ#tWW{u(8g8V2$FmX!QAM zKYeMP_x@>QK8S3@AL^F;8Q`4wkm!^46{-X@G`KJt;Xsj*(RYI(HP?$WI120DZhvI_ zzI`QXme9ap(UV|=HGX8`i^lzPbgY-!Q(8Jit814W>z+iP`t_jYVV|SQ|MseM5^>Qb z{nXeP#=vjl{zRmOl=9nND`($KN5 zpn8jw4aBBz_v;)!=ZgAsD)bAM0yN0}daV-UJ!dQJ*<-<*qkR>EhxUQnQf_%$gR^$IK+T$5bamhv8UUqKRP>vLgKf6kw@ zkS|%{s$-yIQg(XN&M}>(O2lBF#1!`UHz_L>2nIh>SMwp#Eu|0Z7Rt!Ozb-*JD44v5 z2m9Y`slUwUZ3?-i`tfSQa5}yHPw-35^&fZ}_Kcxhiy{AfVGMKte#N7Reh`|^8h{V2 zd&keh(=VEc4bH;1l%czkJ1Hf6L%RBkd9cnYttav;s{VZ1JNi?46EirxQx8a@r^B;F zM4e-oSzisU_vcrz4TUk!<+^83=W^EVyBCxPWlg#N^eXm%`1kK#WnWJa?H?XKi0RK7 zZtUZp5faJ~>my*u1q7`9U1&4yJ+Wa$k(S|QbF;1KBq*7hn8bFn0hz8)z<3%4!qv?r z1R{?<3mzz$NEbCfJE)8Tu~%2(+Tg)1Oz#QKD-(A zO{HUS@O5lL4$B4&?f~YP%UdNNI$|a>{`x~x)!mmeqMi6AzbhTN1VccZ+3(LdmvAxO z07X2R`#(z0;mwz?4od*jU{*^)MH7-{MxOL3%!4<1HYTI}benhi4$Ol8u;(uIF+mDl zKKAx1W}>0m{l@ZRUt&;9vgLUzZLDJaoJ z()%&kX!fZ8@t8vSuD5pWWF>P3 zhgGqwmeqD&ihUDQox;P;;NMaljjm&PZ@&a}9bvm5>Ci{ix%fG1V$VV`Y1-!(5~JxKtp7BQYBj&tD?Mo!!{c}d#^cri-3P>d2nvD8X*=z_V}CJdcuE9U zhKYcS4N3}S0-fes*-LOzv++;WyvEo=Zz{|~Z*HyU*Capw`)zmKz!e0{BS|P)1jTh5 zOQZWHob(4atB%2NDBwbMb0R z7WjPKc}6;${LPL4`sRfO2fDa2_q4awd+3+{t)A%BJ?U`zAN9n4*<9-1>PchAN|v|Z z{P`E>oBg@D{^*8(Bw8K#qxY8|D3&!s6TYsna+7P;QM&lp2yioWAJ;7CLia{g9n0GTs$%XIyeD-SVZ>xJgO z3v%AYshL5^rlNW|W2L$erq7mC>Z08#Mk~s*a-P_bDOF~7SWJeqeD#NN5@rmE*0u;`Rw zhK6N&xw1X*`loSq(GQMjU-pEXbPUL$vkL{hNZ4ZS|E>>D#_E^;W^^`X7n#sPXopR0 z5e<4mg!BpmeGgV+Ve8!e{n=-WO>`ZQYWI-AC-idiJztbw%S`Y*7>t?jLlJTLgaS2w z?x%%B=eA`7g}_ulBHOv46sTI~OMr9RE@!JoFidb{wCg^}tv} z+(Xi?m*}_*VxG)d**)6Ik!B;M@aNunM$TpQ*@hwuRH%UuhwSf+N;J6Ib80%nJ`zI*~`zEP-*%MP9sjaTynl&)`=VC1N9Cv7?9@{KPooEJ03P!soMunJ^ zTkWnZ-!C7e2AR~Y5*3*&JB7K;*!IP|Bu3rxF}*Qs4Y!~DW$`j@HLmutI3@GX=-ATj zk}(OMT9atV*%fHdQIa+|7$<+_RI(Kk{0apawv_$`S#BEOOoBKJdr)A;#387p5r|8V zpLi__3Z(y-F@P$?{R ztFERWy!4bPTAD8OCem?aC;&!UsVbXhxQ#W3CtvBlcm1WHvlt~`?v4-JFY58HDwQ|^ zLp56rVFDRGsxApVcOyIow0q7$0~P!BRTX)BnB{?z_cW5j{(Kk?-hr(|vCPJ=3%ND+ z+$mpI8#`ksmCj2t>f7mWz>fV=G3N+?u%k+S4&9p8FUKue?Yx0);*2L0aT#ePyM4UC zlm3hQr@UgujYKWuxq=y8bNq4Uk-x(!7djER3VgGtO!9%a$9{c_NC_`j43=F|MfdI- zr?OHWGv7ZznhvwwNf?uz7VX}NexkQ&ZS6}n(h^wjntFX8Zke4RadAo+c(uN5Z4S-M zo~2q=B|+thFqqt3ZBwK49{)kyQZ@4ILL|3j;QGL7ElnKGe&o<&A7%sKkcrTmWu!Bk zW}#!0N7<`(mr)MnI+H8qDlW_XD5H@dw*IE9>OO1j%R{lFu^~oO)H?)a38bpNItC}t zN?z>vKFkhHfM2g+gJ4Yo$?aN-M)qru$WlHyY=!oq&6M+ry&|#ckn0DXQ~%_`#%6|; zr-EF$RYTr~aynvT_Ih~toi!a99aHl71iYrI=3Z=V-%v#X-1)!Q{C~N0f%55&f+>a8 zb`(DW$i^h^y#(Vys}9y7uu7^17`_9tl&ZSo-!!~mI!R;KI%x4>%cpeuNY#eq z@eDeqU+@zrnV+X^W>EUxy{@haHIY}TobuVxoiu6(C6x=>ebmuVDYjt4;?}MZulh+2 z5uDE_+Gta~Bl!aClP)Qxo;fg>FUhNQ*t;xWypfc==rwS6|A)tK-~-_Ha#{M_K>-~z zj6WJuB8q_iolH+b{tn$K+lll@=Fa8+*_$n@&wVV=hiWgb$;*{d(CRXF##)%h%=VSf zH2toJTS6c(S3A<@=eIfD&@~togwUnK_#G)?WegOPy(%3hD)j3@r}uh+vHe0`$-)pG ztU++*i9K*Ry$$c4CoK;nWe5>dq0?;I(f@2yGsnrinIY-SZs&8h3uZ)hTx^+ttgo@jbdt5Wo2A0dBXIuV?QlY~0;Yu>h; z9aR-bjK!DGc^d9hdL{Vt*PjF%4sF3pwQ1;DS+&p7?&WdnDB}bgu3~`7bj}ELpfE<{@Q#K;K7CA&Uo{0>4QPmjqxGB7M38NSW`M zEJ_6$g=a~oV_|vKPeM=yB3)FSo<@lq@Zi`8a>Ajxvri>eRXH0Yib|FEbSwvu0LPuf{jI=Vu)_CoaA~nz@5wJZowuy5c?-Pj}96TrcBZY zDtSl|NW-ZadBX`s#ETe4L2H(Trty7csLw)+C&F55G;D0a;nU~Pa&*n$@6Wirui0RQ0EQ?UqA&fbY6(Ib<7P#J;YZdx27|Z*VWPV}(xy<|Bc{+~ zKf~psV4TpOUw5q3gMmfJFwZQ`8VJfqn@Xt+cE$*KRW$zLy{@YC%Jz@jGxZDO5vb1- zRjy@lgQ~9S2e(`%9bRgVt=|~WaYS(bA%?Dr&b4c%*1Gus1Y!^ctScLVmS6YXDfio} zY0s7EaaAZFEY7aS4^p3^(qf(&IhK5VN{IGf;t()hx);u_aQp38kiKD`-2HwNT{;YB za$UxqL0DjE|NP}Er2^0ccv7XOqedGbKgLSp(ArWs#Ta)g>R380HJ4bPQ4yZY1mUtO z0Apd#s7{U~m21`P);aOmUEQm1d5Nk>Q;{?2WId4WBfPA1fUX<_PYoX3v|k%hX+){_ zApMvf1B`GXP{>FsfYv^R{sIHmmZ?WwhTEx?<>-7G-y|??Zm#&Hy%c5pvAgreosTDvqg*e`AqQ@8h3W9RIj=-UZ3+hn1ZyG6kFuNtx#?4?z=R3+Vv&R3IjM zxEUL3WOBLk>)Z;N`WBpFRj&0^(ZeqjFZ`~$Zdv8IS8qlxSK-5U<}&5i#sKayviFju~!u2cy#Pun}E(6@FTDO7@0r4g_zidkgYbW~e_ zn4OUD`V&@^0NdKN#q^5l$~L77*>#V0(!q*yaUBcb3FU%;X1vLWgvbj_K9+p8h7 zm?;#4nsHvUA5>AMXXnc3j(7nEW3kI1EGY8cO0Stwpo*~OzgqvdpN{CpVALY52-y30 zaVh@sc_EO>d<=koB3G1CMm9;APXF%t%Dv0Xg5SW=Ar0LRwSHC4+FWH7mc!JDi|Qf) z2&$K{KdKZxnPM5~$6|>JF$<@oIG-*UY*$+JL^4+>!_MoH6Q+_E_SkfXD&LJOcP5Ir=d*Lpx#>jue5gwyw4dAqIw0Wwx^ zu+MBV>s`2yYjuFY%`9t^CUrQ-$-}B5^C){|?)AZ9z%t6Hd&I4&=WAd)E*0AOhre?R z94!x4T25^Y%NVMq)+L=3=dJ0lbX&3JR^j3b#tRSjuLC%fybW=>0l1nnE{w5sOcc0o1C$`Yk~|4jGq0K+_WuqDC(BlVn7BWn4}&;zZB%A%yDp7Q z(BO~2kbhydqSxqwGK+jcJlcJD%b#%)p})aTHnIPB9q4~g z^Kjz9xs)?Gr)#UDN0b)4OR5I@AO*+#tD{OksJvZqGJHM^os&4~_qck0XPvBSY|oey z)}!Kr0yyLcyt1OBSty;=Lz`(?)b;XiBQTM-$k_mjItnEZF6`RgBd$uN3F{Ol6w8VX zGed7ty{p2l+znK4_0Jxor`O)jc!o3mm;H^5mQ$m9J8q`W?B8Dx1vXXO3TrakRiS2{ z-(LS`grQE5=5-G;Jz`M?c$jAU#F>I2X&A&+IX54k>$Y=yk4Psu6~(hIjw146rI9K!s&NY2{E155!_gh7yE-4$a$yEVq{h~i7KpWP|T zx(&K%G;C14(7n*VxIwzE-|KSmiRC5m4|w)|6#9UpX0fD=)Y~MPZ-J+gmjj)1G)JjA zVPVy)Dx@AD8YA_bK3U8<^C?TuoXLQI^yl0^G}hz7|I3=91R&z+z=Q+*J(&Ux3RLSf z=3$jZcv~r>PIM6P?^ogY)ws}N?9*xT>~Nh*U=TtynZgd;7`VGLK#9`nL4v_2ZziPT z;85bXN@rQEE%4boT2r7wTj7END}h!TXE=H)EyPbZkEtDfSMDmD zEG}T41?lQ>GEcixq!ni81(QR|^!SH(v!MW)`}#Qg>BkN){Y&kdW6_fa3-4s>wMGGz zgNDXGiC+U-1%}@9`A=3JZY?gqf1|Ts)JsMdQ%tsg+K87zIB>RY4b z3gVBZPZ(aEA>9IrolR*ve9|htuZ~M2OniVrfI3Rk#fx@8=0NAwTe>KAyxirWv}e*C zZ)jzG_b(Q3I{uN+rsdNlTO>mav$xSExNklXno%z|ZqIJGXj=lVOQxQB%-h)McO@4$ z&1{fFbauo}7JO|8ENQwwP@|;jy4zUq5k?jxqv``SdVmaSVPM@1I)NZ(25lz1~izZGpKprm;6Jmd&;hr=n%O55QRla8$`)YY%(d$GU~8IMAKk!Z?5BMiU-Nq^3ZJl(JF zU^P+*FI4p=kS}kCeKLc7Kgc-hyD}fm1#0e}0S$hpZ?+#?=%x`XcEiPqDTsR5P7oQV zF0)biY1^(Z-Ib~(2nf8Gzy}7Y>NGmYF%2b1g$qp5gtV#`!~CEGb5gG3sQ6YTpI8g+9qDC?wusu*jA26}n^^e!%pygAU zqn9;3L*rfblH|?V*_~t-k+1SiF7OOvnb0Sx8h6aIS(BlarsGw#!@Kwz;>s{lV0`TM zXWjMjNP~J`^nzH%?aK+pg#*Lfg8cA>k(i$uM?$1ZwU_rqQ@nlA7faVGDm=w#esBNg zJa81F-zq7*VI)i)$h!HXYxexdw4USwWEhiKN?8Zpq+!3`ZTHn}@dR%fhg!ion@7R9 z6G=+aZ{-a##%XC@yy?%B>2$!pNvwBh9vrzY88R5*K5K?YRE5XM;?Yp#4LxE6hYcM<`7n@>@%rm)62JC0-fIcpBA&ZWFKrOala$A@{7Vt_Jb$n^?k#6$oo`joM0xqYKDhRY_9Xv3BcL~nBx1ovUg^yBHad2G=F#W-ps{QyB_ENt| z7k|CDyl8Kf+AFzlE5+qJYD#N=m~(7XAWs>rp2#gP3gQ^ zs6zRFF1*xDI1zBQ-h1E7{M)!|+{phGjo)3Vk5Wj9;b+5bYee8@4Z{=vKg&o4-itZ* zrWMbd+kx?ux@L+y=Qi6&JkDI_>(#i{EX4r2SA|4@?V{JK04q)!LQ(akYWw}qkB>{= z)tJ0%w8}?S@f=ijW>1nT!&K7ys+Tr9wg?B#Qq(n&PO*Q&BD0yH3E`31}jYaKrDSo34WaZv7 z#Y*H}jvuJvUj(_rxBemMmq)cu0DxoCS{zRCQ}HWxritAgEcW=NGOo(X&Az?XHq`rK zqOHx1DYyUX0;BMGelFov_kj(=jo!I$+}Q^A14$u;bp?^>@4)Q;$JkqiMg4C5!!p!R zl0z$9LnGZFNH;Td3R03vNe(UDEg;Pd9g=H3{aeLs^vgOMDdp3(AdTBXPl!Xj^TH{fg zR18GxNK{U{@%JE;e_xUR`CxKjOCKTyXX7F6fxXVn*9hrSSMAAM=yO@QV01fP^~pP? zB`y46FP>`Ip}Hjf-(CP6!~qyVlX~axm75Q`<$_u@dly2e7bqp(56M$&7g0GQ@3w)NOa^tfPNS0~4uUWK zlj4!(j4Z*7QoVM8r>}b4CM04fA4Zl?&Ji=Dzm#9*ket(iTnhnE215zTAC609-U%kD z5BBE(bon)p%6&Osi-yJ*knn=ExMfpu5g1qw+Vq81j!FZAakP4yL;k=4csRXF5GP2W zmj8~eu%b_qct+W`48U!nF%Ub=W;RO09@dE^w}Py|mHRg**m2Yoqk)PFmD*o1ZoH4` zbhEWZv4<)5iIblHINROUk`Z>=n?E<+pF#ANDt@Q1Od2H~Up0#_*teKIIyTj zeF+?y3h|{gKD7(WX}LV3d^Mv0(Uj9?!;=lWpgP`2$LwE{PAxbqab^*PZudm}7Sh#Y z3mJVnr(5LF`O)`TOpWu9sHR`j>nWM)4^5=vb4yIk5GyDl6d~`fOZcE7)lZf( zRyQ{JMy{alsduEg2mY&81_Ldcu(l%J+n0rzpScf-LvnsrFAYG-4_zW=*U+3_NR1Q* zzK_XB4}zflNdD>N3*Bp~FH%XRuYPK*;QiyWUEqcUhFPXpcJE#Y6pt`(a6Jnz2>JOe zs;2wjVseT9D<3D=^rr4V{XBKb^AV89$(>#VFW^J*fd)LD;IO+|r zdLrXpU}E!fh`hz)hZV79yoVSdwNmy&+-qNa{K@)PJC6kq2g07rSL-DsLhW=y4q>>Q zH21#iWFcUl82OwaR(Y>bRe*M2pCZa;A>5zFjih}#Y}qpsVi5O1pB=5PSnY~attyN0 zg(l*-Jx8_+%teC4T{6S%G9 z(B@HVHd4L4G7SJzp3`?_aE6EvzVlP=RM&^e=yXA{^Z^np>Gk@eU-FGdAe|@ zTSLE}sVAyeW1nZ~?!EzY$~4qHCx>aBfHIUZ84^ku>wzM^80Qljv)tb z(QI!Sg?05+elVFZP$|nmz4Nf$Ca_XLsFzry8qeRomB=3<@7JKhonwi#Ke=b_+fkyz zp;ER3xIW=X3aB1I_u%4r;x%-}UFEXnAfMSOYhgTXmJW4)>AJH%-k$RE(JPl`j`l&a zQ{UYYX!|So(ob{H9q-D)pj|pQR1R0wbR*k1w1fNiI^~Z5Hb@GWSPmt1C^~OBs|p(2 zL%w?sK@!{YRnTDZ0Vv$-54bmF;kgQ*=Dyq~-rog&CL+Nip$7w7F$J zB+5BX4A=O3Pee%Go$S%;asCK(mh^Rabr!YgcaTRj6@eYf&F!aeEtkH%&W&{UCqf9G z*}Ym2o!25%9ao=Pl#@&L7qu_9vAGhSx5Bl~82R$z2rN%aW8iRS|FMR27U%(~VR zC=NtQ6)tF5pFK*t7Xih02KQ+7pS}P2XP64}KzqnzrsmcbFy$utk5)mI0-*D<8hjXq z)4Gb_7ur4RFpwV}3_#Dg5r~G0+?jT;<%33LGCHr?EiBwEBQRtEOSfV?o-DMINwMbc8>HQA}%P7^YQ!gP9kX7 z@0XY?=)&da*mF|R`5CSh@Knu|gd)WEe7!Ek5aiCP!^UJ@kWhkhPK@5BEk#^>?-c(t zQh(FA5Mr4U9pumQgLFI7ZX1gYy4eF>-Rp-eu*jR^Mu)K1A4PuI3N=D_7*OD&z2{z- zPete>kF^w1&L2oCrCnwz-yIn!U>wwhUSsxpqTDO`CE!}~TUxwfCYqdSu9Nk{8y`tq zuD8Wqq*~LTW)xO`Ch$yZM5JFIAbu)x<_ekEG>bpqF=GF(dZv@WOA(DZ#+Io&ikZPs zVk73uA`K-B$?x7%CvKYV6&1spH^qofwf@oM{TU6n3!9!4^<+a+x`2|`T-8?*ikA;} zA(j3fc}FgQAa^HO8>Ka{mYHxsS0wYoEjR-HwCxIMk zcAUg+bI!TwJ{^4dP$8P^_L#9fbc97WX|}G9qe_$0k<)wB4K?}G_MkEL_L@V|fTUM2 z0_{B`f;ZDF~*bq&m(H^w~e zNpNT``V{ctb+Ijo<%GrK%kk)RwE4vjx#S+<-k#5I)jsk4OavPAKxoHjGItCUK|_k7 zvH>lFBKr#0^b#9+ZJu?3;kAhSojQp!y_yjYI{?Fydy9#N8>@Ep4IyzhSi}Bv1kj6} z*NKb@!S|9|J1S6X@Cqb=xrk%SZV>1VaX(?-Y2$&@9;UtNPR z3Q(1*YjBZIlJR13bw(EM@`_!6Ub!@aUrUXuG;S=>)}1orcZ9JQZ_D7b?2ut-s#Cn| z{#xjML$%anBcb-yC-u{gyAq>Cv)%7YRfVp9hJ;xI2A+9EKgcCKstIiGXB-Fkz2Uto z&L1&8+c1`Y%+OtKQbx7AWA%VkLYemff-Q>JQU_->G@nHlgTp}=bZYW`y%oh*COUbm zUcc(C^4B9By-mp3c?@aIiz))e-x^NC?Cda>tRlMu0GVYdT~q zrAG%H&}P*|ydPS)e;328FMfT`Y5!L6DX`ybh=#}Nr-3AL^x=8WCxZiGC-YMa{4zFo zR$oH+kt5STlTmzFyom=5lA9>$!mKv;l|KmBgp$B_O*@cjkS-BNgHV_dXh%4L5CAKa zg38Ir|2DVefbqaiMxu(4Rz)5tF06&%SEs|XL2LyiVC)=@aUxeZ2d$YiWv-0wB(cUx zCX=Rz^eB8T>LemfAzQ(q6`);((D6+W8FfzCzq-qz=`GAZb;V{?Gmof}V=MC?Jb9Cm z%&d7Shw?U+SyNG8qzqV3iLz?gr!y$yKa8oKDWfCTIvAykp#SjX21{>)D*8D2T%}Ll z^q^tkiPSD!?yxJ^*9l_`f)}jSyQ|*)Hrz<*p$@`KW^Ravw*4V|N2Tk_Uc>r+A8WV- zXJSgf<9=4QosAj*U5Vw%3H^ZVSVQfO=a;d(`4HRCpp@i3QEe(+sklC9LXY2`{xVm! zzzwUhquz!J{p%PVrLm;|UiE{TFVWn9zE3#Br4FEA-PT-zfW|I%Z zw6a#_9_DfY0^E@}b~63(p^-DL-Q=ne-oIMuRebS{G;Y5K9EX>JJ_kQ>O_|)M3ON5t z)rD*RYX4TyXn9i55Nqus>Ee{h_S|$ts;2@zOPibZMbdG`i+U|dnv~{@nA#9OyroLcUKYAoz!g-(@1KVh~w@W z%5Xs7&1+N$h~`CGBe#V3Wa)=$Ykr~cg(H*iLY+(k9iM(psOwAVqb#2lYP!S0(K7vA zJ6&M{jc{a@qqB*D0UUj;Ua4GSy1B2{o=#3{E|-^k%9w9CIVoxf*vwwL50g&6YIJrF zda+bwwYnyN9MiGi^~3V3f1 z)-BggC_koM*$#|YMS`8)nbIBj1NN~nA3hydE^RcvHxWj38pJcq`UkSfYHpdGTFahd zIdm@PK;4_FXTf(OW$=C4`j2VL*MQ9*2{eQcjEA7HpKJ=HV^F$x6U8~SKTmBA-Ilhh zFs_iw#DI4!dU>8y`y()5={RRs@?-hAPDf;9%0hnkxw}02a?1v%#A?y0)<>(!K- z@Ef@oMMsV{2f}!7+R$sv=`>{An3|<&#t>74ZKswuwgRsrgX#Oq-dm0&R4qg@mVa;a z-G0@P8n|_>^E8vU<7U^%^qJhB0{X_K%TMxY>@6+qd;SJfsz=Q?`UgihM&C6H#nT8b zkfPFPOt*j~fD7O+Py1FewLc_zVIIa4gT9OECI16%JS{Z!Tp)fjO#V< z#lMMW)w_Ch?du?st?~;m9?g1yN;Ht_iem_2+AbR|qE=Mgpf!>Ut%C03= z?qO`|&0OZtcocdad4-YEare8`W!Wz`VgVR741}MW$7?`$D{r>bfH*j^`GJCBv)nSa zV~rjUy9f1V^8LyIGLD}=RhkCG!|vmJBerjABMRnyHxv`lqD(X57GxBa)h8V4GM{BoR{KAv16r?USBb!kf9toiWAxM!l0h`u#PaKFqj+jU@lxfzKZV=Y)(xg{3=9tuTp|7_VrYUSKn`5J#ynn3SK% z3v3sx~N!!?83JQqhhkZpILlearG;pqj7*eul!DD60 zMRSX84=W(#0CWK5k7c|to%;NC(f(Y%eyRS)K79JJ*>zQ2QSc?4FR0@qCR6fTq6^P- zHu*_YdAq~PS_}887-%CE%3|)pQuT~sA@=#ylz4!(HOl~){@xx9HYpYihlIOya}C0- zOjGh9;GozYWwV9Z{=0EjGrz#dNx&s8QEEpXl_3cm)O~xX;UxF6Wq^B2H)Zt8Dstnik*mc;dD{V*W!=tW0VMvtkT0jD|KPZcnXRzMfzs+R3JiY!=&^ePHwU8Sb=8 zF`4+I>ha@B;kgf5Ao{@m<8NZE4O`V8w-S4DcJ#9zaef4_KlS^6G_sI1kx^gyBvSbf z&NRfa{pUMQzWStai}&@r(lgOotfz_6;T@!fn5f1bVshqmHD$l-vjsbBtlHl7}qGL z=`7Wrjv;MOfID3ic<+w%hQZKj=bl`@hm0L=v~ZFiprwK?K)K$&(N|qcT3|4)C8+9U zBD(=wlmr{#;{nb$#$DOkOguuP-;6ROBbaAxBUuD&n{9jEX?$h+6$CM5?}W9tKt9hy zL;qvfgOtzu|K0oLQk^R1(Wf&C*R5+g{v2i#Y(4&{`g{|#=bbsxB}H-G*7!3l!LR-k z^DjJ9@KRJ)C%tf}Nb6AayCkD7LwZ^H=jcD!5@$E)^n>|fDE>HQg{-BxWHJN^Ip90C zcKkm+drJKj!jy13z|gJ71sKLI-#RGz_uff=1==nsa$lOt+k0#_5lP4}YnYqCU!mIM zZ#U52eI!7ETPjd)NYw@AR>cgCQQoR}3=F=c%kx666Z2BlU#|&TQCi+6mrHY5p0Ca4cSnh@t ztGU_`Kgku$AdlO1!Y~c2Gg{}ei6=HOFE1Kty{-SQztwna%Frk|Xd;%;fdrohnvVYI zc=vw)pNvfiXzCNQKCb)UNk_Vqj^JV z_PhL3@0v%a&-S5wvCD5)bb3iv|*_GzjgqkIr+2N#1icH=P`!hILQ$E6B%9_jIc>JHB9#Cb_!S2+j zxnX_eK-wtVJ;_D7T@pfFi#5v=djz!fmvYLU=ncqd3pUw$))?MUMH*(QJpr`X9R4w} z{)|K_orJENqqj3yWOx2E;RU<=*#v-M$l$d(#7$|QuISqS@jc`3DDVD<1){T zOXwB$?k8`lPyEw@0ehH*ZhDX!aOG%@Mj+BC@IQJ7q%}^!?QpAu@gk)iYE%@pM*idl zm0xl>J@QFkQaI_adYgzuNYPW@7V!ztJ`XyuE@yE37U-4)XiW?W1o?vGs`k^Ed8WTI ztL6EE$c_BHz-LE%oy&{LcOERv!;oZ?B?O)PVy7*2ZJZ+?S}Xw8f^id;yMUtz>Q6NC9l_xtRQuYQ>7iUC)A?BNY=>IlUV32^#!Y zcv)FXU+GQEvCf;_EPia;W!|~1T2dRR#Fz{4vEuvX#SDK~C_{RzH9lP|Rk->bp|huR zqAn)DevxZT_o!n1G_JnI&~YnH7qL?L!h`5#Jv}qIjlfH31Kf-h?yBf|23#KRS&n-i zKt7kgsT*sSsw4A>LF3P6UUuiu&BflErNs! zn?v7p(2%NqQJ<3{Kf0*@84Ob*p~@g;fh^8`rMd%^SYOvIz*DRN{;0R^tr4`~X3WC8mqC>#+%^(2IjVA`Ietrc3+(U{T&ovOwnt)7gGxbRfTj1S;ZnJpq7E1NguMw zET>O{Y-BFr_n@0^zZ#Hzxq1i z-~O-KKdkedPO}j297$#KN-QsYQ)J18T5@zP7Rx^NWS_zP zD-zPIh?*8BDgP<~fZC3Efy@mksz?nSm_e|0T`dZ8=33B0JENzmB71KNlB~zKT!V3o zDb?9gW(vD+xuVgU@|>TDTfuh+gZCh$T2+aWk>N*2y*7VnHmaznNH1l~ZThlre`Sx@ zkqt!V+APVB1$}DGxo|Q$?NAu}zAX^}J+IToB`E`4A}+c7oRyj~nQHYyaOJAna#@K{ z&+N-ARBJw5C2$rfO8N48S=rR0s)+((!aR}cKq4VGHs=7F-SwzA>=d-E7ShSfQ;@uN zf-eU%Qc}wu6-2HzN;~eQ8aM|@Q@_;Z*1${Wdz$o1FZZ6dUtNmuP02Ub+owkz+Gud$ z-d5kye+=7n|KAPUGMq~?`d-?w;m`kG{Wqy{)uPUf96%h^`H@jS#S`cCm%A$vqCds& zBlPXFQbX_Q+G*~5r_k}$`PP2uk>WLmH3ENlRZbVqOIRsLtIzqCemeVl`}h_Usnk+G zk5^%1O<~{24))(_)M}v*?84tRAft%Fz~J3rkCduYE8gqL35aZ2#HjtV-nvLJns_pW z^uT`;Uk;Ew9A_QtYaz>5-aM0O-;IFy_KYMh;m0XG$pB}WjQo_zZ}A~x_=N8*U&{0w zFYSi=d%`Kn^@P)dE+csC@x;Q63Gx+Kw&z}cGZYWkyq|?4v|s+AhnBUorub+%pqfCEv0NpM-RPM>gb46+bm18vT7qT5ywDe;> zf;!#x<9UR`m{h+RvCeO1?Q&ep^uPL#pG|w`T#}Rg5=b>s@b>t>g?s~$L=M4DP*HhI zQ7`I5W>}2NFMx1ueUUVq)1q7Mw0!W00mcc?X91_*Ba0SMd^sF^cPeU>p0h2bF5nkS zH8-r<-0Z-Vx>Yv;!~s97)PSicTxl3Q&O>X5vRF&=!zK2)RIZR7yce+&qj=l&)e`vMy!mk zXE6fw25_~pOshQDg8klFoDy8E$K{#~V@U21-M8W zO4}f#LKhqgE~=KnaIAP++l%!|`Vt~ENk|%}*{dwaZcybEYYXz`)LtSljpbm+rt-7? zG5hPcLQ{6S&$ako15tieFelPp-kYqotw0&2B{oTUSH{~t`_%9vb>}8>yV1~b=)c4X zqzj4!AW6D@OpQkXQ5Hbr@U^q0pZC7KUNN@B{K@7kF^_RtYuXg`QS38}+wV43@k&3~ zTnx#vKaRf?-|-)-EV z*QO{|NEI&orey}vd?NDLV0bynaR?Sr9Hccd+W~QBtPbrx=BMIbB_CtP$&m* zJ%8WhEZwpv-C}N#5N9ZPo`M1#O_?2pw8ZuX2V7~Q2%4L^KLwql|dcgUN zf_B@|WqWMN@Tsd4o(8bI)~`W`2O>3pwX$OJgHaA8+I#0k$M!*4r0a!B)T$o8J`d|W z7z#v|NflP4Xit>}B406yFFBdGbL9aq;CWe}%~LS44zBZ*B1bUl%PGh%JeA~>&Fr|` zeYba761nT(4)xpmo%+20AveH?!#jnED@dMI!QLg5@6iutH&OWmxSJ~ZVMq6j%@L6h z5nWTN+S&wu0#*R1&aoBUp+5nx%YB;6Y%|{p*QV@DW)!5$0DRQ7i+>tlBL#rwnHFxC zA@AQZOC~xI^p{mPw~Re?z^9XLgIXM}q)cV7@ac;me8&OJA58ZK`FO~Ou#ve#C#X}a zZ5|(xvgBv#t|wPHclboS87LtVBA~auf^CIR=3Z@}eR3Ygixrt`7vI6CMG6u05k}is z82;4Ud!)?7UQtQvzoS>>iri8LbYL?~**uL*`|3}e3zFZ1UZ#&9eONAjyl^TL&UYP^lg^%rzCZ{bEOeAAn-*!=T8t7D5V{1=%pU<(3=M$L=v48=VP z8sU5=LD8wA9H**77$Dow1^3KbD6$b<^h`7X9b^f4Oi~YnYpbHYjQ;r`D@dtT(B$*0 z6B$E8mw61X6yNHti@SS4v+D9z?Zk>Uho2FU58?quiVUR@!=OJme?F=BD>aSX4V&&u z8S%&_Lysx;75?<@r z>sq0|AJkK!*P`P@(9=)GrRU0KDA$fU6Fon9Pi_6og9{77zI{3oFxQd!pkYDM7={1A z(I*s3A1(UbvZ!xmDLuDzY2xd2?)halycm`G~ok&s{0qP%_*Uqa5A z$86O|f{q{%`aFiF{1uE>^9H0Nx3Z-=ckpK{G=qP}wxsKArj@p{gQUua`*hLQ^NY%~ zpPY-LCT1tb4*wTTVj+ULG8x(tY~RF8v?6CE3FYb=N!L4MCVH8j3eK}#3bgWgJuEi0 zO`oh8>|(Bn%UK?3i}_)=`ywZlSPz!m18rSlmPoKNL8m(M4x^=0DP;FsfMgGxbUp|J zyz~_sNK{Sph3R|s%JSJ9t53-TsU&U*Ml>9ot-`mga{o)w({{l8H>48iRg83*aY=K; z6OmrsyBnTL18v`N_Z^t5 zO|23Rs5demHS^$z3<@6Fbp3R@lwtO)0d*D#?F(o`8p`?UojD*Ec_41IFRdZQnLx;S zK38oT`tPr?d+1sW zP>;Np)!18oRr$6$U4hs;tqpvlyof1D-z^Fj3-%wJMR-$YII8Ob2?U4dckXc*^XwX zUy=P4+f>;-Qdg~O{g3|Zf0Rlv;<&&FXuzrK?!i(T#hrdlvwuNcBE*UfHSU5-b!*@^ zW%O6S#z!h#jE7aN0{TgKqB+c(p<9>v7E26EIYq8qF3(LF9DQRglUxhFFk^lzA`FR* z*;Is*8?-M=e+B=!YxW-b?acG0-&GcZB`eB$}hH%DEoXv9d-P+DonMs4!N2{!&90QH{Uv`leV5{g!mqQ$V{`>O>s ziQ1Yc55|yVPfZc6Uf7Zju|I@c&MK7Pf!Ps_PS3l(VV`E6T@}^4lcf+jvL7cS79t5)p*mgTb zbt|+-_yzZ0z$F8<_harqes2dIpl*8`jF@*?uN91t9AI}9LSaM21KY6983v?s1+zo` z=P&C^$NUuksM%ISQMBvRvSPxCt!H6e7!}d_gq2e>U%G4aMGW2(0y9UJPVdepoV(4z zLndRmk|TpomK6~Otx-Uo$=aU5%k-bRlHKz8dkrEsGKJtv&plm3HXua#BSzU^Lhphk z`EeDAn3=UxAsm34br2Q!+{qN@GRJp&aB({4an4lZpj|3p1@Dv7sqBhEK-a*+Z6OSm zPe<`>hB<9NB*c=TfkR9WCavD9e9c*HMxi0ytIR7a6VIv{p?P5bLOEy)n)*u4Oo4zp zt1VbQIbNp*j1?x2ikWuZiTCB%|5mA_YomRK3@fvtLJAd;;zTcUv^^wq@W_XZrWh}- zwkqf9-nhRu%@{5{1`y3pQ755f?J*UQ%H(L$*A>@!Nif zEh8B(^e}UiSHwmAi-$cA+7=4C)kWMgLrCD>RXVRy7oy8s}bv`hEF|eUvS-5YY z@{=MUbMsN#$vNFR?-3R1xviCuOSrrlyXG6BhXMHs;@7VW#e5a$!?5phlT*g}5h|*w z^kvKqGOPUiTjZG*{=9|i7x(sntxeJPsp#_$cS22f&uOexkXt^GqU+dF<-S32Zo@Ef z`M|2*X4fCUB;{TyuCKzY$C76J-hK=WrJ_tsS#rObDV`4gMJd`XCiZ5}Gobd+I}S;w z{RLvh8CW}1ZxVKFFv}afSp=jh<=&t4jZ{v!iXnA$g;EsZ4v(iSVFePcMfQZc}KkUR8eJ2Odm^b;{8r!z{eR5fvF?E$c%_h+|SY<%Jg=lU8Ga z;wqd2iBcHT{WiYKAqkF!G0kqB^>-|>2e?EuXFu2OPNYw1+}eF?(@icj?i~|sgj3=% zDeO#If~6J~1x#E!DDT)DvB4gld^267>hypQqzeUfHuJgnMUV1{Lyw20upJ}^{HQ&* zt-7AvUr(J}O&@-3jsk$N@ybRMRTE$s^wZFQ*n4q4JM_Ask~#%{L;*N6?|9vJ&MOD` z2g8C$f3SCd2@of{-BvpDreG44$7TdU#?zfsQSNRZ=mBj zmKfTuZ%uBWm5~r*Tk))Jcd%&Cue$l*dB6un1pG;TtfgmGFxk^CqIMTM@LK=)WLfMb zjq*kFp3gwH>U-{Y;v+Xb^p6ocyG`FRT1jA5-Y&VO;v2lTqkTV*Lnn_4j0L`Z=o{Ih zGgw^pszWDne!;C&!Szj1%eih>F3`rYD3;ym{{>U~{>1FRm|D+Ysa*oSJ?0BEW~A4g z{<8J2s28ksh3>*-Yz#{nr&6d=EGAL zzh6A%1BIlIiJMhCl-3#>Ki+S{_g_b`P@zaLYz`5b{+U|qL5i`IfgJ%X4?j-VSC7&( zDyXDO3tLuN*;ovi9;JMor>Xi9-r#NN)kNmo>Qq~QiTM)^+l9tukxt=Iv{On8NSRj$ zA{D(mWHFa>QeAqrl}j|1=Sx5y()4E2WTzO=N?3(|<7B(-5QeqNRJx#iO(Z0vW?Sx! zu64qNj5ftZFR=$70|Pw+gF2*$K1bL!=ZQ9CN`khy#UT3HA$nMrcdOY|qJPYbGpe&E zGKa=AHnwG?K%HTDZ^rK)&(q8dM|S&W96kv$X`RWf{=khE5=cM*kd+1kVJEAD!MNHv?Cp=CY8JXWDa>dgofL%CICn)pUNL4>5J*?t zaAM`Yb1a&Qm7z973j^DI9hCw)9_0UHoIq5=Ki9T5tIuDq^I1&-0?~ z0BO_QhFmwE%RSi{a1-9}SX!Bq*W*2VC`} z_{R3Sr5}@xJ!+oW|2g`txiwRR-x=S`L6%m?et+BDOUqcUW&9mK#?HB?)9;)L{O=Ay zA)RFe@gFA3%rkxH%(wt0?msZQ?B`zKd(fz}uQ4rKZGpH!2yLVaiHjOQuMfFQXjQ60 zoS>~nGW<>;HvJKQpw56B>wQ+=<$8wBJj|%c>q79;9CMwC9yES4@))A`4aLN~`rt>w z4Le#fedkfs{oY;fb`$eF2K0b>$q1VxGP3Q z7jC*L#5VD^coJv%c;5O1i2XN8P>}xsGGv5fJ83#*9Ox=8N<14~GNlG0am3B;0-mYV zjs?x1XsAs05guPWv-7OWXvPv>V)6WxR8g)%MNpa@37Rb1zr$c;Cn`G*2+06#X-cS; zS3f83j%$Hb?MXL-D`9z)-vqWaMshNaA^I85-Bo8gUoLf@w0Wg3vTg|mUD!m(7po(G zif*K|_iKLwuV0o2uvU+Pjz13M;wePqzezkUp330bQj zd{w&F7uo*6IU+@LFM$Ub$*D@6ok)Olz_3leRW2KHf6rai#NX2E$;(Vo-fCL; z{(0D9cU0!kqZXumw$lbip0vv-(q49OnZ=aZc-HlB21e-S9Fs-1UaZ77QsOANScg}< zf^>;uzq0b>EJC@N*^28h5|O1<2wY~kMVDo4CR{o!+Il>J;X%q*n2!SzjVJLJKE@bu zZPjTdujakP`GI7_Yt{q>NdK@|=kMev9bZJv{_UD-xkmLRiQiDHiP~K-*QS9LFX`U& zDlPa^^=zB-PFE7lNxDq8uRX<|T~H(7Kaa=Xum~g6=Wm3WN$p2k3EUCx+T#L@(<=9vqXXNN;iq{Vy3 z%<(Ew;%Wt0^)I^BM?%u1>qXHt){BN|uh0w2^;{3%uPPH*#3*;_lUOZ;zP9iD=I9ps z#`hI!7DP%L=LW(6zxf&U1nbEi;Tl;?gD-1QQzW(!Onm^`XV8`hYme6OS$9OYr>w}n zBzBNq{ZR+*fcV<|rQN_&NaWMZ32tFUZB<}73E<3Qh|-m06EJ}Ah0%BZq!tFSzk12c z{1JNtq5`L%1@Bv?gEGo7`AE}$lfx-mtR;dRI-{KoK2tWuk_jxU;X);#&01r{uy=s z?JQeranf3n9v??R^>nqs{ni2!>-IkUg+AP2z}NVNtT5IRKQZ<{CM=X2$Y zXl|$9io}Vk2!^*Gth9LW&Wg!(3UCim2TIm_!3AFcy5)+MfN*RDqGCHrc?VU6XS8|^ zgc`AVR-p|qBe+9+nk=5(bjCgXusTUQkPB1r)u(WSD{gCz?3Ji-Oy~U(GyaLS)xY83g4PZGBfWLz62PR5va;clZUNx#)?D(So zy?oZr9{7`nc-jG5ESX>SAsK*$z-8H5{YjE=n_s@PJl3+5+GF#NmyO&aaD5BcK+Nc{ z^yTk_$*FX<)|Ss5@{BEl-;gM{5YG(fSC;kBEucPsUOXzp8D-hjboKdD=(6Kw_eCy- z)R56PM-LNf^~sd~^I@^j6l-se%@Y+42c_CBOl z&jb|t*R~hGyA|1cQ-lS3M;jQBl)2sHxW}nYl8v2!tt!(wo5|^Vp>MD?`<^^wF$2RU zwmjZKk@JPi=T3*%df`3Ue7Z7hQrp+n^Fjr!ogTJa>tAnm${sq%PN}9M1v#Z_n||MYCyNeiEoKHX}4^GeyWFVx^bO6+rCoJ@)dp#RJ9ftClb~ z2P#d(JNE)jsH-=?$_A@}g&;vZ14|HwxeGB+52k?e1DV!gR_NN0b8S?y>eeiawv5n* z3Vz=qL0cAELCn^wUFP_K`=7IXHw=IC^S??uWD21~b8Cdr8+^iGPsyM`KFAj}!t;l3 zUbqqqV~Y@#k|Uk!#ht2@;N2E!C<)S+yVhi|CfHCAt^YGj7&5COz77CpvA>ijzH`59UqKd)KF!{&RhBaAp z1H$8zqsv-L-YSpVx&C8~$Lc>cXM5wq-4Li%ykhSqUtu}+1^oG%SJf%f_KXqQWz8Pf zQ0be#hU-P~;N|nHxkwLF*d!vVVf~5av1=bD z?A9@wu%fG_p+eP7EZ86+Ps%$N`D*yxOVM|s zBRLwr!`aR4-piWI^@$UMp2#hIv*OX|{dGZKDKZ!hHm~0k+AgArCW0mrE>^DX&Ylb~ zc9|u;k&)$7!0NHGxuKxPX&eTWh1Xz<#Mj*ky7(}?l<)yEnGQ8sPP&~$=+(r`3o*9> z&*dxUT0HzXB}pi9zjo|~8aIsY{EGS#gggr}0%XGCS_!Y@!hwm<|H{>p zx{~_ibZd{9Jv;w#U-LeD&oY7b+cF6@H2v7^fhv?1G8^m)HXKzmS4BR+F6!A?oob~9 zCKMjTS9H%^5a;p!+sby4*#Bgk|B3SKr!aZVI--HS_qmhu92dx;!$oP!l4om|DoJ_u zl~DZyX$q%|`~p*gtu+=zw`}b@N%`Hy0E7`r7XydtR*Mc!bu+{xL>Vn$Z{2}zkZKCF zj<@EAXhOf8+j3rC4A=eqb(XQ09EiiOu|c*giiP3~ywX2*p2){sqVye%Tw% zp72fwI?R|r+mD`*z1M@N-}+q3y8f+|JF^K&Cxe>Xof&w3-5wdS*r0Q0_xq1^QU|lW z97~B=H_0MFE=xIFeT8&|lCBK;0K!iHPH2L*7AQ7#F0$0DU)j;}JZoBr=&X^b-Ba0B zUXK{%E_mK}IHyc6iQu@%RA5+sWht1OMd0^>cakyMbMX&ehaXrLz3vO*jFL|A=PvYE zHgkoJf=4bl>5to1b|vYmH-N%abBUfWhsluFW0oK<5;W&Kt$hO#7#IMSy48(cIO*Bc)vAk@}r`LyL-{+1KzrJ zDx~R_{I3hZg$9Vqd&87C8F~fhRAr%&QnQh*rJS_jLNQDMTfL!G#~a)z&abC6N-)F_ z`kiV!S!ag@gdTV21nP&rfRxWUXg;8Kf%h_9qCj0C!`I^sKXUzte$%@oMg9$~b^>y> z!nJYRJMs_UfP02+O43^eMs_8jm%_@pXJe-1^R-~wgMv>8h$}`UxjcfWc4?gA8A!P6s4&SD(vOzYT zUbFDZZ?tbAkZFTGtiS2K?n!a2ta_o!D=(_I0`AqrqZ_!0E1I?@~zWpy6q7C#!# zNn6o;BYS>HW_|b44MMuED!}f7;6Uyws9_~QdJLl1Lfj@IniS*p6CsDp$7&NYLa=g8 zC&0&`zi0CA4h^ngzGM%jH-J25aQVc%=dD+E3M z;R%iE%o+FyVT4@l@DbR6^-dzj+G3Gu`S4rshXt4i4`5pZSLih90eR`Fp8%;FQ{Imj zZ$$UNshn|>o|5Jn*>@*0JjpV7H_)25RmBQ8{4}dq4!%h87JNqN^ZltQPbEhgHdfdK zUkh-fA6&!85s&x)GYAfXeBuJM>Q-U@DDto^CBM?qy7XY+3*cV!T^6j``()HvQ_Es@ z)QJ#tBW&6+%j`3Q9MQXEegN^FtkSzscJ0j`3Z(>4XAB&wBltho4w|6-sZ~Tn*mbg- z{Vkw2s968$RNs7*4zL+sesGKn&MiaDJL5mx4ZuU_yQ0voTcS)NumunV+Z^bl;NIlv z=gjt)<|at1>rkIg23CFK3lt(Bbd`v8IXz~(!tBEGmK3kmJqKg&`QJc2RfnxeAdAh*2k zOiM8eFCURnK|25K1uzmKxq$WKVAcboVdREZXxczclgCX48~<<=Y%Xw+f!T^*0y;@_ z=F<`B8dP}+DMQAhN^l?$s&b}P_F2~4ouGQk<9&GI_m3!fny@D^FcAbPN51)UtJbWB87?c#Jix;)wY+z(s|#p@*xuCMCO9bnIscVA z;N3~o*D0Fzt825*6)(~*(9EJ_w-w|(D=7g7PJ+Z_Fc-5In&p%z^xG;PI?6JYZtqy^ zc?!`Jz)F!+-Z8`M5{ZZ%W&&*JGgkcuADJs^>`so07hCG$k53XOJ#U+{F7gJFX`9|j z0i8Fs?vb@bCmA3XMH3!Sp2VfBpYqM(8i0-YEv}^%3LUKxv2WeZ~=$&Ryd^4p+yI>@zh3dp5_`MaY zd;wcmgdVQ>pCJ$dU?{4y%@{+{CkI`(R+|7IXAU8?S1ODgoTBYs3trocQ z&8hP3_C!q$@JhQXbBo^Ps4MOD{5d_vzmn+{JtXey<&6SOF#DI=`JM%L4+AP zq;cqw6e$IyL68zb+HVcleciX$`#jJ0J@5AY^#^X-T5Hbz*y}hWDRQ?|rLd2M#}%9f z&xRqqjtrPcy&63~sxiMZGcX(xWB!C%=NM_MGtMF~zz=rn%8zCX&cs-eHdTIcB8xDX z8d5W^nkAkLq^~Ee*RWAmi&6{(X%K z;n5eYNEPI{C?A?!@m-Wu?V^iu@PnL=dSH~~Hx9MU`gUjc_S4O;U&E8eS)e8Jw|3LH zB6M$1weB^Ry5#-#JK085Z{U{^(UsfiYkl2dkxYl9CpizSCLm|uW=kLWiSBEcU;ABg zT%_-wLxjr&M$6xON!Uh2YIK%7J*oe~fQ;m`xEh?gEPF^u176O(N&BIKn zKcQWoq(;W6;=4d+EA^n9lqFYX_JPeU)?yEu;=5&@PK_EM15WjWhdx9G#aG@=1Ss;# znRv4mx(>^c4_+U{Hw98Q*vyD=zhpH1Jrq6RoO9Z7o4R2`6!VRdr9v74?4FQjDr{s0 zr;#Ar*hi8xB~d!5yg(45gv#lqN_mh+9&`M1C#*=9mkl`PZ#JNqbw|~hp(XteAf_8X z+Z}V<+gSvWb^EhZA#tg%%Ig2X&`?Y?t7H&EYs*9Tb#7dwI9@c<$Pi8)MmLH-i0FxGEIC{2VRDLh1UIN!X^}-&D5pTwEWl4%~(F<-AaN-Ok_=O{p(z zV4f6HA6O|$3USN*SSl8=4|$QPqVX0u4Hx;|+l8vPHBe$mjX4$lJtbWk;hch0l@mlx zz?o;jrgIEkEqb3ksdN%bOz&P}-@2>cvG)a}-`H?@ijszu;su+Cy4?i-2SJsAC?$<= z>FHUqx5tjeb9i_s&XEV4k;iZI?Lj_#7DHHKUqz63XdkZIIRE zo?Q=*aZ4&ILUj3EdEb|2F2~PFSw^%7$EQK+&IlAzz4T8}u{~W~sU7(#Sypx9_+{V@ z*`SBRmzcBon4z0I)Hkq9RldE-DH0I!)HjRBg2!tt8NZ8XAv#rB`&3uJoZO`FH^95U z3k95kd*eNCx`~kE(ik;-%WV#FVz5@_>q+89!p@vr9#vu*Byd(Nz$$h)Dzxhfo59*N znUAYKD|gJO8MkJhc9=1#i8F|&6{-9iCb8B|KlxDyjl^$GdH0pt@cs=(?Zoo1*UrP5 zCyOi<--+L)N%9jRhT@gm=%R}Z<=~uXQ$T1SyRa)V$6JM4O(W{x8eZ2BE;3`<9$&ip z>?vFOUc*=ft1=~kh*ThWa!LL4fTkvV3r`wHpyt{5tifSmIl5csphXCTD2*BqlI+sl zX26xb!{GVF%egQLo@XdW@Bz^mwE@3VGOGIAC)J&lcJ089J#iY`04rTvMmg#}Y+p#l z)Htvn!Mce*?u!Oo#*$?1T)A0vq4cZ;}q8PD5Yy^bFOR?IGO6M*mW}yXYRcHgJ zXH|a!U*I55UIk7A=R9L~FRzC_?_SA&kxUk;3^Ben)hxhsd+x%V+(;HP&hf{iEujRt z(Fv&}L7^qB0aFGyA>-+Il9y`O-Iqu_Or$J?Qwt8RNaJ_s znYxGWYs9(>23I(=hs*`(ba{_uzH=g;heURC7idWgG36m|TFX_^KL88ca z^Un0pcw{J>ueuw3M3Jc5I9oGe((O^5XzCDF00_rBq&$Ood$n*C;k#v7FN%l-k<4ao zV!244RCtwe@DF%!{6ruAgEZSB;BLPx9wrSZ@2S8kr{*P-6}tO1i)L8jz3Ej@~% zsho}edP7hUTMaSfiXigNEcj6MQ}0B(dtka)RGbK7hAw;8JsK1c; ziADqJ(Ex^j7F?x&qa+6Z0}T_b}qw>fNnR!2+^dHk|=1I>#rb%sU4LhxE1~5swyzL zW270J1lJ=Q+-Ccgki6D@rL&LZ&0T)btJbFRpQE@9ke|qn#mR>AIr9UUkywUo*^h$( z9>~+yjT|jU+AvmaHISmW^n5TgFzMkP=Z>qfR1`s$Cp_Ml`Zi9U=AMC@GH^sxrq!4| zOck(CoJE#o9Qwd)2yLFYf7q%&VITtC$l6>ltw7aAjYxQgrCqAEN?9pGwvYEdS*};E|AA zdq)oYqASiI)F3#U$f6<~mz4s*J)9*NFjt#f%z{4NWL8j{2n2luh-7{+Gy?8kPcV># z(YMJbX@AtE`Hms+n>~jju8t!@Kk9$GaHl0 zw>?AF{?wZn%Up8#z}=bN)zZ#!@v!rOtz@M%EppJlG|p-x;F+Cjm&c9UqE9qXjwGPg zPlmt0F-=^U4Y0M``lWEet_DToo1@LcY-3W4wT7VRyEku?QIL=f5{{P?najO~4%s?U zc+u8e86zAN-~Iy!#{+JF-@gy`)jRCJXb_8tn$>HU9vy`MX6OU1=(LRwa`QC-C(Jr( zzN>B99oBNq%b5dO>J>jeK8C}9AK^z9o`BUBHv^53FyQ<~fM+Pfv;~pzV|9-dg_~w4 zb*^J-_&f30R0!a< z+>DP}X^yP@{i$!*aql(6t$ceC`vZA=4|KMbQ?j@0m?c2Id04G=1bM%ezgRb5ev$Sn zTY3q%|Ch5Y;P9uj zOn&^Qvs_5yGA^ILzf`aWdjyOF(MU`L-C@^yz0V!sQA@KV?2Jey^L%UG%(qyY0Y9UN zi5KEKt8E|Txt%r(mTs6%Dl@GoT#mEL`QQNhw#|B^My~2aCDdO3E&n-6^UqT`2r`gl zesIuHCi&hNQj!!c8Q>Nd!z;vq{+StAawVCfRZ40uU!Zx&6!)o-cB*YP)XWD}P^7$msBWJ*6^XX3vfEnRmfsC}PNZq6ozXnNN;gRwAH}b|w&bQxu zBuyuxDDoPMc9jml6dY7JZ$EwEbY61%XFBtCGpxa07G8Zy}=_ud^PPCjY~gTk%*~-+&|{xa zT*i`lk$)+z0{KQdb6A2|J2A^760W%Of?c};7CBcl3E;%-Yv89s6^=)JCL4vm0u5&h z`-Ob;QN+k;w)u#p{2YBmzU#L5hn3~+>X`6F;Qu_5H+E&aK<`-ApUmPLcp8Y`9|kP( z$aTsJCxYJCmP+7CkYXZhXOg%5>)knU=vPj*rkA24Ho~a??})dPtUhh0nWF zwbYU6M!w|PY=kwXMQRh|yG0@dF&3oVN?y+l5jvg5`&QP7Jsd!6JX8Ku&+Gha_NVi_ zoAk@SEPR^*yS$<>zl1~T}jNq&dCUj^IV{OALrUaFqi+ z?G6@(RtWBJ{rmXEDkU^vfL%OO`0%Uajp+)5$+!eDc;+l(*oCWe#`Ol_hrX{u&@4tR zv_73D$DJjdZQQ18&3c8iCEx&FwCWp?Dz7~Nje z;i6fWfOH@iABd-Uwxsd*d!O2$U1g0fzkbuCOJf#T3 zz2_pxJyUk>&!uAE*#y&F5pQcWg^+?3dgdCEhZ9+2a({!c;rKJ)H#?tlwzl5RcA`jr8>Z}?r4rXOE0 z>a&d=== z1r{EpdyiprUTRij4OMB={2v>HOERy~BinX7a~thv{gbW+83%q4@-*2Anp2*V^h;@a z^jVxbR5BHdc_mfi<2S*6%n40|13WW*A+wRhtk81%oSSfs78J9lt{kVs(#>x~DEOVl z#}%)Apz&+u;E`T#4h``}+!#|eH#?H7&d<16RWzphax$7nA0STJkIesvA35~j_>pHq-x~S>v2K2mdH%Zv&D)+0tVT^79hAv_;zBjGh3BaG zuHpS=QD`W?kjnz55icg!6uP{P>ueZp3#ex6tO*K`IvNTO3khKL4lri8dAH$RLDYHQ zgGk)nXgODMXdb9zPGu}C`g4}Us8T3iksvBwQ6GWl29xW!fh%a&3k}#Zg(LmsUc(>0 ziV`fNgniXHU`P+z7hhF=qap`*T@eYoE)OZFyV|qRV zTs1siqv*Ch=#n60CHQZbd`SQKXt<#by)MthzBH&Z8i@GR>~jnpb%Jih&a;C7P+@!%XI>%T2WSGR__z|6VI zM})yo>wc_VOBXIB$HF1!ryK4oDSyJD-NJO#zu<5hCSvjJFF5>}Yw(M;_2iNu3K4o; zp58&3mPWZ;Xf#&s-~pX0na}sB%>(+5L=&Mi6T$Z9(4pGGDjzg1g&;!P zx%^tGnYIVudo8M8lY3klwQ$0UjK&CROzZWn-~p6oQ(N}WDa5!8@;DEFv+wst6NUJ- z6u&y%17c()MChHo28M!fX4-`hSlfKfHe^tz%yDMIr0&ieeFRn=LP=rnAauFlc~AJ4 z=_|ZqG|M*fVq(ilbG~+WSCQ%;&ib3ah|2t+BfTw7Aautg09Tq1LITuLNd0rBSN!c(_ebI1|vi+;KJB!3E+&Gsce!h8lIZ8#Fq z>eOYoSxDLua`5cpmpnwkatbO_*j+G!0e4Ucak>xR3B~+iomaw{?=#%_`XN`RgRLa4 zVF9@2Dgn*o9VdNTn^)cYj0bO&NFf1iA}HUCkB6?sfgyBhjX-@UvvwxQsuPDMnmNf8 z;I5oczYxb#njTSPH2=&7NsdF79WZAw-99zr@mDW5G6_I&Sti37R0o7`xzMsM%ANtq zE51&g-5!kNKfK9T)faGAmGeIj9VcNO%WAn?yR)r*j_9w|r~XY}d7sLAz7l1UZtzV` zd_udny??mHLWJr48dyw^{b`VcETyr?x%tAu3)!A8%2dCRVX1_OkOkOv$IA+-!l+VH zZ1zZh^d9p#O0v0KelE*g&J<4a5dpxa{W6=m6GiVnU^KQ2g7;3_Vg+3&$DfO^YEopv zNssCpzU4dq??H3v@L09U-7bJTTDF2dI*u4m^q&~387IomvE zO#+p*4av$65x?}7ShdjgUgVnXQJP;Pq7WRaQYhv8DU>$LeNU>XuuKeBXts$HNonx- zGg`-ICia}9U#0y6kF9$=q(rK4!~gVnuaO`D(RA(>ZAq;kf3(P9y7y!&w?3lOkZFVV zl;GRWHKMoT#B7eKP3#R(cdVsQrdj&v{MIBm8-$A&E(N(O08G$5vkp%7X)PY_KU9m3 zQQX#?5?^ObcyIk7eVM7#dehi{Yb_iu-`)-?ARh6t^6mYNnU4O){m%k4c2<6>5VKQ0e0o^Hg`@9-w& z0x>v$$+!wFVNISmwD%WFK(|_S84LjsZP5F#7xo-^EErdF#W8?x6)gECV?Y8@VAhE6Zg08j9RK8 zyubuc2LsgV1TDPDApWee2az9hBIzRuCoAutLX;8N0>D_Exx1D>qsHv%Qq?$U8}e{x z>(P(;N%aQtb-qw4bynojdBt z(6R=WEs!3IqoVXy8poq*wU`K%K%T({YB;CWLIvwXk&-KR{fBm?y;|k4UM(Tb+Nc#O z%F-h9aT)qgXfII(qlpDZC#E&EBJDQ^y)R>ozUKPC$!C_BLPd_10<~#z{p5f!Ps{&( znCIW%Y03hKOkF3<<4;*C^dX%A*WT|gRwp9}!VP4D+<8NSpmHTbr0>c=->xM@6S`6K z9^a2M)T$;zXnclr2{U<+PSz61CTYT_X9_ZIzaR8B?u_UNktkWj|><05PFIqD5**JQhGNBBV zXNdQqidP};q2v`oo4n~SGT=)FY5P{ZQJ<+wTYHvlyiNz}EOH^DKfTmto>ytHc^Qpq zAu#g!HpfnbB#3A9@m@WM7k5xY9sS`c0$=$saSq@qaz{}pbWOBG{_7#SNiLw`x>8ORjIokmv2Zd zpwh$kZYGonIP3Sd@4WLhnI7@7tv@KU?I;FZ>%X5k@M&k<0aZIeg~qOaGm*|E0xMs8 z{lu;4nM&_^^#hk*LGN&A5l&ZW-6Z>+1VuN2710$V$BWFnN7lkrf=I-pH$mo)7S;y>03e?*)UgeoMWBN7ti2m$=r=E*mf8B=pHPkM6%{J{DpHP z%KV{!6v%je^LG+F^CTFm^m>gS8#am|7J0Yiv`rG zSO`Q(LXEnc<6QS?^?2}svxsE;fWPfvf zyU+*gqbIRE&f%FWZQODL1XGaRt*#!}aDvS*COQ0mxJC+QT_>c-m2_;0`V*(!F$sfW zNPJAil_dC7F7_Ok;riWYLf;y8a`9tKeaa5M&AIr@hO3YIwD($owTwBLiZg6rGB*T>=Blxj}chb!&bEDWY zW)*q1-Xwqi#;?n}g8O5U@2mW?f@BHrOB!1Q4&TR=p%jizJRJetw1wMeA>!H8V`17u z`37_aoMUEE3B^7UomwT*FX2`Fb>CiwTC?jW>OezkfE7YdL>{EuCicyse&Szeml<%|AGRz5Dtc5Ldxy%cied zgxRH>_oqMAwzKzV?b=uj?_jxRYF=>^kWT=DyC1NMFc1|aLjnOP$5-q8hct3-fl_>M zd_HJO311Ohqbo`wg+s#zFt0fhU*^m_aicpkSObmfQS80gp6iRhTupZ75b zEWS|wPYw$EWfOzM4e(r$eUq$X`k0S0&&hWef{SJcpS||E|0(efO_(Jn;EtzSY&a!% z{woRCkNc}i=Rbo}7A%#Df2X+0jr2}5IDAr*HkKj>#*ySf*Y^fA3;cZut#5_rAr^pJ zR<%qan7m-D$AFq0_Vu1QPYVp1U1~a?*{+OuZ}IM_$sPrUX6M;>DRmplMt0cnYxP>d zdTMzKrG${LEvg};0LE$FRd+M^rtZR4Uk$Cb=n==t3rD!nQ~oq7yaSjb!KwqUcX%{L z9xxhZ@X)AxY0Z0v%M0(K=k1L8-`L890in|ZL-*^6UCoEs zV65Z+9E>fd0D@@665{-ZkPaj&yE|5}DWT-AKSCe<{;y0XWa?12&b&=Nif%I$1Eig#B^2NOnO+ z;Kp6>Lv(T!oQX&daO%fsqkQo0bhwwQ_B&Y%6CLZuv*{2t?>Ue5sU*ucP8+7+O0o26wa{rh5RY#y@~DPE+*3h1TiG zH{jM{RqidVQU@sJ`W^|0ClGfl;{hE-0qGmx{~zfa>_9mA?}5-9$lv_H4ur>6=5Mdy zaSlBDZca;9AtG$zRA7=gW>J?+bxlzPg+l75@?Ft1A4p_NK)r@=f!MyU$N1>dnpt`@ zTT5@Rl2qLHJ%5YT*q?ABIO!f*FgF0w*MA=)fLB9+`zSO3xDgbBgv($WXfr=?qvllm zwR6aIsKE%$y+ZSi+VHp;1Ll0X?{tK$T6xiZm9Nk9o%VD?Lg8d(SK{ZZVeNDtWTzr? zl9fdfO~YIFXxd&dd{9o0B0?5ls&%e5d9KvEd0gIc&-B-y`W4kV7)D*VUdvq^wB8!o z*46ceO zJKm$6f051}siupKg0ubAfAwZ)L}zb-5!A?!CBAaLMGnBTsvv^Qlr~b$8tw|XXU`7h zyPPAsqQbi&n#&K`3)x}7t;?nD4?FJ>{;&~Z&UB2&k4c^#dI4&zju>F@JjJfYL- z`eS6N$b}dW7ObGH#jm~etA!u@mD<5Xv++Fgd^M4VVB_c0nt?b2+Z@2huJuH3jOsQSsl%}OP!mnPlvu~=&g_uA`90J3hYs*cDWS)l&!_jF?u%$VT zccz8YZbk3|!ObTPQEd)Ac#+skSOt*Ll4ES68%4~zQVm2f1i|p`1zz9At(mk5nd;Nq zkc%N<`LK(pK3atz)1bIU?3xV_{Rzj;=vv>?mOdbOf;c^Dx|N9T{(4~g(1iS*VM+M@ z><#(6Uieywx+ftrRojf${-7#ZTyn$p3hY zXG^`R9-dfz8ote1kuJxoqjtscSb84BI=?zYQ?a>G1)yLwP!6%Nv@98~JG|uVwR#<9 zEP0QTpk3MC#vfz8xda#++VvqcKu@3~allypaGYFdqlC1#o24QG7q zQGXb?#=KY=Dm1_-S}i*Awcbr|8V#=MuR{9o>2;nq$8yrf#vglq?aObB5*fMq4oXzG zvL{r%wt+%ReQwJ;`Re#%w?!6&Hv*4-6MYqYwVg4zY0%y=Ec0pL}AGkx#-YfTk@E zW3KT)sjM>%Ordo@im-v`{~Wt_ znL({uC4jbzA!`ceIM$K1`}DfuE;9nBClXdP;>4;!h_&cPw(uvPO~Y+_X!kHn{qP|i z)dN^lBFV3UT>PI!AAyS=B;#1WL^;>vFFO}&?TAx1{SvNlkAbK+_{ydwg<=T7yJWKa zvacyaF%Ym#t?`f~i&!c_BL@tRY8qh-|X2t6~VsF>lo zuRo>BK3lm3Wc}1%0S;6rUJ9rE&FE?013kc8B#xB?dNqSv3LXR>dD*RE3bF!dO}HsN zo!fhL@)t&7I(Q3upl@efg&%rtJ9dwnrSQ;$PLB79zk1yek^JSmU=%?AxkWdiT`}Lh zA!zZB(*Y|M;dDrFhXy2;JOhef)&Ue1Fj6+gwyq9}J|K)l9E(Ar=SULEqli&(UMDuX zWUuT5ot0jAh4b~c3ui52<4vnjAaPAF^*EdoSG)9`b8{at*a2qC_nH~39*KFoR@U#1nlctCKH#lVfYh*>fYe*Y+L+a%^H5fFe zQl=Vu9!`{>KV-^6#VLc?rU8V|5!5XyOmcoVssEuT@d??rN_3 z3hJy^Fb&9EO2qhxLFk1Pc3*wiRJ%U&xcf*G=5b%L_6zV5uo)cl3nAP{${?I;tS`lb zblj^7$OceluG7B5$3i6S6LeA9hzBv!iD;ONCX^6};Yotg<0qoy3@jBckma-{=G!x1#GFJk z`1$t$b@{J!On0)$_DxdS?is%{5m%Fj_9C>ZY|L#D0ut?ILq!BnCWNo{uHNDi5*J<% zvmF**XqiQ}0jj2`_)ko~ZvoLCa|APZC?tqm5c&1@dYb9b7;Kge{465K3=!+S?A_YO z-9f7UzOnqYGAug6xWS33Z-BlA)=6UdVf_k$nSunVN31l4NVWJiSTNREMpso>66ZPM z@s>OSQoGGoA?O?)1Xb zZBN6^^0ny`r@Qwy2f<4PfV@i3;D6GQjhPX*6mkEu$QBg%GE-d7`+q|gXRT8?G~~BJ zlx48;DvDYI&H+Xc;GSicT^_kQ^x(p)1w)^cjjj&vJerYt1EZy}j!~s}TRBS4SlN_J zgb+(8UFS>KfNPKzp!f@91SB?I?xx4i%pW}~{y34$Tu@n=Qt%J2<}Vg*LXpu~yOF9l zD<|64ZhEJfsYO3pzVH|EZC30Ic^>l9bAMg$L3_XLc*7{|XF;z7qp(=B$5p`{k4^{* zX0IHQs+6@qt744Qo>INGN8+o%Ad4oYUMj5O6za*IS>3y~$ThOM0E9=7N)Qaj!xhMq z|B$T^CbJoz+yCD1DsUU?O+L0)d8ExSj#5NbnX%}FQMgD$OFR5*SC0i^UjPzuP@#>% zm0&06F7r2{Wmr;4)n#FlLCHKo-nnezchJ()>$tsZ&$U5grhc*`AKbyWmFcAc9aYBZ z?)NZkXG8e=|Jd2!1cD*Ozm$KfWCI);Xl?A?SK;2Z_WW2k+nJ%{aNm-Vpoz#g=?No` zB>vQN-)!~-EnyOL7VOc)AuQa_y1jA9QxtBI#?6c>^_{uY!&@Kg|9iT_{>cUMAi;t! zaKZk`vJ2Fg;*(HGIwT9t!!S1)5;(p}8H~4t(m~E-9W2H0R#=PPt08{L^6V|JGK2MB zg^v7+T3MKI4LyZv$&uwJ11aXZsKdB`>jLqHmBl4PvS;5gtCex;fxtCsqxO9uLfY`{aDEXm<^auirsa@@cd z>q)|6t;Y|<8jM9Ae~(3^QtO(xsfHIW}AFfy7lcn};_GO(=_*wf3$*JwW03CfTTV45=%x zX^z6$f{kE^&4)XmdRlr>@oc$MdHLvFfVHC?4-i z2D#>b&(Io=^roQi2F~^^m5QMCk8R~u3nMpEhTK9l8=HAF&bQN-@(pzl7(U-8S1hV} zN6NshkH`}~p zv54bMRKVGElV$^6L;v&uR*2NHn7tye5Wei11W4YsSA(I`Yuq**_!+HbNKy5bC}26wTi1NMChBVznE7y$y_2x8Gp>lU^|Ga{H;&;|rq{m&kpuO%KN>4YVytjnk~apCf6 zzat=xw;q-yrDwEd)@@t2@^pWF#$yoBGn6GRdWbU^7mxe;&oK5~98lnyY57V%U90}v zBP;XT%9gD2{k7jbRPiF&ZPF=2S`giJSIT7*L2r`Q6QH9eVFanGZe4%vo{3)kVU~ z7V9IH?&vxKGfp?!dSmi9`gjg~+2b&*IEoyH(lseyuXc}o2mPNVr36}$< z%uAJ3+9>J1o5dx)v@F_Cl>_MIkk%+2(Q~PT*TiqV{jjOkZ2a4)6d)RE+@aFH^gk6O zhUwzFW!t9L+SazZWg@sQ$~W>^&(rGgp8S0L8F2eiecToK9xxZE zA%CCY`+Bpdy>UW{{)Kam`+q`y5M-A$4rfnkmrbG`0Hx8A-pJ8nfvIaj8c@v_NCFzF zXi4{Q{!RSn(IHzHH|grc7({8~nnNS=nuU>DD>GAax`Qd(pef7T8j#qNxS8$tA(dXG^z|d-d z44tRT-ci$Fd3+PZstu1^J>K|nurq@tqWiT)pPQ~9N7c$R>n_}|R)RMR+$rO|sN8^G zr!f(go$N!@qMjy{EWWd5zfI%<*ZU&K-66-EDpu!w=9A0?+942RTrU4W<_U7mgmRX! zV_yK)!gLya&+~6B)^xE0&}4(ht7jcyzlHt|{dBw}7;_MkyYw%G@zi0gU@jO?KQQ?8 z$ng9c=tG#~G?$>er29nKxrv#CQ!CG?p`Kd$PWu*&=-3t0+yHjB(xK@4? z^eX)gmdxM8SKlG=d%p4okN8lAdTx_NAiPok2T!60(XPV9=YG~6)mTj#q#Wv}3?~yu zie^1Y71?*)?y0Pp`aSdITt|X$2t%6RHuT+@1}EDH1n^MeZfrzi&+x~?H% z@EUN_gVm+>)kLQpOrMQ`%L$>dq|Ysw|k3+U>_X8 z`oLBY8DNQ(X)#3a88x7U_DRScBnvdtyIltNpF;gtBmdp(ti1X*^Ahk z3lIgOH}8!oNGAfB$(}r2?7o5+xm*PCsyr~LKk@HaV}M%t7V7gG4Ag z)FQSE!Vm5R8H!$FZBqjlYY!*>S6It{{84ji%9b|=NF)^_5R$v@z*CY~e?6kbCzP;i1(CcBuWhQegZpdAEm1ZF= z`kw~zUo$EQT%WMMs+>V#;#_HIfajrN-=o#XB`nVU#-8!1;$4f_=+DC47F=eH+!pZ` zor|w)8vs`VC?Hb5lj%p`<-=}i%jI5O&g}jEtSXDcc#D2U2k;+_mwNl)HJ-HfgrWUJ zy>NmGPdodC#c}4M8-)`?m4_u`b)NMeGp%A3NaiisR?fHkT~dG5;eC9MlUfbMoy1hK z#kIs3YOPkmoN=yvoEq?-tr3>yX>7}2SW?mM>VPzfucX?Wr;;-YeDL50jU+Kt&HO9} zbrYm&H@L1m6xSnYgH4eIr(c!cXG_~`@~ynFa|X2HBb|p!pUO~=QyqKT{`91yz&xMU z@Ag0sa)IVNTZNifEPw_4Z!$qq!&^`4*R{D2ig$KyVXdYc^QB8SCiA^ySyC2Pl<<& zd_HDG)Vk=K_sEBdNEUX*?At5-pdRv1f-Y&*Y%E~BdKrOxZ3)Z1G(?R*s{gGlcB|lb zb}BT_8+y<%Qg^!voUM)fFZnmjv1j_U;?f1Jp7d%I!6u2cJEm2_qz+7p-&)K4fzPzK zF>HV1b5OS3#lo;}sqVW8dL|P))!oP2D*bPVJz)0Gl8=n5pA&}O`oKl(b%NnFSFfL15W36iz2iN&Dn`=jeHX z7o9Lc@hYA~ov;u`l~s+>D6p#v>U9#lm6T?(;=1QI#sq~gtso`gM?fpk;O-3|R@%IQs(hGh_dk{Y+rnfp%f)L)*V_o6`!$4d$W+ zIT-r4HI{B;>ut*r)f+n5wSlM4SMBR^F}|gHK;A1ysj5mu#Jdn#DU@IWEqx6Rm+#D{ z_I@Djp>{-kbwVa2KY74WtWOD1CqXeCHFf)MMgDJCSx-B;frBH7qa-V(RZ$Po8ON`~ zM8M^_9()dy9pNi79Vd0Uh2-brFiouVZ|3Z|SrWI#J;lEY^gV5pUfyQ|`%&=crif`L zrG}J0dM7O{KaMh`qDwo-GqfnlX1k@`7AIv2y$zn&#(hd4hEgX6A^szm5D`xJu*R#Dn{bnuCA-dJ z{#n}H*cZuVPdH3xs6up7LtBE)@m|6@LHE8z5h1k7L4Ez$kxw>Fr)&HKYo}f#69@YD z2#QH1NEQ^gSUVe9d7J|5OaUxZAe`_&>=pvn{j}M<5m$GYzXWinDWyOrlUqsILg_G; z23X-iqHhZMcL(*}6-_>-{dIlv#;I|-iCj~>{3ToucL#f9!8k7qY22F)(Dv$GBW$es zKOrLkLoh#p9Hc(16Y`0uDMH8QaSWau<2iegPNH6_v2&urbGjqVp6}};&|v)z zm9VJsZMJlw#gge!$m0}t@K9V3n}#lQ`^Q$r8Q)+bO=fR~y)t#LY4x6XxThliuJXHVK)~C-X1H()V3Q3&$DQh? z#iLt`4AqboUJasS7=GMyW0r26JGauuQ>(VPEs|sz5U%Jo^s`< zwHL7Rz{D=Ji&8{Htv<5qbY}Dzwj&>8ej&`hzfX)UN&Ux&y8P@($TNPKvOW`KTa!}t zMCRZ&Wc1p0C*O@y{T8<^lo@hGfA^mg9>eipI!GaS`eMyy!djr zqMh5lcaA)^;!qy%O7tP=$JTzT?7G+TXFfWQ7*38R$fwY(^xK5I&BUknCBOr)u2jcv5+P4X|3p-WA5D2s-TJM0QDHz@GjOmQUXn-_#`? zkNtMvDCJH}0z&+&`de$`yEkU0bvyWb;hVP^qy;3l9ZVH0pLx0kXe+KZ`8YNx2BgVZ z(USe?ZyRqzjDsdnjXU%kORU|~N*_1btIC|pB)oWC~5uHL! zU7fVik#=e{-%*|SI9ymLh@(^`FwS)B=vQ96tPjf5as*ie4VAEboWI<1vteHW14ImS zT38cNx$1hEGWbvN$E?22zc?ht_8SgK6VM+AWS#uGj9kmC@SHh6KP)nq6GX;nR%V{R z)|UizZ3}v9*uiWCyeV{6w#_`^MP&lH*+EA`{Qho!;{ny;JUixNO5~_qU=XIPSfnkVVq9c~j{^-L@Z{D=J@05LMZta^Z{pp~*7sGe#b0Uqw75WP zKb(io^DedPALAIMWth=0$%6mqf9?>3;6~%DjsNvSv5>;hfB(H}DG~12Om2UEmBiMq zQx0E-tp%K9F1N(?W`EWswZtkWG;a@RL_wN_$yhRXHs<4o>Y}27*-3tD^==imj{J`w z2jE%qaMp$|ja)6}r+%T1VY#Kf!Vb*vMNi=#RJ~oYjRfBr0l)7Y_3Hj%N#Jz+na_Qx z8^1yr9G6|ouw1q2RnzB0xKWT{>Z9T3^j8%lCK)Bv$_I(r)&xeLfJPvO$yO^dZ8gpb zjFJ`(Pr%YgRj*60FqX~VB>pxbW=!r8uS)$UwxR{!eaBR0dRS{q{s^$=3Z}F?9BDXB z9z9ME#)jy7;W@bCatEy!_FrZqe)~lAXg>)x%hqU19566MP%lqCx7I=jX9L+=U0`3K z_A!6o>f??m?;5orDT=@M;SdjU#`fAR9UYK$WvsV#F;G2J<^D+bdsV>y=db^Dku)aj z_djH8DqO+0hJ$m(+Q2g{;R9(B=%l)}Iwp2*t7=*O;%FPOB25Wn`AC1GnqQ^LremEJ zX$&o03Tks^`1P3Ym3>}iq9p?nOxxL|_@5=+OUQ0!Gajeye}r3&dWE>l{MoMVB%({k zr^DHXbRgeXGO1*Os5V^q8a2aX+j{T;Y}J)F(UObfhn);sh)IN$L%D~~4O{vtrnvor zTYV-u8Eu>_*=f(3ZmD>%aszBeefQ^6#4-a55k+feGfABumiyt7X|NZnj30B*# zs!R_2Sta^^;|2F)!>j%N1se$fy;zeo{yoyXPXefxEK-c5}1&3Mx;bhPug1ju(-E4EyEjTx}v*We;+X;*cihpXk zJY@ctj_Td0yWa_u>k#9y`?U}QcM)zM|JJ$DM~98S1&HnTe7DN`o@xhKL~}0=iRH3A z-uv7LipkZ-yC(Tc-(nW;rPClxN9s)zU|chN$epq$M6hcc@GZP z+R3Zj;JXlbW>tsHk?&t}bLlddmT)&C^+Honw{)CWPOytyJG4i(Adl~Aw!|iD!ld&| zN9??z)WvY2S!mwJ+4y?QAlI^aHs1M(#*I!I{ zW+8l~7XJis8_hq{;ZFP<=jpg=Di?u*cV`r1JoJa*DkC7R;#M26D8dc zWrHYq{A3h{%)XM@Yqgo1iK$Ref?w;N9ZW#$5elT1E*t?xii?%E*}HSKXS$x-1Pk|0 zAD=h${;U4@JMaKCOZhF|X53pD>=*O=*$N*Wv8jJBRg=gvADtcibNDR2K4no>>6AIU z?i-Gk49GWj&6|oHzByq$jKX}ng1262w%BAxnLbUgl2EHOtJ1x1IVYxmtT|NHZ4U;=@1pvYmA{^zQ-cCqw+JLRS{u(Kso3qS*JM ze0Enqq&Aw=F}tyAkCwGvVd#^j3G!#Vg^-H4Dg7+v5D1i(f#OwVyBt1?B9{L^lwtWBnu3?Y7M<`$XUZ%Qd5Q$JvO!eIe zE#RO^?~Cc-^(ql7fN~|1DkA9gIWj%UOb;}_;R0LjE*J7_tXrl>aW*dh-RBA99hyGd z`o)3P&aVDkT&D)Ca%efu!fVg(P^ zkXAzJhPCMKMjE7Bx;v%W5AXZ?zWwceu5->`z_qyNGv}Cdj5(%zte$LFzR7sT*fU#5 zj3ISw2hL|aOCq8|{UD&{xT7kZdR87xr8iLeL)w+!*JND_Q0Se!c;P3pr}f79HL^bO zKlCWOZr^KW;{3g}UZ~3%F{YVDaR-74+G`}2SK87WOfYDyeK%4LygD!k&a=mK%s4FD z0CZx3A#yLI$QZGbWs2Mn~2{xHtH=JU(HYtz^tM88t`Dd_Zy?Ft82>#L+qp?pMN zbPMG*BC0jRi~}!43`)?;Lix{RPx97{N;%p5e=_uK0=7IO>1c$dm}7G-?j$Bz6uR_w zPgNY_m%`uM?8qkwa_eUDxtZ&Kh_>SXe?(i&T*ZUhXP(-${*&5b>h9F`V)$h#epqv^ z&N=|l-H>b}J@n>op69X0=hs4K6zL?Tj1@JZ>w~<;j^viy+=Wz^p>4oCUP>|(lTF9A zwpYXqMhj|RN&E6`;i~-$o&U#Yp(Ty%#mYF0g;iVL6$|Gla{PGsem4kWBl>F5m_`n| z`0T+RWEE3y{#2)0W2vh~qHO(=eSBSs2(-PIE82wf}(6eog--o}j3S zTOM9xdR+q;?m=cXQS>>jOz11rv9ybz>$wBbqs`oM4LD9 zb~OCguMeP_>9S-3R2j_?g8j|UCwX77({Hak`~I)(2p&XX-M5yU_EYIu>08%YDuJYj z3xK!T_lj?rExoekUE93rD&;V8K5a|j>OF|K0X4_BO+atckTVt&`rXFNK@D?(C*5<3{;oBh%ftL5|2z)UJ169S+w;_!|Jj~%0{*}MH=~VaXN^MrlwLjq zd(JQ!Zs8&%F^TI?mHfWoM0dV-<$K=PU499t&?++#dlen%NvH3fl?Jm3LngUY!4yt3g z=Hh-TGxRQ_tS(NINKg5dChUMaSGkKy$ay-i0ad2}nbIZdAkHMB8e7 zZP2@$`$WG0D1pW{0sqw#dRc9qkFt#+3&_unP-u>UlNiD$eu1S+1<^GnR;j|Wosl27 z2|C=}7dl6JB185qAQ6%K7$#{cDm)%SrgBg>ZAYQN+Fb25w*oR!8U9)m>gU6F!5CZF zgzxOUxwv^rNhOMRUAME&u_(>Yvd$s*}X@%$iiNq=LK=@#(-g)Vtxkzy(+=RiGb zl_WD8_ZW&&`Eswhx zf9PH>u8zS`&rTrqWAI$`O@^S+t9D&6*I+`Sz$KxO3NYh$YQhrPY`_aArUn%wfd{5Z zr*atV50-=CYc$pu?jLhRMm1vG0ccBr&rhSg;mN(Te62GW@6)RvUoF>l}*bn)K?f!P?9oq!#6cH7=jx_^fYWj`aek{QH zm&srmrcYhw|CFNqDPaw?z!!3jKO@3z=QP@;STn%WJ#)5$uBlNhPi0|dc2++qS^@ZU zr7hu40i&vIb}DVojUgfTF#{4}hRkfC|M37UoIqw$V&x=Pm_IkuR|!0^SSQcFGt!ES zPewOeSEi0}^h1$i(%eAXCUNoOWKeTnrYMUv)=n8`pqc{ZJl(X>U2Q^i1jEr=YJ7Xf z64*KoJf?G9JIt!~*7^x0#_j8%(rZ=`K-&YN(@xg1kb2|EL27&k`ZX4C@A!!w&^(zm+Rt!>@xq@mDOW2klkSZ zsYN>=-TY7=Cs0a?c(t`=KIwgWGGEjX3D-|AE&u&QjJq3c1q+j-)z{fz{SwE=%{P^* zW@_Rp{wk_a(c#9}N?(qdl}?Oh2CR}QSxQHS0ONHfepl{8DTb7hy)&)&t=05}yyzC) z#p@+64Y&ZaPsx?|%4<%9ZRK*t-o+ zbX1_|l1j+^H+3`}i-Tz+hWqCET<%?cQVMkn0OGuja6*h<>Bzs+bC6Xg0GEdfZ|~d=3L#Ay}Ar!%?~;bEqJh`*{2<-R<^nbR4W?I(H(= zDH=~Wcv0JncCup)eQ-dLDoo?}S+7CWJIm>eIEWHi@3@-{o;Cd%Y6U5{1n}7HfEWU% zVA!vq-AuOYwut|Z9sx_~+U=_wWw9BSXWMYCmuQmH%#qI?5I#Z|#qf(@(&zgZ*EHCz z|FYyJo)LV&5k_LmlMS-bW}dgGh%4V)po-W$%1FOnS3mFz%NHE>)v1>

Np=bxq!k zM}^#=+AU;1)dyfL9gwneOa8yl*3F*QlgvauI0MI6Xc3dDaS>fKYfv>Of9=yYXwVrX zK&2_Skh!3o>#%K_vYyv`re}qgD5~2r($Z<{ndIOi#>7Xf75txS*W6q*^^~^0Ty@e* zNqpaz*b72|_xx|H)iy?Be;pU>rf|FBZhIv}M0lIS#70Xq^NEFV4&@ik5E~awG*-A= zc?NnESQeU#BE`ZaY=I&*^W;%M2`bhV7;)_*-yOoA1yc zyR8kpGhUeEV+q6LMNd)x^@dS17r2uUI*(%tEE3`^Aak(;Mm4XPw7Rv=xp))aiPVwP zm3h<7Xw?M2@;i8i?&9w`y}X8Ei>Rm5e~FiKF|OD0#9`{be^P(jt~1T~7Ori@L{qJ7 zt-7htF=A|y<4F;}iq-1o1Oak>rhq|8a8bebEPn?-W(mTI-ee9jp3Q;GCV&z3mt+T* zP{HioZUqy%QX_s(ZTpDBe3#Bs(N9V$9lU;Z{EyAz{^b8*vk3eDuvr8lngLaxDLb!5;fuZ>>sb^M=^&wV2!;6JEIEE%=XeW;Sb_XO?9tVG(W=#!Jc?c@H82rc z@sf-KM{UjYk+|}8HY>fDsu??G)j9gd@#tG$tJyfJ@6|ENuhz$A(KjJ2eBBorub%Es z{8uqD#K6BgUo6gxHdk^qQ@CCjkdGE3^ReV=}=rK70zr5GJ7#Yxf1RnD@K1?=8 z;E?`z9vFpqt8{4$g?t6mCDH+qR8wgIrs=4JMfU9AqN$I!y^F|WmnYj9L?vKb#N=n= zvm*UPB-zX2O~AU~WRPB91ztbD!O&Mt<30LpmoH%l&XF;S1Mc8Vi>qqcQ**H zdpBFIqy)>lm|hMd?);rxu{XGug#J;ZtYeSBDouvIvtEg)pDI=7A7do}sudx?Pxn;j zZteUeMn5v%r*cz=M0jx1OD7|L1PQ@l7c#DkeE9BaOu?0*9$m=o{P)0{f4TQ=v;z-n z3y#8x*(Xoqr~PBxG@_yS>osIIJ8vnQynTdXb4h@%j5FY z=y;ySd7f4geG;6g_XCF3>pxs%pOO4phWa_9xcYI+(%1V#j5|DDq0fD)*A*TRlz$f) z(bQ6#e6*VLUbu9Y$l%`SUReBEP(7UT(Wl(^@OTt^5*R-IcXE`+Z$qxv{ul(aHUm~R zpIfAbM~kBapZ)&f^OkCn;kS??rntp@D;K^GY7AM7wf!%h7yQ}$ zNnGy{-Xuwi$zWh>BRJxt*UNG7#TvnTF+l6)UIgnjU+l;wNcm9-$=nt9U{oZK4&S@Za|emXu4x2POK|M-jMg%eLzN^$}ge>h#VFOe2p>^%d}J~=bQA9F;zZ$HIhS-^u1{; z8a3x%{mxrZt!tUbhi|@{KHiygLgn=-7}yhkTE6N$hY`}NpvoQDhQ@SQgpR97GCKf_ zF3Bs^@-f+A`+)lr4z5e$?(0Y*-j^hQdbM0&J=l?lGZDLgD8jr~^loRbbo+~}lK;j> zzBENAM*OdXoNZxJ8Lt`c!%(^~eb&)s%Mp5vVreJaD$-Y2B#Wm7!5d#9wu#(7{^6GV zk;$)1?GQ?r)Du$}jN6ThHf}6C{p~?DQKCvursa!qC+ZlqR@+CLOxBSxEfC}<_3@yT z`ZqQrCV{NwZxLJizl%_0tnrd598Jw_Z&2=!a)+XzSAx>b^1q=bxp^-y& zjU#j{X7||zOo;e&cO=vJf}1o@E+yE(&X1r$Y$C++LjFY)r@g!Xy-9YJ?|zOk1R~X# zxNw=qjePa8c~1nDfaQvr{Rf$ezdZ#LJTJ5Z%W1R}7m|UIMk^jaCv4`_V@9K`Y6e?e z>w{K#&%|ui&xut#%jM)`Ha)TkY)rI!&9MMSa*9JDYc9m^JGN2F-CFI|C(+J^KX{xHSyq z=ZRl5b>z)0wjN$oA#^*d|KpdDOJV58*6G;WrF7Fw{Gv;|#Frm|H=#0oM{r9*jF^vx zV3#;I@I{s3L?gA%TP#C=2|w54t&}V*;*{(6eU2X&wd?)nz}MtdPZDbNh$tAxhTE}| zhe3~8yi7l zc^lNbr#KQu*oh7$@H^>!^R|!LVV&yxxJsWO#(=bYdG5?F`k;oBKaxvZvDu*fEJCsz zQ@R8xPbz8 zHqMlBP3;Y6B7=4nT6*4Lf1*2Sf)X+!rj*NG)Suu!`XG$&;b2<8VAW`xuf3P11rsPSC{zYv?;#+#+ZnYsXScPS2+6nbYhNl!xoe*D7d#hOs=<}`5sK9LDNLoES4$h49mRfO3gwP zx}V&1y3A)Xmsw8apDJe)ba|?jQH%YSlYHKmJt_=i>wYX7d#{pm2dj(j{pQ3b5@YSK z=-d%MhJRqTD^L6VaB@I{GqsTfqJ!7-56({nQ4%O$lWe*Ec=1D$U(cI=*Pl^3vDxfDiWW`;Y)thS?$H1rQ zdtYx|#;LJo(cvfONtA(z{i)4wal5VAthD8yx{u!fb!cw-@e{wgLGDrK+;_0RLye+7 ze*E7UY6J1dqu*T}e4Q?O>&Qe|yf{ID?zUy@j*X3Vq_U7`jKuz)Y%(~qU@H`y(AnQR zZW!6}e*HOcyHvI8!+AJPSv=H%F(H-W0tY5+HH$Zs9au9DN)kqdB(>2K>f~h_)6EB= zBZ3{vvQlK#yYANL7;vd>)hrP*lMu!%{j}T45n!wzo{vxsj!{LF+g#Z&LLEejY2SuJ zitUN%S0ZGgstge-C860VSzOj|f@4G3Gz1JJ%@qGUn%)%?ek9cSL z=jFTOfb^CF5{0GvhnjhtwG*-BmB>E0=FTBoYo%vsTV5}OsI;Z}Rh?oiT%4%n&`s6l z?tC6A(c_$xA$8OICfwex@9Dqq=l#kM8SX7%hDM6mgX;&l?D^0dYEQ&F>f^&;!}Bzf zxi^a}n#?RszIJ;pR?`ki>`i6WgnaK*-`8@@vJH$!-jM~CAD>S6XfQNq?e^oF%SOcN z60(7L3b5m_p&buNXs?5%y)b>;V@3~bEsR3TTGmfJl}Y;KdTuJQI*;Y2&44MOj%(?9 zC9PTLN8p&NKkfl~4L zL;v$aIe#H~lEiV2aQ%>@f_p*O`6PSoV^x^f-)|PqGLAI6Zuqe6V9QzMIreT;@$B)@ zSdU4Ha2!Z*N}`L3i)eMGys+dK)#;>~+3i;3c z^3eD4&^^vYqP~@EYExgO{A7sGkUM+*zG7F3=i}#yO$4jd*x;Gxl#|JUSe0cA*`00u z^MMK!AU17hRpeTOxV6u8AfSpBSf`K7^@jl zT5W+GepeJZ1TTjx7L+agEjiHn;<)##HqV@=hdk52^`UG9M^|`pq_39_uM}w2#nsxq zQ?|aXX7iBTVFtCjG@dK^@cBH_WUF3uQoHwr8LYaZN*;M_JGu}R@|%8Q?D~W<3$s;W zWdA_yF;u6LY2hUCpOV%FO2S0^hKE;RoGf9_gg3(G_zZcS8%Q)XkkMy}8j(UlQXK=b z;p?4NWjqXAPO`<8PitK?ORZoUjj`~b4fJ;plX<6qSPjo;5c~kpaDM`21<-<4qWm+B&N8Q(3wLH}u3ZABSCpN#-j5$)OP)e4R zmjZone0NW2Abp^nQff`a7#Puz`2q-C@>Qa;RCp?Wwn>MjQZV=15M7@T&a@d?G-6U6 z(mpx1gD)wvsW0kBOx?fYr>43jh%~%A{yaTU^vO5=4-?TLlq`?`ap9n(`ev8B9baY# z$2EClTF_b@9W>U)>{Kth&Wv9ww6e>Ct_k<4WjNdUyBl9z0&Z3J`Bg%r6aRy36c#vO z{$B|C3-xanL`9O0NZHYSr)21%A|u{QDd4dB%(FFr_t>sn;~&QcC|DAK@cxflOuSUk z01vPpRVWGP{PQ4?HBmUuO3X0G3TKy1u6$+mrxQD;Oi_DrVmJt(4He(e<@ zn5L}Rke8KGX^3^MQoiaFDi7cFlhxu3jv+ja57q1@3`*s&#tWbb*HkNSkgPsApmhf;L1K$FW8f&e<7eR3%pdmk~zE=j6^y}Vz?24BN zHUWd_wdi$)TS+as_*>IeyuHX{NXg!llNBu1CN%r!zb3(FYRuA z@{Z_&7acu}*F#J8q@laSVY`sKUfactK70UH`%m69N7IMu#zGP-MCO^t`3Kp=0#hyJD}DoLRBD3j4e~H5;*{W9vXR-t zQ`1++r)i4J&h*W`NbfT_KxVq-{f>_S;u-z;Lf0E&G*^cR(;pU{g_Hk0- zMD3O~+0MDVS2{(BpPE3?x9-Q!bFHg8d`G9tc2MK6qswx1wJzIuKFiJ5vB+Pmg}D^0 z5^zc!qSnw66YUsOM#Pq<%LG6Zs&yHn9oMr00(0M@?J#Yd?g^w0yT*tpj(N36-tOS) zmakb(r7MP9_wWq13X^U8wUR6k0aal>pCT*KdtPcQe7MfeZu`9C1t*Kf3i6&Qd~dxG zXVhx!vi`%4&FFuDC=TRdfq$aU&{@DoTah1Z@nI!oTvR6YTPk^YLkoEbcU74GVQNT= zEdD^eacA!j#<=*XZvo{R)?dE*nQ-=|k1BrOoA_*ozi}-dteDEd(%hGTAGJ%$@G4?N~0(cuXOveU88_XzA=PB*T`!%H&~+^ zn$yUsts=!CS12fJVp^&u)!(0EoQpK>LoQX!{Vja5o>!v?Cp5`~2|9|U4JtK?79T26 zYfivX_?VkC`3Q4auCDwXhoAJh**&hDU(sy(FFXu5Pi)C_kaYMftVaj!}L^J#{*!kyPx74)+HUU1qkmuj~FEADtP}SseI*LNA zz%Nau*$W5u8MbwIZHQ=@15a})6w^s3y%dW%`ooGd-+x-A1>=VcE#| z@_Zx+vyeIt^0U;)sE1l?>o)AL^YPhF+Vp7A7fW0Hn2REPOWc*dtp!?D7g8Yg7=*G=nF{^D?3AK6 zLDrFwdsfyn|L*SV&^)_5c&m(IT{$7knC>OB6{>R8SVje?0`gq6h9*G1+X^%ksOlLL z3D?q_YkToq!GB#oWkgsDb77Lq7EIPAar3C(z3~%k7pZ92xS8YH;9zuPCWWTpo-69G zkk@r1Tji{12`5&?%=J4Pq+OhqO!Bp#h6zjC-L6-isp)N#e>?*u%$~yVi1vdOm`&S! z^~Z<0b%q%ii$Y1becuSNqmG`Y%gEoPUlT&`MP3mU^y6Q%(O_qFuPI)WGM9Yq*W=^-e7j+l32WLxuY>LIQx*-Ktnf zo~AVR9LN!d&KynqmJ)_m=)XZr#_x_{km+>IqUYti^j$FJLw$@E`dnBwo8zHe*)ci| zIEP7m9EC>xou4e72OqRtG^g+@yFl&KtE^6Yqt7p|*xL<__E_^yp#GCHhs9{hpKvx< z>Gm(R5P9@HPWvqSDI4a3%16ItHW|gyX_I3pqxZ@Gl8#&$%wp`gN#aU+lNFGL(uIUw z$uc0wrp#!!J16wjM_@mLaxp}gr!z2MRo9M}AugEgH|JlZ9~KP0>>)#{8?ZR(tdU7s z=<)m&!VU=`GF(%~=)wKSsrHygk`o()#?()o&3p1IwgY$v(_g#gn6}?ylR&vf{zb|9 zp;*sipk>hRheP=+@Vc0tCHP0o()|k_W}l*)W~GL7`2;}B_Fs{?TtsucR?DvcDa(B{ zNByBl7@l2sgqD7o_xSnhx%b7ufyleN*$@#YNm$2icFVASI1WB04njB%1`DPD&)dMS z3;38ax)FZZ3oIe$b~!T{wm;_{D`Vh?6?U;^I0Z9u<$jAj0xQn??^yb%?lwYpkNt6OXrez?fCf9Dh)C^iR*8=`{`|~;sEi0y46cN zd(&3gLaX~sbt$tRiVnJg3^uT`GPmZ-AjR599#_%2^-#=sB)(2M6M_RHv}VmS}j0(z% zLUeeER0z@g+`N1?AEX9+YOKa+M;0?Z#EDnN6iovBltp;j=|cfdF?kfOs?H%QlRqdM za5ftjSa>>q{ep_eUkRVneYK^;&gkPa>L+`*&JvEXu=N8PGH94Q9vtSWp3D%ht+OK?_AzVj6}=Lw0;^8gG7^MW~iLgh%?DFVq%(2KvYG(&J;CbNPUs1 zC-pAaeTQ{zs3|#VOGR{FGhXS+$8c>nhgR^{#tt)Zdv{9Cj=vo zL^t|8(TIo-ISP!ZGZaTy411qGUtYhbwfF`63#dI&6bN^W1e9h)ja(MXqar%Y?NDT< zkStB>0++&o5C&D;9%PT4{6lG z-e>R$j3?uLid%X+g)>Tj&7O?r$U5gA$ERcZH`GwaED&9?85E@s2ZnUh zrGB4Pl51AGVNAZmi_3a7Oq)+HCij4;QD#SQiJr^s@`+lRU-%>`QqdnBjn_5aeG}jR z-c~Ke{WHZfAoN?@pKW<9SuFELR+0%4#qHVTm*_mpVKxC?Nip$S3}mgvPF%>GBWIGN zLb5j7D(;}uT(hwImO3YX)Fk~UTCe0XCU^X+s5+C+(|BJ0_x`Ypn_Lpww5xSF9EeX}s_CN7C{3k#9G`QF?;QQJ1K`!jS#2Zhd5W;;t z>&ZT=z;8UrRyBU2l9CW}>Vs$?=pj9DbsNJOOsTw|J0@72DyV);GEzI_($y3-6@;%e zr_5hgH3YUhjYf1D*L1TN>*p%xI%?GOHfv8ZZ-k4?Q<-~jtK8_Y2NkW}a{^fh7i!~5^mlxEiw8pV=Tb6+ekz@gBi zH!$ZDi3q#`ZFh@7=PVQb`7Gd=b$lF&9v1n2DYbn9Xy&wdVdqzEA$h-5z^?mQjt%bo z!pDC)!yCY$46MJoL4Wx9LASW(<0@jpmIlaDyxeF-_sU|Nv57(PJJD%4&PA!dUU}yB zo?BdeN`#vAai^SBIL`0a#6seKLoa=DXMP*~K;i?FSw1f1kMMsZ;JlA>GN5L;a&t9j zp;c;@9RIQ|I(_vAO&RKYSL2_qBd~`a3d%3UK&i-9D~~IdaKU|io-hzUgcOGW2`1E8 zhJ^X{$tQ4?Ma;0Om19yt!(94yh_g>w4ti84_tExK)}i`! zwEU3z^kojyrWNH!_wV8MeOiO66>4;7aKbV0!gv?6wpeBVOIoM*`|iS~B0Zm|Y-Xap z0{xkEPv0cvCl<;8Aj3AEnd)FiBD~_QpoE!78upu#hh3;=yTN{g-!)V$h*Dakk?ai z_X(KC2~~<2rFRo3GlMEnmgmcGfVR*~1|0pZbP@HHm{|%Rei^aex&7mnyM)Oh2PGnm9Jg1!oW~h|jem65^<&n` zq7nZ#fdmIG7j4##2OfI5p9!L!KEi2&%#nGWC1o{Zn#5tAbLV8FEg$8pslHGZazNd+Jon-h&s*MDVEFY#c~!8$LlMRxWba~uvGeUXKDAqgjLw9TrCe)K z|1$zvi5tg7HJuPTLPSb|>2=cSRBpRHLW@2JxA_2FA`ptxcK(Ld(>#rT5BHsAU)_e; zjescrgADMVaIH{UKzs)%Lk6laty|V%B~=20L%R#0?;75rZ*ku!heHZ9yF=-7Z7H5B zzn}+|n0sp1p%z-)ge-QPD>nG{DZnSAA`dc|ZMnv-@H`t&N*Tv-*?S z7y>B8N+++SVAZ<4u>E=O@wVXH=Xhrl=N2By$n6O&rEny){;ylLqF@&=0@iGJ;I}kc z&T~Ob6I6He6;MAZRBFWRt78`G&2VFDKE9EhCSzQPe!@t$5{>($FIu2U;5NOe5XD>5 z@~qIJTPh?UPqxe|A(=uf1*TR3FWv%Ix_g%0FJsa_8QvJVl=!ld(HhZbL1@LPB;hEb z&^LG~U$0MUj`UBAkXp;vE4m?%!gcXmis_^4cFjC9}+IdrX$`oT(u_)Xdn=+i#0;b+Y4^AXU;;2T?;z*mhwj=b2gjgt8a zxSjAP!su&YT1IDve6-B3wWy)spknJJI#nz((eKV3C1mRWjeB4yY=GBz%@PCPqzhf{lx<0L8$)b+@?&sF*IYfsQr%U2Ny97`6^=m@CTU8xA2qQ7HUjE+iFe=EFk+jo zob?0j$)ue{+{c|pfOB<8&_qHPYT|SU7}~Q_!Nw4D>u_D?lU*IbtO>$;Or^)&C?c~S z<<@5V!u&FJ0wt$6_sR?@xn>ps(`JDTAllSeG@DxYBK(VaQ6ka#D(I4qW5-(+&zGqW3n=N)QjEFt-Wz41%cB?ME=JYJJ>W|{HK-@7Ug zOM1LH$Lg-nm{w=hqSk{1{xZ^udgGv5o~h`Yfj7Qta)Y7AA;=nzv#>oPYT+Iy5|Ag( zs#p>65355$ucGOBWqNfxf#Z937thd`t5)v8riou-ipCV*$Z4)(cE#bx!P5K=>5X(( zuy{G9(=!W84g>5*s-ZD4ilJYO+7~ewzsQ4N>^7Fq{^JEO>>6bSsq?$fUfU!wof_f< z%!)%BW_KC}#ZiR+A}igY`hW%$7Xn}V^ZM5)0x})28Fbe3glBDc@{W3i4AHzU$5$Xqpde3_{=k(^@%S81m=u z6rN9A!+hVO{uZ5a_r6PS=B4-6)~6RV$11(Ea!;)rGV?7CF`~@ZHt5y~50Py3?z{e~ zo8Ktl`rQRxz6eU%_7ps{q+s7`)thPR{5v+??@LBn5jp-3*;{rd`KV~+TPz1*Z6Ek< zk@DU}3)-LwJv3b{8q(ZPH`LZ`Jm{V=8|>UQ6Prj-@8upuEEs!YN$i1=R)zXVyz`D} z-fMkuXp%GvT+E`KQ&l_q<$769?uCbthR;A4oTXtdijo*0#Wh&Vb{mW|pa+`u_aIAPK)Je>6)aJ$d-J znstF7VXiJ+5KJ}8Y*>^`I{deM>081TsGE3vXJ*ox%6T}1#xN{2_c#CDt4&1;+!d|g zl)b9fF%q06*L?(A=6$tl)@}f~%e?DZbX55`rB4>WD#>E605^Pl(jX3O;4vt5DH|$K zCu0-v+kly43pMZx##_FHgV~$hH13ADkK5o{EAwiJl-;nTVQdaR3ofYb8Kh$fY%F}O}BzJJ4TIVE{^e$B|43apDWn{W&;xC@#>LZCcow1 zoG2Y^GGSdi9Z<_>*l4f;>GicnUrArXoD2+sKqW%)T$ecerp_dL?e; zqF~9A;2r&lcHThXx4LfKmAEDM$R>*E4!X*ru^oww*WlLf+*)NyEf*P2KvJsH2-iIL z(N2vw6RGRm%koz!GL#+eNiore!2Kmsck)?DLN2vE6a@5N+U{d6={@*<#lt2kP2l?e z_;nbTYiXb`?Q*u*)zP6{li>T))*!{ksRpjwF$b@*{LJ@qKOcD((Sngh9ydxo9>F%RD;Xe z68v$nt&GhSK8Pd!`h(FY9NxHD>#6P?={kyY1s-^ijJ8XZssSZF*YO6tmMaJ}6J+X4 zx?JlCOy9W?y=LVp#_6%wx`0VAokcNyY#cmW6OOX^wc3o8joo;L1w$KrZ6#U91Su&= z+}h8w-6)0V{}%2tA)bd(+pB@TaFj1-XtN3#s`o1KE#+wh=jF~&RWXGa2c=A1AKb1d zZfKbL*#tS2J2Livl$~%Y9b&e4HSe9A=F&3r9R0OLCv_JSAzA1d4%|Oxx#|%E>kv#f zW!X58d8JG0U3{NLaDsFHZ(GD0CrF^QH{uBJB*POGpU4qn-*NLmL>4^x?<6#e9`^F= zOk0_t;TS*LTUIA8#E^G(B&spbRN_U&t3fp8si@m<8P(mGg3Y=Z6~CAwst-FU*z^el zo-w@hdx4luzlZS49VnjMoX?aTKYM}D06id6A>}fN)HpdzeEkR zM1=2NW)|r`@(-h(mC0!K5w3N4V!vn}Ud8;DJ02pcem?-PMrp!jO0zG$|GOsw@&0ep z=;0jw@wBp;E3$Jxjx_t*d$|{+?N)JLKIjLxkw^dv8jF3u`2z|{el_g(GMB%4BTn<~w-on0GQbvuzgy2p)t>^@-p^Z2*4dGz77 zuy)v^g{tBaBU6%t8-$DRB>6|Yq93m1jsou@DEeR&vB4ASVVe|g+0CelT-ssJx*hT+ zm7`meK*I`p;Ns8|ZF$(@K5ORN?#ZcCaoY9u6Y)FQyV2e`*3xNJWAkZVK)?||o#E#E8w&a~(EN$+w*yUU2 zO7w?IJ1fN(9Tvuv)l=IK;xkGhMd|;QBhtiVNKg_+q-o=heX*Hkz!Q6<@BTtbavPVt zz$}Pm3-{WfYHhZNgO7V-cJt-exB}JNm-Hy)Ut=Xyhr4*yrv@xgjs8d!Q(G2q5cxA2 zv}N+s=F9}4kcTgl9j4;-DXg0pj-)Mj)%wJfX{D{R65@V>Gt-CdRGnI>b<6*?U*l^Q z(Y@Inq~voJnLzN9jqmprVS-M8s$3+B3z;5%blK`k{1m}0sa z#U^)kiA_7p2yoAQ2>%9ny`tAJbQKm+E@(3n!I8;CG}u8qY>$Hz!k5%j#XgKuvc^Xq z%w=6_2?V1bMj>A1iF@-Lkj zvcMPVQ%KSgYWOghKx-uh4C5(Xau3AU!Af1isf`L4W==D1?He;}9DfsC9zXPbstKHf z>Tm-u^)2`x`Bq0JhX17g^!}g5%4zDz{rnDd^k}+&#@11Ih|*;O!Xq8>j=`XeO5Ri}U+NK% zkb;r1aqBRNtC=itlt^OXIab{&NQ2;r$G93W42m#{?9cHLO)ly3jdfEoM`TvU8mgZx z^d(=SZCy$7B5*wZg5;HUY8hoTcp5ZA;Wt{V+6X^d!yEvTnpw?`XW!uB`t94VTQzCI zA9_^P^GXNpf*BjGucXVoCO3`}Zv;I#Nas6 ziwD~~$kiFg(hD4TY+~XR#PB1bsv={O^wyB)GHn6(4H&ZwA+C%*5fbGCQbb{4l_$9! znRU7^Rt22A$K(6$%*q|5A9*is1+wjxE^3mYuTrb_!NDV3WK@=LR%z&=YVE0p*yScvRhw@%VQhu3f4x1%0q zkpJ@hsgYT2qF}3UNEuRQf6{?1-We%w3MaPQ-uIcB)^i89b&`G4ELAgNZ&k-Ku2kGf zrqR@Ex|!nDWaj0OD#6sNYgm(-JsG!S*WJy;3furQc!YzTLuIZNOv3Bp~hDU>=P&3kJ7i zPGg{I#lq~nIWleI&7?rj>zyk2WeKJ_+o|LOI3(>nWs4AqavTIL?al*`$)p+5oC1tw zd%xHU2(B%-$V2g{B_qBSF_!7V16|LhEkQucRiHSxGs*-wB&Z@=2&r@~Hn`xj! z&y^%4VWm4o%-(wui<(;O=~|t!;$;)iz$d%AIJGL|+q#AJ zbRCn9Bue&Jv+_N9%-@TvERB$Z{OKf?O{rC2nmkZT^-?SBxk8JDKnaI`W|0l}*I4oE z97Xl`-pnsCNR_{X!kAc7!^hGgf@xEA+;Fa7k*nh+8tonapWB$&Np#$^U|gY|OxXb8 z5kgXXqL8eX?4#?rlm(GM;U--2fp0BH_rF-~CUaC%m3$Q8T0AeAR9=3**%#(c=+%G` z(7wZo4kt=)<{tD$`sp2m)ue<4Mx4Z$V%o)k;hf-t>@mb|&SQte4UNSnb9TSP0ESw* zyFvgGrkkIq(H4Wsu2>FlVIA0}VUHhk_Rk#XGk;|t*MGdi#KUtjm#a_{G1#p*bdQVg zH^0B%xtsXw4fY`zxD~#7hW6p5`?E4qfvT{}-`t&anq^m)ph>bbn)0HbT_cvvxf7lO z(gzH|fj5^i0lIAd;CKpWT~R{jUD`R}da^R&jm(6%Q?((y{&zw)SrpFe&9}pPb7G&& z27^`j0=}bE&dN!B390u|8?fCF7D7}k)I0zKn&^;Ley>c|w@&{m&RpJ}9XI}mIcl8W z%fQ#auyqwYHN0&(JFAvK6YPuv${aw^69C<|jj<6>aJfOmZ)2FZ%E^1a@<$#yv$Fuk zIgbPU)Z=d0E|GGtINqPua^G)8VYOc7UZxv5wz-9egXu9vrWVPx9442`9dI)bzPv^K z@V@RNIlb^DdM47hP$NT$&&6Xe8lVW6+}Jh;jf||;zov^nKPe7`kvJAlSW*`Si^=PS_Sg3s zC_HhQhA?z)O48pA)9T(CfEKwu{~O;juhxRMnckx6hR{ThEDTbZ66BSfGqJoyqw#tn zlF%aiOPPTf>NlDJl6$HP*bA?4YE0+>?n-L|KUmd4R;q&OW%wt4LW4hfk_@w%SN{aqWD zYHr%Q!47vbp&#_UwFK!mV2>cgP4em2#Scg|H|!!_oXOBKpQ`?duM)Lby1Au@9KY@9 zh2;O8RoWZYh=v@bACwgG;$O&d5^9_|sx?aws#|8yX%@JLTQPCPXTO^;7tS~c$Q(0! znV2pSi+7eA@MhQi2b+571$rWj_K2%~cE88q=Z;P~yO4C7dCC8@CVcG13Kpn-150yy z&9K*0F0V9soE1q^;ffp0IxP`QF&MM|GEyPZZ7gje!NW@2$hc8$y`)`*%PfU zCz_??DJ+)VUa36t!g6kenIgZz%_;Zd^&Jm1Z^T%PCj)3LC14&%;**_`-#Pq#_-^n# z7v5b%W>TNSaD2V|x|-v&`~qxz_^x1#xo9a>*`JSEcQM%h;-*LQ^s)gf$957lE8krp zo^0uUcUe93^vk&epDD(<9^`&P*Bx*%KxY}0YMq<*BM>gIr}DrI^0yI7ZdvEnE=I+}+ zngR0_Do?8GBnY2?i%k-Z){x4xyIs63+(^Bs@=Fv?>LVeS1gi!WFrvI(u`qLIK5_%Y z5mu0IwXy|mg+)ZRH?s95#C7<S72z6?i!FBB&1t9rJ12qI+QNyW%(X9Xy^}~Q^$N;u3h97;6n3Ne|KsKqqXakWm%3$`$_lv_ zatzD9FiAb%+?Irp;Jexr0EQ&bO+PCA zVtBkeI_b99apD|17r{9f!RyPg2&sxGe}-k;v?bsuQ{^~7OM=O`Tu2TvV#5Ab_0k%O zO`g%bU7Yicj>^)MxM^|bUD@WJG2V_a6o@=-5_ej)8JCuasuB5zf+Y{V>g+ZJT|VohL*^D|_XbdK40YyfI=`v-9{gvn2d)s-7ogx=*(? zI~k2eO<|-=obI&+6$gkN%6yT|)mr@#q8omXsMgHNF@6N>2OjQ;Qe!EBj^(OSuzz?Q$Ep z*BJP`rh`-*m*9(}5S!cYaLp7&=;K)l*5RBji6VWVZ2)l{PH(@h)#CdUrKG^=;)2KV zhz3GI3`I$6qvwok^Kt z|BRI3qcnxi0^p9snj&^tnW{^BEl%8CnvoclOR`1~V-rYJh2AJVl+|-kGg9V zKN|vD&O-YdBkKewq9#)+#coX+QApI7IjJlZ8VRwt9zURO7BGqE6-OO(Hn3SeDGid) zJ8`3bQqV7rP=197R#Ahc2mtA+O8=uJh$c#1Q0?rdpEN#TK4YHKwiC zH%N{j$^(_2DM0DAzy0&D9VxBrh>yG1`#S4Kj{lzJq1BxKuEB+dyik-okZ;rQWj53j z&MWH26wSoU85Zrvp z7zuw%Aj^xoPgay6=9IlJ#x1D(jUsWJ4_Fz=;+L|`wi9q2q?k-C1+SI^v!%5O%1Mz+ zVBfHJ#ANlzmrKWhnT!;;!weLxx$H4>tp|iO+n9JQ{$N8K1Uy|7AvRMXUkM~HEZXo2 zjmjjuNroLi^SLk@ny!n*6gVmgM<(sI3!^I7S9X39-zPfXQ+z!cp6fWMi51G)rCpWg zlSt}X!Pthuy8R>s{}_Kp6f`kyP23&lvv06PIYRgB^rY% zi=T+wwdLx&_wE#n_rkj@O(1Fja6QMf?n%_B*g4#)Q-($hv(}iEFqsAI`FL|V=wtq*e8`EKLXrOPtykRzW0J`He~y z!Z$R6wG7?s^NO$4sU(_`Eyg9{mG3XdtMQ+7bh!9C*SuKxL^?^LW*xIK^dEGts#d$d zaEJ9q#XP&bIpsCsj0I1(lY~0{Ja1J7j4AcWtd!;|CtFJJ6U0I}K2v=&n)jwFa`+h_ z)9bi*{-PK8ML1FLj%6`f!cZfup0wDv$%=_?D76 z?{~y6MnBjAT)AH!cU+%*cqzZJS(i2YS)H)b)@V}?3-Z9NbcnC{vL;-I!{3qSkb!6l zH<^r$aegmswm~kCCU#_u(|=m=tQbTzp&%6PDfNPklc*A@RsyOd^}6)}{VFB()@?@F zXil`;W$_L@oXcUcNIujC{Z@JgUZFkp^`TBlPm!U5;@#UDsd?0HY^b zoAA|*nmngwGY&RgOq}8lC=v_rv)vy=Qf0pWz&YjH8Ho7R3sb6Hv5+?fTBuA$F%Mc1 zL_j^D&!L?6tD3czB!%4tpcVQnq%dOC3Kp)3^OYw?F&qJpj!ty;wUHjk7gi2|-P@GC z;4@sZ(2M$3`lnA{I!>N(y1k3grf|t}KP%FzB2X2*QUdT|W(k3!$;iPShZW zVi@}Iz37DoRexm!^Hq0e2RjeWolIkKhL%RqN!k9^An&e+7ZO*(T1jvu*$zci?^DUt zhvk?hjyv$&QYYZ>M7yqWWAD*{1c)XkWklx3rMQm`4hyUcoC!LAx)V*z(TiP$7@!b* z{4b@((DufAYgpmGVVi@A|7elSjQ?m68A{4lN8u3XJmeAc2N&fs>FJ=84#%4Ppv7sj zr}=kyfULQ5d4q+^Np53-u@<@QZCh!W*gH1Nnx!75*yLXbCbe^A!2%H~;n~wARmx{6 zpYEN!b~(jl(1a$#G7NaEB0gfH|LhAleS=129#}4oT--QOE{hJ4KAE6mEI{r18P)R~ zOTdP7Y12LHn3-o^)lW+yQEy=~i+$$No5mfchghn*)nZh@c*FFD3{O2Tz zjYLA$bFV1YsxlRFUAz81z!t1jgE}tpvT!0gAIxE*vZ~66PNq@IWb9|dsdo;tL1xfLEVg2UnFkqq8ovxNx(*Fmb_5bm0elBkbWh z(;O614*>7L9Yp5Y%v?uZjRBcDB$>7%*~zajS02G5c$pvAtVDTep%LA&_DOR2Tby7K zGB@t$9Y$!+`{{Z&izkcC$nI(&vcA%$p4;f@HGJ#b5*t4U&6RRHk1|qaY!mJ#6 z78=0}L=+mI0_>&nb4T{K?e^DV8&n#yGo!jB4=*8Wr_7v9+ksj1$*`q$vSHP58dCZW zCPUFXOym!Tzud~aGfcVW_ILX|Aeyz&LRw@sos+pG2iYG+g!2p9BqC7ogUso zU42hCz<|hiw8#!q=t@C))8DqHg|mi!oMi)1^B&yKkf~#x8lBTDETS(IUmYLUlt|>O zhd`qZd516uTHEw1v7h<(DNjt9j7-BeGEcD~3YgVwemV8Pqr3y|JGj zP1k>xXDz6P_L9S@-f_IDva|4vfT13@WU@`vO{qrtjxheOsaC0T7?C({sKoQ+rSyQC z9R~%dHl9_=)Nj8*8Z2qs%_GV7MHqohy`R>*5;?OWGuDVUK@XEo)newzN9USrNk2Xv z_U}5asylGI)!SMEj*)1*0qk?q`cx#tA0j#vYCVvBw`G{oe1j>&Uy`e0T-iT~!4;O; zXyihAD&LVTl#8X0?Whmyh0Rtqvmo(S+MPBh_jX+(PVl~Ef zHw}lWd%ZTstaNs>l4=R=7#sAHjKmGh9<{R(7Xw^yWneJwY^H4L$)V#+ z`Ik$BupDF63&GOm(p43C$flaKh$Q`5*>MmYq2GjK1!L21^(6(l%GV#r)*HS3-bXnM z$D#0jS5324=n{CYI92*1AJkNYxbm2LR~>``x6hdUCrl%at{U!9Yxa)!!{`!W1{SJ9 zR!pV+HM^Y3379dZY4ZKNi6ewb_tkX)N>k-qUHej;&u_$)b)&*yXP@8h@1r{Al5IXz z3nj1F8OVIWU@4z z7=q7v8LxU^HXMdte2T4n)3B);A`rwShWvL9lU}}TOl%_KZWf(hWdb7;gjsGwgekGc zLXI}6ikf|sTxk%8BGIFmN%6p+;gp*b&z!~zCL3B!#BH5Z8$#9!S1^!H6_Sl4cW4dM z+a1;;M97!+lKfGDFd2bU*bM9FMr`Zww_wWpOeT`Jo8|@XuzXh*FND~lswKKO&I@Ic zUNdB<41e*U;Blu^fS+X-XxgO%#4wsH|(dtH=xMmViqoW6d2|WQ-w82U|1= zn71LT8SW}h+lCMhnz`>|wK&rotBdDn$JA+_mLf$cA_p||(v^jnJXXksFpfB-VfF*@ zSYf%{Dz9IkjnV=(Rkz`1?9Ma)^w)E}+w|a)!;a)=A-zHK$`G8zD%Vve!-|^YdyK{k zI|d3+omVmp+Sw~0)Z;kIZ9C5akM(RBWD^wRwfmbbSzi=1qF&C)osIkN5Bm`U=krUc z+x@#HA?<^TVudW$%@BCt4iK#1h->Lb*eQ?_Ms~p2&uF9WadQzoX*vUpJm-oG*HR8{DBm^flY{jY_R zfc1YHC<3z8nOVPd-#)2e5U%W{ZPx+sG-Gm;XTk!6W_tEQL zN2qVKs~yPrYJrBa_9f3)a41EnPocxI)|^yUx{-MOSmF#`y(lAse7$Cwqv7_UDj;D( zzGCu*j*cSe9EZ}vyGcS3T2!5?rN3hV3UCM$=bwPFE4!mcubINuIcz32l@hbdCeDe~ z!K7&>fc@-b_<#&O`^)?_Kji4APIU)_m9En~>iXSYHgAM;ZMRQ3R6$fHQ-8O=ZCxtf zm?p7SD9QoB#s^XPLX3j=2uGfLOM>-aC+?5(^-G0M&!_6<8|2*#b)It|K<@Gi*BXJq z^S%v3VByNwRl}h&3SX%8TZnTxclAC_UuzVL<-g5$(ia_;(^h}0w1~tT+6XhqTj5yP zV01k(Q_9@Ab_eJVpSs-Y=ruS3(>g0V7OkrpO_T(i zU?CYwhAdd!eN$N>$jonfG)IjUz1@daP2QbZ+c4FR<&E+^vojDxKErR~4@_pq|w(yl>%G z+9R@atY@z@Du((AFF}yZ%2)t^0jZjCDhN6 z#5^PLxCM|eLm$xpH`7J&^7g9i_&-ABiY@SK)%(YdXhgfs)S;KzlFL8K>;Dp6wHu3m zNNvn33#8{-3`2fLzM&4)o=4V3xLh5w2GWfpA24mKO0S03(rcFtqsj%osMOM>UzkG` zHrFl_4Oj0f73kh#qSDszCNNn;^K=WoBr9#xL-^)%I zbZ$v*kACeLb+))*$c5RDd8c;gr?9HGY)HE8XMH=W`Z*lP&~LG$_W>t6nLksMnMkVA zs8l)+#+zh2ag=ZYBQN)@zBw}%5$!s}q_bvQCTHf*@_jF}u=#1}w15+R6J7DTH|A{$ z0F%%y!6Ys|dSVzTR-9Z=IBe|uK>!u}LbIVjvH1BJIg~c}PV@8Yx6cHocuIk2^JeX|{Kl(>qD zG5`6tU%b#fNA1jdQ+2ItH_S2O_xJ$K*Iry3x=ZSeXCqdzak_zu)E`5t4w_|Gq2*x; zXD{A@XFiq~4G1rZ8`b)|;c*#DR$#q}GhqoBJN3~CW)V-nZxkU$h+lxfj`lM@ZE5l( zzCGOBEIox@` zCo$3Qwl>!&JN~9v*1-37kA|Im4oDC% za1VZxX*oNlsuik%XQ0Vs>yjfs!g{4w4UO|HjK6vtR&je%AN>Stnm)CmR~b5D!Eu{K z)0qs$^RXs%^S>VMFVE~n$C5RhWImsQZ{?HAw1o$KKiK&d;?%h9Csci3+VRh03fZuP zZ?DGFCEIyvG29t^8D+dtK1WdDL2i1a0&#pXBNb)83#oQlQ}ff+3zYhJRAH1=uk;DLzgZ`g$~f)7hov1_k<2gs6Z3nU7lQJg z6cs#URJ7N1_34~J(2H(#o=_wfYvpEqBiP{1@jm#Q&}IcV@)*+$V2%B!l2QZ+5w*t! zV5V}sV2qL;&9Z3fhOw0EM<)1&P63g{L(KzmA#SH`+2~FlDkpp($nh7NIj5M4h&MLB zxNO;o_&6IFzq6w*V6LwQSYEa1brrsNuF?z&q#h7V)h45N^;;C}-9tBjn>CC0S!^~p z5`H=9J7|n6Bk`tJ>i9$Q+0ta{4(K3Wo>6HUlB!k%`Wh|QGlk+2Um)}5*;n~4T0`~D zw|+52nA;rRv|k_P`r)#j0SoFapA&JuAMC4SV{^x4JEvSkeojZ6C^H#1?YSym^D7=u zSaX)^-8$3SFd2`_HHZ1ZUT?O!OR>#bCsv(7odpLarTkNO)7ulPPUvEty|P?tg7G#T zJAo#v%n!q7b2PX2&esmT5uCe*AAbIAou$`s(RcH^cfz9e0hFp#>)7`=M1+YQW#pBf z397_o2%uQ8t#B$8(njG#_H`&5MO|*dsBXA_iDN~B!f7_?pg08Ok2nOBg!HaMSI>^* zaF65*PeKA{>_jXnFVAEJl99i}I$U&dV{O~Zj`ea!ixeeWC*LqZR}{2l%`DFM-W}a> z{?-2?|G2M8{9sNf>qo9lH|Fq&dRs!l4i4@<*!0`6wDvPHT&?%~f>>2fAZieaBN{|5 zBZIhi+r*KAdY3jWR%thD3xEWunJ^vmzRO5klt&ztK&{E*tYJR(3D=T&s&ARwZfR|> z!#_7wlca!hhtKBC>}^^YE_$U7_vVky7)j@=R!q5+Awd?X;l%Rx21HXhA6C4$%$F;J zN1#J7kPW`idKdU?Df&rqM3tAUlT29Su;mfEKRKLSm%xdk&D;w8Er-!Hei2m8qG_AuNT-_W{GZ z)GwRfl<0BzSJ#eC|B?$yTXrY$Y5H<8&RX9j!4}iQ)}0*?)6+SN(0gE&1so~V8g*Y! z0g^cD3nOp-OVqtrKJwZ?xneW42O$327}GO>Mfugy&e{1BBQFu4`XMFmqF~?2@X13U zLXH3OcsKZ|L71nT@Nc9tgnCc4Oz$Pu0?`^4%|M2kVJ{wyWtc(_RJ%4q3}|3uNTY+E zHB*}xJ%tAq>O@Ae7VwSv9e(^O2na`=xQ|%&kDjt>+xJqeU5XP4TPMd0-6PykFbsOP z-6ceeI$gOh@$|0MZyFw=?O+} zg`O}Ga$z%$i0BT1{ivgfJR*6hA{;_wX4$|P?w1xQmqog#a~8~GQ=qTh2iW@ThB9e5 zZwqmIm-<~ZWtvCS>R~^0dtbE5aK7iKScKy*!V#`25or6;-&CUrtFzMve@!Rpia%0mfL3UpCntIIy zX1o?r9DSKU&q^Oqo#u-hH||%Q(;7JXOPui|dW8_mtWlSy)j3NcWT=!L0ofki5?t1a5;54g&LtG588CvbUhKD@5ll1Ghr2CxzV>(?tJAo>dW&iT>lj6Kk?$ zKf0sXm%>%P0!gw)toO3gJA_1T3;`p?KU^K5)+JVP$Hi?$)mAL1VKrP1Ngo0w&`~2Y zKfk)JM88~7s1GaIK+v}R7N|1CAhj%)4(D)jx!hNK&28w(ehI z8CWrAhyKIaGy%=@W0R4K+k*=Cw`!*KontZ#=BG+xvz4g_NG&QjUKBGZh(ZTOMgF+@lTtkJX z?8bOtE=(1+>o%s3=TktAw85G3BFV14rvix$dB*R0$d^Le{{Kojy{a^GWPbZU;uS9x z8;Fz$*dg|Yl&H%<-UFT~{QD`PdzTbO*PqCe)`lGd?(T)ZYGWkSV4hMmZ+u&k55A$K zAl_bXP7*~xD=6GKR8p9glu`yzPu?{vkoi%}p<5X5ZK_kDMs8JL8nIaXX~TlS`meVp zf%~YD*tA6WIJ1{Q?o>}9D_%pu6|1q!^;V}6BV;rzhW1yYkN+WfFIQ@aEG9zxH||&p z@ZcnOJNfH@3YS&{dg@b-B~b53Oa^xq-(@~%tT0IyN2lT@^kM)#u`4BTmImUH{Y~sM zyq~0+HnM*cz8~`2#Rus;nh_XI>Y+fi5CwH6`I2$Sn9zTi>ed`O1IkzQARfB zl_WyI(Y>=j66YU*>kBNY8+M6R-{z{;vg5Q{RKwcZU`Mtsu3;%mCcng*?0zl_6=6)I znTky(Gx7!^H%@Lf{Vi)+ci4_hoLNDhP8QViF{N7Jd%XVWJIs}{h=3JtQQSpiQR3TY z2Mbg3MKoR*Mspp?j$U#NBecwKd$*-A*YF4yG%T^@>SYhzNa;^>mzErTj@cwE4wt7$ z@pQ-2?+wxWcU8>wm^pRdeHMT#rK~r~q8}V(B4t}tew&SW@*KM!kSENM^tH)IoZ;+n4hd7@@|zV95RLXMDxP%I`AiO1A;zKIFqi{it`b>1C7e|8!Thcz3+h-2oXzBgD;8>0Uox z^_Jx7p|E8mjkr7w5w+2Yr>PA)sx6&V5y9HKc0&L>>QeJB^#Pdr`P(N-Ay-Zs1R;1m z`gde&fDk#S!TlIOb{(pHN=*=FiKeN6RiVoGW`{F5kYX2vrH_}US?3qwl*nOQByl+4 zaI>`$@ZH>iVt^@TK6%5|hr?iv_05I{dL2Fz;a1uhd~eqML7Gdhi*d2rl6O1-K`xbJGsSL|>z(MAw&c5~iW_FK!m9yUJ@s*#L9| zB;hmZzXXh|AIEq9H;c>Y2s{!*>>(AdX#5(oX3 z4qLwLkM2fZc2>-i4tB9LO%8gLNVw5h1ejp{*j1re=>=#|PZf>7H!48&N>@*=1d&$o zpw6**j$xA+=)IhMnTwDwk)bd}BNgZg{nm{7=|Fl;keZ$mN=BB{`}bo?r9d@Ew^I7$ zZ1QlNoZu{38)AH_z-bpv@av-yPZn;}tv~PJ2FL~yiT4JZijstnmvN=pvu3T^Uyca9 zGjq;Eo^#fe(MVL&*SCx4cS~xPCb$!wmPGSJH;pZqgNk~p5nGQS&5i_s2#0N)D9y*} zd2O{$);Uw$)`l!R6PSM`Wx@fbNY0Hv(Nj?D&d!Wk&5cRu6%6}<2BOEqzqqzrSWkzu zp_Na_)$6SPT??G8^qw}Vm=jFxJ4ow$z4q3J6)CY7$VPBxMgAACXuq@OjZ3u98C;g+ z)O8uoP<@o<)-unyf0fX=vZ}G_weAztWy^@_qcG(CgslcWH3$SH)otk?RAQWm9zY!vRnf%^W5+?H^bw=BS zFbgno)d^{V9!Y7&+rCIK7-33-Ptp5Qjy!SV5!k6LZxJqU%cd8-ku%f`Jk%^} zMBc`vR&Mv-Pk9ehs-JIteS1D(6Nrp{pw%Ulpc&I;V|L#JXt!(CP#0>EIbm^Jq7pr7 zHHDd=-xB-O0pLIz{~S<%wT1AhrnV|?_T9XhrMkd=>QV&PN!^aCZ5H!CDk0>guhqAk zh>1>5HL*Hd7ErG_0-;`nXR25NjL$e_zK`@;=APz#y-lp)`w`v&Yt9e0a&;-P66_&D z-&8F!iKc_MCOd;8iLRCYeDnOVoCkQsY?O9VUC+U@t@-bm}%qD)~bwf*XaV8p#BQX0BWm0(((!kYx zo}*^@MsE|v92rEZVV3}|Qw>HdGPYhk8d%nQA_L>~a`zTK{z3`4w&l`{OQge4jT`nI z`^VvmH0@~9Cg}U~SG6*aKH4G<#UA!D@UStJkIV^n`}TaVmFf!}Gy0eq2W%d>9fGX$_1@o86jtgq9Lxv><#^XE^>16EArhCm zR*i|PJTa<;$ z;u96xV<3ccN&1)ad@lJ_O##-cHP3XHe}xGV*UfJ+ff_R znypD1+*Y3!ZRg67MOtpB{Oc1O4|RW#I=*Upagp{M5{j# zQdcWhXNthn3XtIW)7!Ebc;&q}&bV>N^V*N*5@eW3t>NgYCg$z-D+}Lzxc)UKmhw-C zJc8!m$+6~cTIan|nt`TA$au(IykeW<146$}RjUpZDB7W$;qulQ$i9GAAk53~g=z~O zPgbuz+@70&J~$1N-$06#^}At4PlbeB`_QoBv-^$q@!%>@JWl=G{NK|^*z4i&*JsA& zzGtr;{}L4s2|I0~yzouCaSgZ6b;yVg|N16Fw%EBYJA$k5pGm&Xr@0_68!=AFHPDg-I@3{)~7y0Tc(}mt0KgGY@Q&=~-pby)a8q`h9Vh4G{CDV^BeAYv0=9CZJ0=sB2$Fcl;#J$uYS)2- zw8Sv7O-I-v*)LT_y;RUH5mzwLG;E^gv08Oq?^nh21Xp_;{uMItfgD}e`5o(T#JoKA z`cr(Z7Tb^f#>!W@ggfN33|j2Ur#2*hf@MdoSq=%;mU})bP3aGd5ZzX~C8Wp9wGgg_ z%WeP&V|6)%PBho67!%|xjCK84H6vF_`yDRX%pftIl2m9ygaga79e%0ym5WlV;vH|oa+)B1LQeUp8vdGZf3+n_KE zZbH}x`TA*Bw5oEiKf9;p@d2*VN$~xJIH-btFzztOgX2XdB?v`0ZVAJM62Ef8XlS4~ z-re4kQhE~Fuq=aE)imz#j5h^(T2PK*czpeqkqN_K4ygEDzr*+WyY%1vAFvJ1^A?~{ z-Ct@b1Tm}4e(1tzR5xnMsTdkhQq_oXp45rbh6}nWEl=$C?qI^GzaH!dtExABH0vTI z>NvMgKH!<-YW(nU|9f6=C|0m@89rIdv`t?YtW{ke_GKJeEd(xcr5FVyl!&jC)UgY8(X9)Ye)Sw#rie) zUN|_Jh}UXPCCGoU{m;RW!`-*wt3aRge>e9ZiK{UB{`~$VrPkebV|9|gmKtiVwEc}& zZ2uERd(`&|EUFML=Bum3?nd0t3%v$&dgb3T{3kZADmv1Di#q7MMF5a zmEpWE(4B!ONC^bW8g+t7ULM}HBSRT!2&H(lFi3VV0&+3GE`Zxgc#3$@CJ-%zI-5?k$} zTNPo7_W<0IUf2)1pGRO@vQ3=ulcC$n0Bi4t{u$ug(l=JF`I-tVg!(;e|RxV$$+}cjZYpK{h&lhU?-S zd^a|MGp_VZfGnFtc(DnxOJN+lkW-AD$d^&2Q9gC!{>PEcJ88YO&+Naql_NgF`drI7 z1W8%G`xG(#N=PD2*lMDS-tDh6&gdaIFTXPV?hu8eR}dxVne}A;sU%g}OCjhEF*(YE zsMri|DQKCH+aJo)8&4_%inY|8<_mn4*t+!Ur&|wpbV{pOa6>q@ForV8j>4lrD^dTeBSRmH`&8V`*55JPf z`m4JBSMfhah-S~{NHG0m{ohpmgh|aM6!O$u&G5f_hhl@6%rJK(0cp8uMeC%Mb`=zL z(foQ4*|f4jl2yoW1>Qp}Bs7!-GywY@3mhW@do$IFEpJ0tIrbfk=9|Wlx*rKZ@gPm< zB_18`Y{ptmzaiN&@Kfk%TceAtp;yOvmgpdrc-Rdu-G>4dK# zrJ;pQ^+_py)bv}SfpSoH07mHmH!C~X~yVnYNurZ9_z=JcXM#fv)m?c$_8mm}?@M?MUTxBtN zOGs(ayAw&PFjW4)DtW)~rPm`NMv=4uP^v|OHe}QV%ZM63i%3-qowCCCm2({Yl{w8s zn^OFJUigk+oJ79+Y%01_2#^DBSb5+R&PQxsf@ia(}L zPW)4{$h8XUGMD>1UK+6y23*-~Cp}#qc6~W5u}(~{rHW4zfoXD|*wLcxSU%!Iy;tsQ zTIN?P@#{kaZ|GCH@q9*DUVhcs=cGrIBi8!ztNGR6mU(jHU1J+zQOl$ zG;2ipvjy3bomjB^BE-k6)2AtSPz9nVA8!%j4V%7UecN_8q#tV^$<4*}E0I1h0$myo zd1Pefk&myh(PGBnz;&TbRCL{%Y;MaZ<8 z27Iol_0@ zenjl*4PH&9 zk<4;p=~HSnFumC@;U&44u_TL3%=hk%<1MV`|N0O|!HYnNe1@xpKz^P%1ZfsP zlryP{>7ffbRo+J4n#`)5o4*MbwK|(}+Z5FXV!*V6-_skh+c`wHpA#>tVt|=1fI2Fx z&_{N?({xOP9U8uHWckg=H3@0k*}JT*%J(ju1(JXI2^v|t+W~~pSzK{ z$@l9ZUN~OgWQEDAEptBoHKKt+O_AbMP?=txkYY#S(cIPVxe52KygP$Wh_nEyxw|yj z_LW>Uj>CB;;mIerKcb%|Zij_=xX-!&#SFGfEQt)DCm~)6Y>=Ylot0$bgcdCuWW)st zeq1i7>Im#zWIGG@A#^ch5WYt5VPF{QaGehs~OJL)q?c)!Q$SVa%*)(h85V`_^mMn`o!i zi%Hgka-}BJMF;+uzdV>ZL9|UpoXKRK>h?O0x^mrcPQwaIP2H80O8KI~6GzM)?0%4< z*uI#aBGN!3x}i~IM>?!gG{npj^6NO%07kT#;=GTul~^27DP!o8ZO`_MelPBYOJuxP z@%z-wFON_{_R&i(w=fw6M!uP#xFixs%i2b zD_^4FB5l#n4uJlhZZUed>OW`3h?*C06W2%KpU&^|KLZZCbbf;F z-&G6`u3jw-?VS^9jo3coJ6iuqV@8WjY5WiGVnKs^_nPq8k)87*)mSk*Ry*+RJ5LRt zR#HB#oi5hYO5}#3Z-(XCQVAguWjl|StTUYQoKqyVUy=5oTD~rt7m8-$5b1&l&`3u6 z1P9`GlA2VxI(cS+#yK=E^2?)iE}GkyeVJpk>c;75PN+eM>X7;G8g$g%v3UPpylNwu z#mwU7XE(*8BtFxC3N334_-i;4bcH8B4Cqvcp$*Ws+sVup|BaEaTS|6a*IH)Y#SSVt zichP6ErieV5mpTGhV`N^LJf)+6Gi)Z;7A{)+POR)b+$?XXuiXV*Azje*PkNsO;g}| z7r>jeX~CCWroy;MyXD{`!nn!E>{PV?%RLEEn5*B)WL2o&a<=?za~cI(pN>!ko1fio zhE{l$F=7N2WApQ4xrt=yJLJfsIf8s8$`dSF2Jp%871RNIe_rC$RlP3Tv80*tb<0}1 zh8*fQ{%HVhz3u&=4G>k_&V2Avci$eDc5;eU7?OlBNJ+k zxl#e66UPPDdTy;|yvag8$0iC7kmeq5cxW`>pFhrPWCQdXvcZMH*tcgjcV6ZiDVD>T zy!jGpu-et>{J`qNB}+5jAHGm@;&Sf|e3{tM`cvBEHR9#~KIn^JEo zjL541sQ;CW5Ic^s{wB&G2&t!jFu4EL&$ttp1>W0^AGL0~H6sJ3O;{(yogb3T!v6C9 zdw;R?_qL{cuScT(hJ1hl&VM2M8diLsv+#)}Aw3 zHmvxuVeqt1`NL27H_3r8=m2v2806QpuYF9`pfjRHXlTeL_PmKZ>5@16+RVn~1YA%!@8-`>>)w-Ay98^|uc!&~8J4zm`$LN}YhHg^#zc&Gc z*!uj0)k}hroFB_`c3Bk}(JOf84bA%uR(=sGo;-0jVI4S}?7c9>jgetJ{Z^|v7WdNi zwymZ3JBA7q$u6RRZTM)Xt|}&nOq-GErJBZbP|1i&$?k}D$?i3aRB7(j9bKigAB{D!hW29(db{>&=+9uhEkp(($~me~Up zqV0{h_<%X#E+Ln?=9Sc#0iYf;YKy|WXyUAHk-bnn6 zUE9%4uEe6c`G7Mch+PQlN(9ptUo{<29~3YlMUlBOR+qx#zlQ50nPl4QKs#^nYg=zy zZwksk+|M=@wqyv^64gQr&5C)OCzH1{>{KM=E>sFOZn?Rg%_0~!2N%Win&P_uGz3z( zQTTM>iqkfSW9$+Jl5&ikE!OXru*8wyVAL8|)B!1;wc=XT@r+F%+7Sm*>y;C^nQNbO zb3a%6T|BtAl)>xk>p|UfIhTls!1Mb)e>0i-!e`?7f?6W_P083VZDHlQQpPB#lEa`v zr!#rZt+a3Dy1J51ls`NT1WS!M#wunjr;9aOqHgnEmQ-PwnsC~j$)zU9HHw8m=jJX} z$>b4;obRDH1*=k!@h+v+OJVwvZ`zIiMD5-ofl6#grF`$7jt=?unhGk2imrKSB)WoXY2di$}Q3$Y{QVK8OI8seM`UEa#}@nj?f_;nZ#}Tl-}vlzF#x zmF^i+$!cm2yIt+sAN-n4MaA|&4LLF+ytUd!Xq8rS7M~b_E508*QgA(at=dL!_V~RX%yPj zb~q9AQFLloZY~5ebA;t+NhDS=i6U6MBa1L2JI7cesN*Yun@}ylR6X=rjBOVmXPV2^ z@0G-|jPSgHa$|67)ZA>9&TJLmY!$gELj<3MBJ5=Z3F#(0mpvIi`i=6B0%(-rSqt^- zp&SmME1sV!zw?YU>YzT0(^f3?aRh0$Z0c{Zk8^+i%3Z1o#eKl(1+(^&9eTSj0l6p?71Pe17JiikqX1FfZ^tr-_ZtJ6rUviXTo`+mngwy$5xRY z^(3KWk#GGb{~Mi!jzYo@IDIe>M;F8jES&yyBd5?BDl|_?zJ&@^3_iui19d7KATA_)%nx~`EDkf{ z>`JF5zcMAd#5osNEw(LT4|4^A+^Q0i=aLi3ax25&;9XVD*l!4uUq};qByzHOe zH3IJMc+F;&#AFwku6LR-K)MzjZrWH7*VKyy7i!rE+23y60`YmmEt=N+_PD#c%#w7j z_(O)5OaGHE_7;AczpP#lR=+!DI@ww4^lW#Rvv2_jq$s)%*mS+Iw*aHj8u7`&K3A3N zJG8;=i634drr*U8pwOWlf09K5G5nQgu_FFEex+yACUwls!wkcuyv$Kb|L~1W;>JmM z@(Q_A4<~doTSTJhv|n4r1NRpXoNKbPL;C!wbRcg@{f$#`fY2Hu-pIgh=LA9DftR zIR~cLgz8qT1n}#BLCQhFHEa%%WojRN`)~a+6LrtD<1W$0dQHDbjz6CO*E?;vV)=Zi zhmOE_K~dzMiPdYs<7#Y-0u`5R-B8a=Spr@f9J;Q+T;=iP(K|jb8rt_vwR~ zyP1@ZnY<63Q#&-CzhwfQ4%+!^oIX0fm(-3GbuzQ%pin9aRW1?A!NdO`X8x~fT_hlR zeY7Uc{m8W%B~7uK`dKj%yH2S;Y}T?Fjfto{^@5gUG>zqxSgu;GMsjonz# zPipO)+|rd5n*hs={D<=o{P2c)yTxC5jX;p(Dc(Om!b!Y)d2&|Qg_xu2JNX^SfBsH` z$`E64!vTxo@Ha^CtXlnYbQ7 zq2}}NxopP)xowGN^uceQG_DVi4Gf??Nj~-tSl}b%-x5};%)<)za(N3Fv8Xi0rel|~ ztl6>o#?I@<1F9FRorxJZi0Wq-g1(Hh8imG~8lM7QcQVmQSpxpDANP1yE>|Lh zw;Fnq7&faW(lawOrgQ@2ahapVuzR2aKz#PS`AQgzc}!w|_&ud(Us=S4;K-28M@LBq z=IK&Fbt9zQHCsNMEOnM1GYwofA`uQyvichBiMI(;kLEmJa>~nfrIW>dso6}sE4-WG z3V^}^_@o-Ti=Y+>CBsD)2sdvJdI&kR3U&YB0hcvmr5odFU5Ck-0xNHk&G<)7)Yq*k z3K&@7960@@*i8kHZQz^MZa|@=_v6CD`-87l9b3f??Rmq!|7=#9d6x6CZAq!5df!xQ ziWx?C+u?TL$)_N@3BFFhJ!R`pPm;GUy(DN_>`PC(uX2K4%){^98Q>X@UlWEv#vLOm z6>eAs2*Ty9x{#fb7-8s$bjqQinbryPGnE{hFh0!94qiC}jK> zgw-%a(Eo3(`~CF+@Bkdw_2;;)P|}~n@#E}<)I|wa1S{T`hPvyfvAVeTZjYXwGLor{ zFze!Lk4w?oN-GV7Ty665*r9YFAsfKiVb)CMFcsz4eoMp@*Y*lUhemJkZDa(yXvb?y zCb?&2`rl?@FGP|misCTn$SL5kj69cGdxB_is!d*&>qm(i)WUj#X`T~gVtXZdJi57F zIN?R0!OJYQ7feCBH_D(tettVtBDNB8cbOz2o!!{w>Ajm@ zgSKhUCwWPES8pm({={2iwp`RN5!Gt!szKNw4R>5*(!~5`-$q;d$dx@xHq~b6TXjA+ zKBT^-_M_{Q&koP=f`6Mw<=kboWpQgpgu;4qi#?en*8~g4tItBq_R0`K#$|=SM(x|* zD^jI;hvQVA26V=%FSTSsPKUibuO#8Iu(#7zX`H!j^LiGq^oxR#MWe=lAX)12K< zV0^kjWS@ZjoN7WNct_ou_sJ$FjA@)cgR=hFf4%>)FY8?>3y=9MA+8^gBj$@ff=$+3 zo!WuG&c?J6CH(dbC+zu{V?^CXl#0C2)ElcmRAAzt2+tuP&Z-5!eG?B>X*OIW#nDx$ z(#Tv;*9_hnlr*Iq@r|-J!1Lt-GaE85w=m%uVNtFQpewL(Kb^R$v+Nk_v@&R#to+58 zN>rWiTr=lrcI0QEm(v0&{ zGGqe;D5FcB-E&O@4?%;yI zy7a$a_27n(XjkWdvxp#KyO^zrXB`V|oe}E6ZIm>_h~ojPd1wr1wiw9o*k=#$Z+ybd z$53chaaEcBg8LorF|XQM;NWGELmY;i&tyZPUj}h~*kyFvb*{n}DGSub+anMy3y5MJ z&AEDVW;eDAj%qKQpU2h%$hMSeDrtn>mk0dZPiz`-Ta@eowfxGrJo_L3(~OYNh@?pC zzf5Ep(lxu|2CeK8KJl2enH)2}j$~QwZ2uDv6PXYH5EZDXF5Y?oe4TJH`DS!)*O$_a z+kb~gD$1u*IW}om`y(ydBp?Yk6n2)#hD$VrkCg=1v{7<6R*@JKDQN8rgVCN1W%hSb z+srCWy;UEgoOhnwY0{AYI5kN*uNQ#T#p>;Ir(s~7@m8m1;*&mQ)P28ni+IBj3@ZLCR6K^QJNTh9uLmNocyJ zoV5xQIi{;_!WyX=5$~rnKEGa8$*-l5)!fG6Hf%`1s0C6KFGTVDl*y;?_l%ud`r+Rk zo6}0UuQFI+pHAth4P6F$m4D@DeBS96m!#}j>irU`P?eb(ZL(ICcIiHRcO3x12C!HR zgY4_dg`Vl2=a9qq!i6KLL_7sp!21k2iy(xt!kc%UqlAJ&I1NV?>fZD6EeF(r{mhPq zzvxH4z9)8DyrK9064b=$DUFPd3&G|x4=;X*>aIz+03&$tl7hBjwG%O~d|E47@80kH=+d(<8Z zQx6>Qr*f}h5PmjKYDnXL(hFjE!Tlg0ypYBVimhw8AvE)>LKe5W{F@%V?ALoXPEvk` ze)=(Guyzw^IqtfrVG!HnXjeF$5~1@OyY-m&3N!($zhT(yLp6N&n$qp2Hj#Y%ECK7O zrSUvgx=*-0(2>h<^7ryREbTHK{knlGd7csUx@6kARRc^$6u_#L7x8f;MQ+94EMFB? z@sd#H%e88M3NLC*_GrkQf&LCkpF~HSI_2Q=wtRg>2=Ff6k5_xQOm_qzMetX5lx$JN z^9sR11Y%_}$FubBCB`|+aBL0B_cms{m@sWSGMuz%xK55-{URV5QBcpcuXz+Bj@K!w z>*_Q1^5p@O=j$n>@LO7fcJ>6R4s;$0$`~HD%0joFhJI%&n~~oK#XTnsCbTW~ zgU;U?jo*Gb09uL3Pjs)jd*lL-ejmP;Gv(RtGc{ygmS>@7uJSAMgb-w*8sV|WI$Oc~7cdng4P0UOc{}2< z_;bCjCQ`=wOiK50hR_*h4T|qnq2c_c?8#0@k=zJ*U zxh}3?bSniquYI{peJn{y|9Ry*;1CFE8!BCyZd*JR>b}%k`7SeldXX<4SYYY;nhr6r zu(+IzRGgg zQZbyg+QUW3VX4^cdMe%H=7pp6(j_wVcq!Vcy)*02sLAqY#*Qlz#nPYd4ySzHM5oOY z*ZiH=!}C`xt(3J7Q9?>2Z}+8_e*MVucHxkc;V%`_TGx%R#lU#!{@#do7t*rKoV)HK z_WiYM+MsV!V3;OIvHe3TT;NYzC;Kf`zM&hR1%l=GSWIu}IbzDi&}I83D$+22{-p={ zT9DZdGwPxaLYwt}Q7iF`sdxXwtk9@3*xQ#TkyJ#+mI40>SY(k9^GfPJ4fMztcna(r zq4^%4cuX1GtVddSPAP~Yl0x$ALc~Yk22w~Neh>IuA&j+YpQOQX4xMojOXpFH3(JDd zy%T%=^2#QdwOhl?I+(Tk``_gjs~%wGFXQKMMS*fOzu%omOc^IV;4remTLhq&F)(`u zD&l&YKzswATLehbcC&8^%}kTxo&Crd0{N7@dWY=G_LRkOdv+<97pI|kxP`1juXJIK z5mX57*R}_J=n@Dgo9krRg!P}fhPqIYSGkDX`jxT58{~UND5^Z#q#doo=onW6|U zpl*u_1q+Je95sPcz8m``e3izQFdqZV3c)A_G)33WA488ej5ymjc; z%+pL;G?TCV%~(t(%2G_=4qQUMV*+))N`#9eci4reQ*N$)fS)I7+Q@}OTWV=NB@JL5 zujN^?jNWNmDl2c1j+s?fn#S(#t5@CoAXfGLY8GD}M=Vre+0_ex$jZi~O_z^tB<9oP z{j;y%)%tiKUz6bBJ&(#(FfdO4=Va2CVFA0Zeh%waqttKL6xiWh>|DI{Q?Cgbi%Zdb z(pzj)0{JcEjdahPO@(Urb4J)kq?s4o>*iZC8^nZ6IS%k-H4HeHq(C!{WwEp4?c=IU z9?YF{tG|CM4==7{VM>OQ5H-ebok&Y_ah<#vhRwSr)En%xB035Ezt$&7e-KTE9!zbc zA>=VWsH6s7>K}Z$M|Fd8DozLc-n~WVvHglm6Rms{SI+ebOM>gj#nUl!TGLZr0!mB4 zkY1)w49`^NbYzMRsNxuShHuZK9O3=n~rh9$UTSOrlNfxj?KfxK!HFxX8mz54QTVIkMw!?Roswbmx=Gtmh%8I4suel4FnTOW;hlxy}~+Bm2hbb7Mmhq)MgZ(_4My@bnhpf1LN*&2GE->!+Zy zJgLej1O9IkR`uOKdF4r$qE`fRts%jqBsvqP;7F2~WKT-b8BsIdnn_CZ0&(Hk4;rGR zdR)sWT(t6_uI%$axA`Z=X_>FR3hOE(bS2e)0V&-+%o8t}UGM&8pftW-BJuEm9A#b*)nLu| zvl{ZMR%Q$udh53t7U?+P%$Tecc&5IZG*7v{R+DZOZAB>t6p*WENV%mzC5uO?3}yi~ z^!p!kLB<;gexRQla(7m96fL=879PHkuM3Q0#_U)DGHwB}MfG$$aJK=k>QX<-sUA|@ zDBY^y0^A(A5lY=Njj}`sn^W+R;y@KQFFSoUb34Pyl=);%OFKp87iP{aKaSA-Vb|u8 zt0SlE>UN8E56t5pk6icyguN6o)$J#XU4DDRS5!aNx9M)l9;s>z`X-!_DL%WW-@Hs@ zzHDR2PChPpH4OqluG-7v=VY)aei-0)krTaNaWwJe> z@;82r1$MnSF-{Zo^Wv6A(k_mK(Cc1o{ZaShuT9vYBFfm=Mjazj-P8LTU$@dbvttiO zTUgtz%}^C@emUvP%GYMT)^wGm;TL=Xv5~7Z27e~9)YPIw9fpa} zgyP`w3n5wrj+?b7W!I{Nny1S?k6_e76#c19-X8YJ@+^0?ZdlU}y~F&uxy!&3e{x89y8RAj|OOIR)ew^|uS4WWqEZ?u9{bnmYPJ49rKO>es1Qu^ zhapzgU{{0Ml8tAHcmXBG)-+q)ob@9+j2zBRR-KqQHh!28P+B#goaqx&ck|ZRquud> z!HQ#xVpipa*8CO@7>EtNM6R*t+Rxq)&8~~;;Ux?ct;_(0u;E4D^c48c&{}9~=9?>9 z`HvTvoQm}89DQVg;2$c3DG7tCDHZhhJhv$ZpPshVY=|?MxP8k?vyCt|VdJKP=+&Y) zV7`w}{U5G$h#0sBYv!2T+75=nBdkR#AjKUXPU9V`k0bN1cjEqCU^5#R)SIO8j(j{2 zQO#D|O^iWK*Ci47HLl)!U2r7z5e(uQ@Ma5YA-?^C4#$kT*;?Dh+75z?YGB+h&%itQ z%f(uzQ+r9KxB**lm)VqIB(6e&>p8u#14)fU6u)F=s;k80G4DCSBjEu$9+u+(4Q2d% zZLURy-b6Z|JuOv)}#46?-0ygwdNZ?+ACc!)Ia zYhShcLFuDHt2Y?JJUV&zd#prvCsV3~!?Gj(4hsD?+&?fA$up)@W&zy=z~vgcW< z*UPrBGmx3(*QPGeX&D&C&Two)o_~0pHTYE5Rsz4c{OpKgrT3S5Fc@Kc`C>FB=WeN~ zcQ2J@Ug3)ws#?FjZkH%vmA$w&>I8m_n+&abFAsS7q8j9&DGF#U@l*mGWW6AJ%f1Qx zW}RGKfrA<}gOZaRT_6mr$>=8lHWf%%Jv$G@;K3vhq%9GW?1--zDoBLosms93+e6I4 zsO|L0@0bjdlK9Oo2lzNxrY5b3RciESB@P=5IDi0hdnSWvq6xFvmrM{6Sz^1)dtFE^ zh14&W(y!YpM1p3fC^eNVn+Lxgt6bQRsz_ftezekT!M!Tyz^b4mZ4(zV6)kvIt;|^X z1ccBjn_MgHD*6;G&|<|2GJ0K-?#gwhO@mvU{d)9hU!jj6gM84H*3@}|6s<(|A84|a zP8VY&$oaIS@C~#n|h2$4RmEf67>I|NE!4Om&PeZ zP>BLiP{%m7Be;KLm-mUJ>)=jRf5+6~)#L*5k-wXm8WXilNNUAVfUHWFQ^dWiY^TrC z<7&y{BKhL&+y^@saNU_`NL&=HG1+6DI+x6Vr|8irmPta|iuwLB82{cLVz2+zrPxp0 zc%sxv{2L)MSX4yzh@xtN-N@C6rOX<5kIaP%XV66k<8U($CNJoe8|6Fg;OJpM;h#dS zf&*#8Pk#LjNVmWZPA9L9RRQA)h)0sDWVMq0{aM3`oQ4Z92#vK5&P^hu>6wc>825ZV}kvN9(Y3Q|RPWx0%r; z34()%T299&fSphGZEs|{bGoM4fjjHFnjT=jo%Ti-!?uGo$I<>Ht4r9#pDoJl7Z+pF z++Ex8a#E6ey+C6l1McVNVeI^Qe|;XgM<~NlAL8IzGbfHM^-%R@al07Pw_M8{`Axs} zQ%dQ?X7zRHii|z+3x2a40M!qG-a3m7R#@fB?N-qiQsJ}eqlq|++BsAD3Um;>(Hz}tYOx7dXlC5eh8;p`b=bL+MPeMguh6vESmGP63P`s zD#;aOajrh+~ylJ92izTM1c!ieIZb@nm>p=gO}x3S3C_@ciP^^?Lgei zl>j*TnSS{lvTN;l)v$WKXK_A&U6KEjl^ZMHaWH+*RzotZ>CxrJ$d4{g-0|ICukLWbp`euY#ZKe`RsJ zpVbYhecD+umNUgbRgBNotNB28hZbtVxQ{RfQsO=g0j+Y#wD0$84^F@Wn8iu(zyB4( zn9lxl`_E1-hyDk1Qlnb2yaMF)uJcmr%bIO+bZE_c%PiDeD0bu*J7uc z9?pqV+8|4eH0cCMHxKv)%A}F*k1}y-&81W>F$IKT1H*``4#cy6EBI=s%Hj&A0gL9ekr`;Jmzvvtn^yw!=p2Sq&Ang8TqPLN$9ME^0Nm&} zA&)F<5%C{d?e;T|FUfN`sH8A`@w}iZtg1VxwX^qtx}!f7o#to`^GJt z`^TH*q0wJit5?^eO4>vSx5l>_GgYVKjkmNDEm(dd5Vw)Z*$&>vjW-id^2rTPnCo*1 zr%Dz*{rJ_XfB1>MS+Yf-Y|xgo_(M8W#8|}%I(WjcfqsO0U-BQbI)NkM6*tu{-H6Zm zG3zTq_4I*sWFJZ7S%ZNXk`L_FtI@q=4a^=i9I@t6>cwBmF`KK;`%{v`|hRdS6x~M^^$V2RIN0#{e4a2ag`2z z_1Yfmf#&`fm*Id2g7A%F=vD3)+}vpPvj0Hbu)FbXa!_MNV_q5jGcC%%#rJl%zY1BchLP9Gr*cbvsY z6#vd_%r|4ry`cnc5Esfz4`KhkP^;X4D4T;F&(B263*T)^P1!I`Cg<)o`HS2kDM6aK zNdK|%Tes)yVUg4fP7ZF8+!s={Uv#?~i?4=zg6k`<-MH1_^e{+fQ%1NmqKvA%j{s_~ zKf2KH^#^Gu%C1e$wtfD?e~VO{#Q!F3aB;3Wc_o+AG3*`L@ujiDhQtSXT+W&=O2uc5!M|Q_k5HT0bX!FzG7FFjasr}fgj{@au*K3A%1gG?a8evaLg&izX zHSulH(-N(sZ}wI}g!*fIRRP&j&4V3eIfT=b6%j5JI+!-Z7%$$wA0nUJmfeK$MM>&B z6o>C%b1U_)(I67Jk&E!*K0WMfPtcna9^c0MTezFxtuWu!F4|aUg%W5EXc*>EEEpz1 z)hXCRP%HmMr3ml=s{t+_&aIu~Q~cPb)NU`N+DvV)xwA9mCVq={t1<$vW7vUnj8(c? z)5Ejr4GsKu(?HLF=)9GFRwzpPo_Zf>?UD*rZ?$Q#C-}Q6-xPT%x-zYg;OMhh*tIDQ z%`|$4*hLTvswkL)zLL?Ktr&0sA_5=Ms1}xPyJ>G7th#bDMALscNnLkHY>eNRl`G(^ zb5&S;J?R9PsBd@%+!ikkt{z`KT|kt8zD~RO-?`i9s3=KY#MYq4Z6#XBW}aYG#L1nN z1cxiW54}%RJ{Undj4K)${tDafM@`X-j{Nf(`xV(?+*k_cnuB3&19+d^YsykpsQw)7 zmLsiH#)7DB#Yu(r^(4vjYWG$0$6wsioEmOxI=;V7PI1l;dH*=;o1uD+v#e0#L)@w63Bvs1ikPfR@$y@)f*G#K*W@p}sAuzwS_ygk&u2${0 zNPbY&=uhTdd#X|9j(T`6bgaoeuAOLwuZ(M$kr8rp!d3d31kPkI{EJ<>Q5{aaVrLXX z<{YLv-k{d01g5GQ?%`-&w%6-eym6Tj2NFPpo%1 z^8p>E-bg78N<=~YgWS?o1<>8N{MAR(%3z(qha8zEj=wW>1w_LXb<0Q=f8~Fqj&~^CclNv&G9yhQBE&Wl-SiK@_PjW5R(RUfhS8q0z<1EHtXC%)I#v z&?4D9HmoH>-c8{j$sdJfTn80CT!{C^-ZgZlxmPUWEchECbdJttwZBjw^6h*F*1{Of z=Y7ncL}gDN*Tj+&dbjQlU)x{fm-Ez54FOSN9$LN=Azz(W&!}RB*{}1yQb}I`31nro zH4(8LnRw`EY?T?+8!_1QDIAHI=6`ozDYb|94pTbMILpy0wOIw9S3hgNPX71WD$SY=T( z8b;xKT&8ALr`>6>=(i$=3k{O-slZE=IbTp_Bc!>*MDtgcKJFpDwc+6$U z{fQ}3$V>e0!LNgk)-av32g-qCDoCd7vV$HJB4I_l}(B?gt$Tgxmz>-GN@vI^|Q}Qs}7{_RV@g;U&PZ!fC#@_c~xc zC#{7KNspw7f#STvO`(A8%5b&9C}I7%^*yoCID02C?%!Ov(?RFS!BM}w$NU=}t@gOu z*oANTVC@{|^**_;ht8ANg!Yx_bwlQR2B~oNvc0&v`k{Qal{vB@h^ds`&isG;NREAc zj`v%1{#=?)Kd5@(QB1;R$H&?sU=*Wo!1$jug(}⩔AKZtHIqj5~3Z^JR8-seM_XE z95ffB$Sw|FZQ?RYnBjv+ ztK>7cp3%SK^>ma)d52PLA7M>0&!*2S3mu37a#Y+219YpH%o`Le&3$MV$?xQBWy*2! zWm{ohx;2@&#J#Pu1e9|4C0Ziz%_$S}5Q8E?jdpYW!D)f5`Pb?d68V_vd%H>Wzr%UU zjbilJXP}47CJikUm_+E@CNYe2CG>c&t#4<@s+ZDI;L5$^LDh^ns)Lc5;cnL4zVj%R zE`cfvz>3#4#c1z+L!SXX!aG)R45{PKK7ttvKE4ZXJRm3IK8RP}`NsWAY}j#cd0BZe z-K`jQ@gFlnLQn!>#ib`i+E!1QW5ey&Wm>R#4 zuJ9pzg}EaOref>(7XINWTa!`6SKfH;4}4O6-+B);9AZhVqoOJiY^zuK0yd~@$Xv@r zJI6K4b`LD1>6MA_6AThzuvrf0P0Ta)L@C*umQ|XapQE_~oodQ^+#Fh!>tWE1>Q+Ps zPkQ-(4h^V9M*O361;IeXvis6 z5s*nNB-5AV} z&&&j;wK{Rh(868ohNl$yz_w7x3fudu)E>)}s`aWj2BSI|g2atYwEtAwwx$}i%6cTO zqeeaamppt|ld6iLWljEc{_$j0Y z&(I+I3^poPzdGqekZwS{iml)HugLO)EE6#ewbzABs*a>MYSTXHyI(JGxyuIHC3k}! zXLcH$!G#AC3$!&v0^+ZXea|NLZ+Iaauy6n}>b)bjOa7hNumqj)tqL1=yOem8UhT;| zf(5nO-tzVLojF!nRpRHlgJOgutg#ZO2;axgQhfpD@eRe~jRE{-{smNawOiF_0Mr&>;`Nxap}@Sfr@sZmcHCS1Z2wVwWYA)Pp6PnG6FK4Pz!&&Uy3bzMQXQ zG8_)Fnt8jD3i(lGo%nT6%*xO`n*-0I2gk36!c$a!#Ps8{OG{=puZ1u4f{`;6B^F|A zjU2JV`~=&8mk=h?SHi^^BBk%68D#Q-S(!|E#YfunV&7hwJ$RFq*mf4C%sRlm@TzLv z$ino29XA8t6U4}_-@vd}AjQiOwadK6yr@XI;zir~a@vj!*rX6_$-cjS_H+1-tnrd? zO#PDc)o|>(5j5ysm;XbFLJ2vE!zY~AaV%ysZm+{hFH&W>X2Vb`p17U2gO4kBoeA_sQhi8b-cKqq7JE=-| z)G9K}wIrWxo%j*3UfSGjm_5>V`|)@MIe+g$&c1OzMBfunuEI+aB||?sitYUnEJ@2lyAEeqyNEHWo^GdukO44 zSNVA@gmy-+5xZ*GBEBB{dA!3diGwv3RW!IG7Tm8HMyQcQukT5(f0LZnOJM&)vbijm zfoIl~$p{vyfP8*VoM$%`ZGMdSTR>;Q4dmsv#RrOUx(X(uKAEg)!+#6X^91oLoCo*QTVz5!_V+&Ia~=5{T;icc33z^g$9{x5O?d9Pj+nh3)>kq) zzyL8}PUXlkhuF{pO@aBXEKryMyD}3>MX7N^xZk{`Vh7AqY9g(~mA>`emFEOE=cARh z#+h&Cq_T7xNHN6UB@=6S?7&|vmu~2=!)x)ELp|#?DQYd^^3`69g!=!<0zhhyC%Fb1 z4Nj6kfI#^ocwD-nw*BQuFwr{rNv(rBJV;$0@J8sF3UFf*-<8cTAs@L{E39gWIh{5L>yQhM-e{@LHI|Tpg zKU#wtwV272H&olSK?O=?e{1Py6 zivzEnw|GDPW?WSruXEsI`rRUeKOkIKk!S^v?Cz8cVeB6Bdbu(L+R6X%(P#dE7-3EI zoinx%j@bvm{8Z;EORy#ikz!TU$(oMMw#DF7MjH*FX-DH?G#U`EHw+Xn;UW`>0~hhptGW5^g!7!c%6l%j&j9iFRf@RhIU}CQ)@;{Bxe8V? z8Sm>!vzV7l3&n2_jXtvcTZs{^3lLrs_s-wXvBLIvq&TJ%-Ti`%Mf3w4 z9lV4W^5GPKE*q=|Dr}PLu&;EW(n#xc)$dRtl)L=q@#+HMkaY9u~aU% zBDjQMiUPF|_86D2Rn}8W#~!srKcIUpd@K<|vbD_EQ`M$!A>y>~zP1M)R~EU*6itdm zGhsH6Wwsen^^TpRJg3Xg>6F1G#_j?M^M&pySX=IjB|p>s=PNvFqq{%W7Hq`&MM9}H zYMuMTh(%Cur7}o@PM+C=xHMvxPW_dPd^6d)B_8rP!MDZ2#rFM)3E&1KEhC{&1GKh8D>BS!5S7Wkp<&Pju{k&39-_9@=;S&Kku%e3qbZzTF(vMWijm#V< z<;)j>)}LcgYE{ut082%+A~9Rp!&e^fi8XP(%1ut+lTWn@6jCH(8{8T^f-l28t}Yi6 z>;qlDJy`L5TX%K{l$_m0@sD-nTwC?zFJJty*y1-?cn!g4eWFYCWQ(vCmQe13-RCSu z=`T3f6dD7qwg_0e+jZU{Is66C=XJrZLA3kM%=TOooiD^;&?CXSbf1Jahl0!ot#mwP zWb}n>)4yt#!l?f!N917tk8&iTpCYndrF@+SNg*OfW$7P}zZ7loPZGI7Vmo2%?M@}o z9O1p@VF%yes&N$y+IAB2y+)HhT66liKj1yldit>D+(w7!>REMvtcA2ocSLd4Z?3=v zS`WIO3M4N!ES&D++hP^|Ry9%m!g+O*Y1kYvU91|%4_o}Pbde*yZTi7YG$y8V{`f9u z^BX%>hj+!PzdZgkIa3#OY4@$)_!fV{7zIs-Wv~YPhe5DTN6woNHR^J6Kr~n|W-Ulj zisLcEV5k2bp9I`f>XhY7^1JSj7Q(z=vgkFlN-P-W zDmnvAt)fmq>dDV7woT9N!T6?F4FHX*gEB~r4|`RTw*dt?0)}sT_}Wd+x0dqqx40Kz z2ux2s1m068bxJ&pa=}0uO)2*{$qP~drYh)%2)o&Hv!N662e)5g*nl-@6G_ve_54=m z%H5ySZ(k%wsos=lU)ks2jnVql-+S2Z2)#uF@^{YTygc;YpazKdrc%(GvwYyjV6O~t zxOH}k_l)H`HT{sZWIWj&izr0Tq%mR^-}2z3b+|-tJq)@Us8^L0>qermwri#YbqADG;(cHQ=Gk`xG z3Q|I{=GNP^RP;0VevZXC<$(6YL(D?r) zBwV(HQvXqqT&Psa{)cyzz(#-5fby^HGO}BfcqqA%%xwR>>9HFUxYLrYJ7@IK)U#Xd zyDy^cVVbComGQ5ZnCH4&_T=5OD7Uq7P{Co~p@WH6-TED&^N$BC3hkA%6FNkz_t!f) z?^DKymQeLqs<)$)f&CDvJ2kPVg=?c)aiVXq&0*^%sdE|c?XWiogdMwD?+9F@^!}3! zk!Oicy_n1WlRS%tdU6;v5hXF46Y+)dJD){J)2ElG{SsM~M>W;}UtXiT@9lR8eh7|J zfUW%3kR9WTyDw)!Z_ThyqfMfz_0pnLf_)5bQEYvmeiH7hge^zE@%yODQ4Ao-U=p}h z{&RFF0jF+GTcIGqIJ4Huzp{HLbU0EGS=-!JKXGrybEVEZq7Y6&-V%F$WT@O}-c2_N zqR^SOm)Cq z)H%#6S-YIxr9vB9tL$-r8XGvwGbGds<_) zOGvRyn+;$=q{wm038_=|U_A(OQa+FkppqNR6Ev7AakI(m({F!|*6H0!AR&!Zu;*^e znQp#(PfD&oWu$a~bhS(&U{%n1drd3kN3XcoPzF|jmw_9!5N_It)+~qCU&7oXTsf~B z0zOMVy-YWmb-H)p0kxe~eOeG5ehk{{97?GeIk|2`YkAu|)4~LKNT*s)+xhiEn2^7IVytCW8}KzGjB=2?^&e(2jXUJYHrOr0~BDz$98^cF8 z$DHLk+V>2??HN(MsNIE26tRVlB1PN|IT&Vy@oId{=s&nUZ4B0Xg@?DFJ4-4HU0-Nr zm&>C!8&Wnn<7bv*B?@<9dfXAaZ=MeBEURtvD8|HY!5dh2Jc zE&h+?2IAa>_Fv0QQ?OIbx4F=u4&ScA#kK1g8yi0Pjg{X^&ymBn#PZK;$`az$jKq|@ ztckLq*1ubeKOCNZE&kZq&AqfNII$(}|JC^f@xI(~Zp%V$%DdnLs_*$&&mq!2Q47D; zA3NKE9(@*x#|=iF^LM=l6FS28&Isz zlnZd4I?%AAWZt(*R#VS!1En{Iuz(~v)wHCqA(d<a5TAC9_D@^0c@f-2n_-1 zcApE#(i#np7CWTY0gxIn6+NVbuAaR?9wDBG3p6f&J2Yd>fTb7Zurulvqc6T>o2en8 zxeaUo_*x^lSZN3APXh_hv>$U$SBB(B>b%iN5cOp!5({-njLq3>A22YuU_`RZJwxyL zQ^+eg0z%AqM&<8(Bt?6OY}V;~8opCpaNShx$MvL`({~faRHI75W|pc+HArStE__iH zs|PJnO@z~G#d$^p6VYg$xLt8JQMBy!4@!+aS6W6DvHj&Tj!RoQQ(A|1=2naOPHP%w zh?hnue$TvJ9;Ap;#w)ZJo0M(DZMx19;CsZW^q#9SUijM=y2s9!pr&Y(uM*=J>P4}w z^H(`3k6c?*9*(4nGy}^E9%>i?sb z#1xM|(DR@f&$&KG$y2WAFBiij5jWB5lt4r0YD)BZnipkg?N?5nvZ-UoQJ=Q$Cgv>I zU1Q#QWzhAuKOG=gL+4FmN(GZ9P{DN3p9|)02jMA*Thj!ONQ8Lm2K0he8pDTFcxIXIWTvj`u)u!& z^HWbLB=Eo`zwZiR9P~)^<9-T|+#(iPqWw~=5?}uS;su~ynu`Q9$EMhdSuEq`FNyN{I86tB&7AlQy@G=5R zr>koKb)D9dl2sbwgS-idCVniQXA}I`dVhmN?V%!W))*KiF*QRP9c_2E_w9^(zlaGO z2enuTk(+NEyH9(e`q5YlNerFX|Z0P4wN-{kmA46(Noo1&wgpkJslPHb; zg90jLICroU#i2y&&>uG5K_!X7zkgkPcC6g7(t;Cpz)Ti)MbBHPH9EB)xJQr!gMpQC z<SVCu;oB&WY5l)nhZ}hqMaVgYGoZms71Y5fX~W7{JJsQi%tuMT_{}$Ms?1 zUi5>pznxhA3~L(yl4F@;nfij529%bmrQ;X~b)c)Sg*j3I3J3*d>%__s(`iUc03y zjrTpFrjp}wyo7558L4EG)~hPHjDp~6GBo3k<{R{frY5S}6uvPV#_4yG%E5^dO1g~G z$Yy}*e-7h2#4-rpvU5{7d)8to+{Degq`m2}7Qb~}wB8n^u~PM05eCeBrW>=WPdAgA zx@*rB+E{E|yv~R;NZ&o@1J>do{l-|V&JVA1UrD=6La+;8_qG;KzuWji7Nv2kIFOj- z`_HmsR0tGv{9l$8y8mTaAz%c7{_b4zRKDBy@P6jn(Svqkqm5K#VD^o~{!5NKWw*2L`xLJ%VsV;z!U`j(PFN-avg5`Ze zQW5bFk6HU>9$lKkRvje; z($^eIA=4wXbA?+dHu89lcJ}!8ps^h% zkvy+XT2bZxLkwp={fap}!hvw~V5vE)eYeBReSBx0s_PvcVm~nG2isWH*N3#h-$@!l zD-1!u(Gn{zR-FGOx8a9wGXF+Skd5EKS(Nsz7vU#Eu5bC@Yau!~@)bmT<~_njOXCq2 zU4)0}2ujPZ5?_ex+F8=xjr!;4@Myza|}`k#^goXBqo(nL;?8hZc~ z|B1$}+ow$elThJt`&IhYNFi5&xmAoDUQfjV!mUxs#6NQ8#1TTbsHf$Da*^!Airomv zX@9Y_$|H-Qr`~eO=paTm84pZ!YG`?~LCYc7NcVTb-+q=aw%xtgz#7nMM{wnJp`fcN z-&^eB9}VC|9*f%t5-vHMmM5AI&H|`N^%gx`VO71FXGz_c%xd%PS958>pYZ#Un*93i zvL0>ip{ei~NzJ)pkix47OvgBReJW=7m*PrshnuMrs3ZNyd8q#GNrB%d@i0zbADMlh@VfJt z?(0bL>^~k7+XgxrfCXQM!Q=8a&MqFj+X}_97W*lJ}t$`SChpqxhwMAz%~Ws6SVBPLzL!d5g*ssp@WjtIrDQusoPpy)Gp z7y3+ga(;yDmoP7YD7SH%@4>5oBFBLZ^Hx^6Wy{Dldyb1kZw!KGf*RNl87CdY&m7EQj<=2UCgLoW$;$|%8NaczSeLPw1V>t|W(Lfw_!ZJQm4I)%as%a+ z4q<7$W5^Kqr*|JyfgS)GuwP=jo-@HxIdv9TWgIcxKsQX#?k~*?j_?eDgi-{kLN$z> zOcb1mbUOk|VLsZ?zO#ZL#B`?75UQgnQu&E|bXt>^Hiq2wLB139*vpLlC6>u4cvmg= z_ryY*!?9;|)qHlTj^kY(PEmiHSQ|cn<+873z`E|1q8!#DYVJfc^S@)`^bmcJvQdzl+!Q=!}L8Ow#0ceTGjZ;MvVJ^@1CxNFjnjkI9A zHSPi^5-2Zb3Ft25gI<5GF27!v3;c{iOrTD}+HSmVzPuAwnmLFWGQ7|%$pz%10Lm3` ziMyDk;sY&b;TOmA;v3~kfXc)&qG8S@4*hof0erozB4~=+Fi6h%JGMZptM@eh)qYPx z!mcq==AI=-P{{6WL6`cvEbJGG}+leup0&Oy%76I7djbHxKJBom{;;n?r`r1~rc zQh3S1Z~&jz;t#wJafqAkRr|UAmP&JCV8k#uF#Wu2)4c6Ri$8n)&Gff>#3htFGBy(4 zTKCI4$JpIKK}D??OW)eg_-@`~e|w{6ozOJkqy=W4}{ zf7C|x+i6VUbXAw5H)jKJWNJp-$0NDXZ(DMHkWR({H&8rlu7MsxbH}9cPJsl3h4==! zr0NL{SK2gwSHH9Mq&73Gy95T|bYx&w4@H8}xj?&Jux3Tl=9ng1R=btvRg#oX%nmUF2|@o0 z2}?hsJEL3kFSmfdbaTG|ZypGqJQ8(|K&h1_6|B*bsX{Nvj2||2@v4C)k=AbD!5y4f z9lrbZu01qW^aYqjMi3}=k- zA;E;Wy+UA!Et}#9O7@{r9WqW))3U|T_>T;-$}=VY(DpuW_M}Z?mSrGOIfMDO>D4VG9C0;XjE^F;_BZfc3`7<-%xahwqIpl^Q{%TxP9V8KPaZgOaXMMEi(o zV9Y*PNY+Xz)0Dz{bVxd4UARraKWa!X!7200=DlUvKe^hV+x5&> zuiMIM!`stH6Q)Oj5_8u+Xltku=%HC*+080><`ml~y%+G4-fxke-}TQpNu%#^*6wOI z(Gz`G}dx1!26VU}0Ny0sh!X)SV*@`N_8{cmTl&@W~YTN=5{V~9<#{&(&3(_41rY8gB&>Cg5OFz!(KBE-hoKA zD6%4158)3gkuueY=-&7c5DJNU$oGi?xV_t=`2H0nl<)wK53J?Wu8v6`Z*`WH5-R#` zq(_Sot&gba2l@fmZ&67lvgSNI7MPd(XvAkBUnT@N5C<82`sZM1H%g73s|tdI1pvM6 zJgC;q5IsdxNrg?w91}+o_;*ex+WXxYZxtBvu>`iF2nANOf*Yoo>PGesLq2T(QBM^`S-u+ zY=Jp{Qz(_fZ1uOEnWlc?-%3ExFv*yt?sOY;Z|W`JPzi0R&E#+fQhH8pMl!dj+gQnD zSq*Hy2`+^lVfJ|5ma65_`K{78$Q(w`maDwLo3oq-%sg*TC|O~%t-I09?n2$|g~J6V zJQm|fKtt>N>&8*7rYh7_Kw?wzYz~=>^?RtPqpGpDFDbxu@p^jk77; z_kYL*u_G4n8S7o3C}As(;Hcy2c>0bxlhhNK{{8kq&YOP8~)1*TAM5XO* z#O^1b6igC*r+aXTIzogE`p25nuHsHfONc}z{s8}ssMq$Y|Mf%RZB8K}dS|7`_PMQH zHV7H7Zei=C9HzD>h=Hscd|4`y9}V;?^eEfL4db(kn8dj`K8Ctxv&iJjLzg&oAxMq7 zFRltK{d0C?mcd6X{uTl`t)H7viBrlLFQhj95=s>#322Z1O(=B$kb+hVdD}47FKJTz z6Y075S?1$3u(&U^q*|&)(aUYg0q(ZjHg-FF&`b)CT{zpSX#F*K{pp52Wa!&xTc=%I zk$7{j36h&~I|=D4g}unA>_Z%E(9Idj4vVq~ zvw`6G5b_~E+86VYA=OJ1tLxZ-d3EA-8Vyay!%{A`d+$M(9JEE`{(4F!Q2J=zx)$Hm zERiLO)vt!`nLm-an~gp>0`&te1jj!uv!sCCkzMSq$W@8cIx!s^ooO(ZsT({EE$cM9 z$%j5@#)l7+P?y@B%p_M=e1N(OOO+GuL}O_fQpQYb)3;W2^1zr>O@~N9q;7!ZHXy_c z>;dSslSV~X*AP=9S9xtVb|pR~@gY-?WSVD&@-P7==q$B0{*s+%+EHOna=Dd2*jCjo z2vGi(GGUKY?et>r#IAFinFOr2^MV)H1$eHZt4|CsV@!DD=(RKB zzME=%)5diCd3{aF`D5#Ym5Ad;dcf%mefQ!e#+2IG@7ttmAWq!O$x7 zu1mf6rKEtTjqerqI7b1$8kyOCN8Xil#$!;k(#^)a(7^L~o8KqPm((XC;x}zGEN%PN zvyLRo&A-kr!{fibW-x1lCa=h^%Qa2kGi%qIusREupe9(J z_X*fzN;$j&dPGv9B-FJYAJVzqbCR=>c+6cwFX`HG<8Bes2(w57q> zG^`Sc4R31Qhniv6-QxSvCiQK5q3n*tZm85-@Oc%5ft~Q&vStQv0ACs+h&uvt5yVN7 zP=EIT^@!7wVEZ1~ zwm^cnO1F^&zf&BZ{6vcMMZ3Ai3fW{;la|ty=+{y(S1wdz$23Nd@x?Q&xjS(64}MDV zg=m~5t0=L=V$2t1k~!t_ITGV2x7ZPzaG`;~>&b>iep|{-@VTC}pabXj{I4^O(Z|Vd zoyLK?Iq7~w_Bco*83#1!nHb!X8IM+IZ8L6*+Py8j8F4WE1 zgr@2%XRLqtDR!d%uqR+>3RL7pI>!qWMVoelBe1--aw3zq2||V&gf;WxeT^tpk-!}v zt)aPrN6r<1@UG6;aIjVhY=FKy15Qp4K)PKw-t z7R|>zCh}aUOid!~-IVJ*Mp!Zg_{t)GZbVrX|B`($jQ>EQ_7xEPd;OQ z%HwfGOZ6POW<}2o2QIa*iZaz^!U-sKg4dzAA0*Oh8BNCkW@c){f0?@TANWKR&F{jf7GY(4=v0|pjh zj#t=I-}$q$pw&AfS!^(1ldSml=Uq`b=Roc;gXH|CRAnCgLo%6Vey$@G&^;CYs=rV%r zpa6Hfruy+3Pu0u<%-7>%y^zX+OI)(=LlJnP+^G?G7Sw`*urH?6f6|H0d{`jW9deVJZP49xe9_(?iWhqemK-CYi8fGO6*MHV z5}46xiRN7bZX%(D4pB?x`5#=y+Tx&5 zHB5#Ym3NBe34uVDQ->r8b%Q~=MOafhGfCi$__e(hmD7yt2S1OM?re+hry*I_?Q(Gz zeEWu#A%73pb|B93cY+1nLnPng*lwG>-yz}Q-n&+3kXGq*R=tZwg)i{Gu$)np#DUW(FDX5NLA+LJ|0IqwD(2S;dYGO6U$q;n5mKqSevr>+uhieD)3@LUADgul0~iZ_72?ZyODDx!DDL2J_N0g)X&Rz-|*);%7t+&}zV1wN9+1 zqRej%Umo%WWr)1+61nb5s1stjLtFa2|Mtn`ng8Ba^=G(JS(UzJweic>knMp4Hop2i zja~0JVPNKx5;pHTImbt3LuEz)ROMPm_s*)5rLe`kjjxot#7HQ8(hWeHtF=lC_AbXA zIV0j~<^9swc3G!<(Fg3^xR*g-ZG}P(6-@?_$|Qb6Vd4Tzaw+U_;Q~x#C}B4H3lkZe zc0Ewi7;q%Bo=G}6eSB7cS*%{I2ncjs42rzShZzgTonuu9%?<|_H-xB;TqG3zCVL7( z{(e^Xd|L#oV~&8Z>{Y|`nV;t85EK0H3yQiJONW3OQVwa|a<2Ej;>!WOZSjUH-})@R z1RoGh!6r?syDN(bnhf9usTK;f(L#if?Ow!kEwG2{wbbM0M@%V8D9i%Il~HhBn4)LG zQTHphX79en4coQdJQ>WIwfDgFQ=rh;!1lD}ES>(;CxU%GU$;qtKd0>CwblodJXC?V z5NRjf&v6GB9Rrv-PJ)b(G3f9ofLA7yU}T3~v3mgmpY9B9stEZkA0Ks{c81fcyg#7@ zv!s+yXWrG+E?=^ zZtdv{7~9>YsNrC@bMquTn@LljB$?udjv6MK^-x@j+VfZLU9a*NbLIViX zA7eIC27n$IXK8D-vyq@GPCdxluwZ&&k~JJWP%DK^nf)fGJ`Xfu>=08%LIU4WuNrZF zEP@^GwE|6Ms`SB0YkOO-O8WG|e=UNdBe0<5d3;}P$a1eD67%>l=|{^0kk_$DQ*>Bf|Oao9=TZoX64xv9SG;6jyW z?9E5TfEoJ0y^C3(y!0BD<~J;-eL}iTvATB=wC`*XePO|(ppu+4Cd}iXs)6YUnW3kA z;=)3zr+a&6g@FaHtb$9U*6*>9?oQb)a~kTYM?S+Ezc*jPof~*2mNn`SqdvVL9NX@h zi2V7TCBnXl16I0g@?&1Ad_>hMP^N^PwS-+>R+0t^ETF(X1CK}9cDWigMk`RzYeE(k zpg<$o_VHGnqA(MMiRKDW)rC8L}H@oEAuekrc~!kf$X zha6iI5u{X@jfML=E*RIl#B2{Ik@emYUtoQ?p{W0I3cME|>kZnUn&h1fhySFk1173d zAF| zzrOn~c5~&vo#8UnsO8V1M|_#mrl=C`rySBMlj6!N%vG&?Q`{$%nK$SSrKfZF!2oCJ zQTxMV;`dk2>aUEIiQ4Uyr>JYzHZw7NfhB*~2R7(G7I0F}1k1S!M{_lT%@l_X#ee5D zo?KG3Eq~k%aG6LzhU!$~Y(x%6^bKr%zPgv@V$o3QH@DP<{WY$tDz+2 zQ=Y(;`9(cyZD&sL#%cDWD$*zFX3%=^V=^lTU4gc4w0Q(AmLDCQEB1uAjGq6g_zMm% z71@Pc8ch3%nIt|Qi!-AYRxs$UkNran;k$6r&|tbCbb$ezJO$1Ex`)40#(r;N@i~+Q z_Btm}Q!SwrYZ=V!SA2QMiX{AjS)?b6dncOkOVvUghFj`SBy)N)a*=--@qh-zgjsm+ zN3>L=9(PeWS<#Ln`-@*pQrNG?Jfen!>v_gUkmMz>mz~qVwT>>`8%duIVLUbNHt+$V zscjY+hkm!>0GaJU!9~b7pDbD%V5>TRRPc6|CHfDvmJ>qT%WT@%r?s=u6+)k}R67$B zaVa`0*nC>RqMoh({Cj_dqnhLL^ydJ!Ft9V+G}1O_I55|s`{Oa}lw}MtiG2OU4Y5q?Pn7Q z7LL;Q8#g;HHl=}bu;^Q2G3yG#pZkU=y>5b*2MV=Ovc@q(G0yQL^WlI zgpv`7^7{pdB<=T}`pbw93cc+mslVw)Nb24N%Cx+vOF7a-)^n0<4 zm-Wde`N^821G9dKJA0Zeng^v6SDqmi_-tsL7{2pLVj->mB74EVy4HVvMHSC8S$<2b z;CN#UW!BH?Z$?(~frhXY{6-!UH$7>rfhQ~)!`Lj`i4$)s3@}ONSLd(S?iFiz?hgtI zKL8lZ$jyb%9r1+dzhl)uB-;TVW3Z(r4VcY|Z&xMF3AJ^^k2EZ!708{Bx|mXB0g4u! zeNRJ((_2IOs%b0lrUugDBaFR=sHlYOnAuPSp_7zVle_;vj1 zv%@ip0~@mwMy?ZW?w&=~g+rMywD~+QG`u4BgUv^K&QrLo(+TV}G5t=}u65H3o*s`2 z1gc3&@q6&59xFm;%u973jJmVe>k)AlwM-Ocl}t|o{XM>=C!rY+h*MCYE{Xj08C``= z%^f5f|66={>QG}s1xw7<^qhTZWK6lJLzcTk#jHZ}+>VqKz-?aJ?K z4A&S2MSlmB(5yq;Fc#~+Nn`g5N8U|^RcdCpbA%|5Q5GP`ACj`&&VS>xFz3mA{sDlK z$6UXr3oxCVKc&qzq($I4sOIi@*3CWkp0QMOf5r|w-)y$*JVLwqkxo7YaT0+VM7>4_ zt49gA(E8n+jbjfDhX%Rs*F^J$9o5E@tdXoh^^d1R3cfS;gjWNm;C~}Q4K6URwL#y3 zR+kC!L+`!W<_rSpyt|_^h)% z65tbo4qC}}3RGQ4F12iIKxE%~h^wEt zFc)~OTkH}~tKf_2z$vnr>HO5Loz9yA<>a^gra2y`K9OJdHY4A1`#%77Hvx~$0;J!T zCe4|mJH%9H;H2BlXrMM(KkdeCty8tlctr6H-2^=RGJ(9Ib<6y5Wo44@Smh?+(VrwF zVZ69&F>BJ+7}9_8oq=J8@TJR1Vs{E5P!E#eb6}zQ3S<(n5y0LEh}h{#KYcjkOQ+fDdVu zB~V!DTD(>&jKeSv$^79|Q0v5ZOC|AXbKP?Io}5_11*~jbn_Bqw^hsG%#^^*+IJvpf z9%@Vn5S|Q+U7^Kd372@4A7#Im=T&jpb~9`!d@}Q~4>IQ4HS2nL=VUVP+TLk#U9|Z= z8(H$|eB<&H@e@$-l+v4zC1R^l^5kB#;_FOBDdeUq2$P-;!Hb?CG~L_)y*B+`NK-;S3Zsz_K2`3P8Ux()cXKK+uOOgOB@${igaw$K zE=Ynqk!FyV`-f>PnyH<*uM}d2uuL&Y#uq>EJtVjE=g&fs)3|uPd|{$mE|e_ldT-6F zhv}3e{qVLnEnlgD*MD4jWdC4$IFbi95m|vWY%!=9 zGU4FiV{Se&zWK4WT6S|0#5WOm`tHuRqH%ouk+&pRBiBoL&wGM4_Wp1+Mq_hUaCP7v zTzvF!)OXw9URu*f`|UMMY_}X{9E20rXYYN5i{d*AJa7BsHE>yRC2haE@7A3t;5hXC=~zmH3b^(L-8 zKLMHa!HByty2s(?fW^c6LF4gw_S4uQOd#QVtsiuKI5}N$^(C_I-Q~nVl70Z+FU>cC z-0NhT58DsHUP16WU`m#B-Vg8QSTm>h)|k?bSF{x9Y_xH6>!KNEr0(U7u*Cf4E4Baq zYdWDniQNtn-n2#^9-Jr7P~`Bk&sA^ayOle>d0(uyg|>1!sM8KcK!C|KoJH^g`5SGw zN22UQUr%7cAdrm6$03A(zrIi%dsr#eWHuu^1|CAda9uG*DD1jlZsUtBJ~yx~F$xoU z!i6R0FEF@fN*?K({_QL3m(i#WlwCz6!o-deokg^k0>Q(Fuc@kL_c4p)=K8p}NYD%? z(%pPgqj>_Bk{}>}zX1C8vWx`|=TBel6ZrjC3&8iC%DXotK^>P-H-)W&7s5Ou&U4oT zYg#RP%K_})quPGgI-)=S{_VBEb(G^cRPT;OxaY2R`D>c!s65~pes$vWNCaA;id>_c zm`XF~uG{uEM#dppNlCG;!rFg#dwt&;x(25pjU2NKQn5E|Zmt+%qGz0yxFRFX7{9ju zO!vUPr;4Ay)tsHE>#0~s6Il?Fv5j)2 z@(KQvnJ=?8ILnA1@2PNc1&r$P=vt;|Txi`O$S&wrEEz`P?nbP$DFjvB8Zk}v?`b;Z z7}K8{GQrq-r6?#&A#4xzNqSv)Lp?a|z#iV&jct=}#cCAAD~-O$rTQgR0>vF$swCw~ zk*NM%)z0hUN*4DkXLiGsr`0iZ6C%yOx)ea}*FIW|JXw(h{jZmHc+;LbVKlX>d=Wts zY%*42;Kv#8gZ(~g6QpxCCOkXd$PsVk;8C#mN-J)&##b!TfDEOFt8x_9p3f+u-_F62 zHPLo`cN=p}C2}W?C>1GD&fKwf&3;oy-MAB??IJB55_WVOwoqbc@f!3mY{CD5L0P^xJ8 zP$Yd}S(;{2Q2WnVHE$(~f$xv!&{kh>G)34L9VN1SmNEH8F%{cSLJ#L1M@Sl|+Fyr7 z*rrb#Vg{0jXd~U|shEl4A?1Vx29_X{4b=n5Mne09()Xq%u+&#+g}5Bt%CUYwT5Am` zo|2t1^+65~?xQfRk>kO1-IRUpN1+R4g+CLW823{@#ki}=dT@P0vqRyTvl7JrPLaeA zQ$iTMJJZ^&0NEEq96qL+3T$Ost1o&wQ<_ZLke|qshntDtRAVH5GL`oCJk!Gqc@0{d z)x1lsPD@l^CIZZ@vn#m>V)TLO+!leJqdm7Rylb5aif%1FlRp=a#K3PqiiT(QgV~7^wyUj`jm>97UyxftB=|!n5_G zbd}JP9Q5PBz~1|QyAf_<>kGnY+xLYNsDRHV<(I!s^k{fw3qD#s?q=Td~U9L5*g)r zzfAkTZv00WE4B}n1Uiz`tvqfRI3YV%!1dbQ}9pwF5#` zGs7Cc<{(cdTd#U`9xWphrO13gqN&JGbv>`FLnY=})SIpo&rEg&4tzcQ$cv4{oR1lT zdg&s3M{M45!loj^xVkqxW!;bP+uk~N4~UEP91UC#1y07^?WkM2gIm9WLOz;{x5cR<99VCGx$H+sof@oN;K)ZmmBE%zSL#@`{)Fb zW*G&zeYD1u#X}S(xn`;%><;8!uXYWAhbSqMu~Z#zD*6l9fI{eAtX;m&ik?zCV{C~X z#7zpA9jd{-8%OmdqML)H8ZvjfP_q=lmVoSJ6Zptya)$`;w<;g%oUAP508G&A&E;n1=TVQk+p4Z|6?MS|8V`R=NeuD+e8uxnHK=lvBU-kTaIh__!!wb~tIc^bZEGkNlXq%`30B**|~>J>6YPDddcfKIW=I|x?JNgU0A=uo6g3n44g z)Zy*vQ`gnk(y2!mGm^SFFAlP2n~oynx9ZJChU9XEU%EbuxmFtz$&meIr}fW1o=jNy z4CNI@sUX&)UWo22xx2NyZmnU3;pfZA1dHHC(c2V+M}v7jU#qW|2)8?V_}HkvY2Vy^ zsl^Vdkl%6oh8AvVU7_8?y8T@&_zzLxQ$Jo(0vpoX z+zp_jCs?zLZr|y+MMj;11dJ!R8VNTV6wOFdfX{0~();?Z8)zhOoHF4!81p-pEv`;8 zb+-KiOL*SmxL%{(Cr#6;k!{ zlA1`ek$IuNu$qHXsTC_T`U7~ErB%ux+$#}#JJ|wP7AX})3DHdE7JRvb#1B#`y)P}h38#iNDN?|(Y6TBsF&dU!K&V^W=TtVDiM*&`!`VdK=H>hd(tK`{Vu;&4wKYu^7@PX|@~)Fcd9`oHWru}Ps)6hdSRRt7h&v6U zKuM%kLxUML5|w4`+v&J5tJI`Yw;C$(Jxs{g0YN7M@7VERgET3s@nEA2hQqGk`))4? z?c1x8r~)v6p$BcT_k4^y_Nw&d99&N^Gjt{$3XD?kDr`NJ`^9NZ1@z$eXImUCjI}4& z6fxuc{cLZdFoK4P4GCY?v)~{@mvjHzJW3AV%zM&ob~ikX5E?u6;B80T%U6t$s3H04 zP9QrcX{UNAJ(J%_%Llen-$#kT<$f6@p~ACI7mNDo_dPZLOqgM4*j(CA&Dk*Yk{qmeDtfyj)n8*H-q z2pJ;-J{&vggLfIcat|+Lm+B-d7cn~B`tav3Y;1-yGCFYqMDgwhCiqjytW zU;^mWJr>{ZQ8qp2_n-Qz`ZnFtd-C?DCk+Cw-YV;WYsd5hDL<-RYwJ&JG6VkdFPr`I z6GALjm}TNmDG7?TfvH+T|j(H?>` z258E|?ZO5SZmG36H5Q!fnvIOwxU9Mh=)07t7G_6G(Mhbde3LM9%%6yml@fppyR(ew z>od>>H$3Zufb1m_kgKt}8T_vFZmCJZM`9wP-sR*_IGty)LV_qkZ4>Li2NaE#xF4Z= z(WVlx{LD|JH1xsuLMM);XQZX_O;RpaCp-K*IM{U9rFI-u1bI35x2Btz{V$B83%t9LlazKy!djb&L@Sj)ld1mxUl>%w!};)Y!3VKwe?e6(MZ z-OksjDH5(X_Q5dB0Jj>1vZkBFex*^aSpHrQ^yG~l1m_x*g3O0n?qK-LBq6X+Rfg|H zO{NggFh*D^F$z-}{|k6x8aqDNfKTrav%Ep~0-r#xn06$18FA)aYDX;Z25NqN?qNBw zV^{h|U~06T}UsGM2 zCzbya=cBbMKTM0X^i zHy(GvR5s3gRJ`JRA>=24)fNF8^e_X^X(4ADp1zeT{|yXXv&N z?~pV(`)U$~0WFJu8}Y#lWzy^bL4eVbUWgBz_AU~y zq8R&P5hf2&;NQ90lrwql))m<3YZCw=w<}{9eRadUKhqcWT=!3)?`j9vI|$H|dh0)x zL}Y@eE&)df6l1AedH-gv_e<9&$hxiCn6nmEIuwjJJK%3Re+W9yi+@iA8oAX0azE5sL*g6hYY_pJS-p;NhZ1u=}EV%5wRagmODCb+n zLY-~?NC+!L*H5C}AquJy3`k@bNEvXzEpYAIIq=y=XjMh3aKeUxik-M>oMgiJJ@NmF z+cn`pnLbia;PAny`=OO7*<_?it^YyGc)RU;PYO96e#C0C#wI0Payi;$jKDjG3ac3x zv&VJd)bJ$h;^B?G)s3~7^`(D7TmdWvRUXp&=WE3ci~G`Ve8-x^YqpMQ;={gX=gcCK zz@H=bQ(>iV)Q^O0-Tc0e>}e};T&FS|H=oEYl~$^jP!ak>5R&RBfKd$I*XSuW zG|`rcA0qx1Pqo@}St;_FF0PZt1+^@TR0p=~_RC@RLW+JI9(-G$pUHkkH2SDqx_nw* z%1r_s5F=5H!cwU2@7F{)cffl7dT~HBMo{rzK(K^^WPPa5pp?4j1cQtr2>S@(dNN5M z@0}U~)~RSksNkz_5=jWNdI~w6Puxf|*^n^=`5s%JU2hbR5n^U#@JE5ZS;?ri)4Rad zl}q27njc;SzY2fbOy}hKT=vyuY5V;Lpt2KKSdvZJoa&OIXlC747Q&zUjz3|e=ZDIMhtoj>?TOuR`dLc4QDsCV&TD37cJ_wzNyqn~i z@LuYHF%zsH7g8FcZz5Y3y$lAXgKMK@kda`h<^A#!GswLz=DNOW-l9rneIVoatCxfE zzi9wS;C?S8Ik~Y!+hqwLYQ}>f83|c0PJtVv1LjlyEEVhq>Y9^iYf9&9LE+o`XGE5X zC`~^%JNbXlG%XqI3Uf55Yb3spXn5-FP8RW_d}`@kk3>FmZ7d~xhNu1E(~mhOCW|5z zsBjxHosYh$Q70&1=j;SRK(N=_{=4zp0z31vfq>!bUyQ-{V#0!jKji&YQOZPs(XyFt9)7(T}dyx9zxb!j?XbU*197QDxZYW1q< zs9sfP0!ylxg2sdI+?t9@4F=s3Kgh{Xb(Y_nuq~|r8}u|1ASTX1ls|=UBa{RSPW8EH zZXLIKq3h3FGIi8VPX_M*(0RQ!p(hJh7g#N0!~F zlelw6QzX}G@$naf0qj6{mDvQlb-3R z1VMsDEeSt73Q(1qFy=RPbzDN5C7X9QPpx)|pO?G(&(9oow^@93YNNGN&$&viKK~K1 zdX9F^sO?6ooa7k?moD>pE|(m*ojsC6XWBi}8A z0hl!6`>gm4=o877s(AXUAMzdGU?1WztgeA>f32Or6bWzwOCludQ=H_>g`TeLOuokp z<;W6LO7fuFQtgAd5TQ6=iQ1NZP)2QC;Om7lnuS?Xec_~vcYb;Y=UYFI-{~WaBk=ZG z+*>cM85{W#JN;SNy+vW_p-=C$T?keaC zM%a9hJ$1?k*f_VT4BGW>!ZcehZK53gVOFI>l8-_5VqR{G%9MMLl!rtn+wx1+{%>Gu zuOpD}u+z8kkRrJI1dCNyC191weIoC&dP@AT^1y^@axhDP6~oZicJOjtAaKCnZQ*ki zmOuH9$=PyFv@e7+;6&qWpW#9`x!5;#v_Ey~UhE`U7sJ)p3VmfX8?2EPz+At}#g8me zYEcHMJLXzjQu)QCf6)RZJa2w~R2YJ&J|B%x$seCV$fmf4-Y`K}6Q}Hqa!wR0J z4NxC+DF}l}%n(xZJFtQ~Tm}6ISO-}`KV*Ss8$*RGl}-n%iL4J&^xIAQArc60F-CO3 z2DQG_L@CIU4-!ZkFl>woraPijoENh@K@EI#g66&+Ay3y7#Vx4+n29c)qjmT~(U1B29L}b^ZP3_S3z_ zDhn5qnPl7Nn}?KhyuT#%(-;jG5VFF!F@?>%H~elW`ypf0A(XpART@BKdFyqxrj~J) zcXcNbsn@Os2?OAKaK00NjBD(!MnlGb(Zlc%fk(0dUtP~tx-JKd?F)hxE~hV{L~q|g z2^dBUb+brgFHyo7%)5ejuMD#Qx`L}+1=iP>7&>A6pk8wv?6VXmtzN<0h!9+SMQ+K% z+-PS9X24~HUCksO{wVEI*KEg^TSf$4Pb90(D^z#VN)xlwk2atqVAzSM5e!YoziCk@ zRstyBr3QBdJEEUWs|-W>ps}M3E|j%W)ggFfsH_hA3LPvxv(zY29GjZEtfgR2CD$*n zsbgSlI%Pr!MhSpq0DG(V#z*1V8u#<>9|LDRpu$(N7`Kf6?9Kl_qpv6@ytOu_IHV02 z82+8CJ!>@0A;n**Md&7CX9{;QXFI0t#QB%Y!aL!hW7kh7gXUk4KGl-x1m4J1;`@)v z6xDuo?bKIZZSM-2xw|JE-0G`hoNg(LG&q`jc{t}BFlyF&g4X8-!hSR)?~&>57qw~Y zA9lhBp4%`n85`US$s+r4OZxy$tv(PQ8skKfgp8j{3^FY~!F&C{r!FoDxGMcWY`u3p z)zKe6&UNig7uj1~6v--k%MACrMzXWV6;Who?{1L@E$+25vt?&huDvqLs3^Pmo~u5e z-{<>%{Qk=Qhuiy{_jx^E&+$4??p{Rdeo{y}F7Li=i6eipZ@W+eRW}#v58rjxd~tV@ zo_A)YllrAp?mdX>l&_?{_0NvH~v;si48rI32gjFY%jxe1ZyS?RHLQy9d1<=FA%(&MB-a;314n#F;kD3dZBs5 zjrm;ZjYX}oavF1QwnRRNbru-ozo_AX^)+ug{$d>}FdT=%wlJD}|Jw}aO;K_nH| zwp$pFOP(8({$}0n9nX<`Cf+=|>7qS?&>ICvL6R3EQhzcVD^6ym!!AKqx?6r)@lu75 zTtlv2LR2o)F1fGXC(>);ne-kWP->x=tE3P8yh`Kw>u&x?Ixdz=(xW=n8Z!<7wfYZO zi4yz5L%>fk^eiM@1p~R3Kjr(L?$nQ}d2_u8{ZU`n2iD6-3=TA>EjIyYpayG>Z-w|o zT9fV~9uj^F>AH-VUFJFS4=fCG>T2mYUN5IOK>1*D;>CHzi6iYU!V}A)z-N_G%bb4A zI5bS$4XDJ8X4Pc+J&2WGfO!#S{S?&a_VO7+;KIgE_Z9rj^A`S8t~1)F9itz#vht2W zS-X0N=W&8jdNhtbN`U>IyNw(qYwsuM(_f#To z&woCI;(#h3{Cz6d76Wedi+kNHWBxsIYg4=H()F^#1wB0;e1rFPf4-BF!y)3=1Zk__DOKy82LoKQX1`t_dGvxQ8t)9U})j`|!LKPvaNkYQY$;!9G) z%=QauLqf$>xx>H)MZ?l?2BUbm`Bq?obu=x|iK|J{aj9+z)9`vKCW`ANXbj1Hvl8*V z#dfN4H|jbvG=>_9a4Jg0DHNKgSz~~mIrg*#;uKq4SBPpQm$6|6C515wfU3PajVL7bS_>nr%szpDgv8fM}y519wJbR2_w_~KyT zK*b9mN#+ec#v0lP-J`VZF5jY&&X?2`LA2QA*b*q$8#V~BTGa`$twQ-E#*g`b7Pc=O0<`WDR9CoQ&L*_ymf|1f+(fpp+35u+}#sBF=Bv`o83{ z-2dSGfw=noU)Xa)JIpa2h$A7M)H--L#3IFd`6EmDArCV;HOJnB?opWBb-mIEd)VZg zYd{ryVUKsVQemwLs|F6tGbmA&bR3$pzK^)R4=l=q3WH@JGR%_JyB!+yO~zYm?qFX_ zCR@6TXN4w5U}QudU}sY+5|X?g96pALu#@Ub%k z`REGEkA~8PIEFHQ^>&z%g$b;t$eEORbzHrirNK$BDjw1iU|d#t4Jjm&94uB&ZEf8V zEfJN)y}K}_1&8h}$xzn+`6awOywo`QUjP=y(8C@cNjrk8d>g6=sGM(K*>0VZIO}zW zpVkl_+ECnGm$j!1<4zQErS7}o*8Y1#Cz_}I=PI=)U+T*nwkYdp#d%TZep(UT`wFDws^f~NsZi9J(GRduj{vWao%D;!Gm+iosJsM z1lUhr##Zl|2{AI^gjO-*fBem{Y8VzVBWNm-R%Wk1h z^BXo`3B_DTdl5VzS$;S#gE35pt8)gjcMl1LL!~BreWa=3;9kC|!$`;}I{R~4>kU7r zb$B)MI8E=J3%X3BA>iW{xtzA&v+$H zlFDAse$v_vYy3RbauB?>=c!Tt_IJ#j?w{Q2oK53}*Y)3XxtqPl|5RP7(I{VQHbUYA z%n0cHl4h7jPKdGQC0?tuj$BH2e-=~5yNl0#f?NC<^q%l;yYpw`nUuqwS$AvYG4$2! zi*?6?*(<<3uK9B1C@SiK{EuNCS@p#u8QHN9pj4Khqv0f}{C!sZIk)y&I&6b2O%N8_FG@S2=dT(KOC zUhq*^Pk9~kYhEQ?R_2=}_&Ezc2lH15UR|V}W<5}!8*h-qa1zsf1QaI-2qgrp5tuDR zm$03!3EP8cptZStNI-*z##@{Aw2t zmr(Co?p3swq{DNO=RZ%sRhx>fN5?&!sX$TEoXngQA0q9B<|UTn9~Kb`vt>z&OoeP? z;et0SXgieR%){e;q8_WfVpnTQ_g+wRAJ<+4rQboGqVi}mlP4P!8rG=CBxB#8+JI^o44?#TOVj>={%7sMTLU*0>;0<*iqdlr|sWzAjCDS#q( zy?b<1@fNVcFxrS>_jc;yw6cZy)qZ7=aiW6@E7lkBVfGGXqED_v@{WX7~R zVI#AeE zmE+rmehi%#A_9xs>ac~#eBJK22+kfe7#fDa?R%1Uh^FE=J+$}jD3Pkhd={z|*hhqY zRNyCC4=)rlSCzD*3z;^or+VE=tv+NH)yhHICgK>8HK zysi7RhtFxkJbshljrCsfr}gD7`d&z!)_K*=ehta^>N;Z=pwu-)w@d8DA?P>-X8I#T zg@uMKwjRW1<7n%h`>@T&>v_p+`c7dCf!~%ovcw)pDSm2F&|q1-{!d{5OG*9nRyHmZ zUDD|{!BD*B@3(9s11i$qQaz?R|4-7|?S)1PWR8l(9#Ea*iFA>RcX(u_jE+4S?tSat z8{KWz(lYjbyWbRg>;C@9@9)mb3k^EtRgdZG4kI7;C%ykJl|OtNxI!kZ zR|U2RA@l1c@w=hOYN{o;M0qh9BXs?GQM`sI1C9xeS+|=yCJ(^)sskzF1N$;IUVQ$V zoJlpQ{zCxc?=DvISP?)5kWs^=c|D2a-hGwANY`5Y7^`o(Xo`8if&=|m*Cy;%?>ie0|>%U$Z>RNauL0{QWdqUptKi=AF)sqIza zDAlGuxCQ2EVDx6|1z;iMFi$!~6)t#%D^=XL(l!RshgJ4vR7{8eKor;@w*jo656%Bn zTeX+n_+6v>dxCsjee&EH*$%yj1Gu+lSIiuxt zd#n{z+!Yi0qaf(#L8voQsd4xcgi&m#w0ZYvrK|nrs}_9v!-rE+ZeAx}pSd60cfC=* z{rM=M|M9UcU0I7rJHDjau)jNnQ<{GMY~tLXv%f1@>#@>MgcSvi3}l~6GP{$)i7}HD z`&WnNaDCIWoDB1dN`79<44YrJGI9ZXRo4b+a8b9P*G|< zZaKNSvQu`#{N}As#Y-8Mp`mQ2@eNJYqOa?X`F4KuZO<;%r&YWv(|xnBhDlhVP2Abu zYnhU%k4L3&H@{Q(RhujMympQjCaEu7c>20s(o$K%CK4n67IAJ>UOsdGtBkd#XxL}{ zSj4oAQBSyjrziU5298e99Kq~$p@gXIX~7R-xCI*ag3gvU*7Gu$UP1M4oKn0Vt5y>a zVIv{y3?sLrr%=Nr@Ni>rZgcRx(cSO0&7RV)!hE7uox`+f$Qg^IZ4@4 z8|Q!*MeR_H`$Mtye?C0(#@?UA@F#ATcprN^6FYdC*Y~~S-Q|4yPT@EHq;o^wr_M8> ztE&4I!J+m=94e8r?=q>#)CTv|`0H57+hgEmESP!}R{vCf>_UA!6}fe$iyo7&ex1+~ zyfmnhUxRc95^EuOKqiEZhw;`_nDd#1qckz_bnNh}YO>qYg#~7$7`Q}*N>TfLsFg|~ zYPqRgUs`m8>|GzNS7v23^o6u7XqFKV&wZuh zRGZ0W{}z5|{j!=tVkPd)7|*-aTcsH&V_G9YoFSM`drG0pF!ZeLc{jkHb11rpipKq9 z>#l$`&@Y%RFozv{inoLfSqb4<;pW$ysAh*pStL^Yb6LV?Ejd|>Y?r$TwuGT~HVjMH zy@?+|$CiWAJF*B3H#dkGC`FunP_`+MUP~acF`0*ECu+isYy%_^HOU$L} zzjstHPt`P_x7uq~1JLb&`uQJZ!EyeK9ovj$+jKrb9VHkZTv$=#cZSo8wd^d+rB>&x zN${3-t^?Mgs|J3geTD-M@hKOwQ-=PuK73RrYdSOP>lY+N#*b=Jh8W0>YH4DTH$qow zQ+tq8b#K#+^4-SY-8_0Wko#4=oOGmYy(Vl#Q_Yr@H|%WJ#0?33+UtM8go#?18{*cS z*b^@tGL0a^4nGp-lsT*4*;Hp7e&*NYd7hF`Weqs1!v zqMSA&!(r_-3|Iu?S2Mm5x()UQV_bnIF9zGr3x9V7JicKlFEZSrlLvuD(A9(qnlE$l zyuYRcjKwU3Pfdx(!b|W-LGGA#jag8zJinQ@_)Yjtn=?X6_!-z=%P3E;Eqmfwd$Ho< zx6}S|iymsN>8lxmbgR}3h3T0?rQ$_k(sgW(&Z~k;MM4(g*56ZIqFqv`kj7;TXcAAF zcicXd>yp5v%EzYpe41>^xl}iJ_0Z~f{-t3Gg;P0#$j=BC?kcP>iWC1W&dL~`T87LhUjWWbeP?w*g!+YdfnFY!mw;zc|PJ4mwE(c~#YDjEWeXSK!8QYKY=u&FB(z~Xi(!n$u9t|BIl8%JL%8+;u$z_^-LCH z$&Ow%D4=XAP|j}BtyR;NwwB$htj4{lt>4z~pTAI?pR6X$-YUG~d*)$3jGl(7y5AsFu7cEe3ohW{)g6}bev9M zyLl$|E+QIWZg`Jnw4f1Ulx)hlbU`wvTfO)l(KSj{rGR`3O<5Hb>7)<^M51`QlG+}_ zx{vEj)es$tEwrxYs$WZbe_tP%d&Tn};w=9N7T&)K^h;93%jYesNpbSh|eoM{q&0k66c1vC^#LUR?3g1tcah|$6lWZqpnS}BO!Kw$HY)j&Wn&w zL*Uwj+$q0=>qq121mh9bS9wkAFFeE*)AAM*;eZRCD+=@+|Hy7$yH?7`luG(EyN8^XuSd5q8N{See2O%&?8kK8g z+izc=p7T-^j|2ci|;1N9@b+@ zcc!R(o+g+`H4jn&nxj|?(hGR34i-+DPmYJPiiuvxHK#%YZ%KqT0_b_cLNt9|X7Hfv zVU9P{PKgOFqWV5w*V@~q>2(=s#GT_C8@mkw8IE}B-4_ji4%~l(tpxgm_p(o(UG*f5 zBQc$=N3OlLc)ay2X-nskcEuKNtm!V>u3|c_+$Sl{R*d)z&WC0V@x!YKw70Ygtq}z8 z0vuJpaJI|+yWe7?WH*S=$#&;l3%WAV`gUi`#r?wQN~x;t-Ib-%vp6z1VVxKt z9RIQ;qo)WW6ISmA6r-dz&&j@c>t*&kUY|<5Cjb1`h{~o_Yok%_#swl^;P9b^R4sC} z9M#G1-a1Bq5){3Eo(n?h=C<-vTHLd#{#VsjuX zPsdpr45pzvk358tX~vjBdqLOQBEQjEidNe0d!qp6n1r#zjkU5 za(Rih)b+HDA|2?3F>FqXR^ z^2`yVW@P}awr zR@L?4`r`wVWh)HkKAi+)DCenMM(2-Lx48QYEdHYsC?=~Q*APi!5Z0llCzO|V#2{bg zFq-mPBV#LT)eV(0RW1LlCPEg_8tU9qVE(tgMUl#I+4no8-%#k*d*_p_m_+5vRh!w0 zf}G3UT?4;JlwF1M$43gIUP?O}yM2OB-L#9dVm5U?iPKJ0R;Uc=qL8Ov&x$y zBY7#mC8=5ym2X}(rL}6dKsm+bgz+$&wqND99Lhd^85Wlu|N4o;m)&g+&E5UqOc?lM zsg(aBDZK_CTolm=smF+^oypGk7|%4^_^X>`b~&fvjmrsuVanLcxqRoO9jHzD zwxz9;7p)t@y~{bo=c&tkH*+q>gl;QK65RS39A~HG>IabbJ=*2&4`!lWv)QF%(hw6; zA>;Z$T6_{YGtr;4QSFd8{9Ui*5YufkGGFEa_V6@_+DF4;aU7EUxl9^m7zOiH>-tXpmP32Uia!8yd zkZ;YzZMO%6CscaWF45Ah%o|7&KM-Ueyh7K5*fkMpcc*cIoR#yzSqGljL0)4WVGBS* zT9BWFu>Zi>9%=VthaalX!IAZai)t0vST|mzJonnY>up>4&edmvj((qOS?a(Um%OzR zGRG)!#+RUBV#KSq#p(n_W&bwfE-7((ao&8`w~p6WXI|5nA)E1BlVz-lSCX2(fh=|F zJpZtsuPyGJ6+dE!Uk_WBrM_&Zhi&H(n<6Z0( z`CEF0fSH;I*UV#P&9{ZwsjA$1obWi&BI0v@c*zxOrxtJjYu$$&#KDQ&Grt(8w!#2v zL3eu=5_T4$Fd9Ki8y_`m+6wHz`7zPp-w6D6=blNZ&R<<+F6k9K{O0o~1JAlS9`U23 z=^wiRaGMH)nBAP*%Zf>qz3kETP0qPA=P2CbMxBe!rEMw!=cO8fQ7xa7xM!CEb1`Ek zE1az8qI7rhNZ9&D(GixRs#H}^ue1Hhv_@xlGFP?LiDcq z{Zx0}Z?%&Ox@cCKw;O1YM9Cozn(1Ef>arKx>N5j7wgZ)rJHhR&o7MS9z|k@~(TpKz1k z&bLHx|15~BWDO}_=`=MxI9IpPOHAjnG9V{ZcfMU1hO>LeGqd-P(^$wo{yK|h2;#YwiOq}?&2>K#t z=62r1NS%c0qYGFRdfIE|hV-l?TCJdPown*c>JH#dbT&^nn8Dz!^ zJly-*AZ~D12eW4UBIr63Mb zeBgBfBr#0GgTF_*M6*gckR-Hz{%vUuzYB5@i|Z?I>+KYzXwdj0_i{=F@BTH$bIF6@ z{!$_9I@qnr5|P^)6eGL@!-uocKdGSHiUE~rbh9aboC}z4-ZD6AmOhaSu%9{tKeX#5 z`%TTB5doRp@ey|WnyU9Y`4sQI?Dhr7f7!MCggD(KaELrZOrfw7&EwtoJ#AA1o8`bZFLCyS4*({a0M;%lZi_cNS-f7-)8-AppzL57(XEfl14LzIM#3vDrH zjzc_i-##WVRDYc)ghS_$I^0GO$*7u+b53T-b!8aZ>_9lU~Gdc^TY7a^^=gayWm34iEY-O7de zp_}Kj)sI5NntoQAjq$`^IXxK1J1J5SG8#f^)9Hq7QWu|{-*WzO!I77eiK<{Eunoz3>=yW`) zQzldA`G{GiCxR-ygFQv+!vmogCa}>sH_T$zxmbQV=wx3x>E%&&hV^pJkHQR(!jymw z*F0PcgMvidi1q}XBJ1F#)Hj!$7+z18(_r9Oq^6-oxFy6&Ds$J1F6gelIR@v49^M0E1+jKOL}d zt#s~%ApRojad`-j4XuHi(s+>j*BYO|D|`A(Knl%a{nzR)FLVLRw6KWZhla7~eTP&WU2&pD zcS?Wv#plnnJ&B)nRTxlS`O)EF(Opo918Fr`hA*V)u+TBx0j#VN{aQ45>p@%(*g_4H zn`LCCgKz2;Yb&q`&@JgbTCzSECnMoL8s&eh)kL*p%lO4~;PPrKMH?~TV7z&Ce&v~I zPPMw%-ADJtnO{8n^V>s7a;&U=agOXigMm!SE3R0e+$`CKzxZ&9lcQ8V0Vl6`NJ6XD z8e+qm1*Cr|kndUN+}R&Hi)s;4zi*p44EX5utDyuI1{;f&1h3LvmhcK7NY@o^I71;c zxnFDgCbc8PRUIfY;Ii0I$8O#|gOB*O2o5l2G(^O|8tA?ANGP~s1IEmC$}ipK2)7YN zR)aS0%RrY$b?=*j4mcc!5@m}RjIz+Iu+_uE!7vmjb(@j`WOP@F8*GG@e=Mc7`;Jd~ zK$xr*M9cU^fg8r0>;qLz7_Xsf36=&AFIMzw_-u*l$eEEq`9A-h$-|Pf>;hdwEK3$5 z#re*X>$R@)nj7%IHD1!k*iy>zWWOsXeM)i^#QrB&a_g}~)awcl)l$op8@`sB6u$%s zm5T9UlrdZym-{=m&=j2Tzt|uk*4d@iSo>DmtSj;9pEr_ro+2)^29*d*p3x;3+zSOY zdD(}}i3mh=`2xTEpr!Q>No5M^wzz4Nr*|6bpv&;Vv&ku{(YKvupI(Uxc$IX(v>?av z4nJ!;MG{REu{9CSo=OI0lMq_^8=AbLU{NCbB5d(raz$A$UdE>P9>~S3#(gibUmTfT zOD9Y*Ib}A%vu#{kii?S3Zh1Rk=_D4UpE-RuI+-F|Rf|_CXZMRn`8AH~f1{u-JjB9% zgE!BWWzbA|pU)O`Ju5}XPdy*aB3f)0oNz|NU^w9I$Q7vXLob&U>2*RF;!PPbSX;_S zX;zVlOhEYYUeDKMDy(~@O!E*6>5hYVv{4LqKG8FqNVcVm6{3jbrRO49yyryUz);@4 zFd-!IGWEU99QC$wMc+G?a!xUAAr#~h0WjPHeggmjUG4@&T6acG6M|J>42~pYef+t? zUsAxy&xadGL61@CA_cRs|MqGK^pUrtl+b9Rn(LojX{GmazJB8AKNt`B8gWnk^6lLy z5h$w&r4)xExMe4ItZ6ed3S)}1i4OdbUMgX(laA0hYAs++0oOuu$v(GM1#$1A#;qU+ zqzL1C-+KuQ9=`f9QLWB|^r)B8ix9BS322J+FSnvOPDfB@{&6dw&n%Hk@Ugo8pn|`X zI=0-Jj%2QgfMuij`Y%{=o0Rx!c!Vp9Vea&$sR(XBc_`Z}J;bHcu>?mx{xluPNWPtV zWS9(a?%%Z}pw0U#VI6QLs!9|7@$+P6B~NmDnugDPekAx_##F$=geGTPzgR8g)tJY} z#DK8I3sHvv#4FGhkZ-#QQ~j7^U$f*;DHQ-A2{fXbszbC8`I+sbM;*KS1S-% zk+E2Lw+kg(E7NeK2_k5mI?_kH?_iwnhV^Qs%d*zXt!0-pFW@LLtxjLrS#mT&V!6A` z=^R%g@*(KTDXJp_iT}J@AdacXvPKy6pT8g^2M>u>DLZG;d#SukNoK7W>|J2)cW5@G z6^dg$D}r?B=qnL*dzFV5VA7bsH6%a-ZGda3+~OSBQ+XSaSX51ssQO}ih6pyz_V8$Y zAkBKYxfFk!*~9f>QDE_llHi`7n-;i-m>G5yg!$k@gdGyLkYtnoT&}xlF1bm zf6gaQ8Iuq&7$BNLDICGQZ9V)1cW71^Vu;@d7)15I1uYbn4dC4aojhGy?tBu>-f@!+ z{g!Z1yqCW`sLt%)6of06-J8G3{F_J|oYP{mmTt#^x z{zs*Xp+4SKb33&p4_|>a85S?Z7FzlfnEM^AnCZttOhS@YCU>MNsE!IieYaGD&>nS;7So^5xK| z0*sJlxST&uXx>e-514JN!%Sd3X7{+5iXRoctmIhLv$-svl`tDEG~X?7zmW=!&w0KI zyajrIbp{klhHqR>|5?9`ju$-*gN8x`CDNR}k_yr1x{-4zHvVIW= zCRwkpSYN@xiT3f-okC=^E!}7S6^e=^Hipqq21(kP@t+JAC$AV6+s9H~Ld-r~Fque* z0d4`cQHf;C?5LRsQ|6 zfmCzOii4eUy0o(u@6nCtC{0S7+ku_`0F+JE=viqoE~c6Z5)wlcQ9IGRUUfz6?~7%{ z-x|eMLR9aV#yPe9N}T&eMBC@D(^6oD$o`aL#!Zj6?Clc`spmSQ-Jl3f2+mNm`C-yt zO$CXDaAJ{^J9%s(`zT7x0$X4V3pZY_W2lPIfeC9t(u-R2sS3?Vog(@cYC&&VhRKFY(Di0K*OVc}-(8R7jN)lfoFeKvZz;tnP{KNJa z-`TD-;~aakzr-ONMdpl8{yPig@u6v3|9nCc`_2r- z!~NTO4E0xqQN8>UGq(yx|DIoY22ihbVr>rnx7Rax&kG{6S_fOk)t_oqa9Fd7Ruimu zpD#`cI<}1YdHl))nlJiMy)-Z0fL?zQEzdNtJsmZ@|X!dfax zym8{fsKcRBUR?g&VXFGb7}w{Gf~GmM9Z1;Ll)bW^@oO>QKN@-YyclH6zV=HeL)oSKz)WzbF|gl1KlXdq;^=PB ztVFZaCs@NRjxi@`pXw=p=#PV}&ATEa#D9ww8{4U312qQl@&6eL0#8bgx5bdf|9u(D z6Bv56 zZKsp=RO{iH+YWlA{1&0;SxX520n0_7JYI9(YT9}?E{@UbjF89(z`?R5fFX!mCt>sl z99EIyRyObEUz|b47Vg!NGSwM&HC#T9r#69$QJdXob8PhwDy2dQhAmd)t+m_J zzSK-D`=v+KtoQK+)k-Rr;t27oYMR@H=+1i8#zM*m_Z1>`;1mdlAdw3Z*8cS#{Zx}= zultO%Zqv2Rh}Fol8t`~&X&hPc0S*F7|QN5mQybL)N1dw)+d=DO_5Ne17sto2v*(Z&-yjuf}Uq}~pA5rp%Kx9%h_BN#OZgWL!L zOk`;L>jp*R^y2*^jG$-9zsS#dmS`{ehZP6?g3f+t94FG8rt~Z6p1Vpa$tT8AzY|yY5XX zt?P5Y?+WL5I5O@JQV-oi&w(Dx+~(Lzcu(O}6O#VDL2~rH!GE`67B-M`1TsfgYbZ2= zaX^DrQRXd|c-H*bT{+PzWcmihxGWeWAllm_-@CB2DOK#rUv`C9UEYU>`9g(Nd|P8Y zb)aaHcV63+5O=qgYwu0~82X$$Q`8$XbJs*0m=XAmvlyg zSZ(4$j$v~3PpAuJ9gth&Dg@=b0)ejy&mnCEb3h{YcfkyA!?VOpcctB zS;y*>f}W6AH&m*A%{BRmMhn5LkCgRJR>tkS2tmmh{|zsD{%3V~b4#e@mTSXb{rbxt zX!@LMA`df@>+5c_p86s%L!mPzkt<_At>U?OA_t?g+TcjnW192wJTB^59f=CJdXa*N zphN*z`dE3Db5lQ`$f?{oH{+Kpaly2W>OK1-gvcy)Izn!#gO$*N%j9`gh9Es^X$}7a zMNY}WnM9)fBZ9TxF|0Mg<>!aXQ#mJHV*?w?`>$PPifW=%tSElG$<(yFbRnyLenbw2 zD^HO>WB9^M2Wh9jVCGbjajukgbxVdFt1dYm7ej4a^|q7SXSP2dvT5BMrTI?;J3Ue? z_**;JVC0!+)*n~qxFOWA3vx21tZWTif$41Tl-4wS+=>`Vu|bzmb!Ix-$Y`RmR2L@% zxzAI|OI59z-ZOLzk3t!j3i-##KU1w1A)`;!=%Khw579!C>z!1GuTZUTD%TKZMfN$o zG!Fu2qXiN<4$Z)=tuXgx#qx5_!n1PkKTssMF){cq?LG>7oo6W>^J@V@XeD~&(mm1* z;%cLAIaTm-6DT%b;E(J%Ve{I<6vxVkuA5{Mpasd5m0P?!3pbUO^29vqqX9fN@)`Njm=9TCVr1pyP&oC$e_m z>vD95H&*Df4g4ZgtSx5E;kttw(M{;qz(s0KM`nU^_zePDncl=Xv)EKU%PKtkWAtnZ zQIU1$5|<-wY9Mu5syfvGbU%Zt&71q(3(CsC%rM^t>Hf*Y`I0Jz4B^kl7cLyxaMB!X zxpA-dmV)6=Bp@TPTVzDhYv@Fz#Ks;0UR%X*q6~X83ix+j0kth zh;+)8mc$|C2A9~5jq%`8`6_Q!+O8mmvtd7ZMyFd9+diqe(bQa|(=4t_+Bo~kU4D*I zPjXTD#YY9s=wJjJsi0$<00YQLMe`%%tu@_ff0Yi?tJFa9`=mO{6`=Nr!FSR&W)b4^ z+;`+H`6{Teh4nl!2=>CSQiPfp$#!x#Lxug44L z7v!K4n2IXkfxKYjVkVUF)Ax*#(?=hwBml1eDGpt6lMDDO#O-s(bSzTv$)7!#^!phB5x*aImx}&KA9BLHaE&K1N6rw zIJ-Kj`sY{y6!v96B{T-54u~ljWPf<@)r=iu@kcE)Le`5gp%cG^dcLZ1we|U(rJFiy zc;Rlg$DS9gn1S~SQFGy=ElTRXx)O~byu-VZ%!K3-k`&S;WX5+<3lrhr2^W6v!x&x8 z&+7W7$AKluRR#qufOUl1(RG~fxXbs8%7^Qt>-g&kyzxc^3ZZpFR_u&1c?UMz3>R-) z!afURTOhHpo%?;t6#xX*mlFBjSVBTwQk>YRjj8;JOF~O@TQJb~IFxhA?m-u^;+)Q# z%{9@jUYe&CK#AS@c#OYyp~aSkAUvY7U6`G4CjO#={9%wCW%|DUdGy*wtnJX?0iR42 zR7|h`)*^mi^14Eakhx@T6(9eC8ZdIHej&JbqCz^7ZCYKV`W&vdD&&*fW3ojT#camJ z1adnR79=Q;$Cp-UF)i-*#7~u==t=B9JB?bN5`^3+ zB>#SN;-Det)wJvHr)1ZGjppdhM9$gs;~QJa-eKcrgefi~jImsU?_S&Jcv%4LC_rK- z8QT`FVf*d-$v}Ts=MsmAdFHh@-na?fGpU6VcOo40M>-`h)3g>;>RuW0M zTBEG}$pnh|%zc>?7Ou_b@k8Ah^Pa$t{T(l{n%sMatkJE!a^(C0OI?G`Sv&QB0;7qq zJ(CoA=Dn`~-e5gRF^&U(@E}hnWy@U!*B^M)r8u6=-#p0^C*>;&yf=$MJkx%I!c-ZW zD)`m(@XQF@%e0!x5k9GH%7xJoy&A3~Z7PN!E)HB+ukG2+b&m~S9E904I`%hg zEU_&A`kK9Pwezl(?MPV+T=2)Qd&-JGZ4Z;Mntu`Smmp|m-zT2HIwdfscL&>hR3n0N zdTHZ-N-5nNp1bcT#=L_CTGOxVfd9K^NQE5We|c9ePn#k0=;)*=>koO0+nLtA9+C;6 znY_D}$n?h!0^i=EAioHY%B94a5qiGiI}~OJVw3N+vuj3NCIn2` zN+d8$k0qIEYnh;v&`qqk54_Gt`cb}~x*+_)sosdkC0y*K(3x#Ha{Q{!`Tk*P=o|cUhFR&H7K+A*5g&9QqFM(TKo&+)l>IB0kZ7__P59M&rEe|4`!&j_W3Od1@jePW zARxLlEl!EtSquf7u zHz3_bHdIzF@W=Me5;DEiS@`41?9&M5zZQF(F1#51a%$$Hd~@J8yyF{(b&MYCm4LEI zjuPPR-t)>TDK7aiE75B&;+UEGuHBK7(tKC^^2K=T#(Xkgh*;+Jwztrr@0`LWbxUav7dc$5AtpgV2Z)J0hS}b% zM2(InIXoPJMHd<_L*)xiRRiHYs!des2-R59B`liz$SyWS2U%ZVo){=GpTfk_@W2gG z1SQ3BJ_7UCfX$V)Pw&GICFPEKe>}FRDA-Y)ShW<)$GgmKpN=aGuym(%F^uFwlL?fZ zPqe7G_o?E^Cqfu5j}XVZsdSI)#PP1)K>SQb0Y>Q+0Wrt5A|)M^yOKK2#j$4iGf9o6 zZ)en{t#6KC*_6>kH%sH_!!EPw;IFb10*5j9EIjiqz0!UcuLGseeq=J*oB25vk_b%{dp8# z-EuVw+*XulkjH#-lymv|_esn4_gl)iux4)Ll`(FJ;1>S=;H3M@ErfQ;QYf>Z(9^co zr#hNVRLsJe8sQ7F!kWST1uDrz%$G7H+C)7HDdK#JC4-pSgj3&=iSBnh=h;->ag^Uy z+Fe{`%MgCHySR98u-_LUDPuQUCyM2R-eo5?qD{+usnpV5J0J)RX8) zT2ku=5Pq0&OcL(Q4hmf_Q0U~|3b)Gs^s=6;%2gEksm55tXNlCE8U6w;ma3!{6=JB^FjA>D46 zF$`Z&bhUS*#9&#z!8w$?%|q?&;zWONe}z__Jxry^Hjo;R<={nf&8bz#_mSRgpU<>@ z(}9&5FMQ;d?#cChsg5vuU_?959A7-{e%=xv8gNAv^uRwU1byvWx)Xb2yS3ns!QZ@- z&t_cF-RP-Hdnps3VO+;3m4-x|U*B4MyKr82UFJb?KZ$*r+Lv1-nI{gvfoo|@aNVi^ z=J1ZwL0gP>BZk{vL+6QQ0fGU;=%9ulc~?<7N6@r3>&5UPx|N>G@PCDa3}Uflz%fCBe(HU)^0SD(ma%PZo9DjB&OQF0 zNA}ObFna%D!f2#LC&PcYAE=Ns&Ofe7Zrn)!&9r(Y$ld<#iX(Xp){2%L^020Hdr~xO zg4hdKgJbp3n2?O5-nZj!{NpdUR!s9yoyFWeX2GK38kdrm;7lcT$qt6OfsX>Ep`@a$ zNQnrDmJ&n+B_#wlT0lxV1SJFs2^l2Vtr zKhOQVc;4^Dw%_;nz0P&cxz06g*+{KLM-$yDv>5{#V5C__OL7gGF>eGSW0d299Amer zsP$z+Ndz85$veh_Y~KI^8GHb#7O@*#e>_d!!k88-06cKgQp}y*z($}GJmPb5FTY?x z-e+xmrYsfSkAgu|Tf(spB=2P3)9^-AdXYOj56JwyXH*$c*}i2x>gkKe4if;pu544k z!(%HK8m07a;G!8!arbk}qbR-&f971QJ5+q~5{2Trfwd(evVDBAk2brPqYjlEp78A`R?s8dEB_FrVQ69qaEd;)Q;T zR*&puwV%ab2Dn4PA$u(n&fi*sv<+EO_Iv%mgQ>aGIe`q*v7_7Y(7lvQVW`DG=q<`^ zqMB!0IS~C@bPfj&x7Kh(vW7*tQQ2CY?f~2x)XSNkDS+N|UUd;ti37cEb)%qjHz}WW zZsG$G6B8Jd6+U_Aq4T%p!7Y%x_rKtv*ls^92h-;+xliGSp}yk!(Z6Qatz*}j>Fk7V zb0fYmL1baJU6LZiWF1Y9-G1pD3qP-9p5+d58@A-Gu(X6iGm^Y#-VMb=4p-*6$KPp% z;UoSn`6M+>vA{1F3iIiG=@+E&A{cnKdJXFWejb5c!w@_TlTq^bAkw_8)*pY+D?8Yt z-?V)sP?YX&5$H=NFwX$|{f_~txqY}eOj@-zdh>POQ~KCjtMTrCFby#+TKy^0`!L0` zyjFY^TYn1vQ7Wvq@Wa0`2?UJIXiiv(y{~|)M~_r>S=#XbuxtB3$uQi5fnb4bVcZBU z5V+@QiP3O^edUhVnN*1cy)$Z z3q&lYPwYCuw0rA$B)SwcijPZEcVjUz4CFqUm%pVG$gpkd80CL$QAXCTz(2@7fAMW@ z6n0OWfY!>xKi^uL-+cYEXnhi_RQxMOuO_Jj`lra$^;w3VYt~)DU%F;0d*9Yso+26_ z3TMA!?_(u>2Y3*LwKh4u|C4d)XS@lynu2X|c8M8&EkV8geleVHyu!E}WQ(_j=>JV2 zr*pNaL{P4Fd8Ag(()*#N z5ELga4SHDtyT$E3c#nDvve1osJid1Yda4PZ{k)s`{j9-FB=ZX(1@!dA$47Gs9F;AT zfo?)*Ac5@es;nMr|`t8{;lg^G3w^^k&V<{c*FgcB=5E#^S+}KD-L`AS-Sn77Z4(5*L5dm7+gi1pZUb@woDy?P^}Nu?QMq(BZrW>(1mZt%s~6dvoTg=6#}(j zWinU2@S7E=N#pQXP1Moh4k#H0u5m2mcdny~hg7cNO`sxyxm;V!LuQ+Hh&s z3!m=s#-9Hy7gEjG6f(<{GR`r%PkI1<*>fhlI$4-_&_lQZZZK%EK$}IA-+irCL#|^@ zoq|JZ2QV;#r;13KEKkd^E2R6`ujzou3X{v7 zMAz$jlNnGj3qq`EP5qDtwSm*$wR_PB~9!t;W-htKycEEl8(vPmr_gjmGPFVb671SS-<7d z6U!hH-z}xEt>^>)}%I70iIXHZ(daN0lhV|)HYgNx2AI6TviS7%cJT|<_Hm5jEUV@l42<2WJ}SOD#h_Jaj#HSuV;gG9Nh4Xga-Zr zf-v*oSC~~fNlPbTyMep-c!Ld@fFlf#tYCGoYi?I{vN0g; zS*P7=qu;-?kpdgc=8dC!@Z;zHV1#iwtg)8{_j;_F3`x#T*OB2%6d4=KsVot~BSQ9T zfIF)LviQY7(8&Gb#TLO<1Z<071<4>tIso^pmK=a8BKtjiDHH#4;_r^`73lsK?==&e&;Ss6n$H|WMQ82% zJ>35n)cw&(pJV1s@h^5J)*+kcqSEMTMS`YQ3ba0VltkJkBTM*I+3g)v{i-Unm0*Z? z?kq&n51e;D@j71pc7SBm6CfRE^>17xKGz?jh9ojUC04i`@Fgrf)OZ62QlmESPh-|R z8>>jIViiBF+;2Lw$x)aPFe&4g3lOy)qVtrvUw0HW^xM6!<~=LAqygGL;C2t?);$ce znl8+4`nR!ug|`STeWkFoU+WQWg(JrXBa$i_??2$IAWI((g@^}WHRc&B=Cu>GvkBgU zV%I;mT5(ShPim(L^okG<;sW#%L|i=7O)0LVdL1YdxmSsF%@eVvOmCD2*C}42r(C79 z5&CyM;S^>Jgsw7k=GF`|X3w1U_h_MF4(zg10O2A+FlJkiggJUgYV?!wdLG#Pl(o^; zv9kj?KDhwS@l;Z4K`h0I5Cr_(v27w-V2?06yj#D}GrrS+!TP3bVdl-` z(sV3>?y{$@k`hg*rgpZXA6RK_{izsMsCk}eHs6k}uK83s&enQFxvN)hVPb zNV5ql4EBQ~30+v#P$o)qW-%r>0ketl!dFwMT>09%(?T89NE=Uvaof)n<;r znzZ@4AW2>O##j)f$W)r@B7+s{|8}RvD&Nk-5mX2rToFXAu5DkmAhR)6X(L+9hv#|mMMY}aYj%NGn45kOr^k;>%|f_M^*TdV|c0l2G*>D1YJ z8qN^B2I777+u1UYP}`5|KCrJ_LF?>?p51xmQ06_|w7VC3QT#ybru%`!lvjMWB`+#( zvFa!Wx(R>7uwdv@_Hk|oqBsv0GFgXXy0<;-4?=5=L)VK92!96Y*<13v|7wFDCt}nl zi^^c3ydnkA!sxJ^FF{{4?vHS^nmE1ypHEklT}UbNvKt=eh`+_<2UjJv@+GzfsSLD1 z&-jog!WFbsPuZinW6#!~Bk{!s(G>ODZT|E2oP3>14m9iNa@B1n%ApH=G`ZKeakO}A zU%SOBL*zcXU6^x_g1E)%;Jn(91{i1eV{Y$fFP%N-MV|bd|K2QLba6As2klk|H4^t& z@_10@tC+H(^Kx+Bixx6vzb6xdS4+#vPyF27ClarY#XifceqYX*)_v8DVP*1`0}Up1 z&)$%VBX06rty38+bAQe80MOHHt^o{Hv~>kQdzAQ#QdzZbC$a(^a_3)~mT~+qLE^;v zuba%C9?06s+~Js+6yTt2<^?U2{l(xH@=Ujk5oO-ng%2@Gw(rwD1I&n9rICXLT9m{K zU)(F_mgE3?pnP6y!!Jw29kaL~BFPvyw19hI#9!vY;Q_f8 zqP4v+P;azNDL6KWp@ww}pb!WeYoJ~*i)ed6RYy9D(SA!9TUX!*qLs88!mpnTA_xw|naIl{qa zDl*X-T4m-x$9I3x!5vlptd+=5wRexA{v=~f zL26E8HB9FLv2-*7C5!6Q$B|w_Xk%CEyz==R#o~kDxzFD9d>|Wd`Em&$7y6BoXOrXBQs-zB?5oIwBftarmZ`NqG)i|BeM;gJhxj8ua~BBgrRB zEF4~p;oL;m>;Sl((nYc(ma9jo(;0X~`n&m<2c2aV<3VEJx9HT$H2}$%S9jTKtjj0J z#(>N*GEb+X-U5#}utLpundy^h=tW}T$4YLLxrt3>!Z8k}@C{!)I)T=BMK726Dk9{d zMgDC^;@<-EvZn}Cl7gcoR-#{R$N7Pt+`}ennUSdo*?(yqZN~Qo_>)jHLB=D3oDjda z{-yO}l8BTQmdvzj7@`OQy)!$V23>HQK3f9mt&VaXBsd)U4%jqOv(LLO{K zs;jEsR+zT#*P0#SMXEZD#1~O0C0|YaWC_};ID*OpPfjZ2jKL(hK*1^+{!VB}-3y0Ga|ai{baFZ0dV{fH;# zLDa9AymOrytPYU)vI7p-kpb;;5d?-m854(Flldf3xM^>yvqcAvuRi*0UvguXz1SrQslBNrS#Uc8`0d*`z zxeRVJRxr|tVe3~chrRazmqnIV zgxLoLh9WX7v{2Aq6HGD0(2GIb{>&-^PJa`^7%pDfmqjmqkv}f~&O+}huORi$j82Y( zeT}rd-95!y5iL#{k+8k0Zik#y^{_@TgXqE4ZX1~I1y z`#k%Y+$Ut;nbq=^<9D`1X9$HmyqfOWswJHW*C@4HjH|@8Y_HB}2pE^+Qi5+yAARv| zF$t=DUYJ-J*+;!S@^!g%sp!8I2+jRJ*jDlLha?4sr}XGP5F%G$neBEB^sy?tyr#q%Z0XQ4DE}9=k?k7Z-cwyj7o`pWT)C% ztfwc5tGaT{#O!LO-{CPqyH$&o)W<=<0_$!>R#%kJz~Jw>c5c*XNX`$UHx0NXuF;J{ z;~Rg1O6QXpuWWB{3Mm0)N`Ai@3!(F7_Ks>xuy}-l7TnsDSqKA*c460SC=mwGl#)T9 zbTn}0sf;8pD#aWwXvs}f3QbQ|a-kW(#0r>Lt|b|F@Npq;fFi1=>29?1KJA1O(VzSy z@n8WVWEC^LffuT3H6lJ(Y%al+bF9Qj$>M1pEGiKDT}zK@tH2I|XyR!>tgxEoSG&?9 zdd6h^HPyzyiU)IrdVCFzg3OBCOmPiIlZ;qN`@XBjKLYkApfqf577x}*WH)n}_5FW3 zpT~#1)K-v%(|%5=EH)z`0SU(H z3!i0n)Wwsm(8XG==dY))M^rd#acZVhi<7;vd@;3U*>AC)k4ry#L-sQQ_Acn)*{}4D zdZPms1gUm0(Jr_XY!*R0JN5WOuCeVs6X7vx=vevjhrP!mBoYdY#T{m)M5RGlNmx!@ z)-9a+X7C98AIQTCHA_X@=k>ZpfTaa}g&?@tHMhZvlzhaovi zpzS7HsxDW}4S&CRxR6+F-^(PEQpYdEJJS0Yry1tAcz^&W+r%jK!-elqtb#m&&Y@QP z`mwr8KkqZ}%HgO$2+x)*9pBRQr>nmPbza!dRc$Qj(e=?wN!;%)CN+Jr{N9^G(YL;o zFb=($l^eSFp<6oJ?d+}QpRtj%7<44jGw#JCEYnU)onkYU9|hy3-~dd)uAMx{%xKI>V^G$pzRoKu8hEo`VrLS}$y=+i|edKr>=Z}Wuj zX4a#b4>lC}hABo>@3QcYj{^-2B z;WnNY`-h8|le(*S)*jZb>i-|S-1+ryfgpcPuM+O@a%S;bd)K4$#j&v>nu{H~9N&nJ zF5U|+S0MI!&iWa*q&#Tfb+r0dNy7um`>8Wp2xr1?0!q)prbr^e_%5x_ zRAV@6jUGBbU(qrxRIP>&q!@CjqItHV1!x^?f z14swT6$#MC?moS#z&7(Q{z<}iKcxU+0Mclwej-1C8hAh|?>-khiJ~}PhNc0B@V)5A z3aJJh&X(5GLW{(*LT`I9IxQJ8a<~MDWj=BFVH})V%{ptMo~#!$Q@PV~5}0m}#)swn zt=s}^si~=7vEHXgR6Ff3MciogRNG47nH*0w2isW0rKq8g9HnZ;1L{l-wXVNrp|!{Q z`8zG62D^$2iQjE=l9$^6?=c?|F^^AqLV0r7lIZNv($6JUHe%rwVE6TxTL8jRG8Yb7 z!3j;hI-!W@+gWb`>nhstPyzE;gt5y;|6c-R<+>_LS6S*6x+stj1Ygf}lqY zB>_(#FvNF<(nOX0-7!(NwRcW8uIj6|?IU`@oVIbRg9cB`(Ha5Y1FI%V^{5DIVLl}k z@nmL;7ID)r001?BzLvUW0sfp}hJH8GS@Fw(tN)DJwq^SF8nagtNIbM4{4*>oo`&p4#JtBg?AH=iw9*nU34ZPH6#`?jT z#u6a#)+TYxN*pg^&0C(-;-sWY>S!S&rr*i?-adl%!9nSQ#q%Ql?uCgMp;x^~moQ|5 zL7l{6I7}#5?Oipp<0OGYK)Yy!}G6q0vHNx`?w)iVt!>oW@5fPcW{)T)yuC@9H0C_3(?+G z!0YcEO3V)k;=7+fC7%+#xoFAbeNthUSI+uohXR*eV?1j2xQF+OV1B@e638foy%u6& zFR9x`A0Fhga}!8x(nCoS=SHsR7%gwX(G!x8K?NEU%DbC9!hkJ}0Vv!0uf#@mNF2%v zL}(V<`V#?<5rIdy@^O$8I-)qzzeZBT-ueO;RvgUGI|B}(MLjh=h~6l(lej^{+&eeH zm3D4ZOP)w2KSESh@d5zPV{bHo9H)C-@r7{^Rn_*3okXb@(;K1|2j_iD$oF#aYF%*T zI|X`68^Rx&=<(y)e*!R28Uj~G64^wA zkaa7u@7O(xj;OhEq_7s41rBp~L)J56;*~fwj(DP|LtPgi zK0KHeyMt?yYC8L$6D1A~Kbk+a08s_i^2lfE(I^@^G(9@}P#-C3opZTTr|*b-Otq8k zET+%+s@tn7ao#1qn;Y6Kfx0}=VG=1NqY!Q&i*DcPsV=&{TpGAZ{7}o@>-tTY5?#-O zdaQDOsH)+Q${UL(bD!5R6Q;e25-{TY$)#0>HPRSnu6bd?Sve1 z1pE=Q+GM#m!bnyrOIbub^izZN>wLQFmrwYf9fiRN>RnL?%&Iwv`o5`-lNE>9POr!p z^}?2Y@dt_}7VfKrz;}-izf(q!W7nd{u<0kS#!BJ682Dm`{V4}I(R<&2G>9=RaO#D8 z-DPXmtzR35yJyxx?_s>I7rG1t8;QAp^KLWLS}de(%W<8Am{|^ZE55h9U`xV$Y=B3N zS4*m-a|Yf{;|+@8=t${#%ZmZ_{3=cUk;EQy!YnS4MwgaYy=S?RjQL0|Q#0Y1Jj9b9 z1h=td=o!ZJv(Y8CqTCbxA5L_qi|$cGTvM%NJ{=_hr%Xt zV|XHc`VJbK>0C$8bqXTlFJeA=mFNlHNeC(s4)_RQZ-B3+M<{u&HpXYWWa5n9L#Rdm zUnSK6oRZRAikCs!qx8X5@%ceJKgJU%U+JlW_gbGCDx>vBd%S=}^DpAN0iaVNV<*6C zm}xXIBq99C#_pc-je^1c_u-q-sI}@(5XA=YSFaWCO~y5*h>h^yS?r&bk+wYpck611 zx1?SM;~oGU8JOJC@5fUij)jc&_3Ug-*7!!#1K&~wFjxTH0$7Kt`-fG&Yt3?obzrmJp`mbSV zgh;A`Uy#<)lP<6V+i50>a-LQ`aJKOKVkr)e?UGBX7@>ZF?l=m;^ezJe**cI#Db8Wh zE?}SG1F)wjnVKT=I>b^cCk_GHgv>>!adc4FeWjVB-Gs#sr;9_x5fo3}?3NR_pi+iL z%TQP^HN|DBL*h`Ie}EoxS`CA=sf)vVC3+5iDrNZN+r2RJ;PAirx#xv%EV53KPsrbmKKuJ{9anOfK4{-z=bz444~1grUOlzn znu*>Y*ijE%K|l&3M8TVWsx+N{-relkNnT35nYP9?>kFg;V&Sy~_Ed%K;ms#x?AW>% z-ZWC{M8|Hb@k(yJgiBd-cgbhH7zz!MZ}F%)jmK&pm6Oa1AV0YN#9il7QXgJ-2SQJR zE#xtGbm%j!<3`GWX=Co1`TLy(K{RE8at;Sp)pK*j!=xi=FK(|U%PTeYhsPc|mR z*JJbMVyy(%W11J}L3m8AE-mkst1in-K>YC~)6Q}vE?meH;3|9lCcbVMudFfeJBg-v z+A)K6c0XAU*$chwRg5kA=`--_NnUwH33bR@N5&!36X(J$B#sq8C7?neP6VYYO$N}u zTQ~fX(4D*HmIpzmqU%Nd+m+^pN;R~oG(-h47+ysk^h@wN2)naVx6b_MW}KpA%*m^7 zLKi+H==xiURm~ZLt^}UQ=Jdfbbkv;A3)Qy7lUvzjk~V6l6o18fF%~l5mmx7u9yN>z zZpj~s`vAW7X}rG11V5GvO@x zHH1*wep%Azt6?Dxlk%H)iE83M2?c4hh~!O>?rboxJhe_bIpyJs;}eXXOafNtRbIMq z?L4A~H@WU8I9r<9Tc}b%;sUS zD^whQI_nOxS)!q8)hCE7h*lL3B)`rXi_#s_d2~S)3!i(gDWFFrlm_Qa=+FpILH!gQ z`>QIvmZriwWnW0obUJT;_4_Es{h`PI5UzFP=-Hdq@#vk%Yxnl~z4t3SH{tF$`kP~t zaT3*w>qEk!U*(4nubic-KIQb{+=9pYW zy|uu@$gr)O565CcW}QzFcpfH069B|mjfv?jw}w}qOtnDZD?A4!qj9->)amj%=3^ja zxOyl=y~7q<0uvup(~H~SZFv>+O~Ax1L=iE)TH3r6yAOL?!f@`jWgI3$K0Y&fOVNsw z)~b4?A3IRE>9iJ|sY@l-4%Ku}28XY0+}_D~Dk1E5kO&WYADEc(cZrC$1-WDwzQPbD zS}~pc>^Fxm$Wqo-z2V{_PgV`je@N3sn>Gxnvd{{xaejOXYBAk-cLU!-3N;sNG9*GM zF#Q8$y5@!AhmPcfvFCNc;&Px*v~?nZb*wS2kb5f^Ag|L}6gGOi0`xKbJ*87P=vIMaifS9X`w_` z=&c3PsMog_ECA~9eDl`nCutVY*?Un;BBL; z&7c9+`QcmNbaoSi5@Uo5fAhH~FO78Hq*T9UlMQ$;|F6RUa~KEh5n=z8!o0oO|=DwjulZ;a2#fKdCM_X~YzOb>7Vdg>Z;15s6v z&wjo7IWBJR(?QW=`TfTTgi&K3(NX6C#QLj{GQ}dJ)Z19%$%GrDMHm)XAqdCc0{L-? zUgYEwa&3sF$wqjLH^aPb=xbm3ISroIwlhBMjZs&5SmxNNuTjk&(s8P&2L`;{xN=(A z`f0uNR992gl`P|3lJ|3NKvC_%W|F5-#QlbPaw=b0pt-(U{$ihX{m;h;2UuLhA|=ob zS8s#RsN^xgqgV%PeEm#klbofGqVVI#c?5HSapF-EEYdPDnU4q>V)4|&Kwmv9c1I0- zGO(thnnfP%yZ}xcg(@Qab!^zu2%9)c{c8O6%su{ewpIwB>jlhv8Xw~1AuY1v!yCQ6 zfJ(q8^?GzKW^%}ty%v;+cb3I>n}UpsC+?~r+`=T)hTr$fv&>NXX19^3)f2kv62aKb zvaF0piPvTCzT{qk;Ha2Yr`HBFsbZ72r$)$jyl@`BP*2vgA4OA*^(igDqa-d+SsD;h z?*NaX1A4fbGFRf1atqsl1*d%}8$7EbN&Lt$UG2d}fKJt~m426G$lm+@{G=cmqM~s& zZ#}LL(;65L@ShSl3U!7&;{ao%mI{d!oURyiBqK_!;gY3hJ?)|7$S?&szciZye8;zb zUymKaY<6wT3I1flg7litykiz;o!HGY6m9G-BdSs^(j>0Wvxn}Kjk^C=tTKe0{%krZ z2X`n&cWJkz}aKGGp z^h6uMyuVEHtY%rM>F`FmAusN||N9cF$eD8i^C0Eq+D|52sS9;jMN)8g^>$&l@zBlY zro@oy%9TJUw{b$@icp2)8={vh1oZ8})z8c4|J0?H|ENo!q6!49Kt4Njd=i(j?o_|M zZi#xAw2!;@B3@5Ma~30@1y7xl^HF%m2z-= zfT5N=uFDE95z(baNi;WDMkid9;7HzMYWk^z>zAYN!G&xVBtfEz@)U0lj9qx1!7#P)MB;JM|UzcrTj66$OvNL()#-qi&A1OYb-7K-!Gxy zI^vvUJ9rP*R_M79yNNyUz7?kyYj+5bg0*$ic6{*no@yIVq*c?y|G}02qpF`I;Q3>T zy}j-WWiMJF)-k*Bf9*LYjCr{q`{`bPWv>zdRT<&cq-X%4@abE@v5k%OA(~D-OZgd7g|DIepF1lnEI_oElRWevthX*{=BhLVUP( zF7*8AeGQ-4!&YOKZ`H*!OTumkwnx^#>wh0{PgKMqyTEQg^3s#2$F3BzcP1It#5NW& z&?mh-Mr>(e z$=gk)22*_TMX-?vDhvJNkMr`WR(-4%?wYl_RTK45i5A}hg%BL!b{0r=Bu0{uf^llO+txV%o3I!jSYpwCFpR={A z69~7tZv|8)eRy($N4zy)1@jgMv_x+;eMh=?&{0-zW=aF)YAG&t{n>}Do3E93wc)US z!AMIp20ce{s&dN4=2v_WAKI;(AN2(Zi_^F6856z*;JvzqK!`B$z)CQou@z~{nsjKX z#EAc3Q#j$9#)NLJfod;uKR7ixmHNK?4B%<+Z)k!ue4l4W8>WVs-CwmmhdsQjeIO#Y zuuH1m-P+b3$u#un)3Y1wGS_CXeD;5(TPgMbnulioi6M1f|1l4Beq}nt_elurI$G* z&y{p8@`mRxTv9Z1^;|jld2>4}y%Mpu?x{Rvs;~GXCt6YEm_RN(u(6-eY%wd3Z|LK@ z{Xm$->iwN}ZDFNN>_ngwR=zFbo;*o|y&jNFQva84+wNGa^>SB7yH@+5Zzm2AhxA{I zfxq!g@7acVQ3_hpoHu|y3X&x1*@c$oyqh2#C;(bz79e!LyTzPN0x|08RGYjB(YBx!J-|8WYT19&Gy8u8+0+$Uxc>My>ng?Sc_1TRv&U((FqM6et-geJ*k5B<9Fd9Z#3 zRqG68EQXC8;Ld(h25Kmu-Q6^!g-&SF-q}~S>~|&m4UdKyT?{#gf=|Ud1T5|i_ z3mzAeRm6%!YesYbFzw+rlASs=Lp~ewZ{=U>V`Ymw$>TEU35C##eE8`+@)LlZVuCjX z1ak~kRA5q%@9ZY`2_rGmV65Az2SKP)@&g-O>UQ93UuVknt&(~jz0MY}dc^BT!hnOE zZWcwj@WVwjvk0#D7@S7>G7w6S2;kWFo(mL9ABcrhk%TGyuQ7@OJ$vu2_>psBDL_{evKM zYEi3@@Gn`Rz%IU;d*r?F@JHbXk0BA1_Chpy;OHSpRm+uD^Sf^pX5J*$t3S7WFyxdgec^KJ~Qfb>f&B zHE%Rli8Ib?CHKUTCeycR@%Zz8#D*_W!Owz|3-o%|yw%gx5Pq5Ga1c+frv9Dmj{u{Y zTdyq@oPo*fls8py0TLUtO&yKtkRnvTHokN<0`0nm#lc5SydMQgjm_TpBEY2Qx0YMwJC5 zNbnncc#_IpjK@AnGRL@tmW8hLfECT&b2#dQ{q7a_-aq68|DNjPgd6MPu(Jq=V!8leKC z@(=+S`%@m{i4?*BBX}WUB{b^A{Lbm$Frph!wegqV zMn{N`0VB%g;TjuRZLVG8!dEAM>W7|mk+E$gZgl>+4(R&p*%c9hQL?+(A?LuO$%nfeN<3Tu5|!06|TLPjr(VD_cmuU{d%?q;C`qAxW9UYZlji|_!8}h zUp}ha22I&0lv&Gg)iOR&i3q9rWAIJ!Q=DJ{`~Rr}q`J6Ci0MR%!V7hdS)U5Zs z^arP`KXV)%oz+2yKpksr=P?VABoWX_agtbdT7Bv5y$eL(T;e2hc^V$GqmyUJ6|k?W z`Fz$$i&_!4UoRwTc}y2iz?A6(Q17C@z?sLKa-rdbT$}l#&v=xqA~Za&M&bKJ+(=naMmO^*t2tmy(=rQP zf(~<)1AY8j4bzDdHG7YjpP)T-tdvHCEnvs#eCEgJLgjYTW1cOUJcNcGEG&X2mh(ysRV z1NM9WH2vC8`7|_t{Y5(pnPK&iu^3|jfgf;ds7RwrL)SyYgOoGUOwAt?!Cewb4fTiR zZZ&_VyK0P-jz4q|Wt_yhB(?4L3Jvu<2d0a|eKe(`t2hLnE7`X4yzk{$foX49-X2Vn zYUXKjYV$ze))(mL+Oq0%j-? zE~MwK_E~J|PDFTxIagJw9YgY;A^0wG&CXwn3ZI+#`xa=QuvywqRQ&&^8)W}+U)w_< zg9nw2ME+0!}>UbwIN(lxp&o`}CCZv*w#HY#SCaRFLEwy(7y&uGtBmo>mtO7OY) zXHJyhBDAVacL*j*B4w?xjM-qq*-jUGGcekz^iFw;-KwuvER z@EmpkH!ita(ml~3v&wUIwq`DE$54J;mU}{??rZIn*Skx*8>dicF$^>uIzXNGY}tX8 ziMa72%yh%ep(sVuJl!D)tJoaxycA$;svV*65W|@bBYoNPB#NL~1ue}^L>O&eMB%GU zGSwYRpr`C+S`M?f)aC+?-80@?EeE)eZ6e4KHsHjJXJ3qDWA;D9(T@ugx@;rhppVS# zx2Gg1aR>!P#;A9`5W{ejR-*4eNCs!-(wsbz>{KUy_}f{Ri@*RM9#UxbO6KKX>c88+ zehCs0Y$UA*Gi9$S#0jqc7#>!7P=u9EqLyEQ&*L% zki)}zC}2pkM6(nV>cmyO9@arz`khb#WL5IwZmSbHQi|AWU74qzC&op!g^L}lR19NS zx%daslc)`d`MEn2F*)auziMNN{LTRt+caxyw*4FQuNakt0e93RuqdUC@$|Rpoa=~z z0L@e01c?9tN8&Q1N9&h?*&lq#gjiGF-kcdNq-K-Oh*19U*pG+K1Z5sn_Zr$F-^5+V zh1a+^hgR6CMJ_ceg$ciDykP*c^f07^ZPh>QyVYWIPi?;YFLz^c$+T#v@p8-rCf8^$_j*tw|UG&EUuu z`0$huJ>s|G5GDaOI#sEer>LIWBxmf{>zj0s`UFv=GLhMOAYfTIDuA#>wRbE+3lyP}wc zH1}c2D^79bqjyfC;RDIl4);4Xe1Fd>r>#1KxK z8wxf7F~#N5Pwp~oT?-XqVs2}w6p{pJG|vafmKiWiT}-E+KV#3Ryct2#ytH}L8?pbP z;`>-&X(H9OBe4Ni*{6u3vKWqp8{g=1M|>c)f+>q`ksmx6Yq2a%tD#+++Gy-!d!Mdy z?wD|I+}57^Vg6qJ{7F z*Z&z5z%lbqD1K0+Nx(R!e+~E^!)WFkdDMelz8ak(B(l8yqya|RUZ{TcPaz{YUiP=I z?|jrXHFto2kdmaJ{p$M7hnBGj#V^UTG3mW@Z8p^o7n9CTI$z01uQs2|T!46VA)z;C>qA|_TJ@1XzX$I0S_Q=L$jQF!v-g=&>6 za31FN!Ph06-^OS`JZgK4V>X1CplzW-yjU#1C*bsO;3$%`z@c4|X;VQJ9L=~*jR)GG zV!TUe1Vq4yUsakpz{rkm#vxs7`Z^`5R&DkPJ@BZj7msTd!7U7~cN8MoW*3($nL0Pw zYw#K<4nF~eH#)Hq5Py<2)scG9Mvb!<6QnQC!FXgx5ZBM?!m~i1P=;Udb8TM_I)KB8 zppl0gg~b3=H4Im`DrSIjo%U#ivcsvb3CDwsoxBeAR`Tvj>D_%p`;1TCIJ0kGP~-t)mM@Yhd5 z#SN$Fv!v-;n@Mw#zcXN0{ihUmyXm4T1M(%)I>BDMZEIeAF<;&W~(fa|b^!w;^Sh0o65aVsqiA z<^gAGxx=LEVkVg(D_xUe=7|?Jl8@l}2c_4Ay{$nrmkiFkwSB@Z*Rw?L&W2MH_g#s( z3Y9p%_a^%pz67%5Hgz&|w4I9yt4A5GM*2j83&A2V&y0LaJ`00qYZ2J5rnvV$cO=nO zEhr$ThUu8WccscFm9yi1nz7zlQq~7&kUU5#-3cY`W*mDcq`LX@Nn1oiNf3W;_sAvE z8P}Pqdi-nr;p&5;jU83yEf}XsD;C@b2FshWX8PlgoGxPR&F;6G32Hiz2H!g=7LpFN zfcu1WK}!)z-KZC-8bte$x)kU_vOIB=eE-uG^Yoa)QK-dJcb=Z`B*DYiMF+D_7vdSd z`9{K`MjM~`3siIt;zy+-ZNtINlx21zcal@Ox+l%%dU2DX7j^Jg2(Ff= z$&X$9>+m5~nUUraH&9BK0@~Gu+&d8x|qGx{5-OMi7M0z93YStK}EpLr} z|7rO~&gk4?MaG3xk6Cgx?dV;aWleKy5ZC|{Ojs5z4+W2VwJd624V_uvnYPtz^Tdi2 z)84vBmnvH=PhYVp*{D3+*bu!7fzq3C6!!ItcF;=ilRgBZKgFLuZLxW!<+vf$!c9l_ z=?%NNh+Ew#%}JJZ_yokM`MSUIo?7r#4kz?r$AV6J15EL9=864W+{ zmiAh%yFvmq4WViM*1OekM}-6(W4{a0#6j_t1o@Xw%*sPYpiG&bvn|myyOjsQ{yR}w zK?lGz&fRy?CamqQA@it&>9hs^Sns~OnXNx=0&rJ?C)9Q1Bi5JNI-%|!Mh63~&VR0) zXv(O~x4ch2i&}T17!hjHOI+MAJ-yG%l_MB6V_s5IG8R_S;*AXr8jrx4n&ix$~M`5n+qBis$4nb(>lh{y;uT6(vGD zrk9Q(-dlm%;SYd$vDVNEa~nt|bJ+FXMRqa+P!DaSfiDDEenZdbozwFzz9sxBhq0z0 zk{*HV*4p6@5T|B;@-U3YaZ|7k9D+t`EDavfG$~S$)=gSm?GYDf;Z!H9s5mJ4C@Rp&WeS0u$snHsbS@*!g^KpAnbx z6F}&+;IV}Ulw649%CCHxcY2u(^%l3h_xGk|k9 zB7D)0-kp7tEj(vO2vcl!GrhW+mIHCob)VKeM=;LfTz20g(J`~x456!HWd|7T*pgGh zOH~{sz(!;xLEel8^zp)iT%i*On%?`pJ=2>Vx0%OTD8uHF&0m{ux_+i{@w6mD6qn8H zz2I-KF@?XOcxh?T^d+z?C5*jpH9-W3xPb6V!pFS*b|n;>LAYD-}K6 zox2(j-IF~6F~SW@P)3kZn1?nPR@imfB~Q++5U6q>g7Y-Gmg`U>5#6~OhUa_qGg(j4 z9jc<3s& zVc;St{urth1(UyB(rj`-1fQ@Eb@`C65=OKXlL%IsG3F89p5{4~Td2QRGV?MBGrUrr zl^8-DeB4WdXcLbZCBGIe?)Oa<(YbBkCOSqtN>A`CK!69mc-`xAj3pvJELa($Q(9;m zh=4e#C&q;f*3o%kVP5=q!c!fLzmAYOz7Y)S)s$Q;@;61;kdp~^t9-9v6ZN6u}()Z88AVgYkZhZ5oPuWhWH2h05%8K`M63-(~ zxNM4VXxozRmxZ*7SIn%#y($UHP|q8d{jvN7

0ybUymw)5FscP zZr3HVbTS=WK?jI;jf{|`($9H*@>UG_nBWv7eA<4vD)jBd*gK2K2}l}C@K_NPNrXEX z0$sBgE>VjTsK&T;9aBr99k^APPvWinZsUXNdH06S{WkEcDq zgMiPt`H^}tZlFcO1k*AS zv8(yiL*oNYa*dl!W=rBjc0;N)S$Hzrx2?lmxjWZbC2fn=#*-?UcGY*BMkF=r)*VhxKT%)OlWsW zUTB~Nmg^x%(o<0|>CRylS{%inD>P>(wDAjE-a_IL)(w42UDLNo6ZdfZS^3q!g&oR0L%VEJwAF4TwA|>( z1w1~b5KtF=UsmC%9kI<}Gxo49PFNKN$vIGi_0S2%*@T3X%UqV>NB~PP#OF&l3D>0f zi-);&Ye<0GxA~Y1!LaANb@R<4vB@Fsm$RvsL=Wv(WnlV#mcIkTLLeirLPxz(&W!~V zik)y3^r3|+d?vaurYI-y4JQJaec9lP+=Rp6J~!>oMH{8V3y_^;_NzOzd^nSZ*oLQQ zcIJHreNbkI$JWN?r^ajjNuvwD<$a;HK^HoJ1Gz=EXY_=g3aT>(^;6Vx-s!3Hua|=MLV02d5&Urv1f+DAR$fjAK zd4z3)1(~un8}Fv1W9a8+p2`qVkt`Hr4ZX%+J|>dqu}0e9&0cDjEgVl@V^AA9aSG}o z&h>ng=;(8^x1u%o7mQ;L4Fq;GKN#JcSTw`-SmU;}+YC);<^9v050w&tQQtTFmt3w=hcU=5Gz9!Dl=qX6K_>6Ibm zxw1NWQC~A{3L#K<6GMN58JalvZ2ltzaYFQ~eo)P`-GxrUG>E%Vtkr&6ythlDgY9WJ zD&qD`EkDy_6A?*}ja=3rrwH(89s(_@E`~4?bH6FRcck)N15;B0YbA=O`-4 zrzNBhfwJClHMhNov_m&99WDJ(FAMaM>>aRoKNF|%e8)tYwJ3c7THM@7Z#ce@rV+g!miDacZZ4-ui@ayr9&AgOfKh49ynZ2W61&0MURQ(}?uTcxm7d#guEiN#r| z8WV->kD7CFgS(-WiEC1NFR}OoHL7>!!=1Lx&{xxX7azUz2 zEv6WJ)-#}y+U(E!xonPReweV9Rqp9UG8wkj9?Y(vQ2)PTs-vR?D*x!aa1>$`@LulX(WvFTx7$Q}}_U>TWne|87)nnUtXQt~(#|hF`@@E~7 zOY2}nVOgU0GPGq%IW6x`vsy$%9rqA0h<2j}(!5Le zzsJ@O^D>{e7+u<4X^WOKITYhumrv&t#d5vGWT{ON(_7%P0ZinSM^CIBw#4*Su}}Pa z1e0T#yAIg{6bN=2VXIch2r6&C8`{~gjTc1?k_b1M0}kC>)eKPdVN`~A=Q_SEgxoQj zuX%ADjD<;(#lNBPoD0V~gyLT##x5&0J9RZEnM4Gsye7G!_fjJ98~L`3v;Bj&pJ|eV zs^~!Zv`U&QY)2LZc2aI3G<%xoQp&O^lI(ppIgsdxk8Hmu3jngEDA`0Lh|U+)6fiYb zw^N<XP`prNsRWmDTE{#KcWP(t>i`6o~SBCVg<{;LYNmLSzuH z+imKkoeCRZLK!6}^RdTI)?muX=`U?Hcyjsw?p=Zlh>QOY$8K|BY`;=pY)uT0-oTeQ zZH#ILDRhJkpp-@s7?BE(7r*{SIMiOnTR`q zQEyln%dxTRL$(7t`A>5k4dT?jgr@{{6q4_7LI&qRlk!h9mwDOKp^yxv?-|lfkz6J+ z859x^Gt%%GEo4&^fIiYaLZC1G)efFM&{uciah@w@9}?E#(hfIU&?6@q8)BtON`b>5 z#AN}aTLz@70Nx$Jqb%oC2f6MLkObH|EIK{Zu*(_(00Aq}b-%{7jkT^rpk1 zL6YF9k;YU0rpGh~?z95qGJlEa-%Af^+?A?11M#edrIz5tr?^uP zLTLH%PYGYj&S%0kt)s^VkR^1>*0167H2L)WSMONcD^9QL`Lp*{b4>xPV#yU(#Fi)25HBn1BcIkwE(jlri4MHAjIwBQS>(M z7OM!dLPyfpJs?Gc&u0&zbExII+zw-()}9{_Ab8+hCM8f|qiEsAm`ay|Pm1Qs^KcH}lms7b=U2}w{J=TS!7^7qH^JQtBv}aPp`q;07i{xB(?Qp}_WTSyZt^$l z-eDuGQ+w*xxQvJ|poXsw_sL*@tM)#0G2cR%nM@^au;ZTer&0wMlwh& zlOvYbiaDJu3=e$Sio2vclpJE3p8f8{nOF~#Mk}r&8=^GURCx#2`PdclbmIVsk9#Hx zRU{J~IxEH;(^#a|-mQ>;DoJ)AX@(HOZu@)yHdktPF76VCE{YXe}KC#hp` z|Dd7^Fbt%Wn8F&v_|9KQcC}Z*%Tyw?oySl*w2#_Ah?ci^gsPs$)b#k)2GEKks%JdZ zTEI*3>Hdd=pefv^am|pprl>>Pv=F|&h$7kEswd6Cbb6UJfmZ`FJ-E7PO=P7I!a}q^w;+=nUgAwc;h(UKHFiLwD5y4UIS@0D43%hCOu4T#TT{G zA7qi9m&o`AT$7DNyK!=~eDqM6jgroTzPZJbzifm&rtTe{*MZlh_9BH)^ix8*LqlKL zq#xkBz?~8NgTdkG_;d%LI(-iJE_r{07y_L#1W)G z@#F-3w`w0M=BqEZm9P9Ad{m$cuv`h_v$+%q?&|lh9nqkz5q^K9xo35CwZ-EV-KJCZ zdfBTJw6|yZn+~?~71K5y?gu0Yakl?lW&fZ0a#2$q`Ig$8yDe0Ivc#`r&(ye4-K1qD zmPq^%+^bS{Y^tfz-GrY6h(5URP;{H7+YC)hW_hJ{5we_Ql5Zmh>f<{Uv#jbXrWc&Umo6Md%81T-fW_X zr9P6Dj0Xs>4{?osjN9T*@K&Dr=GZTXqVmG%t*k}l`gW#o;xj5?DqKNaIWM@FUl&-s z%A!$|-Ye(N&QU_D4SY~i#qyNum;RcH4NcqiZxLVLp(SoVQK*tI?x+f$6ZGjfSE`a# z#OmkH7_&XUM)qQ>L738yd$Z#vSRdElA`x3XkT?;M3GO1sDZuH znDh2{ebdR|bZAJO_bd>>i9`S=8^YkhU8dVIJO?O#AJtuFQ0>aBue0IV6SS;0#)QaF zKAb>sThg6Dq)(rHTBu{m*zF@eyZ)qxodt5`R{R}0)TYX&s_g5553mk>=Y*<8WymfO zR~)1!|8S}nZcV+}fCs_FIh@UC<=Q}&3&OvX_une8o~ci@k;+zqlVeU_m54VzJ5?JR zd?kS3H}oz*YtRG{&T~R?Pj?Je-#$(`%zPAKrv1Gs;LB@tOauyO;jZORDV=4@KhJjy zaQ(La`2rXK3m(0|_T3>~|KtuXJmNnn;9{~yvyx$-2-HAa%MH5~>CmR zQ^2Y(Bqm2y&XyF*3!WrStpMth5|OsI$zD9NHd7qI?|nX{l1;e|OCn8v#I?uS*HYop zGF(Jp=j75=@bg@{?z*dc!K1%VAzq3A%h~xWoq53#%e(CSA-trn22wp7O*Yk(9GQ@P z{tw68QojkqfMtY3XhhVWX$!dO9x?Cf9j2@zH-I(c6CHsku2U|RtbDJJvC z_EvyuHG_;g&#za?{So>VuIAlPtr6d^b}T>fS4aY<5`wq08jt!z&Y}@C#}@YXNt{7@ zoos>BK6AM6cOA$=u6t&7~s&oy!H>Fy?6q_ zR$4!HfuogHIwicANx9HDu}UVY*w9xoxdLg*O`KdSyKgdno=StGVDH`7x@fsfcI1wIJIV`76`GBJ{s~Q zpMVNg;(M=XoiOZPLptZL8#_DZnTJX(^r-Ql%psbY`LFw2#dcKo-7j;PVrgWEr?uEa z=HoqN66L{GN38+W`m@RNFE1Y?+182XdSx5n4|BzjhNc&0eyY`(4NxNQ<}GU9Zy?qt zE&X4{<3Z8oJ zQg|K{4pzx#`kRVuE!nLa%lwBOx>eHF><-Nan%7+r4eIv)m7u2n)v&#Il&$k%>94>2 z*VbH(33VG^?>V2j?r;wre@}#JScxHBh@*5A6W%3Y&|YWk2NS~ySWouSc- z$aDIM=l6@R3nylOKku?N2`$=%77sgk-r_&tfR<+^Z!{XwoWDB6eE#A?XgzU$Lbz4B z<-CDlP8={00_oM2nKD&w6IhJB2`jU)Br+HS3V+uPxv(XlqZouU>QazVkJz)r%Hj)p zD3It2mYeAAoH+DKeaZbyH+y4Ib(ibudm_avB=#&(oG;!U<#LKGFvr86 zCx^rhX`UoTpawCP?c1XRVX%D^FXQ~^H+SAeo&K`cfSW%98{Lp=r_wpd)lUQvnGX7~ z6g3zyiMkURN*@0B?oX1ITWZF^0g5bybSO8hc1=o7<}eR`LAaqq)mm^y>W>85Pc{&` z$;a!kh=J3sLpHx#H?;IVSZ3CcXD&a7W|o;XxXdH%+?+%HCQezh(7Hn|9T)B@?59d- zFWjpf069CNEiJ#eG#@`+`A~hfJ09YdxU^8)DsZxJU~#sqPU3Bm;KVs3xF-L<;cgfg zmznRv@J-6qB{yu*9-%?N9&75{8My`P>p6u_7ODvL4U&?Y5VA7e+q4#NSqpt*qWYqk zcw>Ra6%?|}>r*g!xer%kntBlbmo^y}Hh)YX-^q6_jY)%GV=-0JE|xai>AUp_=j?A_ZnfIqH% zHX@9#L&5??Y?5S#sNacvGO)p0i~|l^@+mk=yY{(#yI~YX>fNV zd*@J4>9Wj14bO6(C;|Iyb>dG9oun3$||UD*x%n1<|75Yw%X%4*y{>m6J$t)Z*4iE_Bq z(}vfeQ~rh4OJAcY8b6r?A}0KVqBA)_0$<9V;golLmpQ}q;Jdnw>~*#?dfe{oR(JOx zWgnR*-eo5}i_3gjygt(0tfxM=-VK&fsZ{lR?`ex11h!hwi(IsDSC^2z)?W*$m^NYzxWPP`te_%{1-g+xI6N z0~BRb(1{vjEp(sOa~o$Y2BER3j)aA0ZnTD{TcgmN$3MkX4~naGzH~{{v!nL~3g*n5 z0#!^tc5r@g!Q#T$WqmfazB!SybAq(2yaN@4>J5Ge8gwB4&tX&9f}MZ{T(*wG|Hn{L z6XDM`J~nw|QU8tVb@5A4lL!N6qZFZUl1kWPa*N%uB~O01G?)ojUS6^B#p)C1WK~$r49>|VbW-A zzIf1TP|TIExU5i{R!Q}DG#1c06EV{UDCd?hFQqb$`O9|#Zlj0ln6;uWz+QCy#^IB> zFWr-%B_WcA6Dx#y!u(9mCUZ$j^bx+)^v=FCr);1I{4ib+ne*DIDcBCukg zpa*$Uv69YT9^5%<5vNMT=&~1#hUP#eyulBZ^oHfWnLA0LI3rZ#?#wQMV8L$1Sus7n ztjl-DTXzsORG=uK;XBX@BrK$%jLpxXuE|&LJdjS{ao|zFdeL5K164F+*NJ8L4RZc~ zR$%3*VbAPOt@s6=fzU4Vk%0-jTyb0X7At$~5k}2+6rDy{ZH|zZ1F09Abg$oPxaHE2 z7sv@2+ThqFGksqpbBeUx2Criy@F8^ru+h}Z^5Dk1nQTRUD&5efO)k76`y*~Vk2@Z{ z=7EFrX+77YY+oMcdVHcTyCSuFc>k3qWNAm{KG=7)lf}O*g1pwec6Ba*IeQ|}_MnHx zUkf2H3?6)+agvxLFdM*d=gsc60)LtfPBea5Cb`0r=w1&U30Xlh@05xi^+v$edjboYXbMB1md~~pv&D2_@m=$l3v3!Pi=5-%4xdzzWOmJVfB}+7 zAKmP!D{9`B5OSnD_Oz|VW%Li`pLxjpn+quX-?#w2iV__y=5v?O0JAjm6PSMl54^(q z&L$|=Z_o!cN1y&Gi!(1tGxpDp`#19XexK_z~-0vl_IuiOakL`Xs=je7u z8Dj+2>iwFk(F-G5sHkP=B1wJVEXs1EnW?W07CGUFr}gVrys`7oIvaQ02pc;nBaU`s z&MgM*L-8IdN?I(xmBb`%s~C%{pU8Y{FaAvOmE{!DM|WuWr>}X2udj0ydC~P8)2+Pq zn=c(H^{60Tu>)t@#$E&gH@Ia2YXeWLZ%iR?UZu# zPV8*tsK##VC=2<)tnai5*O2&L@O^#A>WKkDc$quQY*51{CF;+-@pWe4f)#2@DK~51H}lAzk`&4y!+*_ z>z0SE5#JYgk`(oSQpQHRP06`_JZ{B>9E5x{s!M*l(R*Xe^@3g8$hnm`aOB~~W&b|u zZ>a9yi#ikjY)1xso31QL6c2ERcDfNT_P1e>Z<5B?V~wpJv8AI6B&9Xe?)7ujkqF5e z;_{vfO~0s{+TUA)z3Hq@Z-(F7{y^;tpA(7~thYQO5cX!7k){f`%OPs~)sE<3Inf(J z8n^|dHRs4DC@6T~ei>O4tmI9bpOhb{lHNI^ic*lnt zA4B6W-t(2qgi~TzbR-ODI>>yRm5mGnybOz)9@c$|#)af@mFd4Ax+605q>$xMoj3zd zLnn{`ryg&!MAWStEk+X&%b0CtCclL!!d+zVcwy&(`636WHzF3YC}y(8iTGhtg|7{d z@;tO%y1DD+-`_jz+u?GxUh4ZGR&Jnf6t5p|{Ns6}UhF(E<8}wP*;mD2TCoLSUT2rj zVs!D!PuLY~)irfkir~IXzMvGkjRvEMFsNFO<|R%SMP+%Y_9mes_nnL8Vw^YJMMT_H z_fw)^&W0_wR^T>h->;?Ad3F-u1ePB*JrZ_6onajRC8Gb)MgXWo^Ojon&f@0bRJeKV zKP2P#_5#?I1n3FLyPo`g0@&(3kpo-H2>9wiD0n6KD-tJ!9T%S+ew_Ygb42=OZtE8E zOPag6Sy_2mn8-e+;4R(?>jF6hWV|}E8{yU5dY-`$xjNX)w9-f$i=zy8|CA#NDn6jU zk0QD^P8#{;I=K-8Rq{`T>Py0HFAK>{ZutG!KjBKNs&QlPbaGT#`~Cr&U6JTCl8R5o z5VW_FDVTe@A{D=8$DOQL#kM=}oI$xZe?>5IV2-3^G|xux+pq)D8=b$cdgByDZuH&)}edXNpl?B;&5;tNR?A)+#XaJ!m>hE*aNMwwt-wOD>Q>? zVQ*3-1DT$`IJT2H?0HRzf|A}3Opo;i*dNyAzY};uLbS^rb|6-p9KJddO^B{3+#aZ4 zE86f`l*1gKTu<{1)b!8y=!G;E@(5e#n#5plW+B#_pQ0WKo9iWvR5baGIsO@p+bWtbwHuAr=!6}!Fyx%dphh$2apZ;anWGQ`sOO0Vkj1$b!(ZyIq_5>ycIS8T`9A@ti^ziX6scDu z^W_a-34IX>1uWTA(Hg zrL_p}6{lbk&NgnbgM7LFX>}~lJQJT?qKInc10#OmZX7L7WCSF*%p&HiRm5uk zW={xL^HL6^VNQB6+inYWPOFrF7fX#GdbO)&{904y$48S+yKa#ni=Pw~(v8Ar_H3HN zDtM)xA8F>}&WxKB$i)^8zw?N^4=MhJ+iHs!&>*o{sdj+a`%v%;w;N=fwz~HKvmF!c znAGpva%b9n$m+@ra^TLk38GcM-;_0$fRow`cqpa}sYxMTIb4sLuPm6!#C6bvfe|%- znCrg=pPvcewd7zS+{*W&1M0uo*#rpzht91VN^pgzOyxEsjM z(-yX~e(5wR@^!$6{DbjGwn_d+TIZj^wU)1`E9nL8d(?=`7k$=oEj8#f>qn4)?arro zJ+7h5NV_R|8F$Ml{o*2Nr1#6q&y$^AbW|)O&4b0`a6D-?xeTd_?SgCV-{OLvHB%}`FY2aNymkNzXtR8R@Y|7-5 zmrLhm9ZoaZxW6Vv%ZHEZp7dd+#1HWB*lwK}W7*VaW8tq84G-p{R)suOs;K6o$C|Qv z3Ul5F=h^QR9^Vd5LFF8tBx-x{ExybVWX$3X4iD1a`K*w|>(P`IiVT7)Kdzc?{`kg*j0Io2&0~`| zY?Y2)oLmsdWYtEH;=P#@-HmzbVUtO1U5LuaV9*Bt*su6e^A%~Awe?@mD&TuGPCIX{ zu%i$<*zk?T=9GAAyg2XWw`PJ6R#IDAxOm)JMkuo{B+}5i^pY;-^an{Gr)c&Zv!c?s zmD|{<6{ySyPSY|NsC%@LNdnw?u_wjWVeA`0g;OJC@=O1)VW0$G0(AS5=~f!t4URQI zl^x3uFBJ$$H=bB2S@wf?yR8=iBO)4Xrc2gfXR5efg!6%+wdXXGwflw0EZdEX zLlH0TxiILII1gMSPbgk^)(Fw$FB*I4XN}q{{3ZU#7WrU;<*Z2b7CT0T!8bP7&nwpo z$w8!e|AX*rRuu2{sD9)=JAzRkQzBet&*ndOEbBBmkihHFt|u{k*-T3&YfJTi9Fp=| z(NZsiGfr__$XZnPYJb_c0HxKCjgtcqD$_u>DAXV=!L%U2Z*P z4klQK4N?V}fcQG}w0S%p4(4Y@49h5|iv^%rOO*wh`e^))))>?=F}9&a`=AzP8q@uO zvgJe%ajo&75y3}n3kp~Xsm*PmA;I=1RXis5o+>sIjmI*6zWioMGH%pL9Csed>puG6(+w542iF#lw=Xd!x)nJd{wDt6<>;=R{Gf-8{X zk^O7x_aEnK3d&6_rVBj=TIPp>UoQqr^CU=R-eLI9UHog<5<;tQu1c9s{$XDK6>+QT(KK~_+Wq$i7k_Er*U(irtxskvdi76lAFPbzAU9@JP3#~cK*)SVZ%#c zBg}B$9Y;E?p+a7U7v5pdCrE%FMN6e%c94grqBIZpzFFh68wzjybV^*ZG zd&8LrnIrw94WCdjabVqpHf{~r66WS0==thm=BfFl>$u9Cy+5EYrUO0C)@##Vmdm(F`O!5DVU2>u#-%p163cq7zkeV@>AQDHqL z!WnK$T~xxpI{FYKgN!M3Kdg~ef3G6Zx@Z;kj(buIYW}da-pVk!i-Th=%`B?=Kb8V_; zPUz-=N3TVrN%wUMDdEc(;?G6u8o773%_N-qy_r%5E8e$MjQ3NaY4@R3pAbnMAdL1@ zjfZ8{lt%lHTeCk`!96V)UD|0sl{Rpv1`LI-nBp&XAb0<7Q2wtg$|DUMIEWCeyI%jN zVCEkW1@xF51C`6Iw-cDJd6W#ic^*NNJb>T74Is~X+myFWxJBGp>pzhGxF} zqfI-rlfxzW(2~1NKC{G=$zx`9P6~Vi5b5u4#wQRHPCPX~iYgn(rMvj4->>*R3X>?^ zcu6D;^s61d0RQdM8rcktmrjT}}?3q!8uK(n7mt%kW2>@MqRWQM^^eT1yF~y8REHYj#P2N*o*vd#->N#_Yf|ACNhp$;GOUEH%k>= z8#@#~E_)@6il<+>7srhIM7ED{C6Tas-2qQ{=vDc5aTlxHR3P|1oi@FcV=nmbvcse< zG25Q4n?&&ZM5T-JHE;VAnrR|)>ncn6x(&?lRlj&p80-)vAY?E1jenx&O@K>6QuaW0 zR-t%t>=y?++aeT5^NUk=Ijs7sAmbxNgJ|bTTLL(xl)^&W=p^UfYBHGYkKD$IbSx*U z1Fe^u{J{C|ukHgYDPi{1H1qG$u6Xt#>BIKAY0vu_aoLbw^@7zjDaiiBCC(1L$1!yC z^6gLGSaJU&5H6+cXr-;-e%m{?|GhF*f_b?Xk-Md_BREOFH3TMSBkkSjfW$qr_qhrs z_s0yfB)v^vxR-2+@>SkD?K#z?f7?^#lw@Vsl;F|nUO&=a7`VIzbN`Z1r1eMm zf$7aZSI&Nj0d@W#Qav>rYOhu`NWL^_B!AoHVj2?q#= zTuCR1^=SW^`<;2y|N3F2nHNf179NGQ0sm2S5UCdE>IOr7k5^!Z)*2J#RYZAt|Ml)F zx72jBC|`qP#Ps7F*jg&ab-SGMna^HVW$$?KpHEx)64)BMA6U)#g^FxlD4^3^tCu|| zm@{uaLCR~?{a`#T-bwrK-ub;>KM-Iae;RWc`_Go*QnGrlw0EkqfB*gbx-86fHGva= ze(TBfJile^$}3K$3*t+t-AGDzQy{9IcYdS#pD%2tC%ZA_p1Dl^8uG6S;}evLehqYe zp7AqeC4Ka&)?VjLocJ`6Ki=52gE{eoqnmaO!B)TRIWkeF!srpxXeVtxA`0{}c| zKdCV74aL4>kuf*xJ};Av{lGelze_(Sh_xQMOi+TQcgC&H4vpLAKlx)L{v+l8DCvLh h`2Sz;*_system", + "user": "<|im_start|>user", + "assistant": "<|im_start|>assistant", + "sep_style": "CHATML", + "sep": "<|im_end|>", + "stop_str": ["<|im_end|>", "<|im_start|>"] +} +``` + +```bash +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --chat-template ./my_model_template.json +``` + +## Jinja Format + +You can also use the [Jinja template format](https://huggingface.co/docs/transformers/main/en/chat_templating) as defined by Hugging Face Transformers. + +```bash +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --chat-template ./my_model_template.jinja +``` diff --git a/docs/backend/function_calling.ipynb b/docs/backend/function_calling.ipynb new file mode 100644 index 0000000..99d11f5 --- /dev/null +++ b/docs/backend/function_calling.ipynb @@ -0,0 +1,544 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tool and Function Calling\n", + "\n", + "This guide demonstrates how to use SGLang’s [Funcion calling](https://platform.openai.com/docs/guides/function-calling) functionality." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OpenAI Compatible API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Launching the Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "import json\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0\" # qwen25\n", + ")\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that `--tool-call-parser` defines the parser used to interpret responses. Currently supported parsers include:\n", + "\n", + "- llama3: Llama 3.1 / 3.2 (e.g. meta-llama/Llama-3.1-8B-Instruct, meta-llama/Llama-3.2-1B-Instruct).\n", + "- mistral: Mistral (e.g. mistralai/Mistral-7B-Instruct-v0.3, mistralai/Mistral-Nemo-Instruct-2407, mistralai/\n", + "Mistral-Nemo-Instruct-2407, mistralai/Mistral-7B-v0.3).\n", + "- qwen25: Qwen 2.5 (e.g. Qwen/Qwen2.5-1.5B-Instruct, Qwen/Qwen2.5-7B-Instruct) and QwQ (i.e. Qwen/QwQ-32B). Especially, for QwQ, we can enable the reasoning parser together with tool call parser, details about reasoning parser can be found in [reasoning parser](https://docs.sglang.ai/backend/separate_reasoning.html)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Tools for Function Call\n", + "Below is a Python snippet that shows how to define a tool as a dictionary. The dictionary includes a tool name, a description, and property defined Parameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define tools\n", + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"state\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", + " \" in, e.g. 'CA' which would mean 'California'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"state\", \"unit\"],\n", + " },\n", + " },\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What's the weather like in Boston today? Output a reasoning before act, then use the tools to help you.\",\n", + " }\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize the Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize OpenAI-like client\n", + "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", + "model_name = client.models.list().data[0].id" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Non-streaming mode test\n", + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.1,\n", + " top_p=0.95,\n", + " max_tokens=1024,\n", + " stream=False, # Non-streaming\n", + " tools=tools,\n", + ")\n", + "print_highlight(\"Non-stream response:\")\n", + "print(response_non_stream)\n", + "print_highlight(\"==== content ====\")\n", + "print(response_non_stream.choices[0].message.content)\n", + "print_highlight(\"==== tool_calls ====\")\n", + "print(response_non_stream.choices[0].message.tool_calls)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Handle Tools\n", + "When the engine determines it should call a particular tool, it will return arguments or partial arguments through the response. You can parse these arguments and later invoke the tool accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "name_non_stream = response_non_stream.choices[0].message.tool_calls[0].function.name\n", + "arguments_non_stream = (\n", + " response_non_stream.choices[0].message.tool_calls[0].function.arguments\n", + ")\n", + "\n", + "print_highlight(f\"Final streamed function call name: {name_non_stream}\")\n", + "print_highlight(f\"Final streamed function call arguments: {arguments_non_stream}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Streaming mode test\n", + "print_highlight(\"Streaming response:\")\n", + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.1,\n", + " top_p=0.95,\n", + " max_tokens=1024,\n", + " stream=True, # Enable streaming\n", + " tools=tools,\n", + ")\n", + "\n", + "texts = \"\"\n", + "tool_calls = []\n", + "name = \"\"\n", + "arguments = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " texts += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.tool_calls:\n", + " tool_calls.append(chunk.choices[0].delta.tool_calls[0])\n", + "print_highlight(\"==== Text ====\")\n", + "print(texts)\n", + "\n", + "print_highlight(\"==== Tool Call ====\")\n", + "for tool_call in tool_calls:\n", + " print(tool_call)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Handle Tools\n", + "When the engine determines it should call a particular tool, it will return arguments or partial arguments through the response. You can parse these arguments and later invoke the tool accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Parse and combine function call arguments\n", + "arguments = []\n", + "for tool_call in tool_calls:\n", + " if tool_call.function.name:\n", + " print_highlight(f\"Streamed function call name: {tool_call.function.name}\")\n", + "\n", + " if tool_call.function.arguments:\n", + " arguments.append(tool_call.function.arguments)\n", + "\n", + "# Combine all fragments into a single JSON string\n", + "full_arguments = \"\".join(arguments)\n", + "print_highlight(f\"streamed function call arguments: {full_arguments}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define a Tool Function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# This is a demonstration, define real function according to your usage.\n", + "def get_current_weather(city: str, state: str, unit: \"str\"):\n", + " return (\n", + " f\"The weather in {city}, {state} is 85 degrees {unit}. It is \"\n", + " \"partly cloudly, with highs in the 90's.\"\n", + " )\n", + "\n", + "\n", + "available_tools = {\"get_current_weather\": get_current_weather}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Execute the Tool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "call_data = json.loads(full_arguments)\n", + "\n", + "messages.append(\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"\",\n", + " \"tool_calls\": {\"name\": \"get_current_weather\", \"arguments\": full_arguments},\n", + " }\n", + ")\n", + "\n", + "# Call the corresponding tool function\n", + "tool_name = messages[-1][\"tool_calls\"][\"name\"]\n", + "tool_to_call = available_tools[tool_name]\n", + "result = tool_to_call(**call_data)\n", + "print_highlight(f\"Function call result: {result}\")\n", + "messages.append({\"role\": \"tool\", \"content\": result, \"name\": tool_name})\n", + "\n", + "print_highlight(f\"Updated message history: {messages}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Send Results Back to Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "final_response = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.1,\n", + " top_p=0.95,\n", + " stream=False,\n", + " tools=tools,\n", + ")\n", + "print_highlight(\"Non-stream response:\")\n", + "print(final_response)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print(final_response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Native API and SGLang Runtime (SRT)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "import requests\n", + "\n", + "# generate an answer\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Qwen/Qwen2.5-7B-Instruct\")\n", + "\n", + "messages = get_messages()\n", + "\n", + "input = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + " tools=tools,\n", + ")\n", + "\n", + "gen_url = f\"http://localhost:{port}/generate\"\n", + "gen_data = {\n", + " \"text\": input,\n", + " \"sampling_params\": {\n", + " \"skip_special_tokens\": False,\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0.1,\n", + " \"top_p\": 0.95,\n", + " },\n", + "}\n", + "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", + "print_highlight(\"==== Reponse ====\")\n", + "print(gen_response)\n", + "\n", + "# parse the response\n", + "parse_url = f\"http://localhost:{port}/parse_function_call\"\n", + "\n", + "function_call_input = {\n", + " \"text\": gen_response,\n", + " \"tool_call_parser\": \"qwen25\",\n", + " \"tools\": tools,\n", + "}\n", + "\n", + "function_call_response = requests.post(parse_url, json=function_call_input)\n", + "function_call_response_json = function_call_response.json()\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print(function_call_response_json[\"normal_text\"])\n", + "print_highlight(\"==== Calls ====\")\n", + "print(\"function name: \", function_call_response_json[\"calls\"][0][\"name\"])\n", + "print(\"function arguments: \", function_call_response_json[\"calls\"][0][\"parameters\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sglang as sgl\n", + "from sglang.srt.function_call_parser import FunctionCallParser\n", + "from sglang.srt.managers.io_struct import Tool, Function\n", + "\n", + "llm = sgl.Engine(model_path=\"Qwen/Qwen2.5-7B-Instruct\")\n", + "tokenizer = llm.tokenizer_manager.tokenizer\n", + "input_ids = tokenizer.apply_chat_template(\n", + " messages, tokenize=True, add_generation_prompt=True, tools=tools\n", + ")\n", + "\n", + "sampling_params = {\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0.1,\n", + " \"top_p\": 0.95,\n", + " \"skip_special_tokens\": False,\n", + "}\n", + "\n", + "# 1) Offline generation\n", + "result = llm.generate(input_ids=input_ids, sampling_params=sampling_params)\n", + "generated_text = result[\"text\"] # Assume there is only one prompt\n", + "\n", + "print(\"=== Offline Engine Output Text ===\")\n", + "print(generated_text)\n", + "\n", + "\n", + "# 2) Parse using FunctionCallParser\n", + "def convert_dict_to_tool(tool_dict: dict) -> Tool:\n", + " function_dict = tool_dict.get(\"function\", {})\n", + " return Tool(\n", + " type=tool_dict.get(\"type\", \"function\"),\n", + " function=Function(\n", + " name=function_dict.get(\"name\"),\n", + " description=function_dict.get(\"description\"),\n", + " parameters=function_dict.get(\"parameters\"),\n", + " ),\n", + " )\n", + "\n", + "\n", + "tools = [convert_dict_to_tool(raw_tool) for raw_tool in tools]\n", + "\n", + "parser = FunctionCallParser(tools=tools, tool_call_parser=\"qwen25\")\n", + "normal_text, calls = parser.parse_non_stream(generated_text)\n", + "\n", + "print(\"=== Parsing Result ===\")\n", + "print(\"Normal text portion:\", normal_text)\n", + "print(\"Function call portion:\")\n", + "for call in calls:\n", + " # call: ToolCallItem\n", + " print(f\" - tool name: {call.name}\")\n", + " print(f\" parameters: {call.parameters}\")\n", + "\n", + "# 3) If needed, perform additional logic on the parsed functions, such as automatically calling the corresponding function to obtain a return value, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How to support a new model?\n", + "1. Update the TOOLS_TAG_LIST in sglang/srt/function_call_parser.py with the model’s tool tags. Currently supported tags include:\n", + "```\n", + "\tTOOLS_TAG_LIST = [\n", + "\t “<|plugin|>“,\n", + "\t ““,\n", + "\t “<|python_tag|>“,\n", + "\t “[TOOL_CALLS]”\n", + "\t]\n", + "```\n", + "2. Create a new detector class in sglang/srt/function_call_parser.py that inherits from BaseFormatDetector. The detector should handle the model’s specific function call format. For example:\n", + "```\n", + " class NewModelDetector(BaseFormatDetector):\n", + "```\n", + "3. Add the new detector to the MultiFormatParser class that manages all the format detectors." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/backend/hyperparameter_tuning.md b/docs/backend/hyperparameter_tuning.md new file mode 100644 index 0000000..ca2f52b --- /dev/null +++ b/docs/backend/hyperparameter_tuning.md @@ -0,0 +1,63 @@ +# Hyperparameter Tuning + +## Achieving Peak Throughput + +Achieving a large batch size is the most important thing for attaining high throughput. + +When the server is running at full load, look for the following in the log: + +```Decode batch. #running-req: 233, #token: 370959, token usage: 0.82, gen throughput (token/s): 4594.01, #queue-req: 317``` + +### Tune Your Request Submission Speed + +`#queue-req` indicates the number of requests in the queue. If you frequently see `#queue-req == 0`, it suggests you are bottlenecked by the request submission speed. + +A healthy range for `#queue-req` is `50 - 500`. + +On the other hand, do not make `#queue-req` too large because it will also increase the scheduling overhead on the server, especially when using the default longest-prefix-match schedule policy (`--schedule-policy lpm`). + +### Tune `--schedule-conservativeness` + +`token usage` indicates the KV cache memory utilization of the server. `token usage > 0.9` means good utilization. +If you frequently see `token usage < 0.9` and `#queue-req > 0`, it means the server is too conservative about taking in new requests. You can decrease `--schedule-conservativeness` to a value like 0.3. +The case of server being too conservative can happen when users send many requests with a large `max_new_tokens` but the requests stop very early due to EOS or stop strings. + +On the other hand, if you see `token usage` very high and you frequently see warnings like +`decode out of memory happened, #retracted_reqs: 1, #new_token_ratio: 0.9998 -> 1.0000`, you can increase `--schedule-conservativeness` to a value like 1.3. +If you see `decode out of memory happened` occasionally but not frequently, it is okay. + +### Tune `--dp-size` and `--tp-size` + +Data parallelism is better for throughput. When there is enough GPU memory, always favor data parallelism for throughput. Refer to [sglang router](../router/router.md) for a better data parallelism rather than using `dp_size` parameter. + +## Avoid out-of-memory by Tuning `--chunked-prefill-size`, `--mem-fraction-static`, `--max-running-requests` + +If you see out of memory (OOM) errors, you can try to tune the following parameters. + +- If OOM happens during prefill, try to decrease `--chunked-prefill-size` to `4096` or `2048`. +- If OOM happens during decoding, try to decrease `--max-running-requests`. +- You can also try to decrease `--mem-fraction-static`, which reduces the memory usage of the KV cache memory pool and helps both prefill and decoding. + +## Enabling cache for `torch.compile` + +To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, `torch.compile` will automatically cache the FX graph and Triton in `/tmp/torchinductor_root`, which might be cleared according to the [system policy](https://serverfault.com/questions/377348/when-does-tmp-get-cleared). You can export the environment variable `TORCHINDUCTOR_CACHE_DIR` to save compilation cache in your desired directory to avoid unwanted deletion. You can also share the cache with other machines to reduce the compilation time. + +SGLang uses `max-autotune-no-cudagraphs` mode of `torch.compile`. The auto-tuning can be slow. +If you want to deploy a model on many different machines, you can ship the `torch.compile` cache to these machines and skip the compilation steps. This is based on [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html). + +*Examples*： + +1. Generate the cache by setting `TORCHINDUCTOR_CACHE_DIR` and running the model once. + + ```bash + TORCHINDUCTOR_CACHE_DIR=/root/inductor_root_cache python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile + ``` + +2. Copy the cache folder to other machines and launch the server with `TORCHINDUCTOR_CACHE_DIR`. + +## Tune `--schedule-policy` + +If the workload has many shared prefixes, use the default `--schedule-policy lpm`. Where `lpm` stands for longest prefix match. + +When you have no shared prefixes at all or you always send the requests with the shared prefixes together, +you can try `--schedule-policy fcfs`. Where `fcfs` stands for first come first serve. This policy has a lower scheduling overhead. diff --git a/docs/backend/native_api.ipynb b/docs/backend/native_api.ipynb new file mode 100644 index 0000000..72b65c6 --- /dev/null +++ b/docs/backend/native_api.ipynb @@ -0,0 +1,515 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SGLang Native APIs\n", + "\n", + "Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce these following APIs:\n", + "\n", + "- `/generate` (text generation model)\n", + "- `/get_model_info`\n", + "- `/get_server_info`\n", + "- `/health`\n", + "- `/health_generate`\n", + "- `/flush_cache`\n", + "- `/update_weights`\n", + "- `/encode`(embedding model)\n", + "- `/classify`(reward model)\n", + "- `/start_expert_distribution_record`\n", + "- `/stop_expert_distribution_record`\n", + "- `/dump_expert_distribution_record`\n", + "\n", + "We mainly use `requests` to test these APIs in the following examples. You can also use `curl`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate (text generation model)\n", + "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](./sampling_params.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/generate\"\n", + "data = {\"text\": \"What is the capital of France?\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Model Info\n", + "\n", + "Get the information of the model.\n", + "\n", + "- `model_path`: The path/name of the model.\n", + "- `is_generation`: Whether the model is used as generation model or embedding model.\n", + "- `tokenizer_path`: The path/name of the tokenizer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/get_model_info\"\n", + "\n", + "response = requests.get(url)\n", + "response_json = response.json()\n", + "print_highlight(response_json)\n", + "assert response_json[\"model_path\"] == \"meta-llama/Llama-3.2-1B-Instruct\"\n", + "assert response_json[\"is_generation\"] is True\n", + "assert response_json[\"tokenizer_path\"] == \"meta-llama/Llama-3.2-1B-Instruct\"\n", + "assert response_json.keys() == {\"model_path\", \"is_generation\", \"tokenizer_path\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Server Info\n", + "Gets the server information including CLI arguments, token limits, and memory pool sizes.\n", + "- Note: `get_server_info` merges the following deprecated endpoints:\n", + " - `get_server_args`\n", + " - `get_memory_pool_size` \n", + " - `get_max_total_num_tokens`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get_server_info\n", + "\n", + "url = f\"http://localhost:{port}/get_server_info\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Health Check\n", + "- `/health`: Check the health of the server.\n", + "- `/health_generate`: Check the health of the server by generating one token." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/health_generate\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/health\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Flush Cache\n", + "\n", + "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# flush cache\n", + "\n", + "url = f\"http://localhost:{port}/flush_cache\"\n", + "\n", + "response = requests.post(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update Weights From Disk\n", + "\n", + "Update model weights from disk without restarting the server. Only applicable for models with the same architecture and parameter size.\n", + "\n", + "SGLang support `update_weights_from_disk` API for continuous evaluation during training (save checkpoint to disk and update weights from disk).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# successful update with same architecture and size\n", + "\n", + "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", + "data = {\"model_path\": \"meta-llama/Llama-3.2-1B\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.text)\n", + "assert response.json()[\"success\"] is True\n", + "assert response.json()[\"message\"] == \"Succeeded to update model weights.\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# failed update with different parameter size or wrong name\n", + "\n", + "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", + "data = {\"model_path\": \"meta-llama/Llama-3.2-1B-wrong\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "print_highlight(response_json)\n", + "assert response_json[\"success\"] is False\n", + "assert response_json[\"message\"] == (\n", + " \"Failed to get weights iterator: \"\n", + " \"meta-llama/Llama-3.2-1B-wrong\"\n", + " \" (repository not found).\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Encode (embedding model)\n", + "\n", + "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings.html#openai-apis-embedding) and will raise an error for generation models.\n", + "Therefore, we launch a new server to server an embedding model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)\n", + "\n", + "embedding_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-7B-instruct \\\n", + " --host 0.0.0.0 --is-embedding\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# successful encode for embedding model\n", + "\n", + "url = f\"http://localhost:{port}/encode\"\n", + "data = {\"model\": \"Alibaba-NLP/gte-Qwen2-7B-instruct\", \"text\": \"Once upon a time\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "print_highlight(f\"Text embedding (first 10): {response_json['embedding'][:10]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Classify (reward model)\n", + "\n", + "SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)\n", + "\n", + "# Note that SGLang now treats embedding models and reward models as the same type of models.\n", + "# This will be updated in the future.\n", + "\n", + "reward_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "PROMPT = (\n", + " \"What is the range of the numeric output of a sigmoid node in a neural network?\"\n", + ")\n", + "\n", + "RESPONSE1 = \"The output of a sigmoid node is bounded between -1 and 1.\"\n", + "RESPONSE2 = \"The output of a sigmoid node is bounded between 0 and 1.\"\n", + "\n", + "CONVS = [\n", + " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE1}],\n", + " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE2}],\n", + "]\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n", + "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False)\n", + "\n", + "url = f\"http://localhost:{port}/classify\"\n", + "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n", + "\n", + "responses = requests.post(url, json=data).json()\n", + "for response in responses:\n", + " print_highlight(f\"reward: {response['embedding'][0]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(reward_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Capture expert selection distribution in MoE models\n", + "\n", + "SGLang Runtime supports recording the number of times an expert is selected in a MoE model run for each expert in the model. This is useful when analyzing the throughput of the model and plan for optimization." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "expert_record_server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(f\"http://localhost:{port}/start_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "url = f\"http://localhost:{port}/generate\"\n", + "data = {\"text\": \"What is the capital of France?\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())\n", + "\n", + "response = requests.post(f\"http://localhost:{port}/stop_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "response = requests.post(f\"http://localhost:{port}/dump_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "import glob\n", + "\n", + "output_file = glob.glob(\"expert_distribution_*.csv\")[0]\n", + "with open(output_file, \"r\") as f:\n", + " print_highlight(\"Content of dumped record:\")\n", + " for line in f:\n", + " print_highlight(line.strip())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(expert_record_server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Skip Tokenizer and Detokenizer\n", + "\n", + "SGLang Runtime also supports skip tokenizer and detokenizer. This is useful in cases like integrating with RLHF workflow." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer_free_server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --skip-tokenizer-init\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-3.2-1B-Instruct\")\n", + "\n", + "input_text = \"What is the capital of France?\"\n", + "\n", + "input_tokens = tokenizer.encode(input_text)\n", + "print_highlight(f\"Input Text: {input_text}\")\n", + "print_highlight(f\"Tokenized Input: {input_tokens}\")\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"input_ids\": input_tokens,\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 256,\n", + " \"stop_token_ids\": [tokenizer.eos_token_id],\n", + " },\n", + " \"stream\": False,\n", + " },\n", + ")\n", + "output = response.json()\n", + "output_tokens = output[\"output_ids\"]\n", + "\n", + "output_text = tokenizer.decode(output_tokens, skip_special_tokens=False)\n", + "print_highlight(f\"Tokenized Output: {output_tokens}\")\n", + "print_highlight(f\"Decoded Output: {output_text}\")\n", + "print_highlight(f\"Output Text: {output['meta_info']['finish_reason']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(tokenizer_free_server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/offline_engine_api.ipynb b/docs/backend/offline_engine_api.ipynb new file mode 100644 index 0000000..22ec2e5 --- /dev/null +++ b/docs/backend/offline_engine_api.ipynb @@ -0,0 +1,231 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Offline Engine API\n", + "\n", + "SGLang provides a direct inference engine without the need for an HTTP server, especially for use cases where additional HTTP server adds unnecessary complexity or overhead. Here are two general use cases:\n", + "\n", + "- Offline Batch Inference\n", + "- Custom Server on Top of the Engine\n", + "\n", + "This document focuses on the offline batch inference, demonstrating four different inference modes:\n", + "\n", + "- Non-streaming synchronous generation\n", + "- Streaming synchronous generation\n", + "- Non-streaming asynchronous generation\n", + "- Streaming asynchronous generation\n", + "\n", + "Additionally, you can easily build a custom server on top of the SGLang offline engine. A detailed example working in a python script can be found in [custom_server](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/custom_server.py).\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Advanced Usage\n", + "\n", + "The engine supports [vlm inference](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py) as well as [extracting hidden states](https://github.com/sgl-project/sglang/blob/main/examples/runtime/hidden_states). \n", + "\n", + "Please see [the examples](https://github.com/sgl-project/sglang/tree/main/examples/runtime/engine) for further use cases." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Offline Batch Inference\n", + "\n", + "SGLang offline engine supports batch inference with efficient scheduling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# launch the offline engine\n", + "import asyncio\n", + "import io\n", + "import os\n", + "\n", + "from PIL import Image\n", + "import requests\n", + "import sglang as sgl\n", + "\n", + "from sglang.srt.conversation import chat_templates\n", + "from sglang.test.test_utils import is_in_ci\n", + "from sglang.utils import async_stream_and_merge, stream_and_merge\n", + "\n", + "if is_in_ci():\n", + " import patch\n", + "\n", + "\n", + "llm = sgl.Engine(model_path=\"meta-llama/Meta-Llama-3.1-8B-Instruct\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-streaming Synchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Hello, my name is\",\n", + " \"The president of the United States is\",\n", + " \"The capital of France is\",\n", + " \"The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print(\"===============================\")\n", + " print(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Synchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\n", + " \"temperature\": 0.2,\n", + " \"top_p\": 0.9,\n", + "}\n", + "\n", + "print(\"\\n=== Testing synchronous streaming generation with overlap removal ===\\n\")\n", + "\n", + "for prompt in prompts:\n", + " print(f\"Prompt: {prompt}\")\n", + " merged_output = stream_and_merge(llm, prompt, sampling_params)\n", + " print(\"Generated text:\", merged_output)\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-streaming Asynchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "print(\"\\n=== Testing asynchronous batch generation ===\")\n", + "\n", + "\n", + "async def main():\n", + " outputs = await llm.async_generate(prompts, sampling_params)\n", + "\n", + " for prompt, output in zip(prompts, outputs):\n", + " print(f\"\\nPrompt: {prompt}\")\n", + " print(f\"Generated text: {output['text']}\")\n", + "\n", + "\n", + "asyncio.run(main())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Asynchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "print(\"\\n=== Testing asynchronous streaming generation (no repeats) ===\")\n", + "\n", + "\n", + "async def main():\n", + " for prompt in prompts:\n", + " print(f\"\\nPrompt: {prompt}\")\n", + " print(\"Generated text: \", end=\"\", flush=True)\n", + "\n", + " # Replace direct calls to async_generate with our custom overlap-aware version\n", + " async for cleaned_chunk in async_stream_and_merge(llm, prompt, sampling_params):\n", + " print(cleaned_chunk, end=\"\", flush=True)\n", + "\n", + " print() # New line after each prompt\n", + "\n", + "\n", + "asyncio.run(main())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/openai_api_completions.ipynb b/docs/backend/openai_api_completions.ipynb new file mode 100644 index 0000000..b874283 --- /dev/null +++ b/docs/backend/openai_api_completions.ipynb @@ -0,0 +1,534 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Completions\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/api-reference).\n", + "\n", + "This tutorial covers the following popular APIs:\n", + "\n", + "- `chat/completions`\n", + "- `completions`\n", + "- `batches`\n", + "\n", + "Check out other tutorials to learn about [vision APIs](https://docs.sglang.ai/backend/openai_api_vision.html) for vision-language models and [embedding APIs](https://docs.sglang.ai/backend/openai_api_embeddings.html) for embedding models." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chat Completions\n", + "\n", + "### Usage\n", + "\n", + "The server fully implements the OpenAI API.\n", + "It will automatically apply the chat template specified in the Hugging Face tokenizer, if one is available.\n", + "You can also specify a custom chat template with `--chat-template` when launching the server." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parameters\n", + "\n", + "The chat completions API accepts OpenAI Chat Completions API's parameters. Refer to [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat/create) for more details.\n", + "\n", + "Here is an example of a detailed chat completion request:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a knowledgeable historian who provides concise responses.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"Tell me about ancient Rome\"},\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Ancient Rome was a civilization centered in Italy.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"What were their major achievements?\"},\n", + " ],\n", + " temperature=0.3, # Lower temperature for more focused responses\n", + " max_tokens=128, # Reasonable length for a concise response\n", + " top_p=0.95, # Slightly higher for better fluency\n", + " presence_penalty=0.2, # Mild penalty to avoid repetition\n", + " frequency_penalty=0.2, # Mild penalty for more natural language\n", + " n=1, # Single response is usually more stable\n", + " seed=42, # Keep for reproducibility\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Streaming mode is also supported." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stream = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Say this is a test\"}],\n", + " stream=True,\n", + ")\n", + "for chunk in stream:\n", + " if chunk.choices[0].delta.content is not None:\n", + " print(chunk.choices[0].delta.content, end=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Completions\n", + "\n", + "### Usage\n", + "Completions API is similar to Chat Completions API, but without the `messages` parameter or chat templates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " prompt=\"List 3 countries and their capitals.\",\n", + " temperature=0,\n", + " max_tokens=64,\n", + " n=1,\n", + " stop=None,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parameters\n", + "\n", + "The completions API accepts OpenAI Completions API's parameters. Refer to [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions/create) for more details.\n", + "\n", + "Here is an example of a detailed completions request:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " prompt=\"Write a short story about a space explorer.\",\n", + " temperature=0.7, # Moderate temperature for creative writing\n", + " max_tokens=150, # Longer response for a story\n", + " top_p=0.9, # Balanced diversity in word choice\n", + " stop=[\"\\n\\n\", \"THE END\"], # Multiple stop sequences\n", + " presence_penalty=0.3, # Encourage novel elements\n", + " frequency_penalty=0.3, # Reduce repetitive phrases\n", + " n=1, # Generate one completion\n", + " seed=123, # For reproducible results\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Outputs (JSON, Regex, EBNF)\n", + "\n", + "For OpenAI compatible structed outputs API, refer to [Structured Outputs](https://docs.sglang.ai/backend/structured_outputs.html#OpenAI-Compatible-API) for more details.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batches\n", + "\n", + "Batches API for chat completions and completions are also supported. You can upload your requests in `jsonl` files, create a batch job, and retrieve the results when the batch job is completed (which takes longer but costs less).\n", + "\n", + "The batches APIs are:\n", + "\n", + "- `batches`\n", + "- `batches/{batch_id}/cancel`\n", + "- `batches/{batch_id}`\n", + "\n", + "Here is an example of a batch job for chat completions, completions are similar.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import time\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "requests = [\n", + " {\n", + " \"custom_id\": \"request-1\",\n", + " \"method\": \"POST\",\n", + " \"url\": \"/chat/completions\",\n", + " \"body\": {\n", + " \"model\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " \"messages\": [\n", + " {\"role\": \"user\", \"content\": \"Tell me a joke about programming\"}\n", + " ],\n", + " \"max_tokens\": 50,\n", + " },\n", + " },\n", + " {\n", + " \"custom_id\": \"request-2\",\n", + " \"method\": \"POST\",\n", + " \"url\": \"/chat/completions\",\n", + " \"body\": {\n", + " \"model\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is Python?\"}],\n", + " \"max_tokens\": 50,\n", + " },\n", + " },\n", + "]\n", + "\n", + "input_file_path = \"batch_requests.jsonl\"\n", + "\n", + "with open(input_file_path, \"w\") as f:\n", + " for req in requests:\n", + " f.write(json.dumps(req) + \"\\n\")\n", + "\n", + "with open(input_file_path, \"rb\") as f:\n", + " file_response = client.files.create(file=f, purpose=\"batch\")\n", + "\n", + "batch_response = client.batches.create(\n", + " input_file_id=file_response.id,\n", + " endpoint=\"/v1/chat/completions\",\n", + " completion_window=\"24h\",\n", + ")\n", + "\n", + "print_highlight(f\"Batch job created with ID: {batch_response.id}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "while batch_response.status not in [\"completed\", \"failed\", \"cancelled\"]:\n", + " time.sleep(3)\n", + " print(f\"Batch job status: {batch_response.status}...trying again in 3 seconds...\")\n", + " batch_response = client.batches.retrieve(batch_response.id)\n", + "\n", + "if batch_response.status == \"completed\":\n", + " print(\"Batch job completed successfully!\")\n", + " print(f\"Request counts: {batch_response.request_counts}\")\n", + "\n", + " result_file_id = batch_response.output_file_id\n", + " file_response = client.files.content(result_file_id)\n", + " result_content = file_response.read().decode(\"utf-8\")\n", + "\n", + " results = [\n", + " json.loads(line) for line in result_content.split(\"\\n\") if line.strip() != \"\"\n", + " ]\n", + "\n", + " for result in results:\n", + " print_highlight(f\"Request {result['custom_id']}:\")\n", + " print_highlight(f\"Response: {result['response']}\")\n", + "\n", + " print_highlight(\"Cleaning up files...\")\n", + " # Only delete the result file ID since file_response is just content\n", + " client.files.delete(result_file_id)\n", + "else:\n", + " print_highlight(f\"Batch job failed with status: {batch_response.status}\")\n", + " if hasattr(batch_response, \"errors\"):\n", + " print_highlight(f\"Errors: {batch_response.errors}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It takes a while to complete the batch job. You can use these two APIs to retrieve the batch job status or cancel the batch job.\n", + "\n", + "1. `batches/{batch_id}`: Retrieve the batch job status.\n", + "2. `batches/{batch_id}/cancel`: Cancel the batch job.\n", + "\n", + "Here is an example to check the batch job status." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import time\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "requests = []\n", + "for i in range(20):\n", + " requests.append(\n", + " {\n", + " \"custom_id\": f\"request-{i}\",\n", + " \"method\": \"POST\",\n", + " \"url\": \"/chat/completions\",\n", + " \"body\": {\n", + " \"model\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"{i}: You are a helpful AI assistant\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Write a detailed story about topic. Make it very long.\",\n", + " },\n", + " ],\n", + " \"max_tokens\": 64,\n", + " },\n", + " }\n", + " )\n", + "\n", + "input_file_path = \"batch_requests.jsonl\"\n", + "with open(input_file_path, \"w\") as f:\n", + " for req in requests:\n", + " f.write(json.dumps(req) + \"\\n\")\n", + "\n", + "with open(input_file_path, \"rb\") as f:\n", + " uploaded_file = client.files.create(file=f, purpose=\"batch\")\n", + "\n", + "batch_job = client.batches.create(\n", + " input_file_id=uploaded_file.id,\n", + " endpoint=\"/v1/chat/completions\",\n", + " completion_window=\"24h\",\n", + ")\n", + "\n", + "print_highlight(f\"Created batch job with ID: {batch_job.id}\")\n", + "print_highlight(f\"Initial status: {batch_job.status}\")\n", + "\n", + "time.sleep(10)\n", + "\n", + "max_checks = 5\n", + "for i in range(max_checks):\n", + " batch_details = client.batches.retrieve(batch_id=batch_job.id)\n", + "\n", + " print_highlight(\n", + " f\"Batch job details (check {i+1} / {max_checks}) // ID: {batch_details.id} // Status: {batch_details.status} // Created at: {batch_details.created_at} // Input file ID: {batch_details.input_file_id} // Output file ID: {batch_details.output_file_id}\"\n", + " )\n", + " print_highlight(\n", + " f\"Request counts: Total: {batch_details.request_counts.total} // Completed: {batch_details.request_counts.completed} // Failed: {batch_details.request_counts.failed}\"\n", + " )\n", + "\n", + " time.sleep(3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here is an example to cancel a batch job." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import time\n", + "from openai import OpenAI\n", + "import os\n", + "\n", + "client = OpenAI(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "requests = []\n", + "for i in range(5000):\n", + " requests.append(\n", + " {\n", + " \"custom_id\": f\"request-{i}\",\n", + " \"method\": \"POST\",\n", + " \"url\": \"/chat/completions\",\n", + " \"body\": {\n", + " \"model\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"{i}: You are a helpful AI assistant\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Write a detailed story about topic. Make it very long.\",\n", + " },\n", + " ],\n", + " \"max_tokens\": 128,\n", + " },\n", + " }\n", + " )\n", + "\n", + "input_file_path = \"batch_requests.jsonl\"\n", + "with open(input_file_path, \"w\") as f:\n", + " for req in requests:\n", + " f.write(json.dumps(req) + \"\\n\")\n", + "\n", + "with open(input_file_path, \"rb\") as f:\n", + " uploaded_file = client.files.create(file=f, purpose=\"batch\")\n", + "\n", + "batch_job = client.batches.create(\n", + " input_file_id=uploaded_file.id,\n", + " endpoint=\"/v1/chat/completions\",\n", + " completion_window=\"24h\",\n", + ")\n", + "\n", + "print_highlight(f\"Created batch job with ID: {batch_job.id}\")\n", + "print_highlight(f\"Initial status: {batch_job.status}\")\n", + "\n", + "time.sleep(10)\n", + "\n", + "try:\n", + " cancelled_job = client.batches.cancel(batch_id=batch_job.id)\n", + " print_highlight(f\"Cancellation initiated. Status: {cancelled_job.status}\")\n", + " assert cancelled_job.status == \"cancelling\"\n", + "\n", + " # Monitor the cancellation process\n", + " while cancelled_job.status not in [\"failed\", \"cancelled\"]:\n", + " time.sleep(3)\n", + " cancelled_job = client.batches.retrieve(batch_job.id)\n", + " print_highlight(f\"Current status: {cancelled_job.status}\")\n", + "\n", + " # Verify final status\n", + " assert cancelled_job.status == \"cancelled\"\n", + " print_highlight(\"Batch job successfully cancelled\")\n", + "\n", + "except Exception as e:\n", + " print_highlight(f\"Error during cancellation: {e}\")\n", + " raise e\n", + "\n", + "finally:\n", + " try:\n", + " del_response = client.files.delete(uploaded_file.id)\n", + " if del_response.deleted:\n", + " print_highlight(\"Successfully cleaned up input file\")\n", + " if os.path.exists(input_file_path):\n", + " os.remove(input_file_path)\n", + " print_highlight(\"Successfully deleted local batch_requests.jsonl file\")\n", + " except Exception as e:\n", + " print_highlight(f\"Error cleaning up: {e}\")\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/openai_api_embeddings.ipynb b/docs/backend/openai_api_embeddings.ipynb new file mode 100644 index 0000000..38543fa --- /dev/null +++ b/docs/backend/openai_api_embeddings.ipynb @@ -0,0 +1,191 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Embedding\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/embeddings).\n", + "\n", + "This tutorial covers the embedding APIs for embedding models, such as \n", + "- [intfloat/e5-mistral-7b-instruct](https://huggingface.co/intfloat/e5-mistral-7b-instruct) \n", + "- [Alibaba-NLP/gte-Qwen2-7B-instruct](https://huggingface.co/Alibaba-NLP/gte-Qwen2-7B-instruct) \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize. Remember to add `--is-embedding` to the command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "embedding_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-7B-instruct \\\n", + " --host 0.0.0.0 --is-embedding\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, json\n", + "\n", + "text = \"Once upon a time\"\n", + "\n", + "curl_text = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", + " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-7B-instruct\", \"input\": \"{text}\"}}'\"\"\"\n", + "\n", + "text_embedding = json.loads(subprocess.check_output(curl_text, shell=True))[\"data\"][0][\n", + " \"embedding\"\n", + "]\n", + "\n", + "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "text = \"Once upon a time\"\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/v1/embeddings\",\n", + " json={\"model\": \"Alibaba-NLP/gte-Qwen2-7B-instruct\", \"input\": text},\n", + ")\n", + "\n", + "text_embedding = response.json()[\"data\"][0][\"embedding\"]\n", + "\n", + "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "# Text embedding example\n", + "response = client.embeddings.create(\n", + " model=\"Alibaba-NLP/gte-Qwen2-7B-instruct\",\n", + " input=text,\n", + ")\n", + "\n", + "embedding = response.data[0].embedding[:10]\n", + "print_highlight(f\"Text embedding (first 10): {embedding}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Input IDs\n", + "\n", + "SGLang also supports `input_ids` as input to get the embedding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Alibaba-NLP/gte-Qwen2-7B-instruct\")\n", + "input_ids = tokenizer.encode(text)\n", + "\n", + "curl_ids = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", + " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-7B-instruct\", \"input\": {json.dumps(input_ids)}}}'\"\"\"\n", + "\n", + "input_ids_embedding = json.loads(subprocess.check_output(curl_ids, shell=True))[\"data\"][\n", + " 0\n", + "][\"embedding\"]\n", + "\n", + "print_highlight(f\"Input IDs embedding (first 10): {input_ids_embedding[:10]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/openai_api_vision.ipynb b/docs/backend/openai_api_vision.ipynb new file mode 100644 index 0000000..e4fb768 --- /dev/null +++ b/docs/backend/openai_api_vision.ipynb @@ -0,0 +1,291 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Vision\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision).\n", + "This tutorial covers the vision APIs for vision language models.\n", + "\n", + "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](https://docs.sglang.ai/references/supported_models): \n", + "- [meta-llama/Llama-3.2-11B-Vision-Instruct](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) \n", + "- [lmms-lab/llava-onevision-qwen2-72b-ov-chat](https://huggingface.co/lmms-lab/llava-onevision-qwen2-72b-ov-chat) \n", + "- [Qwen/Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct)\n", + "- [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it)\n", + "- [openbmb/MiniCPM-V](https://huggingface.co/openbmb/MiniCPM-V)\n", + "- [deepseek-ai/deepseek-vl2](https://huggingface.co/deepseek-ai/deepseek-vl2)\n", + "\n", + "As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize.\n", + "\n", + "**Remember to add** `--chat-template llama_3_vision` **to specify the [vision chat template](https://docs.sglang.ai/backend/openai_api_vision.html#Chat-Template), otherwise, the server will only support text (images won’t be passed in), which can lead to degraded performance.**\n", + "\n", + "We need to specify `--chat-template` for vision language models because the chat template provided in Hugging Face tokenizer only supports text." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "vision_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-11B-Vision-Instruct \\\n", + " --chat-template=llama_3_vision\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL\n", + "\n", + "Once the server is up, you can send test requests using curl or requests." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "curl_command = f\"\"\"\n", + "curl -s http://localhost:{port}/v1/chat/completions \\\\\n", + " -d '{{\n", + " \"model\": \"meta-llama/Llama-3.2-11B-Vision-Instruct\",\n", + " \"messages\": [\n", + " {{\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {{\n", + " \"type\": \"text\",\n", + " \"text\": \"What’s in this image?\"\n", + " }},\n", + " {{\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {{\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " }}\n", + " }}\n", + " ]\n", + " }}\n", + " ],\n", + " \"max_tokens\": 300\n", + " }}'\n", + "\"\"\"\n", + "\n", + "response = subprocess.check_output(curl_command, shell=True).decode()\n", + "print_highlight(response)\n", + "\n", + "\n", + "response = subprocess.check_output(curl_command, shell=True).decode()\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"meta-llama/Llama-3.2-11B-Vision-Instruct\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"What’s in this image?\"},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " \"max_tokens\": 300,\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Llama-3.2-11B-Vision-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"What is in this image?\",\n", + " },\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " max_tokens=300,\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multiple-Image Inputs\n", + "\n", + "The server also supports multiple images and interleaved text and images if the model supports it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Llama-3.2-11B-Vision-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\",\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\",\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have two very different images. They are not related at all. \"\n", + " \"Please describe the first image in one sentence, and then describe the second image in another sentence.\",\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " temperature=0,\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(vision_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chat Template\n", + "\n", + "As mentioned before, if you do not specify a vision model's `--chat-template`, the server uses Hugging Face's default template, which only supports text.\n", + "\n", + "We list popular vision models with their chat templates:\n", + "\n", + "- [meta-llama/Llama-3.2-Vision](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) uses `llama_3_vision`.\n", + "- [Qwen/Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) uses `qwen2-vl`.\n", + "- [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it) uses `gemma-it`.\n", + "- [openbmb/MiniCPM-V](https://huggingface.co/openbmb/MiniCPM-V) uses `minicpmv`.\n", + "- [deepseek-ai/deepseek-vl2](https://huggingface.co/deepseek-ai/deepseek-vl2) uses `deepseek-vl2`.\n", + "- [LlaVA-OneVision](https://huggingface.co/lmms-lab/llava-onevision-qwen2-7b-ov) uses `chatml-llava`.\n", + "- [LLaVA-NeXT](https://huggingface.co/collections/lmms-lab/llava-next-6623288e2d61edba3ddbf5ff) uses `chatml-llava`.\n", + "- [Llama3-LLaVA-NeXT](https://huggingface.co/lmms-lab/llama3-llava-next-8b) uses `llava_llama_3`.\n", + "- [LLaVA-v1.5 / 1.6](https://huggingface.co/liuhaotian/llava-v1.6-34b) uses `vicuna_v1.1`." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/patch.py b/docs/backend/patch.py new file mode 100644 index 0000000..ce7ba37 --- /dev/null +++ b/docs/backend/patch.py @@ -0,0 +1,53 @@ +import weakref + +import nest_asyncio + +nest_asyncio.apply() + +from sglang.utils import execute_shell_command, reserve_port + +DEFAULT_MAX_RUNNING_REQUESTS = 200 +DEFAULT_MAX_TOTAL_TOKENS = 20480 + +import sglang.srt.server_args as server_args_mod + +_original_post_init = server_args_mod.ServerArgs.__post_init__ + + +def patched_post_init(self): + _original_post_init(self) + if self.max_running_requests is None: + self.max_running_requests = DEFAULT_MAX_RUNNING_REQUESTS + if self.max_total_tokens is None: + self.max_total_tokens = DEFAULT_MAX_TOTAL_TOKENS + self.disable_cuda_graph = True + + +server_args_mod.ServerArgs.__post_init__ = patched_post_init + +process_socket_map = weakref.WeakKeyDictionary() + + +def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): + """ + Launch the server using the given command. + If no port is specified, a free port is reserved. + """ + if port is None: + port, lock_socket = reserve_port(host) + else: + lock_socket = None + + extra_flags = ( + f"--max-running-requests {DEFAULT_MAX_RUNNING_REQUESTS} " + f"--max-total-tokens {DEFAULT_MAX_TOTAL_TOKENS} " + f"--disable-cuda-graph" + ) + + full_command = f"{command} --port {port} {extra_flags}" + process = execute_shell_command(full_command) + + if lock_socket is not None: + process_socket_map[process] = lock_socket + + return process, port diff --git a/docs/backend/quantization.md b/docs/backend/quantization.md new file mode 100644 index 0000000..3a229f8 --- /dev/null +++ b/docs/backend/quantization.md @@ -0,0 +1,152 @@ +# Quantization + +SGLang supports various quantization methods, including offline quantization and online dynamic quantization. + +Offline quantization loads pre-quantized model weights directly during inference. This is required for quantization methods +such as GPTQ and AWQ, which collect and pre-compute various statistics from the original weights using the calibration dataset. + +Online quantization dynamically computes scaling parameters—such as the maximum/minimum values of model weights—during runtime. +Like NVIDIA FP8 training's [delayed scaling](https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/examples/fp8_primer.html#Mixed-precision-training-with-FP8) mechanism, online quantization calculates the appropriate scaling factors +on-the-fly to convert high-precision weights into a lower-precision format. + +**Note: For better performance, usability and convenience, offline quantization is recommended over online quantization.** + +If you use a pre-quantized model, do not add `--quantization` to enable online quantization at the same time. +For popular pre-quantized models, please visit [ModelCloud](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2) +or [NeuralMagic](https://huggingface.co/collections/neuralmagic) collections on HF for some +popular quality validated quantized models. Quantized models must be validated via benchmarks post-quantization +to guard against abnormal quantization loss regressions. + +## Offline Quantization + +To load already quantized models, simply load the model weights and config. **Again, if the model has been quantized offline, +there's no need to add `--quantization` argument when starting the engine. The quantization method will be parsed from the +downloaded Hugging Face config. For example, DeepSeek V3/R1 models are already in FP8, so do not add redundant parameters.** + +```bash +python3 -m sglang.launch_server \ + --model-path hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 \ + --port 30000 --host 0.0.0.0 +``` + +Take note, if your model is **per-channel quantized (INT8 or FP8) with per-token dynamic quantization activation**, you can opt to include `--quantization w8a8_int8` or `--quantization w8a8_fp8` to invoke the corresponding CUTLASS int8_kernel or fp8_kernel in sgl-kernel. This action will ignore the Hugging Face config's quantization settings. For instance, with `neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic`, if you execute with `--quantization w8a8_fp8`, the system will use the `W8A8Fp8Config` from SGLang to invoke the sgl-kernel, rather than the `CompressedTensorsConfig` for vLLM kernels. + +```bash +python3 -m sglang.launch_server \ + --model-path neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic \ + --quantization w8a8_fp8 \ + --port 30000 --host 0.0.0.0 +``` + +### Examples of Offline Model Quantization + +#### Using [GPTQModel](https://github.com/ModelCloud/GPTQModel) + +```bash +# install +pip install gptqmodel --no-build-isolation -v +``` + +```py +from datasets import load_dataset +from gptqmodel import GPTQModel, QuantizeConfig + +model_id = "meta-llama/Llama-3.2-1B-Instruct" +quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit" + +calibration_dataset = load_dataset( + "allenai/c4", data_files="en/c4-train.00001-of-01024.json.gz", + split="train" + ).select(range(1024))["text"] + +quant_config = QuantizeConfig(bits=4, group_size=128) # quantization config +model = GPTQModel.load(model_id, quant_config) # load model + +model.quantize(calibration_dataset, batch_size=2) # quantize +model.save(quant_path) # save model +``` + +#### Using [LLM Compressor](https://github.com/vllm-project/llm-compressor/) + +```bash +# install +pip install llmcompressor +``` + +Here, we take quantize `meta-llama/Meta-Llama-3-8B-Instruct` to `FP8` as an example to elaborate on how to do offline quantization. + +```python +from transformers import AutoTokenizer +from llmcompressor.transformers import SparseAutoModelForCausalLM +from llmcompressor.transformers import oneshot +from llmcompressor.modifiers.quantization import QuantizationModifier + +# Step 1: Load the original model. +MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct" + +model = SparseAutoModelForCausalLM.from_pretrained( + MODEL_ID, device_map="auto", torch_dtype="auto") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +# Step 2: Perform offline quantization. +# Step 2.1: Configure the simple PTQ quantization. +recipe = QuantizationModifier( + targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"]) + +# Step 2.2: Apply the quantization algorithm. +oneshot(model=model, recipe=recipe) + +# Step 3: Save the model. +SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic" +model.save_pretrained(SAVE_DIR) +tokenizer.save_pretrained(SAVE_DIR) +``` + +Then, you can directly use the quantized model with `SGLang`, by using the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path $PWD/Meta-Llama-3-8B-Instruct-FP8-Dynamic \ + --port 30000 --host 0.0.0.0 +``` + +## Online Quantization + +To enable online quantization, you can simply specify `--quantization` in the command line. For example, you can launch the server with the following command to enable `FP8` quantization for model `meta-llama/Meta-Llama-3.1-8B-Instruct`: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --quantization fp8 \ + --port 30000 --host 0.0.0.0 +``` + +Our team is working on supporting more online quantization methods. SGLang will soon support methods including but not limited to `["awq", "gptq", "marlin", "gptq_marlin", "awq_marlin", "bitsandbytes", "gguf"]`. + +SGLang also supports quantization methods based on [torchao](https://github.com/pytorch/ao). You can simply specify `--torchao-config` in the command line to support this feature. For example, if you want to enable `int4wo-128` for model `meta-llama/Meta-Llama-3.1-8B-Instruct`, you can launch the server with the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --torchao-config int4wo-128 \ + --port 30000 --host 0.0.0.0 +``` + +SGLang supports the following quantization methods based on torchao `["int8dq", "int8wo", "fp8wo", "fp8dq-per_tensor", "fp8dq-per_row", "int4wo-32", "int4wo-64", "int4wo-128", "int4wo-256"]`. + +Note: According to [this issue](https://github.com/sgl-project/sglang/issues/2219#issuecomment-2561890230), `"int8dq"` method currently has some bugs when using together with cuda graph capture. So we suggest to disable cuda graph capture when using `"int8dq"` method. Namely, please use the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --torchao-config int8dq \ + --disable-cuda-graph \ + --port 30000 --host 0.0.0.0 +``` + +## Reference + +- [GPTQModel](https://github.com/ModelCloud/GPTQModel) +- [LLM Compressor](https://github.com/vllm-project/llm-compressor/) +- [Torchao: PyTorch Architecture Optimization](https://github.com/pytorch/ao) +- [vLLM Quantization](https://docs.vllm.ai/en/latest/quantization/) diff --git a/docs/backend/sampling_params.md b/docs/backend/sampling_params.md new file mode 100644 index 0000000..4db0f86 --- /dev/null +++ b/docs/backend/sampling_params.md @@ -0,0 +1,282 @@ +# Sampling Parameters + +This doc describes the sampling parameters of the SGLang Runtime. It is the low-level endpoint of the runtime. + +If you want a high-level endpoint that can automatically handle chat templates, consider using the [OpenAI Compatible API](./openai_api_completions.ipynb). + +## `/generate` Endpoint + +The `/generate` endpoint accepts the following parameters in JSON format. For in detail usage see the [native api doc](./native_api.ipynb). + +* `text: Optional[Union[List[str], str]] = None` The input prompt. Can be a single prompt or a batch of prompts. +* `input_ids: Optional[Union[List[List[int]], List[int]]] = None` Alternative to `text`. Specify the input as token IDs instead of text. +* `sampling_params: Optional[Union[List[Dict], Dict]] = None` The sampling parameters as described in the sections below. +* `return_logprob: Optional[Union[List[bool], bool]] = None` Whether to return log probabilities for tokens. +* `logprob_start_len: Optional[Union[List[int], int]] = None` If returning log probabilities, specifies the start position in the prompt. Default is "-1" which returns logprobs only for output tokens. +* `top_logprobs_num: Optional[Union[List[int], int]] = None` If returning log probabilities, specifies the number of top logprobs to return at each position. +* `stream: bool = False` Whether to stream the output. +* `lora_path: Optional[Union[List[Optional[str]], Optional[str]]] = None` Path to LoRA weights. +* `custom_logit_processor: Optional[Union[List[Optional[str]], str]] = None` Custom logit processor for advanced sampling control. For usage see below. +* `return_hidden_states: bool = False` Whether to return hidden states of the model. Note that each time it changes, the cuda graph will be recaptured, which might lead to a performance hit. See the [examples](https://github.com/sgl-project/sglang/blob/main/examples/runtime/hidden_states) for more information. + +## Sampling params + +### Core Parameters + +* `max_new_tokens: int = 128` The maximum output length measured in tokens. +* `stop: Optional[Union[str, List[str]]] = None` One or multiple [stop words](https://platform.openai.com/docs/api-reference/chat/create#chat-create-stop). Generation will stop if one of these words is sampled. +* `stop_token_ids: Optional[List[int]] = None` Provide stop words in form of token ids. Generation will stop if one of these token ids is sampled. +* `temperature: float = 1.0` [Temperature](https://platform.openai.com/docs/api-reference/chat/create#chat-create-temperature) when sampling the next token. `temperature = 0` corresponds to greedy sampling, higher temperature leads to more diversity. +* `top_p: float = 1.0` [Top-p](https://platform.openai.com/docs/api-reference/chat/create#chat-create-top_p) selects tokens from the smallest sorted set whose cumulative probability exceeds `top_p`. When `top_p = 1`, this reduces to unrestricted sampling from all tokens. +* `top_k: int = -1` [Top-k](https://developer.nvidia.com/blog/how-to-get-better-outputs-from-your-large-language-model/#predictability_vs_creativity) randomly selects from the `k` highest-probability tokens. +* `min_p: float = 0.0` [Min-p](https://github.com/huggingface/transformers/issues/27670) samples from tokens with probability larger than `min_p * highest_token_probability`. + +### Penalizers + +* `frequency_penalty: float = 0.0`: Penalizes tokens based on their frequency in generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of penalization grows linearly with each appearance of a token. +* `presence_penalty: float = 0.0`: Penalizes tokens if they appeared in the generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of the penalization is constant if a token occured. +* `repetition_penalty: float = 0.0`: Penalizes tokens if they appeared in prompt or generation so far. Must be between `0` and `2` where numbers smaller than `1` encourage repeatment of tokens and numbers larger than `1` encourages sampling of new tokens. The penalization scales multiplicatively. +* `min_new_tokens: int = 0`: Forces the model to generate at least `min_new_tokens` until a stop word or EOS token is sampled. Note that this might lead to unintended behavior for example if the distribution is highly skewed towards these tokens. + +### Constrained decoding + +Please refer to our dedicated guide on [constrained decoding](./structured_outputs.ipynb) for the following parameters. + +* `json_schema: Optional[str] = None`: JSON schema for structured outputs. +* `regex: Optional[str] = None`: Regex for structured outputs. +* `ebnf: Optional[str] = None`: EBNF for structured outputs. + +### Other options + +* `n: int = 1`: Specifies the number of output sequences to generate per request. (Generating multiple outputs in one request (n > 1) is discouraged; repeat the same prompts for several times offer better control and efficiency.) +* `spaces_between_special_tokens: bool = True`: Whether or not to add spaces between special tokens during detokenization. +* `no_stop_trim: bool = False`: Don't trim stop words or EOS token from the generated text. +* `ignore_eos: bool = False`: Don't stop generation when EOS token is sampled. +* `skip_special_tokens: bool = True`: Remove special tokens during decoding. +* `custom_params: Optional[List[Optional[Dict[str, Any]]]] = None`: Used when employing `CustomLogitProcessor`. For usage see below. + +## Examples + +### Normal + +Launch a server: + +``` +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --port 30000 +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, +) +print(response.json()) +``` + +Detailed example in [send request](./send_request.ipynb). + +### Streaming + +Send a request and stream the output: + +```python +import requests, json + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + "stream": True, + }, + stream=True, +) + +prev = 0 +for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) +print("") +``` + +Detailed example in [openai compatible api](https://docs.sglang.ai/backend/openai_api_completions.html#id2). + +### Multi modal + +Launch a server: + +``` +python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-7b-ov --chat-template chatml-llava +``` + +Download an image: + +``` +curl -o example_image.png -L https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n" + "<|im_start|>assistant\n", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, +) +print(response.json()) +``` + +The `image_data` can be a file name, a URL, or a base64 encoded string. See also `python/sglang/srt/utils.py:load_image`. + +Streaming is supported in a similar manner as [above](#streaming). + +Detailed example in [openai api vision](./openai_api_vision.ipynb). + +### Structured Outputs (JSON, Regex, EBNF) + +You can specify a JSON schema, regular expression or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request. + +SGLang supports two grammar backends: + +- [Outlines](https://github.com/dottxt-ai/outlines) (default): Supports JSON schema and regular expression constraints. +- [XGrammar](https://github.com/mlc-ai/xgrammar): Supports JSON schema, regular expression, and EBNF constraints. + - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md) + +Initialize the XGrammar backend using `--grammar-backend xgrammar` flag +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ +--port 30000 --host 0.0.0.0 --grammar-backend [xgrammar|outlines] # xgrammar or outlines (default: outlines) +``` + +```python +import json +import requests + +json_schema = json.dumps({ + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], +}) + +# JSON (works with both Outlines and XGrammar) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, +) +print(response.json()) + +# Regular expression (Outlines backend only) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Paris is the capital of", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "regex": "(France|England)", + }, + }, +) +print(response.json()) + +# EBNF (XGrammar backend only) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Write a greeting.", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "ebnf": 'root ::= "Hello" | "Hi" | "Hey"', + }, + }, +) +print(response.json()) +``` + +Detailed example in [structured outputs](./structured_outputs.ipynb). +### Custom Logit Processor +Launch a server with `--enable-custom-logit-processor` flag on. +``` +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --port 30000 --enable-custom-logit-processor +``` + +Define a custom logit processor that will always sample a specific token id. +```python +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor + +class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + # Check that the number of logits matches the number of custom parameters + assert logits.shape[0] == len(custom_param_list) + key = "token_id" + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[key]] = 0.0 + return logits +``` + +Send a request +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "custom_logit_processor": DeterministicLogitProcessor().to_str(), + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 32, + "custom_params": {"token_id": 5}, + }, + }, +) +print(response.json()) +``` diff --git a/docs/backend/send_request.ipynb b/docs/backend/send_request.ipynb new file mode 100644 index 0000000..6b51b42 --- /dev/null +++ b/docs/backend/send_request.ipynb @@ -0,0 +1,259 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sending Requests\n", + "This notebook provides a quick-start guide to use SGLang in chat completions after installation.\n", + "\n", + "- For Vision Language Models, see [OpenAI APIs - Vision](../backend/openai_api_vision.ipynb).\n", + "- For Embedding Models, see [OpenAI APIs - Embedding](../backend/openai_api_embeddings.ipynb) and [Encode (embedding model)](../backend/native_api.html#Encode-(embedding-model)).\n", + "- For Reward Models, see [Classify (reward model)](../backend/native_api.html#Classify-(reward-model))." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "# This is equivalent to running the following command in your terminal\n", + "\n", + "# python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --host 0.0.0.0\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, json\n", + "\n", + "curl_command = f\"\"\"\n", + "curl -s http://localhost:{port}/v1/chat/completions \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\", \"messages\": [{{\"role\": \"user\", \"content\": \"What is the capital of France?\"}}]}}'\n", + "\"\"\"\n", + "\n", + "response = json.loads(subprocess.check_output(curl_command, shell=True))\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "# Use stream=True for streaming responses\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + " stream=True,\n", + ")\n", + "\n", + "# Handle the streaming output\n", + "for chunk in response:\n", + " if chunk.choices[0].delta.content:\n", + " print(chunk.choices[0].delta.content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Native Generation APIs\n", + "\n", + "You can also use the native `/generate` endpoint with requests, which provides more flexiblity. An API reference is available at [Sampling Parameters](../references/sampling_params.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"The capital of France is\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 32,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests, json\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"The capital of France is\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 32,\n", + " },\n", + " \"stream\": True,\n", + " },\n", + " stream=True,\n", + ")\n", + "\n", + "prev = 0\n", + "for chunk in response.iter_lines(decode_unicode=False):\n", + " chunk = chunk.decode(\"utf-8\")\n", + " if chunk and chunk.startswith(\"data:\"):\n", + " if chunk == \"data: [DONE]\":\n", + " break\n", + " data = json.loads(chunk[5:].strip(\"\\n\"))\n", + " output = data[\"text\"]\n", + " print(output[prev:], end=\"\", flush=True)\n", + " prev = len(output)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/separate_reasoning.ipynb b/docs/backend/separate_reasoning.ipynb new file mode 100644 index 0000000..756ecba --- /dev/null +++ b/docs/backend/separate_reasoning.ipynb @@ -0,0 +1,427 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reasoning Parser\n", + "\n", + "SGLang supports parsing reasoning content our from \"normal\" content for reasoning models such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1).\n", + "\n", + "## Supported Models\n", + "\n", + "Currently, SGLang supports the following reasoning models:\n", + "- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `` and `` tags.\n", + "- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `` and `` tags." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", + "\n", + "### Launching the Server" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the `--reasoning-parser` option." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from openai import OpenAI\n", + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that `--reasoning-parser` defines the parser used to interpret responses. Currently supported parsers include:\n", + "\n", + "- deepseek-r1: DeepSeek R1 series and QwQ (e.g. deepseek-ai/DeepSeek-R1, Qwen/QwQ-32B)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI Compatible API\n", + "\n", + "Using the OpenAI compatible API, the contract follows the [DeepSeek API design](https://api-docs.deepseek.com/guides/reasoning_model) established with the release of DeepSeek-R1:\n", + "\n", + "- `reasoning_content`: The content of the CoT.\n", + "- `content`: The content of the final answer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize OpenAI-like client\n", + "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", + "model_name = client.models.list().data[0].id\n", + "\n", + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What is 1+3?\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Non-Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=False, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True},\n", + ")\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(response_non_stream.choices[0].message.reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(response_non_stream.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=True, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True},\n", + ")\n", + "\n", + "reasoning_content = \"\"\n", + "content = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " content += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.reasoning_content:\n", + " reasoning_content += chunk.choices[0].delta.reasoning_content\n", + "\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Optionally, you can buffer the reasoning content to the last reasoning chunk (or the first chunk after the reasoning content)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=True, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True, \"stream_reasoning\": False},\n", + ")\n", + "\n", + "reasoning_content = \"\"\n", + "content = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " content += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.reasoning_content:\n", + " reasoning_content = chunk.choices[0].delta.reasoning_content\n", + "\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The reasoning separation is enable by default when specify . \n", + "**To disable it, set the `separate_reasoning` option to `False` in request.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=False, # Non-streaming\n", + " extra_body={\"separate_reasoning\": False},\n", + ")\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(response_non_stream.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SGLang Native API " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "input = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + ")\n", + "\n", + "gen_url = f\"http://localhost:{port}/generate\"\n", + "gen_data = {\n", + " \"text\": input,\n", + " \"sampling_params\": {\n", + " \"skip_special_tokens\": False,\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.95,\n", + " },\n", + "}\n", + "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(gen_response)\n", + "\n", + "parse_url = f\"http://localhost:{port}/separate_reasoning\"\n", + "separate_reasoning_data = {\n", + " \"text\": gen_response,\n", + " \"reasoning_parser\": \"deepseek-r1\",\n", + "}\n", + "separate_reasoning_response_json = requests.post(\n", + " parse_url, json=separate_reasoning_data\n", + ").json()\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(separate_reasoning_response_json[\"reasoning_text\"])\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(separate_reasoning_response_json[\"text\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sglang as sgl\n", + "from sglang.srt.reasoning_parser import ReasoningParser\n", + "from sglang.utils import print_highlight\n", + "\n", + "llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "input = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + ")\n", + "sampling_params = {\n", + " \"max_new_tokens\": 1024,\n", + " \"skip_special_tokens\": False,\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.95,\n", + "}\n", + "result = llm.generate(prompt=input, sampling_params=sampling_params)\n", + "\n", + "generated_text = result[\"text\"] # Assume there is only one prompt\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(generated_text)\n", + "\n", + "parser = ReasoningParser(\"deepseek-r1\")\n", + "reasoning_text, text = parser.parse_non_stream(generated_text)\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_text)\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Supporting New Reasoning Model Schemas\n", + "\n", + "For future reasoning models, you can implement the reasoning parser as a subclass of `BaseReasoningFormatDetector` in `python/sglang/srt/reasoning_parser.py` and specify the reasoning parser for new reasoning model schemas accordingly." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "class DeepSeekR1Detector(BaseReasoningFormatDetector):\n", + " \"\"\"\n", + " Detector for DeepSeek-R1 model.\n", + " Assumes reasoning format:\n", + " ()*(.*)\n", + " Returns all the text before the tag as `reasoning_text`\n", + " and the rest of the text as `normal_text`.\n", + "\n", + " Args:\n", + " stream_reasoning (bool): If False, accumulates reasoning content until the end tag.\n", + " If True, streams reasoning content as it arrives.\n", + " \"\"\"\n", + "\n", + " def __init__(self, stream_reasoning: bool = False):\n", + " # DeepSeek-R1 is assumed to be reasoning until `` token\n", + " super().__init__(\"\", \"\", True, stream_reasoning=stream_reasoning)\n", + " # https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599\n", + "\n", + "\n", + "class ReasoningParser:\n", + " \"\"\"\n", + " Parser that handles both streaming and non-streaming scenarios for extracting\n", + " reasoning content from model outputs.\n", + "\n", + " Args:\n", + " model_type (str): Type of model to parse reasoning from\n", + " stream_reasoning (bool): If Flase, accumulates reasoning content until complete.\n", + " If True, streams reasoning content as it arrives.\n", + " \"\"\"\n", + "\n", + " DetectorMap: Dict[str, BaseReasoningFormatDetector] = {\n", + " \"deepseek-r1\": DeepSeekR1Detector\n", + " }\n", + "\n", + " def __init__(self, model_type: str = None, stream_reasoning: bool = True):\n", + " if not model_type:\n", + " raise ValueError(\"Model type must be specified\")\n", + "\n", + " detector_class = self.DetectorMap.get(model_type.lower())\n", + " if not detector_class:\n", + " raise ValueError(f\"Unsupported model type: {model_type}\")\n", + "\n", + " self.detector = detector_class(stream_reasoning=stream_reasoning)\n", + "\n", + " def parse_non_stream(self, full_text: str) -> StreamingParseResult:\n", + " \"\"\"Non-streaming call: one-time parsing\"\"\"\n", + " ret = self.detector.detect_and_parse(full_text)\n", + " return ret.reasoning_text, ret.normal_text\n", + "\n", + " def parse_stream_chunk(self, chunk_text: str) -> StreamingParseResult:\n", + " \"\"\"Streaming call: incremental parsing\"\"\"\n", + " ret = self.detector.parse_streaming_increment(chunk_text)\n", + " return ret.reasoning_text, ret.normal_text\n", + "```" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/backend/server_arguments.md b/docs/backend/server_arguments.md new file mode 100644 index 0000000..3d2aae8 --- /dev/null +++ b/docs/backend/server_arguments.md @@ -0,0 +1,195 @@ +# Server Arguments + +## Common launch commands + +- To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 2 +``` + +- To enable multi-GPU data parallelism, add `--dp 2`. Data parallelism is better for throughput if there is enough memory. It can also be used together with tensor parallelism. The following command uses 4 GPUs in total. We recommend [SGLang Router](../router/router.md) for data parallelism. + +```bash +python -m sglang_router.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --dp 2 --tp 2 +``` + +- If you see out-of-memory errors during serving, try to reduce the memory usage of the KV cache pool by setting a smaller value of `--mem-fraction-static`. The default value is `0.9`. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --mem-fraction-static 0.7 +``` + +- See [hyperparameter tuning](hyperparameter_tuning.md) on tuning hyperparameters for better performance. +- For docker and Kubernetes runs, you need to set up shared memory which is used for communication between processes. See `--shm-size` for docker and `/dev/shm` size update for Kubernetes manifests. +- If you see out-of-memory errors during prefill for long prompts, try to set a smaller chunked prefill size. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --chunked-prefill-size 4096 +``` +- To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, the cache path is located at `/tmp/torchinductor_root`, you can customize it using environment variable `TORCHINDUCTOR_CACHE_DIR`. For more details, please refer to [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html) and [Enabling cache for torch.compile](https://docs.sglang.ai/backend/hyperparameter_tuning.html#enabling-cache-for-torch-compile). +- To enable torchao quantization, add `--torchao-config int4wo-128`. It supports other [quantization strategies (INT8/FP8)](https://github.com/sgl-project/sglang/blob/v0.3.6/python/sglang/srt/server_args.py#L671) as well. +- To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments. +- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`. +- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](custom_chat_template.md). +- To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph` + +```bash +# Node 0 +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 4 --dist-init-addr sgl-dev-0:50000 --nnodes 2 --node-rank 0 + +# Node 1 +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 4 --dist-init-addr sgl-dev-0:50000 --nnodes 2 --node-rank 1 +``` + +Please consult the documentation below to learn more about the parameters you may provide when launching a server. + + +## Model and tokenizer + +* `model_path`: Path to the model that will be served. +* `tokenizer_path`: Defaults to the `model_path`. +* `tokenizer_mode`: By default `auto`, see [here](https://huggingface.co/docs/transformers/en/main_classes/tokenizer) for different mode. +* `load_format`: The format the weights are loaded in. Defaults to `*.safetensors`/`*.bin`. +* `trust_remote_code`: If `True`, will use locally cached config files, otherwise use remote configs in HuggingFace. +* `dtype`: Dtype used for the model, defaults to `bfloat16`. +* `kv_cache_dtype`: Dtype of the kv cache, defaults to the `dtype`. +* `context_length`: The number of tokens our model can process *including the input*. Note that extending the default might lead to strange behavior. +* `device`: The device we put the model, defaults to `cuda`. +* `chat_template`: The chat template to use. Deviating from the default might lead to unexpected responses. For multi-modal chat templates, refer to [here](https://docs.sglang.ai/backend/openai_api_vision.ipynb#Chat-Template). **Make sure the correct** `chat_template` **is passed, or performance degradation may occur!!!!** +* `is_embedding`: Set to true to perform [embedding](./openai_api_embeddings.ipynb) / [encode](https://docs.sglang.ai/backend/native_api#Encode-(embedding-model)) and [reward](https://docs.sglang.ai/backend/native_api#Classify-(reward-model)) tasks. +* `revision`: Adjust if a specific version of the model should be used. +* `skip_tokenizer_init`: Set to true to provide the tokens to the engine and get the output tokens directly, typically used in RLHF. Please see this [example for reference](https://github.com/sgl-project/sglang/blob/main/examples/runtime/token_in_token_out/). +* `json_model_override_args`: Override model config with the provided JSON. +* `delete_ckpt_after_loading`: Delete the model checkpoint after loading the model. + + +## Serving: HTTP & API + +### HTTP Server configuration + +* `port` and `host`: Setup the host for HTTP server. By default `host: str = "127.0.0.1"` and `port: int = 30000` + +### API configuration + +* `api_key`: Sets an API key for the server and the OpenAI-compatible API. +* `file_storage_path`: Directory for storing uploaded or generated files from API calls. +* `enable_cache_report`: If set, includes detailed usage of cached tokens in the response usage. + +## Parallelism + +### Tensor parallelism + +* `tp_size`: The number of GPUs the model weights get sharded over. Mainly for saving memory rather than for high throughput, see [this blogpost](https://pytorch.org/tutorials/intermediate/TP_tutorial.html#how-tensor-parallel-works). + +### Data parallelism + +* `dp_size`: Will be deprecated. The number of data-parallel copies of the model. [SGLang router](../router/router.md) is recommended instead of the current naive data parallel. +* `load_balance_method`: Will be deprecated. Load balancing strategy for data parallel requests. + +### Expert parallelism +* `enable_ep_moe`: Enables expert parallelism that distributes the experts onto multiple GPUs for MoE models. +* `ep_size`: The size of EP. Please shard the model weights with `tp_size=ep_size`, for detailed benchmarking refer to [this PR](https://github.com/sgl-project/sglang/pull/2203). If not set, `ep_size` will be automatically set to `tp_size`. +* `enable_deepep_moe`: Enables expert parallelism that distributes the experts onto multiple GPUs for DeepSeek-V3 model based on deepseek-ai/DeepEP. + +## Memory and scheduling + +* `mem_fraction_static`: Fraction of the free GPU memory used for static memory like model weights and KV cache. If building KV cache fails, it should be increased. If CUDA runs out of memory, it should be decreased. +* `max_running_requests`: The maximum number of requests to run concurrently. +* `max_total_tokens`: The maximum number of tokens that can be stored into the KV cache. Use mainly for debugging. +* `chunked_prefill_size`: Perform the prefill in chunks of these size. Larger chunk size speeds up the prefill phase but increases the VRAM consumption. If CUDA runs out of memory, it should be decreased. +* `max_prefill_tokens`: Token budget of how many tokens to accept in one prefill batch. The actual number is the max of this parameter and the `context_length`. +* `schedule_policy`: The scheduling policy to control the processing order of waiting prefill requests in a single engine. +* `schedule_conservativeness`: Can be used to decrease/increase the conservativeness of the server when taking new requests. Highly conservative behavior leads to starvation, but low conservativeness leads to slowed-down performance. +* `cpu_offload_gb`: Reserve this amount of RAM in GB for offloading of model parameters to the CPU. + +## Other runtime options + +* `stream_interval`: Interval (in tokens) for streaming responses. Smaller values lead to smoother streaming, and larger values lead to better throughput. +* `random_seed`: Can be used to enforce more deterministic behavior. +* `watchdog_timeout`: Adjusts the watchdog thread's timeout before killing the server if batch generation takes too long. +* `download_dir`: Use to override the default Hugging Face cache directory for model weights. +* `base_gpu_id`: Use to adjust first GPU used to distribute the model across available GPUs. +* `allow_auto_truncate`: Automatically truncate requests that exceed the maximum input length. + +## Logging + +* `log_level`: Global log verbosity. +* `log_level_http`: Separate verbosity level for the HTTP server logs (if unset, defaults to `log_level`). +* `log_requests`: Logs the inputs and outputs of all requests for debugging. +* `log_requests_level`: Ranges from 0 to 2: level 0 only shows some basic metadata in requests, level 1 and 2 show request details (e.g., text, images), and level 1 limits output to 2048 characters (if unset, defaults to `0`). +* `show_time_cost`: Prints or logs detailed timing info for internal operations (helpful for performance tuning). +* `enable_metrics`: Exports Prometheus-like metrics for request usage and performance. +* `decode_log_interval`: How often (in tokens) to log decode progress. + +## Multi-node distributed serving + +* `dist_init_addr`: The TCP address used for initializing PyTorch's distributed backend (e.g. `192.168.0.2:25000`). +* `nnodes`: Total number of nodes in the cluster. Refer to how to run the [Llama 405B model](https://docs.sglang.ai/references/llama_405B.html#run-405b-fp16-on-two-nodes). +* `node_rank`: Rank (ID) of this node among the `nnodes` in the distributed setup. + + +## LoRA + +* `lora_paths`: You may provide a list of adapters to your model as a list. Each batch element will get model response with the corresponding lora adapter applied. Currently `cuda_graph` and `radix_attention` are not supported with this option so you need to disable them manually. We are still working on through these [issues](https://github.com/sgl-project/sglang/issues/2929). +* `max_loras_per_batch`: Maximum number of LoRAs in a running batch including base model. +* `lora_backend`: The backend of running GEMM kernels for Lora modules, can be one of `triton` or `flashinfer`. Defaults to be `triton`. + +## Kernel backend + +* `attention_backend`: The backend for attention computation and KV cache management. +* `sampling_backend`: The backend for sampling. + +## Constrained Decoding + +* `grammar_backend`: The grammar backend for constraint decoding. Detailed usage can be found in this [document](https://docs.sglang.ai/backend/structured_outputs.html). +* `constrained_json_whitespace_pattern`: Use with `Outlines` grammar backend to allow JSON with syntatic newlines, tabs or multiple spaces. Details can be found [here](https://dottxt-ai.github.io/outlines/latest/reference/generation/json/#using-pydantic). + +## Speculative decoding + +* `speculative_draft_model_path`: The draft model path for speculative decoding. +* `speculative_algorithm`: The algorithm for speculative decoding. Currently only [Eagle](https://arxiv.org/html/2406.16858v1) is supported. Note that the radix cache, chunked prefill, and overlap scheduler are disabled when using eagle speculative decoding. +* `speculative_num_steps`: How many draft passes we run before verifying. +* `speculative_num_draft_tokens`: The number of tokens proposed in a draft. +* `speculative_eagle_topk`: The number of top candidates we keep for verification at each step for [Eagle](https://arxiv.org/html/2406.16858v1). +* `speculative_token_map`: Optional, the path to the high frequency token list of [FR-Spec](https://arxiv.org/html/2502.14856v1), used for accelerating [Eagle](https://arxiv.org/html/2406.16858v1). + + +## Double Sparsity + +* `enable_double_sparsity`: Enables [double sparsity](https://arxiv.org/html/2408.07092v2) which increases throughput. +* `ds_channel_config_path`: The double sparsity config. For a guide on how to generate the config for your model see [this repo](https://github.com/andy-yang-1/DoubleSparse/tree/main/config). +* `ds_heavy_channel_num`: Number of channel indices to keep for each layer. +* `ds_heavy_token_num`: Number of tokens used for attention during decode. Skip sparse decoding if `min_seq_len` in batch < this number. +* `ds_heavy_channel_type`: The type of heavy channels. Either `q`, `k` or `qk`. +* `ds_sparse_decode_threshold`: Don't apply sparse decoding if `max_seq_len` in batch < this threshold. + +## Debug options + +*Note: We recommend to stay with the defaults and only use these options for debugging for best possible performance.* + +* `disable_radix_cache`: Disable [Radix](https://lmsys.org/blog/2024-01-17-sglang/) backend for prefix caching. +* `disable_cuda_graph`: Disable [cuda graph](https://pytorch.org/blog/accelerating-pytorch-with-cuda-graphs/) for model forward. Use if encountering uncorrectable CUDA ECC errors. +* `disable_cuda_graph_padding`: Disable cuda graph when padding is needed. In other case still use cuda graph. +* `disable_outlines_disk_cache`: Disable disk cache for outlines grammar backend. +* `disable_custom_all_reduce`: Disable usage of custom all reduce kernel. +* `disable_mla`: Disable [Multi-Head Latent Attention](https://arxiv.org/html/2405.04434v5) for Deepseek model. +* `disable_overlap_schedule`: Disable the [Overhead-Scheduler](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#zero-overhead-batch-scheduler). +* `enable_nan_detection`: Turning this on makes the sampler print a warning if the logits contain `NaN`. +* `enable_p2p_check`: Turns off the default of allowing always p2p check when accessing GPU. +* `triton_attention_reduce_in_fp32`: In triton kernels this will cast the intermediate attention result to `float32`. + +## Optimization + +*Note: Some of these options are still in experimental stage.* + +* `enable_mixed_chunk`: Enables mixing prefill and decode, see [this discussion](https://github.com/sgl-project/sglang/discussions/1163). +* `enable_dp_attention`: Enable [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models) for Deepseek models. +* `enable_torch_compile`: Torch compile the model. Note that compiling a model takes a long time but have a great performance boost. The compiled model can also be [cached for future use](https://docs.sglang.ai/backend/hyperparameter_tuning.html#enabling-cache-for-torch-compile). +* `torch_compile_max_bs`: The maximum batch size when using `torch_compile`. +* `cuda_graph_max_bs`: Adjust the maximum batchsize when using cuda graph. By default this is chosen for you based on GPU specifics. +* `cuda_graph_bs`: The batch sizes to capture by `CudaGraphRunner`. By default this is done for you. +* `torchao_config`: Experimental feature that optimizes the model with [torchao](https://github.com/pytorch/ao). Possible choices are: int8dq, int8wo, int4wo-, fp8wo, fp8dq-per_tensor, fp8dq-per_row. +* `triton_attention_num_kv_splits`: Use to adjust the number of KV splits in triton kernels. Default is 8. +* `enable_flashinfer_mla`: Use the attention backend with flashinfer MLA wrapper for deepseek models. When providing this argument, `attention_backend` argument is overridden. +* `flashinfer_mla_disable_ragged`: Disable usage of ragged prefill wrapper for flashinfer mla attention backend. Should be used when `enable_flashinfer_mla` is turned on. diff --git a/docs/backend/speculative_decoding.ipynb b/docs/backend/speculative_decoding.ipynb new file mode 100644 index 0000000..061f415 --- /dev/null +++ b/docs/backend/speculative_decoding.ipynb @@ -0,0 +1,305 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Speculative Decoding\n", + "\n", + "SGLang now provides an EAGLE2-based speculative decoding option. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines.\n", + "\n", + "**Note:** Currently, Speculative Decoding in SGLang is compatible with radix cache and chunked prefill.\n", + "\n", + "### Performance Highlights\n", + "\n", + "- Official EAGLE code ([SafeAILab/EAGLE](https://github.com/SafeAILab/EAGLE)): ~200 tokens/s\n", + "- Standard SGLang Decoding: ~156 tokens/s\n", + "- EAGLE Decoding in SGLang: ~297 tokens/s\n", + "- EAGLE Decoding in SGLang (w/ `torch.compile`): ~316 tokens/s\n", + "\n", + "All benchmarks below were run on a single H100." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EAGLE Decoding\n", + "\n", + "To enable EAGLE speculative decoding the following parameters are relevant:\n", + "* `speculative_draft_model_path`: Specifies draft model. This parameter is required.\n", + "* `speculative_num_steps`: Depth of autoregressive drafting. Increases speculation range but risks rejection cascades. Default is 5.\n", + "\n", + "* `speculative_eagle_topk`: Branching factor per step. Improves candidate diversity, will lead to higher acceptance rate, but more lead to higher memory/compute consumption. Default is 4.\n", + "\n", + "* `speculative_num_draft_tokens`: Maximum parallel verification capacity. Allows deeper tree evaluation but will lead to higher GPU memory usage. Default is 8." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 \\\n", + " --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE Decoding with `torch.compile`\n", + "\n", + "You can also enable `torch.compile` for further optimizations and optionally set `--cuda-graph-max-bs`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --mem-fraction 0.6 \\\n", + " --enable-torch-compile --cuda-graph-max-bs 2\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE Decoding via Frequency-Ranked Speculative Sampling\n", + "\n", + "By employing a truncated high-frequency token vocabulary in the draft model, Eagle speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, checkout [the paper](https://arxiv.org/pdf/arXiv:2502.14856).\n", + "\n", + "In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency token in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency token by directly downloading these token from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset).\n", + "\n", + "Thanks for the contribution from [Weilin Zhao](https://github.com/https://github.com/Achazwl) and [Zhousx](https://github.com/Zhou-sx). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \\\n", + " --mem-fraction 0.7 --cuda-graph-max-bs 2 --dtype float16 \n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-3 Decoding\n", + "\n", + "You can enable EAGLE-3 decoding by setting `--speculative_draft_model_path: EAGLE3`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --speculative-algorithm EAGLE3 \\\n", + " --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 32 --mem-fraction 0.6 \\\n", + " --cuda-graph-max-bs 2 --dtype float16\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "EAGLE process is as follows:\n", + "\n", + "- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. \n", + "- The next token is then sampled from $p_{k+2}=\\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again.\n", + "- EAGLE-2 additionally uses the draft model to evaluate how probable certain branches in the draft tree are, dynamically stopping the expansion of unlikely branches. After the expansion phase, reranking is employed to select only the top `speculative_num_draft_tokens` final nodes as draft tokens.\n", + "- EAGLE-3 removes the feature prediction objective, incorporates low and mid-layer features, and is trained in an on-policy manner.\n", + "\n", + "This enhances drafting accuracy by operating on the features instead of tokens for more regular inputs and passing the tokens from the next timestep additionaly to minimize randomness effects from sampling. Furthermore the dynamic adjustment of the draft tree and selection of reranked final nodes increases acceptance rate of draft tokens further. For more details see [the paper](https://arxiv.org/abs/2406.16858)." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/backend/structured_outputs.ipynb b/docs/backend/structured_outputs.ipynb new file mode 100644 index 0000000..acb2620 --- /dev/null +++ b/docs/backend/structured_outputs.ipynb @@ -0,0 +1,827 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Structured Outputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can specify a JSON schema, [regular expression](https://en.wikipedia.org/wiki/Regular_expression) or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request.\n", + "\n", + "SGLang supports three grammar backends:\n", + "\n", + "- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints.\n", + "- [XGrammar](https://github.com/mlc-ai/xgrammar)(default): Supports JSON schema, regular expression, and EBNF constraints.\n", + "- [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints.\n", + "\n", + "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", + "\n", + "To use Outlines, simply add `--grammar-backend outlines` when launching the server.\n", + "To use llguidance, add `--grammar-backend llguidance` when launching the server.\n", + "If no backend is specified, XGrammar will be used as the default.\n", + "\n", + "For better output quality, **It's advisable to explicitly include instructions in the prompt to guide the model to generate the desired format.** For example, you can specify, 'Please generate the output in the following JSON format: ...'.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OpenAI Compatible API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "import os\n", + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON\n", + "\n", + "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Please generate the information of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\n", + " \"name\": \"foo\",\n", + " # convert the pydantic model to json schema\n", + " \"schema\": CapitalInfo.model_json_schema(),\n", + " },\n", + " },\n", + ")\n", + "\n", + "response_content = response.choices[0].message.content\n", + "# validate the JSON response by the pydantic model\n", + "capital_info = CapitalInfo.model_validate_json(response_content)\n", + "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ebnf_grammar = \"\"\"\n", + "root ::= city | description\n", + "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", + "description ::= city \" is \" status\n", + "status ::= \"the capital of \" country\n", + "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", + "\"\"\"\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=32,\n", + " extra_body={\"ebnf\": ebnf_grammar},\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " extra_body={\"regex\": \"(Paris|London)\"},\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool_get_current_weather = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"state\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", + " \" in, e.g. 'CA' which would mean 'California'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"state\", \"unit\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "tool_get_current_date = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_date\",\n", + " \"description\": \"Get the current date and time for a given timezone\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"timezone\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", + " }\n", + " },\n", + " \"required\": [\"timezone\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", + "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"\"\"\n", + "# Tool Instructions\n", + "- Always execute python code in messages that you share.\n", + "- When looking for real time information use relevant functions if available else fallback to brave_search\n", + "You have access to the following functions:\n", + "Use the function 'get_current_weather' to: Get the current weather in a given location\n", + "{tool_get_current_weather[\"function\"]}\n", + "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", + "{tool_get_current_date[\"function\"]}\n", + "If a you choose to call a function ONLY reply in the following format:\n", + "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", + "where\n", + "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", + "end_tag => ``\n", + "Here is an example,\n", + "{{\"example_name\": \"example_value\"}}\n", + "Reminder:\n", + "- Function calls MUST follow the specified format\n", + "- Required parameters MUST be specified\n", + "- Only call one function at a time\n", + "- Put the entire function call reply on one line\n", + "- Always add your sources when using search results to answer the user query\n", + "You are a helpful assistant.\"\"\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=messages,\n", + " response_format={\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\">> |\.\.\. " +copybutton_prompt_is_regexp = True + +autodoc_preserve_defaults = True +navigation_with_keys = False + +autodoc_mock_imports = [ + "torch", + "transformers", + "triton", +] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3.12", None), + "typing_extensions": ("https://typing-extensions.readthedocs.io/en/latest", None), + "pillow": ("https://pillow.readthedocs.io/en/stable", None), + "numpy": ("https://numpy.org/doc/stable", None), + "torch": ("https://pytorch.org/docs/stable", None), +} + +html_theme = "sphinx_book_theme" + + +nbsphinx_prolog = """ +.. raw:: html + + +""" diff --git a/docs/deploy.py b/docs/deploy.py new file mode 100644 index 0000000..75b7ea7 --- /dev/null +++ b/docs/deploy.py @@ -0,0 +1,22 @@ +# Deploy the documents + +import os +from datetime import datetime + + +def run_cmd(cmd): + print(cmd) + os.system(cmd) + + +run_cmd("cd $DOC_SITE_PATH; git pull") + +# (Optional) Remove old files +# run_cmd("rm -rf $ALPA_SITE_PATH/*") + +run_cmd("cp -r _build/html/* $DOC_SITE_PATH") + +cmd_message = f"Update {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" +run_cmd( + f"cd $DOC_SITE_PATH; git add .; git commit -m '{cmd_message}'; git push origin main" +) diff --git a/docs/developer/development_guide_using_docker.md b/docs/developer/development_guide_using_docker.md new file mode 100644 index 0000000..963d04f --- /dev/null +++ b/docs/developer/development_guide_using_docker.md @@ -0,0 +1,52 @@ +# Development Guide Using Docker + +## Setup VSCode + +Download `code` from `Https://code.visualstudio.com/docs/?dv=linux64cli` + +```bash +wget https://vscode.download.prss.microsoft.com/dbazure/download/stable/fabdb6a30b49f79a7aba0f2ad9df9b399473380f/vscode_cli_alpine_x64_cli.tar.gz +tar xf vscode_cli_alpine_x64_cli.tar.gz + +# https://code.visualstudio.com/docs/remote/tunnels +./code tunnel +``` + +## Setup Docker Container + +The following startup command is an example for internal development by the SGLang team. You can **modify or add directory mappings as needed**, especially for model weight downloads, to prevent repeated downloads by different Docker containers. + +❗️ **Note on RDMA** + + 1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them but keeping them there does not harm. Thus, we enable these two flags by default in the commands below. + 2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +### H100 + +```bash +# Change the name to yours +docker run -itd --shm-size 32g --gpus all -v /opt/dlami/nvme/.cache:/root/.cache --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` + +### H200 + +```bash +docker run -itd --shm-size 32g --gpus all -v /mnt/co-research/shared-models:/root/.cache/huggingface --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` + +## Profile + +```bash +# Change batch size, input, output and add `disable-cuda-graph` (for easier analysis) +# e.g. DeepSeek V3 +nsys profile -o deepseek_v3 python3 -m sglang.bench_one_batch --batch-size 1 --input 128 --output 256 --model deepseek-ai/DeepSeek-V3 --trust-remote-code --tp 8 --disable-cuda-graph +``` + +## Evaluation + +```bash +# e.g. gsm8k 8 shot +python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8 +``` diff --git a/docs/developer/release_process.md b/docs/developer/release_process.md new file mode 100644 index 0000000..e817a25 --- /dev/null +++ b/docs/developer/release_process.md @@ -0,0 +1,18 @@ +# PyPI Package Release Process + +## Update the version in code +Update the package version in `python/pyproject.toml` and `python/sglang/__init__.py`. + +## Upload the PyPI package + +``` +pip install build twine +``` + +``` +cd python +bash upload_pypi.sh +``` + +## Make a release in GitHub +Make a new release https://github.com/sgl-project/sglang/releases/new. diff --git a/docs/developer/setup_github_runner.md b/docs/developer/setup_github_runner.md new file mode 100644 index 0000000..55240e7 --- /dev/null +++ b/docs/developer/setup_github_runner.md @@ -0,0 +1,49 @@ +# Set Up Self-Hosted Runners for GitHub Action + +## Add a Runner + +### Step 1: Start a docker container. + +You can mount a folder for the shared huggingface model weights cache. The command below uses `/tmp/huggingface` as an example. + +``` +docker pull nvidia/cuda:12.1.1-devel-ubuntu22.04 +# Nvidia +docker run --shm-size 128g -it -v /tmp/huggingface:/hf_home --gpus all nvidia/cuda:12.1.1-devel-ubuntu22.04 /bin/bash +# AMD +docker run --rm --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.4.4.post3-rocm630 /bin/bash +# AMD just the last 2 GPUs +docker run --rm --device=/dev/kfd --device=/dev/dri/renderD176 --device=/dev/dri/renderD184 --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.4.4.post3-rocm630 /bin/bash +``` + +### Step 2: Configure the runner by `config.sh` + +Run these commands inside the container. + +``` +apt update && apt install -y curl python3-pip git +export RUNNER_ALLOW_RUNASROOT=1 +``` + +Then follow https://github.com/sgl-project/sglang/settings/actions/runners/new?arch=x64&os=linux to run `config.sh` + +**Notes** +- Do not need to specify the runner group +- Give it a name (e.g., `test-sgl-gpu-0`) and some labels (e.g., `1-gpu-runner`). The labels can be editted later in Github Settings. +- Do not need to change the work folder. + +### Step 3: Run the runner by `run.sh` + +- Set up environment variables +``` +export HF_HOME=/hf_home +export SGLANG_IS_IN_CI=true +export HF_TOKEN=hf_xxx +export OPENAI_API_KEY=sk-xxx +export CUDA_VISIBLE_DEVICES=0 +``` + +- Run it forever +``` +while true; do ./run.sh; echo "Restarting..."; sleep 2; done +``` diff --git a/docs/frontend/__init__.py b/docs/frontend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/docs/frontend/choices_methods.md b/docs/frontend/choices_methods.md new file mode 100644 index 0000000..30a0a18 --- /dev/null +++ b/docs/frontend/choices_methods.md @@ -0,0 +1,77 @@ +# Choices Methods in SGLang +This doc describes the choices methods supported by SGLang. + +The optional `choices_method` arg determines how options supplied to SGLang's `choices` primitive are selected. Only the `RuntimeEndpoint` backend supports the `choices_method` arg. Other backends, such as `OpenAI`, have bespoke selection implementations due to API limitations. + +## Methods + +### Token Length Normalized + +Token length normalized is the default SGLang choices method. It selects the option with the highest average logprob across all of its tokens. + +Usage example (alternatively, simply omit the `choices_method` arg): +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.token_length_normalized, + ) + ) +``` + + +This can perform poorly if an option contains many tokens, where its later tokens are predicted with high confidence based on its earlier tokens. For instance, even strong models will fail the above example if the specified options are `["Paris", "Antidisestablishmentarianism"]`. + +### Greedy Token Selection + +Greedy token selection simply selects the option with the highest logprob for its initial token. For overlapping options where one option is a subset of a longer option, the logprobs of the shorter option are extended using its average logprob for comparison against the longer option. + +Usage example: +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.greedy_token_selection, + ) + ) +``` + +This can perform poorly if an option misleads the model down a bad path based on an attractive initial token. For instance, greedy selection will result in an incorrect response for this example: +```python +@sgl.function +def us_president_example(s): + s += sgl.user("Name a US president.") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["Donald Duck", "Millard Fillmore"], + choices_method=sgl.greedy_token_selection, + ) + ) +``` + +### Unconditional Likelihood Normalized + +Unconditional likelihood normalized selects the option with the highest average token logprob once normalized by the unconditional token logprobs, as described in [this EleutherAI blogpost](https://blog.eleuther.ai/multiple-choice-normalization/). This method incurs an additional LLM call to obtain the unconditional likelihoods. + +Usage example: +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.unconditional_likelihood_normalized, + ) + ) +``` diff --git a/docs/frontend/frontend.ipynb b/docs/frontend/frontend.ipynb new file mode 100644 index 0000000..7bc07f6 --- /dev/null +++ b/docs/frontend/frontend.ipynb @@ -0,0 +1,464 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SGLang Frontend Language" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SGLang frontend language can be used to define simple and easy prompts in a convenient, structured way." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "import os\n", + "\n", + "from sglang import assistant_begin, assistant_end\n", + "from sglang import assistant, function, gen, system, user\n", + "from sglang import image\n", + "from sglang import RuntimeEndpoint, set_default_backend\n", + "from sglang.srt.utils import load_image\n", + "from sglang.test.test_utils import is_in_ci\n", + "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the default backend. Note: Besides the local server, you may use also `OpenAI` or other API endpoints." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Usage\n", + "\n", + "The most simple way of using SGLang frontend language is a simple question answer dialog between a user and an assistant." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", max_tokens=512))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "state = basic_qa(\"List 3 countries and their capitals.\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-turn Dialog\n", + "\n", + "SGLang frontend language can also be used to define multi-turn dialogs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def multi_turn_qa(s):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", + " s += assistant(gen(\"first_answer\", max_tokens=512))\n", + " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", + " s += assistant(gen(\"second_answer\", max_tokens=512))\n", + " return s\n", + "\n", + "\n", + "state = multi_turn_qa()\n", + "print_highlight(state[\"first_answer\"])\n", + "print_highlight(state[\"second_answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Control flow\n", + "\n", + "You may use any Python code within the function to define more complex control flows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def tool_use(s, question):\n", + " s += assistant(\n", + " \"To answer this question: \"\n", + " + question\n", + " + \". I need to use a \"\n", + " + gen(\"tool\", choices=[\"calculator\", \"search engine\"])\n", + " + \". \"\n", + " )\n", + "\n", + " if s[\"tool\"] == \"calculator\":\n", + " s += assistant(\"The math expression is: \" + gen(\"expression\"))\n", + " elif s[\"tool\"] == \"search engine\":\n", + " s += assistant(\"The key word to search is: \" + gen(\"word\"))\n", + "\n", + "\n", + "state = tool_use(\"What is 2 * 2?\")\n", + "print_highlight(state[\"tool\"])\n", + "print_highlight(state[\"expression\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parallelism\n", + "\n", + "Use `fork` to launch parallel prompts. Because `sgl.gen` is non-blocking, the for loop below issues two generation calls in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def tip_suggestion(s):\n", + " s += assistant(\n", + " \"Here are two tips for staying healthy: \"\n", + " \"1. Balanced Diet. 2. Regular Exercise.\\n\\n\"\n", + " )\n", + "\n", + " forks = s.fork(2)\n", + " for i, f in enumerate(forks):\n", + " f += assistant(\n", + " f\"Now, expand tip {i+1} into a paragraph:\\n\"\n", + " + gen(\"detailed_tip\", max_tokens=256, stop=\"\\n\\n\")\n", + " )\n", + "\n", + " s += assistant(\"Tip 1:\" + forks[0][\"detailed_tip\"] + \"\\n\")\n", + " s += assistant(\"Tip 2:\" + forks[1][\"detailed_tip\"] + \"\\n\")\n", + " s += assistant(\n", + " \"To summarize the above two tips, I can say:\\n\" + gen(\"summary\", max_tokens=512)\n", + " )\n", + "\n", + "\n", + "state = tip_suggestion()\n", + "print_highlight(state[\"summary\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Constrained Decoding\n", + "\n", + "Use `regex` to specify a regular expression as a decoding constraint. This is only supported for local models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def regular_expression_gen(s):\n", + " s += user(\"What is the IP address of the Google DNS servers?\")\n", + " s += assistant(\n", + " gen(\n", + " \"answer\",\n", + " temperature=0,\n", + " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", + " )\n", + " )\n", + "\n", + "\n", + "state = regular_expression_gen()\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use `regex` to define a `JSON` decoding schema." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "character_regex = (\n", + " r\"\"\"\\{\\n\"\"\"\n", + " + r\"\"\" \"name\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"house\": \"(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)\",\\n\"\"\"\n", + " + r\"\"\" \"blood status\": \"(Pure-blood|Half-blood|Muggle-born)\",\\n\"\"\"\n", + " + r\"\"\" \"occupation\": \"(student|teacher|auror|ministry of magic|death eater|order of the phoenix)\",\\n\"\"\"\n", + " + r\"\"\" \"wand\": \\{\\n\"\"\"\n", + " + r\"\"\" \"wood\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"core\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"length\": [0-9]{1,2}\\.[0-9]{0,2}\\n\"\"\"\n", + " + r\"\"\" \\},\\n\"\"\"\n", + " + r\"\"\" \"alive\": \"(Alive|Deceased)\",\\n\"\"\"\n", + " + r\"\"\" \"patronus\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"bogart\": \"[\\w\\d\\s]{1,16}\"\\n\"\"\"\n", + " + r\"\"\"\\}\"\"\"\n", + ")\n", + "\n", + "\n", + "@function\n", + "def character_gen(s, name):\n", + " s += user(\n", + " f\"{name} is a character in Harry Potter. Please fill in the following information about this character.\"\n", + " )\n", + " s += assistant(gen(\"json_output\", max_tokens=256, regex=character_regex))\n", + "\n", + "\n", + "state = character_gen(\"Harry Potter\")\n", + "print_highlight(state[\"json_output\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batching \n", + "\n", + "Use `run_batch` to run a batch of prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def text_qa(s, question):\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", + "\n", + "\n", + "states = text_qa.run_batch(\n", + " [\n", + " {\"question\": \"What is the capital of the United Kingdom?\"},\n", + " {\"question\": \"What is the capital of France?\"},\n", + " {\"question\": \"What is the capital of Japan?\"},\n", + " ],\n", + " progress_bar=True,\n", + ")\n", + "\n", + "for i, state in enumerate(states):\n", + " print_highlight(f\"Answer {i+1}: {states[i]['answer']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming \n", + "\n", + "Use `stream` to stream the output to the user." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def text_qa(s, question):\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", + "\n", + "\n", + "state = text_qa.run(\n", + " question=\"What is the capital of France?\", temperature=0.1, stream=True\n", + ")\n", + "\n", + "for out in state.text_iter():\n", + " print(out, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complex Prompts\n", + "\n", + "You may use `{system|user|assistant}_{begin|end}` to define complex prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def chat_example(s):\n", + " s += system(\"You are a helpful assistant.\")\n", + " # Same as: s += s.system(\"You are a helpful assistant.\")\n", + "\n", + " with s.user():\n", + " s += \"Question: What is the capital of France?\"\n", + "\n", + " s += assistant_begin()\n", + " s += \"Answer: \" + gen(\"answer\", max_tokens=100, stop=\"\\n\")\n", + " s += assistant_end()\n", + "\n", + "\n", + "state = chat_example()\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-modal Generation\n", + "\n", + "You may use SGLang frontend language to define multi-modal prompts.\n", + "See [here](https://docs.sglang.ai/references/supported_models.html) for supported models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ask a question about an image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def image_qa(s, image_file, question):\n", + " s += user(image(image_file) + question)\n", + " s += assistant(gen(\"answer\", max_tokens=256))\n", + "\n", + "\n", + "image_url = \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + "image_bytes, _ = load_image(image_url)\n", + "state = image_qa(image_bytes, \"What is in the image?\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/frontend/patch.py b/docs/frontend/patch.py new file mode 100644 index 0000000..b39951d --- /dev/null +++ b/docs/frontend/patch.py @@ -0,0 +1,49 @@ +import weakref + +from sglang.utils import execute_shell_command, reserve_port + +DEFAULT_MAX_RUNNING_REQUESTS = 200 +DEFAULT_MAX_TOTAL_TOKENS = 20480 + +import sglang.srt.server_args as server_args_mod + +_original_post_init = server_args_mod.ServerArgs.__post_init__ + + +def patched_post_init(self): + _original_post_init(self) + if self.max_running_requests is None: + self.max_running_requests = DEFAULT_MAX_RUNNING_REQUESTS + if self.max_total_tokens is None: + self.max_total_tokens = DEFAULT_MAX_TOTAL_TOKENS + self.disable_cuda_graph = True + + +server_args_mod.ServerArgs.__post_init__ = patched_post_init + +process_socket_map = weakref.WeakKeyDictionary() + + +def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): + """ + Launch the server using the given command. + If no port is specified, a free port is reserved. + """ + if port is None: + port, lock_socket = reserve_port(host) + else: + lock_socket = None + + extra_flags = ( + f"--max-running-requests {DEFAULT_MAX_RUNNING_REQUESTS} " + f"--max-total-tokens {DEFAULT_MAX_TOTAL_TOKENS} " + f"--disable-cuda-graph" + ) + + full_command = f"{command} --port {port} {extra_flags}" + process = execute_shell_command(full_command) + + if lock_socket is not None: + process_socket_map[process] = lock_socket + + return process, port diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..085e722 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,65 @@ +SGLang Documentation +==================== + +SGLang is a fast serving framework for large language models and vision language models. +It makes your interaction with models faster and more controllable by co-designing the backend runtime and frontend language. +The core features include: + +- **Fast Backend Runtime**: Provides efficient serving with RadixAttention for prefix caching, zero-overhead CPU scheduler, continuous batching, token attention (paged attention), speculative decoding, tensor parallelism, chunked prefill, structured outputs, and quantization (FP8/INT4/AWQ/GPTQ). +- **Flexible Frontend Language**: Offers an intuitive interface for programming LLM applications, including chained generation calls, advanced prompting, control flow, multi-modal inputs, parallelism, and external interactions. +- **Extensive Model Support**: Supports a wide range of generative models (Llama, Gemma, Mistral, QWen, DeepSeek, LLaVA, etc.), embedding models (e5-mistral, gte, mcdse) and reward models (Skywork), with easy extensibility for integrating new models. +- **Active Community**: SGLang is open-source and backed by an active community with industry adoption. + +.. toctree:: + :maxdepth: 1 + :caption: Installation + + start/install.md + +.. toctree:: + :maxdepth: 1 + :caption: Backend Tutorial + + references/deepseek + backend/send_request.ipynb + backend/openai_api_completions.ipynb + backend/openai_api_vision.ipynb + backend/openai_api_embeddings.ipynb + backend/native_api.ipynb + backend/offline_engine_api.ipynb + backend/server_arguments.md + backend/sampling_params.md + backend/hyperparameter_tuning.md + +.. toctree:: + :maxdepth: 1 + :caption: Advanced Features + + backend/speculative_decoding.ipynb + backend/structured_outputs.ipynb + backend/function_calling.ipynb + backend/separate_reasoning.ipynb + backend/custom_chat_template.md + backend/quantization.md + +.. toctree:: + :maxdepth: 1 + :caption: Frontend Tutorial + + frontend/frontend.ipynb + frontend/choices_methods.md + +.. toctree:: + :maxdepth: 1 + :caption: SGLang Router + + router/router.md + +.. toctree:: + :maxdepth: 1 + :caption: References + + references/general + references/hardware + references/advanced_deploy + references/performance_tuning diff --git a/docs/references/accuracy_evaluation.md b/docs/references/accuracy_evaluation.md new file mode 100644 index 0000000..84c1edd --- /dev/null +++ b/docs/references/accuracy_evaluation.md @@ -0,0 +1,60 @@ +# Measuring Model Accuracy in SGLang + +This guide shows how to evaluate model accuracy using SGLang's [built-in benchmarks](https://github.com/sgl-project/sglang/tree/b045841baeff37a5601fcde23fa98bd09d942c36/benchmark). Please include accuracy on crucial benchmarks in your PR if you make modifications on the model side, like the kernel and model architecture. + +## Benchmarking Model Accuracy + +This is a reference workflow for the [MMLU benchmark](https://github.com/sgl-project/sglang/tree/main/benchmark/mmlu). For more details or other benchmarks, please refer to the README in each specific benchmark folder under [sglang/benchmark](https://github.com/sgl-project/sglang/tree/b045841baeff37a5601fcde23fa98bd09d942c36/benchmark). + +```bash +# Step 1: Download the dataset +bash download_data.sh + +# Step 2: Launch the server +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen2.5-Math-1.5B-Instruct \ # Model selection + --port 30000 \ # Network configuration + --mem-fraction-static 0.8 # Memory optimization + +# Step 3: Run the benchmark script +python3 bench_sglang.py --nsub 10 # Test 10 subjects + +# Step 4: Extract the accuracy +cat result.jsonl | grep -oP '"accuracy": \K\d+\.\d+' +``` + +## Customizing Benchmark Scripts + +Some benchmark implementations may differ from ours, causing accuracy discrepancies. To match [[Qwen2.5-Math]](https://github.com/QwenLM/Qwen2.5-Math)'s reported 76.8% GSM8K accuracy, customization is required. + +```python +# The GSM8K benchmark script includes few shot examples for evaluation by default. +# Here we exclude them. +for i in range(len(lines[num_shots:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) +``` + +```python +@sgl.function +def few_shot_gsm8k(s, question): + # System prompt given in https://github.com/QwenLM/Qwen2.5-Math + s += sgl.system("Please reason step by step, and put your final answer within \\boxed{}.") # Include system prompt + s += few_shot_examples + question + # Stopwords given in evaluation/math_eval.py of the Qwen2.5-Math repo + s += sgl.gen( + "answer", max_tokens=2048, stop=["Question", "Assistant:", "", "<|im_end|>", "<|endoftext|>"] + ) +``` + +These adjustments should return the desired accuracy. + +## Extending Evaluation Capabilities + +1. **Contribute New Benchmarks** + * Follow our [contribution guidelines](../references/contribution_guide.md) to add new test scripts +2. **Request Implementations** + * Feel free to open an issue describing your evaluation needs +3. **Use Alternative Tools** + * [OpenCompass](https://opencompass.org.cn) + * [LM Evaluation Harness](https://github.com/EleutherAI/lm-evaluation-harness) diff --git a/docs/references/advanced_deploy.rst b/docs/references/advanced_deploy.rst new file mode 100644 index 0000000..06a1002 --- /dev/null +++ b/docs/references/advanced_deploy.rst @@ -0,0 +1,7 @@ +Multi-Node Deployment +========================== +.. toctree:: + :maxdepth: 1 + + multi_node.md + deploy_on_k8s.md diff --git a/docs/references/amd.md b/docs/references/amd.md new file mode 100644 index 0000000..c4e53eb --- /dev/null +++ b/docs/references/amd.md @@ -0,0 +1,146 @@ +# SGLang on AMD + +This document describes how to set up an AMD-based environment for [SGLang](https://github.com/sgl-project/sglang). If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues) on the SGLang repository. + +## System Configuration + +When using AMD GPUs (such as MI300X), certain system-level optimizations help ensure stable performance. Here we take MI300X as an example. AMD provides official documentation for MI300X optimization and system tuning: + +- [AMD MI300X Tuning Guides](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) + - [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/vllm-benchmark.html) + - [AMD Instinct MI300X System Optimization](https://rocm.docs.amd.com/en/latest/how-to/system-optimization/mi300x.html) + - [AMD Instinct MI300X Workload Optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) + +**NOTE:** We strongly recommend reading these docs and guides entirely to fully utilize your system. + +Below are a few key settings to confirm or enable for SGLang: + +### Update GRUB Settings + +In `/etc/default/grub`, append the following to `GRUB_CMDLINE_LINUX`: + +```text +pci=realloc=off iommu=pt +``` + +Afterward, run `sudo update-grub` (or your distro’s equivalent) and reboot. + +### Disable NUMA Auto-Balancing + +```bash +sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing' +``` + +You can automate or verify this change using [this helpful script](https://github.com/ROCm/triton/blob/rocm_env/scripts/amd/env_check.sh). + +Again, please go through the entire documentation to confirm your system is using the recommended configuration. + +## Installing SGLang + +For general installation instructions, see the official [SGLang Installation Docs](../start/install.md). Below are the AMD-specific steps summarized for convenience. + +### Install from Source + +```bash +git clone https://github.com/sgl-project/sglang.git +cd sglang + +pip install --upgrade pip +pip install sgl-kernel --force-reinstall --no-deps +pip install -e "python[all_hip]" +``` + +### Install Using Docker (Recommended) + +1. Build the docker image. + + ```bash + docker build -t sglang_image -f Dockerfile.rocm . + ``` + +2. Create a convenient alias. + + ```bash + alias drun='docker run -it --rm --network=host --privileged --device=/dev/kfd --device=/dev/dri \ + --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -v $HOME/dockerx:/dockerx \ + -v /data:/data' + ``` + +If you are using RDMA, please note that: + +1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. +2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. +3. Launch the server. + + **NOTE:** Replace `` below with your [huggingface hub token](https://huggingface.co/docs/hub/en/security-tokens). + + ```bash + drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path NousResearch/Meta-Llama-3.1-8B \ + --host 0.0.0.0 \ + --port 30000 + ``` + +4. To verify the utility, you can run a benchmark in another terminal or refer to [other docs](https://docs.sglang.ai/backend/openai_api_completions.html) to send requests to the engine. + + ```bash + drun sglang_image \ + python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 4000 \ + --random-input 128 \ + --random-output 128 + ``` + +With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang’s machine learning capabilities. + +## Examples + +### Running DeepSeek-V3 + +The only difference when running DeepSeek-V3 is in how you start the server. Here's an example command: + +```bash +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3 \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +[Running DeepSeek-R1 on a single NDv5 MI300X VM](https://techcommunity.microsoft.com/blog/azurehighperformancecomputingblog/running-deepseek-r1-on-a-single-ndv5-mi300x-vm/4372726) could also be a good reference. + +### Running Llama3.1 + +Running Llama3.1 is nearly identical to running DeepSeek-V3. The only difference is in the model specified when starting the server, shown by the following example command: + +```bash +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +### Warmup Step + +When the server displays `The server is fired up and ready to roll!`, it means the startup is successful. diff --git a/docs/references/benchmark_and_profiling.md b/docs/references/benchmark_and_profiling.md new file mode 100644 index 0000000..37c4545 --- /dev/null +++ b/docs/references/benchmark_and_profiling.md @@ -0,0 +1,142 @@ +# Benchmark and Profiling + +## Benchmark + +- Benchmark the latency of running a single static batch without a server. The arguments are the same as for `launch_server.py`. + Note that this is a simplified test script without a dynamic batching server, so it may run out of memory for a batch size that a real server can handle. A real server truncates the prefill into several batches, while this simplified script does not. + + ```bash + python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 + ``` + +- Benchmark offline processing. This script will start an offline engine and run the benchmark. + + ```bash + python3 -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 + ``` + +- Benchmark online serving. Please use `sglang.launch_server` to launch a server first and run the following command. + + ```bash + python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + ``` + +## Profile with PyTorch Profiler + +[Pytorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) is a convenient basic tool to inspect kernel execution time, call stack, and kernel overlap and occupancy. + +- To profile a server + + ```bash + # set trace path + export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + + # start server + python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct + + # send profiling request from client + python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile + ``` + + Please make sure that the `SGLANG_TORCH_PROFILER_DIR` should be set at both server and client side, otherwise the trace file cannot be generated correctly . A secure way will be setting `SGLANG_TORCH_PROFILER_DIR` in the `.*rc` file of shell (e.g. `~/.bashrc` for bash shells). + +- To profile offline + + ```bash + export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 + ``` + +- View Traces + + Trace files can be loaded and visualized from: + + 1. https://ui.perfetto.dev/ (any browser) + 2. chrome://tracing (Chrome browser only) + + If browser cannot open trace file due to its large size, + client can generate a small trace file (<100MB) by controlling number of prompts and lengths of prompt outputs. + For example, when profiling a server, + + ```bash + python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 2 --sharegpt-output-len 100 --profile + ``` + + This command sets the number of prompts to 2 with `--num-prompts` argument and limits the length of output sequences to 100 with `--sharegpt-output-len` argument, which can generate a small trace file for browser to open smoothly. + +## Profile with Nsight + +[Nsight systems](https://docs.nvidia.com/nsight-systems/) is an advanced tool that exposes more profiling details, such as register and shared memory usage, annotated code regions and low-level CUDA APIs and events. + +1. Prerequisite: + + Install using apt, or run inside a [NVIDIA Docker container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags) or [SGLang Docker container](https://github.com/sgl-project/sglang/tree/main/docker). + + ```bash + # install nsys + # https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html + apt update + apt install -y --no-install-recommends gnupg + echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu$(source /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d .)/$(dpkg --print-architecture) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list + apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub + apt update + apt install nsight-systems-cli + ``` + +2. To profile a single batch, use + + ```bash + nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node python3 -m sglang.bench_one_batch --model meta-llama/Meta-Llama-3-8B --batch-size 64 --input-len 512 + ``` + +3. To profile a server, e.g. + + ```bash + # launch the server, set the delay and duration times according to needs + # after the duration time has been used up, server will be killed by nsys + + nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node -o sglang.out --delay 60 --duration 70 python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + + # client + python3 -m sglang.bench_serving --backend sglang --num-prompts 1000 --dataset-name random --random-input 1024 --random-output 512 + ``` + + In practice, we recommend users to set `--duration` argument to a large value. Whenever user wants the server to stop profiling. Firstly run: + + ```bash + nsys sessions list + ``` + + to get the session id in the form of `profile-XXXXX`, then run: + + ```bash + nsys stop --session=profile-XXXXX + ``` + + to manually kill the profiler and generate `nsys-rep` files instantly. + +4. Use NVTX to annotate code regions, e.g. to see their execution time. + + ```bash + # install nvtx + pip install nvtx + ``` + + ```python + # code snippets + import nvtx + with nvtx.annotate("description", color="color"): + # some critical code + ``` + +## Other tips + +1. You can benchmark a model using dummy weights by only providing the config.json file. This allows for quick testing of model variants without training. To do so, add `--load-format dummy` to the above commands and then you only need a correct `config.json` under the checkpoint folder. +2. You can benchmark a model with modified configs (e.g., less layers) by using `--json-model-override-args`. For example, you can benchmark a model with only 2 layers and 2 kv heads using: + + ```bash + python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 --load-format dummy --json-model-override-args '{"num_hidden_layers": 1, "num_key_value_heads": 1}' + ``` + +3. You can use `--python-backtrace=cuda` to see python call stack for all CUDA kernels, as in PyTorch Profiler. (Caveat: this can cause inaccurately long kernel runtimes for CUDA event based timing) +4. For more arguments see [Nsight Systems User Guide](https://docs.nvidia.com/nsight-systems/UserGuide/index.html). diff --git a/docs/references/contribution_guide.md b/docs/references/contribution_guide.md new file mode 100644 index 0000000..3d5e331 --- /dev/null +++ b/docs/references/contribution_guide.md @@ -0,0 +1,46 @@ +# Contribution Guide + +Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. + +## Setting Up & Building from Source + +### Fork and Clone the Repository + +**Note**: New contributors do **not** have the write permission to push to the official SGLang repo. Please fork the repository under your GitHub account, then clone your fork locally. + +```bash +git clone https://github.com//sglang.git +``` + +### Install Dependencies & Build + +Refer to [Install SGLang from Source](https://docs.sglang.ai/start/install.html#method-2-from-source) documentation for more details on setting up the necessary dependencies. + +## Code Formatting with Pre-Commit + +We use [pre-commit](https://pre-commit.com/) to maintain consistent code style checks. Before pushing your changes, please run: + +```bash +pip3 install pre-commit +pre-commit install +pre-commit run --all-files +``` + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. +- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. + +## Running Unit Tests & Adding to CI + +SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework. For detailed instructions on running tests and adding them to CI, please refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). + +## Writing Documentation & Running Docs CI + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). + +## Tips for Newcomers + +If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. Also check out this [code walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. + +If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://join.slack.com/t/sgl-fru7574/shared_invite/zt-2um0ad92q-LkU19KQTxCGzlCgRiOiQEw). + +Thank you for your interest in SGLang. Happy coding! diff --git a/docs/references/deepseek.md b/docs/references/deepseek.md new file mode 100644 index 0000000..9a079b9 --- /dev/null +++ b/docs/references/deepseek.md @@ -0,0 +1,166 @@ +# DeepSeek Usage + +SGLang provides several optimizations specifically designed for the DeepSeek model to boost its inference speed. This document outlines current optimizations for DeepSeek. Additionally, the SGLang team is actively developing enhancements for [DeepSeek V3](https://github.com/sgl-project/sglang/issues/2591). + +## Launch DeepSeek V3 with SGLang + +SGLang is recognized as one of the top engines for [DeepSeek model inference](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3). To run DeepSeek V3/R1 models, the requirements are as follows: + +| Weight Type | Configuration | +|------------|-------------------| +| **Full precision FP8**
*(recommended)* | 8 x H200 | +| | 8 x MI300X | +| | 2 x 8 x H100/800/20 | +| **Full precision BF16** | 2 x 8 x H200 | +| | 2 x 8 x MI300X | +| | 4 x 8 x H100/800/20 | +| | 4 x 8 x A100/A800 | +| **Quantized weights (AWQ)** | 8 x H100/800/20 | +| | 8 x A100/A800 | +| **Quantized weights (int8)** | 16 x A100/800 | +| | 32 x L40S | + + + +Detailed commands for reference: + +- [8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) +- [8 x MI300X](https://docs.sglang.ai/references/amd.html#running-deepseek-v3) +- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes) +- [4 x 8 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) +- [8 x A100 (AWQ)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) +- [16 x A100 (int8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) +- [32 x L40S (int8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) + +### Download Weights + +If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. Please refer to [DeepSeek V3](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base#61-inference-with-deepseek-infer-demo-example-only) official guide to download the weights. + +### Caching `torch.compile` + +The DeepSeek series have huge model weights, it takes some time to compile the model with `torch.compile` for the first time if you have added the flag `--enable-torch-compile`. You can refer [here](https://docs.sglang.ai/backend/hyperparameter_tuning.html#try-advanced-options) to optimize the caching of compilation results, so that the cache can be used to speed up the next startup. +### Launch with One node of 8 H200 + +Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended). **Note that Deepseek V3 is already in FP8. So we should not run it with any quantization arguments like `--quantization fp8 --kv-cache-dtype fp8_e5m2`.** Also, `--enable-dp-attention` can be useful to improve for Deepseek V3/R1's throughput. Please refer to [Data Parallelism Attention](https://docs.sglang.ai/references/deepseek.html#multi-head-latent-attention-mla-throughput-optimizations) for detail. + +### Running examples on Multi-node + +- [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). + +- [Serving with two H200*8 nodes and docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker). + +- [Serving with four A100*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes). + +## Optimizations + +### Multi-head Latent Attention (MLA) Throughput Optimizations + +**Description**: [MLA](https://arxiv.org/pdf/2405.04434) is an innovative attention mechanism introduced by the DeepSeek team, aimed at improving inference efficiency. SGLang has implemented specific optimizations for this, including: + +- **Weight Absorption**: By applying the associative law of matrix multiplication to reorder computation steps, this method balances computation and memory access and improves efficiency in the decoding phase. + +- **Flashinfer MLA Wrapper**: By providing `--enable-flashinfer-mla` argument, the server will use MLA kernels customized by Flashinfer. More details can be referred to [this document](https://docs.flashinfer.ai/api/mla.html). Under long input scenarios, flashinfer mla can improve performance significantly. Optimized triton kernels will be used when flashinfer mla is turned off. + +- **FP8 Quantization**: W8A8 FP8 and KV Cache FP8 quantization enables efficient FP8 inference. Additionally, we have implemented Batched Matrix Multiplication (BMM) operator to facilitate FP8 inference in MLA with weight absorption. + +- **CUDA Graph & Torch.compile**: Both MLA and Mixture of Experts (MoE) are compatible with CUDA Graph and Torch.compile, which reduces latency and accelerates decoding speed for small batch sizes. + +Overall, with these optimizations, we have achieved up to **7x** acceleration in output throughput compared to the previous version. + +

+ Multi-head Latent Attention for DeepSeek Series Models +

+ +**Usage**: MLA optimization is enabled by default, to disable, use `--disable-mla`. + +**Reference**: Check [Blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [Slides](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/lmsys_1st_meetup_deepseek_mla.pdf) for more details. + +### Data Parallelism Attention + +**Description**: This optimization involves data parallelism (DP) for the MLA attention mechanism of DeepSeek Series Models, which allows for a significant reduction in the KV cache size, enabling larger batch sizes. Each DP worker independently handles different types of batches (prefill, decode, idle), which are then synchronized before and after processing through the Mixture-of-Experts (MoE) layer. + +

+ Data Parallelism Attention for DeepSeek Series Models +

+ +With data parallelism attention enabled, we have achieved up to **1.9x** decoding throughput improvement compared to the previous version. + +

+ Data Parallelism Attention Performance Comparison +

+ +**Usage**: +- This optimization is aimed at improving throughput and should be used for scenarios with high QPS (Queries Per Second). It can be enabled by `--enable-dp-attention` for DeepSeek models. +- Since v0.4.4, DP and TP attention can be flexibly combined. For example, to deploy DeepSeek-V3/R1 on 2 node with 8*H100, you can specify `--tp 16` and `--dp 2`, which means for attention part there are 2 DP groups, and in each DP group there are 8 TP groups. + +**Reference**: Check [Blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models). + +### Multi Node Tensor Parallelism + +**Description**: For users with limited memory on a single node, SGLang supports serving DeepSeek Series Models, including DeepSeek V3, across multiple nodes using tensor parallelism. This approach partitions the model parameters across multiple GPUs or nodes to handle models that are too large for one node's memory. + +**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-2-h208) for usage examples. + +### Block-wise FP8 + +**Description**: SGLang implements block-wise FP8 quantization with two key optimizations: + +- **Activation**: E4M3 format using per-token-per-128-channel sub-vector scales with online casting. + +- **Weight**: Per-128x128-block quantization for better numerical stability. + +**Usage**: Turn on by default for DeepSeek V3 models. + +### Multi-token Prediction +**Description**: SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](https://docs.sglang.ai/backend/speculative_decoding.html#EAGLE-Decoding). With this optimization, the decoding speed can be improved by **1.8x** for batch size 1 and **1.5x** for batch size 32 respectively on H200 TP8 setting. + +**Usage**: +Add arguments `--speculative-algorithm`, `--speculative-draft-model-path`, +`--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: +``` +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --speculative-algorithm EAGLE --speculative-draft-model-path lmsys/DeepSeek-V3-0324-NextN --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 --trust-remote-code --tp 8 +``` +- The draft model are available at huggingface: [lmsys/DeepSeek-V3-0324-NextN](https://huggingface.co/lmsys/DeepSeek-V3-0324-NextN), [lmsys/DeepSeek-R1-NextN](https://huggingface.co/lmsys/DeepSeek-R1-NextN). It can also be exported from original DeepSeek-V3/R1 model with [export_deepseek_nextn.py](https://github.com/sgl-project/sglang/blob/main/scripts/export_deepseek_nextn.py) script. +- The best configuratin for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. +- Currently when using flashinfer mla wrapper (`--enable-flashinfer-mla`) and speculative decoding together, the `--speculative-eagle-topk` parameter should be set to `1`. +- To enable DeepSeek MTP for large batch sizes (>32), there are some parameters should be changed (Reference [this discussion](https://github.com/sgl-project/sglang/issues/4543#issuecomment-2737413756)): + - Adjust `--max-running-requests` to a larger number. The default value is `32` for MTP. For larger batch sizes, you should increase this value beyond the default value. + - Set `--cuda-graph-bs`. It's a list of batch sizes for cuda graph capture. The default captured batch sizes for speculative decoding is set [here](https://github.com/sgl-project/sglang/blob/49420741746c8f3e80e0eb17e7d012bfaf25793a/python/sglang/srt/model_executor/cuda_graph_runner.py#L126). You can include more batch sizes into it. + + +### Reasoning Content for DeepSeek R1 + +See [Separate Reasoning](https://docs.sglang.ai/backend/separate_reasoning.html). + +## FAQ + +1. **Question**: What should I do if model loading takes too long and NCCL timeout occurs? + + **Answer**: You can try to add `--dist-timeout 3600` when launching the model, this allows for 1-hour timeout. diff --git a/docs/references/deepseek.rst b/docs/references/deepseek.rst new file mode 100644 index 0000000..b45383a --- /dev/null +++ b/docs/references/deepseek.rst @@ -0,0 +1,6 @@ +Multi-Node Deployment +========================== +.. toctree:: + :maxdepth: 1 + + deepseek.md diff --git a/docs/references/deploy_on_k8s.md b/docs/references/deploy_on_k8s.md new file mode 100644 index 0000000..6fb1afb --- /dev/null +++ b/docs/references/deploy_on_k8s.md @@ -0,0 +1,345 @@ +# Deploy On Kubernetes + +This document is for deploying a RoCE network-based SGLang two-node inference service on a Kubernetes (K8S) cluster. + +[LeaderWorkerSet (LWS)](https://github.com/kubernetes-sigs/lws) is a Kubernetes API that aims to address common deployment patterns of AI/ML inference workloads. A major use case is for multi-host/multi-node distributed inference. + +SGLang can also be deployed with LWS on Kubernetes for distributed model serving. + +Please see this guide for more details on deploying SGLang on Kubernetes using LWS. + +Here we take the deployment of DeepSeek-R1 as an example. + +## Prerequisites + +1. At least two Kubernetes nodes, each with two H20 systems and eight GPUs, are required. + +2. Make sure your K8S cluster has LWS correctly installed. If it hasn't been set up yet, please follow the [installation instructions](https://github.com/kubernetes-sigs/lws/blob/main/site/content/en/docs/installation/_index.md). + +## Basic example + +For the basic example documentation, refer to [Deploy Distributed Inference Service with SGLang and LWS on GPUs](https://github.com/kubernetes-sigs/lws/tree/main/docs/examples/sglang). + +However, that document only covers the basic NCCL socket mode. + +In this section, we’ll make some simple modifications to adapt the setup to the RDMA scenario. + +## RDMA RoCE case + +* Check your env: + +```bash +[root@node1 ~]# ibstatus +Infiniband device 'mlx5_bond_0' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe64:c79a + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_1' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe6e:c3ec + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_2' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe73:0dd7 + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_3' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe36:f7ff + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet +``` + +* Prepare the `lws.yaml` file for deploying on k8s. + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: sglang +spec: + replicas: 1 + leaderWorkerTemplate: + size: 2 + restartPolicy: RecreateGroupOnPodRestart + leaderTemplate: + metadata: + labels: + role: leader + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostIPC: true + containers: + - name: sglang-leader + image: sglang:latest + securityContext: + privileged: true + env: + - name: NCCL_IB_GID_INDEX + value: "3" + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --mem-fraction-static + - "0.93" + - --torch-compile-max-bs + - "8" + - --max-running-requests + - "20" + - --tp + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20000 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --host + - "0.0.0.0" + - --port + - "40000" + resources: + limits: + nvidia.com/gpu: "8" + ports: + - containerPort: 40000 + readinessProbe: + tcpSocket: + port: 40000 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: model + mountPath: /work/models + - name: ib + mountPath: /dev/infiniband + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: model + hostPath: + path: '< your models dir >' # modify it according your models dir + - name: ib + hostPath: + path: /dev/infiniband + workerTemplate: + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostIPC: true + containers: + - name: sglang-worker + image: sglang:latest + securityContext: + privileged: true + env: + - name: NCCL_IB_GID_INDEX + value: "3" + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --mem-fraction-static + - "0.93" + - --torch-compile-max-bs + - "8" + - --max-running-requests + - "20" + - --tp + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20000 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + resources: + limits: + nvidia.com/gpu: "8" + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: model + mountPath: /work/models + - name: ib + mountPath: /dev/infiniband + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: ib + hostPath: + path: /dev/infiniband + - name: model + hostPath: + path: /data1/models/deepseek_v3_moe +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-leader +spec: + selector: + leaderworkerset.sigs.k8s.io/name: sglang + role: leader + ports: + - protocol: TCP + port: 40000 + targetPort: 40000 + +``` + +* Then use `kubectl apply -f lws.yaml` you will get this output. + +```text +NAME READY STATUS RESTARTS AGE +sglang-0 0/1 Running 0 9s +sglang-0-1 1/1 Running 0 9s +``` + +Wait for the sglang leader (`sglang-0`) status to change to 1/1, which indicates it is `Ready`. + +You can use the command `kubectl logs -f sglang-0` to view the logs of the leader node. + +Once successful, you should see output like this: + +```text +[2025-02-17 05:27:24 TP1] Capture cuda graph end. Time elapsed: 84.89 s +[2025-02-17 05:27:24 TP6] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP0] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP7] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP3] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP2] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP4] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP1] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP5] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24] INFO: Started server process [1] +[2025-02-17 05:27:24] INFO: Waiting for application startup. +[2025-02-17 05:27:24] INFO: Application startup complete. +[2025-02-17 05:27:24] INFO: Uvicorn running on http://0.0.0.0:40000 (Press CTRL+C to quit) +[2025-02-17 05:27:25] INFO: 127.0.0.1:48908 - "GET /get_model_info HTTP/1.1" 200 OK +[2025-02-17 05:27:25 TP0] Prefill batch. #new-seq: 1, #new-token: 7, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0 +[2025-02-17 05:27:32] INFO: 127.0.0.1:48924 - "POST /generate HTTP/1.1" 200 OK +[2025-02-17 05:27:32] The server is fired up and ready to roll! +``` + +If it doesn’t start up successfully, please follow these steps to check for any remaining issues. Thanks! + +### Debug + +* Set `NCCL_DEBUG=TRACE` to check if it is a NCCL communication problem. + +This should resolve most NCCL-related issues. + +***Notice: If you find that NCCL_DEBUG=TRACE is not effective in the container environment, but the process is stuck or you encounter hard-to-diagnose issues, try switching to a different container image. Some images may not handle standard error output properly.*** + +#### RoCE scenario + +* Please make sure that RDMA devices are available in the cluster environment. +* Please make sure that the nodes in the cluster have Mellanox NICs with RoCE. In this example, we use Mellanox ConnectX 5 model NICs, and the proper OFED driver has been installed. If not, please refer to the document [Install OFED Driver](https://docs.nvidia.com/networking/display/mlnxofedv461000/installing+mellanox+ofed) to install the driver. +* Check your env: + + ```shell + $ lspci -nn | grep Eth | grep Mellanox + 0000:7f:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:7f:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:c7:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:c7:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:08:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:08:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:a2:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:a2:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + ``` + +* Check the OFED driver: + + ```shell + ofed_info -s + OFED-internal-23.07-0.5.0: + ``` + +* Show RDMA link status and check IB devices: + + ```shell + $ rdma link show + 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 + 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 + 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 + 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 + + $ ibdev2netdev + 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 + 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 + 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 + 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 + ``` + +* Test RoCE network speed on the host: + + ```shell + yum install qperf + # for server： + execute qperf + # for client + qperf -t 60 -cm1 rc_rdma_write_bw + ``` + +* Check RDMA accessible in your container: + + ```shell + # ibv_devices + # ibv_devinfo + ``` + +## Keys to success + +* In the YAML configuration above, pay attention to the NCCL environment variable. For older versions of NCCL, you should check the NCCL_IB_GID_INDEX environment setting. +* NCCL_SOCKET_IFNAME is also crucial, but in a containerized environment, this typically isn’t an issue. +* In some cases, it’s necessary to configure GLOO_SOCKET_IFNAME correctly. +* NCCL_DEBUG is essential for troubleshooting, but I've found that sometimes it doesn't show error logs within containers. This could be related to the Docker image you're using. You may want to try switching images if needed. +* Avoid using Docker images based on Ubuntu 18.04, as they tend to have compatibility issues. + +## Remaining issues + +* In Kubernetes, Docker, or Containerd environments, we use hostNetwork to prevent performance degradation. +* We utilize privileged mode, which isn’t secure. Additionally, in containerized environments, full GPU isolation cannot be achieved. + +## TODO + +* Integrated with [k8s-rdma-shared-dev-plugin](https://github.com/Mellanox/k8s-rdma-shared-dev-plugin). diff --git a/docs/references/faq.md b/docs/references/faq.md new file mode 100644 index 0000000..eb43ee6 --- /dev/null +++ b/docs/references/faq.md @@ -0,0 +1,15 @@ +# Frequently Asked Questions + +## The results are not deterministic, even with a temperature of 0 + +You may notice that when you send the same request twice, the results from the engine will be slightly different, even when the temperature is set to 0. + +From our initial investigation, this indeterminism arises from two factors: dynamic batching and prefix caching. Roughly speaking, dynamic batching accounts for about 95% of the indeterminism, while prefix caching accounts for the remaining portion. The server runs dynamic batching under the hood. Different batch sizes can cause PyTorch/CuBLAS to dispatch to different CUDA kernels, which can lead to slight numerical differences. This difference accumulates across many layers, resulting in nondeterministic output when the batch size changes. Similarly, when prefix caching is enabled, it can also dispatch to different kernels. Even when the computations are mathematically equivalent, small numerical differences from different kernel implementations lead to the final nondeterministic outputs. + +To achieve more deterministic outputs in the current code, you can add `--disable-radix-cache` and send only one request at a time. The results will be mostly deterministic under this setting. + +We are still investigating the root causes and potential solutions. In the short term, we may introduce a "deterministic mode" that uses more padding to address the variance caused by dynamic batching. This mode will be more deterministic but slower. + +We have two issues to track our progress: +- The deterministic mode is tracked at [https://github.com/sgl-project/sglang/issues/1729](https://github.com/sgl-project/sglang/issues/1729). +- The per-request random seed is tracked at [https://github.com/sgl-project/sglang/issues/1335](https://github.com/sgl-project/sglang/issues/1335). diff --git a/docs/references/general.rst b/docs/references/general.rst new file mode 100644 index 0000000..8ea335d --- /dev/null +++ b/docs/references/general.rst @@ -0,0 +1,14 @@ + +General Guidance +========== + +.. toctree:: + :maxdepth: 1 + + supported_models.md + contribution_guide.md + troubleshooting.md + faq.md + learn_more.md + modelscope.md + production_metrics.md diff --git a/docs/references/hardware.rst b/docs/references/hardware.rst new file mode 100644 index 0000000..0500e54 --- /dev/null +++ b/docs/references/hardware.rst @@ -0,0 +1,7 @@ +Hardware Supports +========== +.. toctree:: + :maxdepth: 1 + + amd.md + nvidia_jetson.md diff --git a/docs/references/learn_more.md b/docs/references/learn_more.md new file mode 100644 index 0000000..d5c430e --- /dev/null +++ b/docs/references/learn_more.md @@ -0,0 +1,3 @@ +# Learn more + +You can find more blogs, slides, and videos about SGLang at [https://github.com/sgl-project/sgl-learning-materials](https://github.com/sgl-project/sgl-learning-materials). diff --git a/docs/references/modelscope.md b/docs/references/modelscope.md new file mode 100644 index 0000000..4740c27 --- /dev/null +++ b/docs/references/modelscope.md @@ -0,0 +1,28 @@ +# Use Models From ModelScope + +To use a model from [ModelScope](https://www.modelscope.cn), set the environment variable `SGLANG_USE_MODELSCOPE`. + +```bash +export SGLANG_USE_MODELSCOPE=true +``` + +We take [Qwen2-7B-Instruct](https://www.modelscope.cn/models/qwen/qwen2-7b-instruct) as an example. + +Launch the Server: +```bash +python -m sglang.launch_server --model-path qwen/Qwen2-7B-Instruct --port 30000 +``` + +Or start it by docker: + +```bash +docker run --gpus all \ + -p 30000:30000 \ + -v ~/.cache/modelscope:/root/.cache/modelscope \ + --env "SGLANG_USE_MODELSCOPE=true" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 30000 +``` + +Note that modelscope uses a different cache directory than huggingface. You may need to set it manually to avoid running out of disk space. diff --git a/docs/references/multi_node.md b/docs/references/multi_node.md new file mode 100644 index 0000000..0d9abc1 --- /dev/null +++ b/docs/references/multi_node.md @@ -0,0 +1,92 @@ +# Multi-Node Deployment + +## Llama 3.1 405B + +**Run 405B (fp16) on Two Nodes** + +```bash +# replace 172.16.4.52:20000 with your own node ip address and port of the first node + +python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct --tp 16 --dist-init-addr 172.16.4.52:20000 --nnodes 2 --node-rank 0 + +# replace 172.18.45.52:20000 with your own node ip address and port of the second node + +python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct --tp 16 --dist-init-addr 172.18.45.52:20000 --nnodes 2 --node-rank 1 +``` + +Note that LLama 405B (fp8) can also be launched on a single node. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 --tp 8 +``` + +## DeepSeek V3/R1 + +Please refer to [DeepSeek documents for reference](https://docs.sglang.ai/references/deepseek.html#running-examples-on-multi-node). + +## Multi-Node Inference on SLURM + +This example showcases how to serve SGLang server across multiple nodes by SLURM. Submit the following job to the SLURM cluster. + +``` +#!/bin/bash -l + +#SBATCH -o SLURM_Logs/%x_%j_master.out +#SBATCH -e SLURM_Logs/%x_%j_master.err +#SBATCH -D ./ +#SBATCH -J Llama-405B-Online-Inference-TP16-SGL + +#SBATCH --nodes=2 +#SBATCH --ntasks=2 +#SBATCH --ntasks-per-node=1 # Ensure 1 task per node +#SBATCH --cpus-per-task=18 +#SBATCH --mem=224GB +#SBATCH --partition="lmsys.org" +#SBATCH --gres=gpu:8 +#SBATCH --time=12:00:00 + +echo "[INFO] Activating environment on node $SLURM_PROCID" +if ! source ENV_FOLDER/bin/activate; then + echo "[ERROR] Failed to activate environment" >&2 + exit 1 +fi + +# Define parameters +model=MODEL_PATH +tp_size=16 + +echo "[INFO] Running inference" +echo "[INFO] Model: $model" +echo "[INFO] TP Size: $tp_size" + +# Set NCCL initialization address using the hostname of the head node +HEAD_NODE=$(scontrol show hostname "$SLURM_NODELIST" | head -n 1) +NCCL_INIT_ADDR="${HEAD_NODE}:8000" +echo "[INFO] NCCL_INIT_ADDR: $NCCL_INIT_ADDR" + +# Launch the model server on each node using SLURM +srun --ntasks=2 --nodes=2 --output="SLURM_Logs/%x_%j_node$SLURM_NODEID.out" \ + --error="SLURM_Logs/%x_%j_node$SLURM_NODEID.err" \ + python3 -m sglang.launch_server \ + --model-path "$model" \ + --grammar-backend "xgrammar" \ + --tp "$tp_size" \ + --dist-init-addr "$NCCL_INIT_ADDR" \ + --nnodes 2 \ + --node-rank "$SLURM_NODEID" & + +# Wait for the NCCL server to be ready on port 30000 +while ! nc -z "$HEAD_NODE" 30000; do + sleep 1 + echo "[INFO] Waiting for $HEAD_NODE:30000 to accept connections" +done + +echo "[INFO] $HEAD_NODE:30000 is ready to accept connections" + +# Keep the script running until the SLURM job times out +wait +``` + +Then, you can test the server by sending requests following other [documents](https://docs.sglang.ai/backend/openai_api_completions.html). + +Thanks for [aflah02](https://github.com/aflah02) for providing the example, based on his [blog post](https://aflah02.substack.com/p/multi-node-llm-inference-with-sglang). diff --git a/docs/references/nvidia_jetson.md b/docs/references/nvidia_jetson.md new file mode 100644 index 0000000..1dc4850 --- /dev/null +++ b/docs/references/nvidia_jetson.md @@ -0,0 +1,76 @@ +# Apply SGLang on NVIDIA Jetson Orin + +## Prerequisites + +Before starting, ensure the following: + +- [**NVIDIA Jetson AGX Orin Devkit**](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/) is set up with **JetPack 6.1** or later. +- **CUDA Toolkit** and **cuDNN** are installed. +- Verify that the Jetson AGX Orin is in **high-performance mode**: +```bash +sudo nvpmodel -m 0 +``` +* * * * * +## Installing and running SGLang with Jetson Containers +Clone the jetson-containers github repository: +``` +git clone https://github.com/dusty-nv/jetson-containers.git +``` +Run the installation script: +``` +bash jetson-containers/install.sh +``` +Build the container: +``` +CUDA_VERSION=12.6 jetson-containers build sglang +``` +Run the container: +``` +docker run --runtime nvidia -it --rm --network=host IMAGE_NAME +``` +* * * * * + +Running Inference +----------------------------------------- + +Launch the server: +```bash +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --device cuda \ + --dtype half \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 +``` +The quantization and limited context length (`--dtype half --context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../backend/server_arguments.md). + +After launching the engine, refer to [Chat completions](https://docs.sglang.ai/backend/openai_api_completions.html#Usage) to test the usability. +* * * * * +Running quantization with TorchAO +------------------------------------- +TorchAO is suggested to NVIDIA Jetson Orin. +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --device cuda \ + --dtype bfloat16 \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 \ + --torchao-config int4wo-128 +``` +This enables TorchAO's int4 weight-only quantization with a 128-group size. The usage of `--torchao-config int4wo-128` is also for memory efficiency. + + +* * * * * +Structured output with XGrammar +------------------------------- +Please refer to [SGLang doc structured output](../backend/structured_outputs.ipynb). +* * * * * + +Thanks to the support from [shahizat](https://github.com/shahizat). + +References +---------- +- [NVIDIA Jetson AGX Orin Documentation](https://developer.nvidia.com/embedded/jetson-agx-orin) diff --git a/docs/references/performance_tuning.rst b/docs/references/performance_tuning.rst new file mode 100644 index 0000000..6cc20e0 --- /dev/null +++ b/docs/references/performance_tuning.rst @@ -0,0 +1,7 @@ +Performance Tuning +==================== +.. toctree:: + :maxdepth: 1 + + benchmark_and_profiling.md + accuracy_evaluation.md diff --git a/docs/references/production_metrics.md b/docs/references/production_metrics.md new file mode 100644 index 0000000..7183158 --- /dev/null +++ b/docs/references/production_metrics.md @@ -0,0 +1,173 @@ +# Production Metrics + +SGLang exposes the following metrics via Prometheus. The metrics are namespaced by `$name` (the model name). + +An example of the monitoring dashboard is available in [examples/monitoring/grafana.json](../examples/monitoring/grafana.json). + +Here is an example of the metrics: + +``` +$ curl http://localhost:30000/metrics +# HELP sglang:prompt_tokens_total Number of prefill tokens processed. +# TYPE sglang:prompt_tokens_total counter +sglang:prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 8.128902e+06 +# HELP sglang:generation_tokens_total Number of generation tokens processed. +# TYPE sglang:generation_tokens_total counter +sglang:generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.557572e+06 +# HELP sglang:token_usage The token usage +# TYPE sglang:token_usage gauge +sglang:token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.28 +# HELP sglang:cache_hit_rate The cache hit rate +# TYPE sglang:cache_hit_rate gauge +sglang:cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.007507552643049313 +# HELP sglang:time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE sglang:time_to_first_token_seconds histogram +sglang:time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2.3518979474117756e+06 +sglang:time_to_first_token_seconds_bucket{le="0.001",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_to_first_token_seconds_bucket{le="0.06",model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.0 +sglang:time_to_first_token_seconds_bucket{le="0.08",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.25",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 27.0 +sglang:time_to_first_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 140.0 +sglang:time_to_first_token_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 314.0 +sglang:time_to_first_token_seconds_bucket{le="7.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 941.0 +sglang:time_to_first_token_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1330.0 +sglang:time_to_first_token_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1970.0 +sglang:time_to_first_token_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2326.0 +sglang:time_to_first_token_seconds_bucket{le="25.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2417.0 +sglang:time_to_first_token_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2513.0 +sglang:time_to_first_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +sglang:time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds +# TYPE sglang:e2e_request_latency_seconds histogram +sglang:e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.116093850019932e+06 +sglang:e2e_request_latency_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:e2e_request_latency_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="0.8",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="1.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="2.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.0 +sglang:e2e_request_latency_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 10.0 +sglang:e2e_request_latency_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11.0 +sglang:e2e_request_latency_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 14.0 +sglang:e2e_request_latency_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 247.0 +sglang:e2e_request_latency_seconds_bucket{le="40.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 486.0 +sglang:e2e_request_latency_seconds_bucket{le="50.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 845.0 +sglang:e2e_request_latency_seconds_bucket{le="60.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1513.0 +sglang:e2e_request_latency_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +sglang:e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +# HELP sglang:time_per_output_token_seconds Histogram of time per output token in seconds. +# TYPE sglang:time_per_output_token_seconds histogram +sglang:time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 866964.5791549598 +sglang:time_per_output_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_per_output_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 73.0 +sglang:time_per_output_token_seconds_bucket{le="0.015",model_name="meta-llama/Llama-3.1-8B-Instruct"} 382.0 +sglang:time_per_output_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 593.0 +sglang:time_per_output_token_seconds_bucket{le="0.025",model_name="meta-llama/Llama-3.1-8B-Instruct"} 855.0 +sglang:time_per_output_token_seconds_bucket{le="0.03",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1035.0 +sglang:time_per_output_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1815.0 +sglang:time_per_output_token_seconds_bucket{le="0.05",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11685.0 +sglang:time_per_output_token_seconds_bucket{le="0.075",model_name="meta-llama/Llama-3.1-8B-Instruct"} 433413.0 +sglang:time_per_output_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 4.950195e+06 +sglang:time_per_output_token_seconds_bucket{le="0.15",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.039435e+06 +sglang:time_per_output_token_seconds_bucket{le="0.2",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.171662e+06 +sglang:time_per_output_token_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.266055e+06 +sglang:time_per_output_token_seconds_bucket{le="0.4",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.296752e+06 +sglang:time_per_output_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.312226e+06 +sglang:time_per_output_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.339675e+06 +sglang:time_per_output_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.357747e+06 +sglang:time_per_output_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.389414e+06 +sglang:time_per_output_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +sglang:time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +# HELP sglang:func_latency_seconds Function latency in seconds +# TYPE sglang:func_latency_seconds histogram +sglang:func_latency_seconds_sum{name="generate_request"} 4.514771912145079 +sglang:func_latency_seconds_bucket{le="0.05",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.07500000000000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.1125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.16875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.253125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.3796875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.56953125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.8542968750000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="1.2814453125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="1.9221679687500002",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="2.8832519531250003",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="4.3248779296875",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="6.487316894531251",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="9.730975341796876",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="14.596463012695313",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="21.89469451904297",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="32.84204177856446",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="49.26306266784668",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="+Inf",name="generate_request"} 14007.0 +sglang:func_latency_seconds_count{name="generate_request"} 14007.0 +# HELP sglang:num_running_reqs The number of running requests +# TYPE sglang:num_running_reqs gauge +sglang:num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 162.0 +# HELP sglang:num_used_tokens The number of used tokens +# TYPE sglang:num_used_tokens gauge +sglang:num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct"} 123859.0 +# HELP sglang:gen_throughput The generate throughput (token/s) +# TYPE sglang:gen_throughput gauge +sglang:gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct"} 86.50814177726902 +# HELP sglang:num_queue_reqs The number of requests in the waiting queue +# TYPE sglang:num_queue_reqs gauge +sglang:num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2826.0 +``` + +## Setup Guide + +To setup a monitoring dashboard, you can use the following docker compose file: [examples/monitoring/docker-compose.yaml](../examples/monitoring/docker-compose.yaml). + +Assume you have sglang server running at `localhost:30000`, to start the server, ensure you have `--enable-metrics` flag enabled: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ +--port 30000 --host 0.0.0.0 --enable-metrics +``` + +To start the monitoring dashboard (prometheus + grafana), cd to `examples/monitoring` and run: + +```bash +docker compose -f compose.yaml -p monitoring up +``` + +Then you can access the Grafana dashboard at http://localhost:3000. + +### Grafana Dashboard + +In a new Grafana setup, ensure that you have the `Prometheus` data source enabled. To check that, go to `http://localhost:3000/connections/datasources` and ensure that `Prometheus` is enabled. + +If not, click `Add data source` -> `Prometheus`, set Prometheus URL to `http://localhost:9090`, and click `Save & Test`. + +To import the Grafana dashboard, click `+` -> `Import` -> `Upload JSON file` -> `Upload` and select [grafana.json](../examples/monitoring/grafana.json). + +### Troubleshooting + +#### Check if the variables are created + +The example dashboard assume you have the following variables avaliable: +- `model_name` (name: `model_name`, label: `model name`, Data source: `Prometheus`, Type: `Label values`) +- `instance` (name: `instance`, label: `instance`, Data source: `Prometheus`, Type: `Label values`) + +If you don't have these variables, you can create them manually. + +To create a variable, go to dashboard settings, `Variables` -> `New variable`. + +You should be able to see the preview the values (e.g. `meta-llama/Llama-3.1-8B-Instruct` for `model_name`). + +#### Check if the metrics are being collected + +Run `python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5` to generate some requests. + +Then you should be able to see the metrics in the Grafana dashboard. diff --git a/docs/references/supported_models.md b/docs/references/supported_models.md new file mode 100644 index 0000000..790e9a8 --- /dev/null +++ b/docs/references/supported_models.md @@ -0,0 +1,145 @@ +# Supported Models + +## Generative Models +- Llama / Llama 2 / Llama 3 / Llama 3.1 / Llama 3.2 / Llama 3.3 +- Mistral / Mixtral / Mistral NeMo / Mistral Small 3 +- Gemma / Gemma 2 / Gemma3 +- Qwen / Qwen 2 / Qwen 2 MoE / Qwen 2 VL / Qwen 2.5 VL / Olympic Coder +- DeepSeek / DeepSeek 2 / [DeepSeek 3](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3) +- OLMoE +- [LLaVA-OneVision](https://llava-vl.github.io/blog/2024-08-05-llava-onevision/) + - `python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-7b-ov --port=30000 --chat-template=chatml-llava` + - `python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-72b-ov --port=30000 --tp-size=8 --chat-template=chatml-llava` + - Query the server with the [OpenAI Vision API](https://platform.openai.com/docs/guides/vision). See examples at [test/srt/test_vision_openai_server.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server.py) +- LLaVA 1.5 / 1.6 / NeXT + - `python -m sglang.launch_server --model-path lmms-lab/llama3-llava-next-8b --port=30000 --tp-size=1 --chat-template=llava_llama_3` + - `python -m sglang.launch_server --model-path lmms-lab/llava-next-72b --port=30000 --tp-size=8 --chat-template=chatml-llava` + - Query the server with the [OpenAI Vision API](https://platform.openai.com/docs/guides/vision). See examples at [test/srt/test_vision_openai_server.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server.py) +- Yi-VL +- StableLM +- Command-R +- DBRX +- Grok +- ChatGLM +- InternLM 2 +- Exaone 3 +- BaiChuan2 +- MiniCPM / MiniCPM 3 / MiniCPM-v / MiniCPM-o +- XVERSE / XVERSE MoE +- SmolLM +- GLM-4 +- Phi-3 / Phi-4 +- Phi-3-Small +- IBM Granite 3 +- Janus-Pro-1B / Janus-Pro-7B +- Deepseek-VL2 / Deepseek-VL2-small +- Gemma 3 (it) + +## Embedding Models + +- LlamaEmbeddingModel +- Mistral embedding models +- Qwen embedding models + - `python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-7B-instruct --is-embedding` +- Multi-modal embedding models + - `python -m sglang.launch_server --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct --is-embedding --chat-template gme-qwen2-vl` +- CLIP + - `python -m sglang.launch_server --model-path openai/clip-vit-large-patch14-336 --is-embedding` + +## Reward Models + +- LlamaForSequenceClassification + - `python -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --is-embedding` +- Gemma2ForSequenceClassification + - `python -m sglang.launch_server --model-path Skywork/Skywork-Reward-Gemma-2-27B-v0.2 --is-embedding` +- InternLM2ForRewardModel + - `python -m sglang.launch_server --model-path internlm/internlm2-7b-reward --is-embedding --trust-remote-code` +- Qwen2ForRewardModel + - `python -m sglang.launch_server --model-path Qwen/Qwen2.5-Math-RM-72B --is-embedding --trust-remote-code --tp-size=4` +- Qwen2ForSequenceClassification + - `python -m sglang.launch_server --model-path jason9693/Qwen2.5-1.5B-apeach --is-embedding --trust-remote-code` +## How to Support a New Language Model + +To support a new model in SGLang, you only need to add a single file under [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). +You can learn from existing model implementations and create new files for the new models. +For most models, you should be able to find a similar model to start with (e.g., starting from Llama). + +## How to Support a New vLM + +To support a new vision-language model (vLM) in SGLang, there are several key components in addition to the standard +LLM. + +1. **Register your new model as multimodal**: Extend `is_multimodal_model` in [ + `model_config.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/configs/model_config.py) to + return True for your model. +2. **Process Images**: Define a new `Processor` class that inherits from `BaseProcessor` and register this + processor as your model's dedicated processor. See [ + `multimodal_processor.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/managers/multimodal_processor.py) + for more details. +3. **Handle Image Tokens**: Implement a `pad_input_ids` function for your new model, in which image tokens in the prompt + should be expanded and replaced with image-hashes, so that SGLang can recognize different images for + `RadixAttention`. +4. Replace Multi-headed `Attention` of ViT with SGLang's `VisionAttention`. + +You can refer [Qwen2VL](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/qwen2_vl.py) or other +vLMs. These models demonstrate how to properly handle both multimodal and textual inputs. + +You should test the new vLM locally against hf models. See [`mmmu`](https://github.com/sgl-project/sglang/tree/main/benchmark/mmmu) for an example. + +### Test the correctness + +#### Interactive debugging +For interactive debugging, you can compare the outputs of huggingface/transformers and SGLang. +The following two commands should give the same text output and very similar prefill logits. + +- Get the reference output by `python3 scripts/playground/reference_hf.py --model-path [new model] --model-type {text,vlm}` +- Get the SGLang output by `python3 -m sglang.bench_one_batch --correct --model [new model]` + +#### Add the model to the test suite +To make sure the new model is well maintained in the future, it is better to add it to the test suite. +You can add it to the `ALL_OTHER_MODELS` list in the [test_generation_models.py](https://github.com/sgl-project/sglang/blob/main/test/srt/models/test_generation_models.py) and run the following command to test it. + +For example, if the model is Qwen/Qwen2-1.5B +``` +ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others +``` + +### Port a model from vLLM to SGLang +Another valuable resource is the [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models). vLLM has extensive coverage of models, and SGLang reuses vLLM's interface and some layers to implement the models. This similarity makes it easy to port many models from vLLM to SGLang. + +To port a model from vLLM to SGLang, you can compare these two files [SGLang Llama Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama.py) and [vLLM Llama Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py). This comparison will help you understand how to convert a model implementation from vLLM to SGLang. The major difference is the replacement of Attention with RadixAttention. The other parts are almost identical. Specifically, + - Replace vllm's `Attention` with `RadixAttention`. Note that you need to pass `layer_id` all the way to `RadixAttention`. + - Replace vllm's `LogitsProcessor` with SGLang's `LogitsProcessor`. + - Replace Multi-headed `Attention` of ViT with SGLang's `VisionAttention`. + - Replace other vLLM layers with SGLang layers (e.g., `RMSNorm`, `SiluAndMul`). + - Remove `Sample`. + - Change `forward()` functions, and add `forward_batch`. + - Add `EntryClass` at the end. + - Please ensure the new implementation uses **only SGLang components and does not rely on any vLLM components**. + +### Registering an external model implementation + +In addition to the methods described above, you can also register your new model with the `ModelRegistry` before launching the server. This approach is useful if you want to integrate your model without needing to modify the source code. + +Here is how you can do it: + +```python +from sglang.srt.models.registry import ModelRegistry +from sglang.srt.entrypoints.http_server import launch_server + +# for a single model, you can add it to the registry +ModelRegistry.models[model_name] = model_class + +# for multiple models, you can imitate the import_model_classes() function in sglang/srt/models/registry.py +from functools import lru_cache + +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {} + ... + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) + +launch_server(server_args) +``` diff --git a/docs/references/troubleshooting.md b/docs/references/troubleshooting.md new file mode 100644 index 0000000..5a7646a --- /dev/null +++ b/docs/references/troubleshooting.md @@ -0,0 +1,14 @@ +# Troubleshooting + +This page lists some common errors and tips for fixing them. + +## CUDA out of memory +If you see out of memory (OOM) errors, you can try to tune the following parameters. +- If OOM happens during prefill, try to decrease `--chunked-prefill-size` to `4096` or `2048`. +- If OOM happens during decoding, try to decrease `--max-running-requests`. +- You can also try to decrease `--mem-fraction-static`, which reduces the memory usage of the KV cache memory pool and helps both prefill and decoding. + +## CUDA error: an illegal memory access was encountered +This error may be due to kernel errors or out-of-memory issues. +- If it is a kernel error, it is not easy to fix. Please file an issue on the GitHub. +- If it is out-of-memory, sometimes it will report this error instead of "Out-of-memory." Please refer to the above section to avoid the OOM. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..1a7e5d4 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,20 @@ +ipykernel +ipywidgets +jupyter_client +markdown>=3.4.0 +matplotlib +myst-parser +nbconvert +nbsphinx +pandoc +pillow +pydantic +sphinx +sphinx-book-theme +sphinx-copybutton +sphinx-tabs +nbstripout +sphinxcontrib-mermaid +urllib3<2.0.0 +gguf>=0.10.0 +sphinx-autobuild diff --git a/docs/router/router.md b/docs/router/router.md new file mode 100644 index 0000000..26fa942 --- /dev/null +++ b/docs/router/router.md @@ -0,0 +1,165 @@ +# Router for Data Parallelism + +Given multiple GPUs running multiple SGLang Runtimes, SGLang Router distributes the requests to different Runtimes with its unique cache-aware load-balancing algorithm. + +The router is an independent Python package, and it can be used as a drop-in replacement for the OpenAI API. + +## Installation + +```bash +pip install sglang-router +``` + +Detailed usage of the router can be found in [launch_router](https://github.com/sgl-project/sglang/blob/main/sgl-router/py_src/sglang_router/launch_router.py) and [launch_server](https://github.com/sgl-project/sglang/blob/main/sgl-router/py_src/sglang_router/launch_server.py). Also, you can directly run the following command to see the usage of the router. + +```bash +python -m sglang_router.launch_server --help +python -m sglang_router.launch_router --help +``` + +The router supports two working modes: + +1. Co-launch Router and Runtimes +2. Launch Runtimes and Router separately + +## Co-launch Router and Runtimes + +This will be a drop-in replacement for the existing `--dp-size` argument of SGLang Runtime. Under the hood, it uses multi-processes to launch multiple workers, wait for them to be ready, then connect the router to all workers. + +```bash +python -m sglang_router.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --dp-size 4 --host 0.0.0.0 +``` + +After the server is ready, you can directly send requests to the router as the same way as sending requests to each single worker. + +Please adjust the batchsize accordingly to archieve maximum throughput. + +```python +import requests + +url = "http://localhost:30000/generate" +data = {"text": "What is the capital of France?"} + +response = requests.post(url, json=data) +print(response.json()) +``` + +## Launch Runtimes and Router Separately + +This is useful for multi-node DP. First, launch workers on multiple nodes, then launch a router on the main node, and connect the router to all workers. + +```bash +python -m sglang_router.launch_router --worker-urls http://worker_url_1 http://worker_url_2 +``` + +## Dynamic Scaling APIs + +We offer `/add_worker` and `/remove_worker` APIs to dynamically add or remove workers from the router. + +- `/add_worker` + +Usage: + +```bash +curl -X POST http://localhost:30000/add_worker?url=http://worker_url_1 +``` + +Example: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30001 + +curl -X POST http://localhost:30000/add_worker?url=http://127.0.0.1:30001 + +# Successfully added worker: http://127.0.0.1:30001 +``` + +- `/remove_worker` + +Usage: + +```bash +curl -X POST http://localhost:30000/remove_worker?url=http://worker_url_1 +``` + +Example: + +```bash +curl -X POST http://localhost:30000/remove_worker?url=http://127.0.0.1:30001 + +# Successfully removed worker: http://127.0.0.1:30001 +``` + +Note: + +- For cache-aware router, the worker will be removed from the tree and the queues. + +## Fault Tolerance + +We provide retries based for failure tolerance. + +1. If the request to a worker fails for `max_worker_retries` times, the router will remove the worker from the router and move on to the next worker. +2. If the total number of retries exceeds `max_total_retries`, the router will return an error. + +Note: + +- `max_worker_retries` is 3 and `max_total_retries` is 6 by default. + +## Routing Strategies + +### Cache-Aware Load-Balancing Router + +The native router combines two strategies to optimize both cache utilization and request distribution: + +1. Cache-Aware Routing (Approximate Tree) +2. Load-Balancing Routing (Shortest Queue with Balance Thresholds) + +The router dynamically switches between these strategies based on load conditions: + +- Uses load balancing when the system is imbalanced +- Uses cache-aware routing when the system is balanced + +A system is considered imbalanced if both conditions are met: + +1. (max_load - min_load) > balance_abs_threshold +2. max_load > balance_rel_threshold * min_load + +***Cache-Aware Routing (Approximate Tree)*** + +When the workers are considered to be balanced, the router maintains an approximate radix tree for each worker based on request history, eliminating the need for direct cache state queries on each worker. The tree stores raw text characters instead of token IDs to avoid tokenization overhead. + +Process: + +1. For each request, find the worker with the highest prefix match. + + - If match rate > cache_threshold, route the request to the worker with highest match (likely has relevant data cached) + - If match rate ≤ cache_threshold, route the request to the worker with smallest tree size (most available cache capacity) + +2. Background maintenance: Periodically evict least recently used leaf nodes on the approximate tree to prevent memory overflow. + +***Load-Balancing (Shortest Queue)*** + +For unbalanced systems, this strategy tracks pending request counts per worker and routes new requests to the least busy worker. This helps maintain optimal load distribution across workers. + +## Configuration Parameters + +1. `cache_threshold`: (float, 0.0 to 1.0, default: 0.5) + - Minimum prefix match ratio to use highest-match routing. + - Below this threshold, the request will be routed to the worker with most available cache space. + +2. `balance_abs_threshold`: (integer, default: 32) + - Absolute difference threshold for load imbalance detection. + - The system is potentially imbalanced if (max_load - min_load) > abs_threshold. + +3. `balance_rel_threshold`: (float, default: 1.0001) + - Relative ratio threshold for load imbalance detection. + - The system is potentially imbalanced if max_load > min_load * rel_threshold. + - Used in conjunction with `balance_abs_threshold` to determine the final imbalance state. + +4. `eviction_interval`: (integer, default: 60) + - Interval in seconds between LRU eviction cycles for the approximate trees. + - Background thread periodically evicts least recently used nodes to maintain tree size. + +5. `max_tree_size`: (integer, default: 16777216) + - Maximum nodes on the approximate tree. + - When exceeded, LRU leaf nodes are evicted during the next eviction cycle. diff --git a/docs/serve.sh b/docs/serve.sh new file mode 100644 index 0000000..049f767 --- /dev/null +++ b/docs/serve.sh @@ -0,0 +1,3 @@ +# Clean and serve documentation with auto-build +make clean +make serve diff --git a/docs/start/install.md b/docs/start/install.md new file mode 100644 index 0000000..fff91d0 --- /dev/null +++ b/docs/start/install.md @@ -0,0 +1,160 @@ +# Install SGLang + +You can install SGLang using any of the methods below. + +For running DeepSeek V3/R1, refer to [DeepSeek V3 Support](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3). It is recommended to use the latest version and deploy it with [Docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) to avoid environment-related issues. + +It is recommended to use uv to install the dependencies for faster installation: + +## Method 1: With pip or uv + +```bash +pip install --upgrade pip +pip install uv +uv pip install "sglang[all]>=0.4.4.post3" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python +``` + +**Quick Fixes to Common Problems** + +- SGLang currently uses torch 2.5, so you need to install flashinfer for torch 2.5. If you want to install flashinfer separately, please refer to [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html). Please note that the FlashInfer pypi package is called `flashinfer-python` instead of `flashinfer`. + +- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: + 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. + 2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. + +- If you encounter `ImportError; cannot import name 'is_valid_list_of_images' from 'transformers.models.llama.image_processing_llama'`, try to use the specified version of `transformers` in [pyproject.toml](https://github.com/sgl-project/sglang/blob/main/python/pyproject.toml). Currently, just running `pip install transformers==4.48.3`. + +## Method 2: From source +``` +# Use the last release branch +git clone -b v0.4.4.post3 https://github.com/sgl-project/sglang.git +cd sglang + +pip install --upgrade pip +pip install -e "python[all]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python +``` + +Note: SGLang currently uses torch 2.5, so you need to install flashinfer for torch 2.5. If you want to install flashinfer separately, please refer to [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html). + +If you want to develop SGLang, it is recommended to use docker. Please refer to [setup docker container](https://github.com/sgl-project/sglang/blob/main/docs/developer/development_guide_using_docker.md#setup-docker-container) for guidance. The docker image is `lmsysorg/sglang:dev`. + +Note: For AMD ROCm system with Instinct/MI GPUs, do following instead: + +``` +# Use the last release branch +git clone -b v0.4.4.post3 https://github.com/sgl-project/sglang.git +cd sglang + +pip install --upgrade pip +cd sgl-kernel +python setup_rocm.py install +cd .. +pip install -e "python[all_hip]" +``` + +## Method 3: Using docker +The docker images are available on Docker Hub as [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). +Replace `` below with your huggingface hub [token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 +``` + +Note: For AMD ROCm system with Instinct/MI GPUs, it is recommended to use `docker/Dockerfile.rocm` to build images, example and usage as below: + +```bash +docker build --build-arg SGL_BRANCH=v0.4.4.post3 -t v0.4.4.post3-rocm630 -f Dockerfile.rocm . + +alias drun='docker run -it --rm --network=host --device=/dev/kfd --device=/dev/dri --ipc=host \ + --shm-size 16G --group-add video --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + -v $HOME/dockerx:/dockerx -v /data:/data' + +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + v0.4.4.post3-rocm630 \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 + +# Till flashinfer backend available, --attention-backend triton --sampling-backend pytorch are set by default +drun v0.4.4.post3-rocm630 python3 -m sglang.bench_one_batch --batch-size 32 --input 1024 --output 128 --model amd/Meta-Llama-3.1-8B-Instruct-FP8-KV --tp 8 --quantization fp8 +``` + +## Method 4: Using docker compose + +
+More + +> This method is recommended if you plan to serve it as a service. +> A better approach is to use the [k8s-sglang-service.yaml](https://github.com/sgl-project/sglang/blob/main/docker/k8s-sglang-service.yaml). + +1. Copy the [compose.yml](https://github.com/sgl-project/sglang/blob/main/docker/compose.yaml) to your local machine +2. Execute the command `docker compose up -d` in your terminal. +
+ +## Method 5: Using Kubernetes + +
+More + +1. Option 1: For single node serving (typically when the model size fits into GPUs on one node) + Execute command `kubectl apply -f docker/k8s-sglang-service.yaml`, to create k8s deployment and service, with llama-31-8b as example. + +2. Option 2: For multi-node serving (usually when a large model requires more than one GPU node, such as `DeepSeek-R1`) + Modify the LLM model path and arguments as necessary, then execute command `kubectl apply -f docker/k8s-sglang-distributed-sts.yaml`, to create two nodes k8s statefulset and serving service. +
+ + + +## Method 6: Run on Kubernetes or Clouds with SkyPilot + +
+More + +To deploy on Kubernetes or 12+ clouds, you can use [SkyPilot](https://github.com/skypilot-org/skypilot). + +1. Install SkyPilot and set up Kubernetes cluster or cloud access: see [SkyPilot's documentation](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html). +2. Deploy on your own infra with a single command and get the HTTP API endpoint: +
+SkyPilot YAML: sglang.yaml + +```yaml +# sglang.yaml +envs: + HF_TOKEN: null + +resources: + image_id: docker:lmsysorg/sglang:latest + accelerators: A100 + ports: 30000 + +run: | + conda deactivate + python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 30000 +``` +
+ +```bash +# Deploy on any cloud or Kubernetes cluster. Use --cloud to select a specific cloud provider. +HF_TOKEN= sky launch -c sglang --env HF_TOKEN sglang.yaml + +# Get the HTTP API endpoint +sky status --endpoint 30000 sglang +``` +3. To further scale up your deployment with autoscaling and failure recovery, check out the [SkyServe + SGLang guide](https://github.com/skypilot-org/skypilot/tree/master/llm/sglang#serving-llama-2-with-sglang-for-more-traffic-using-skyserve). +
+ +## Common Notes +- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) is the default attention kernel backend. It only supports sm75 and above. If you encounter any FlashInfer-related issues on sm75+ devices (e.g., T4, A10, A100, L4, L40S, H100), please switch to other kernels by adding `--attention-backend triton --sampling-backend pytorch` and open an issue on GitHub. +- If you only need to use OpenAI models with the frontend language, you can avoid installing other dependencies by using `pip install "sglang[openai]"`. +- The language frontend operates independently of the backend runtime. You can install the frontend locally without needing a GPU, while the backend can be set up on a GPU-enabled machine. To install the frontend, run `pip install sglang`, and for the backend, use `pip install sglang[srt]`. `srt` is the abbreviation of SGLang runtime. +- To reinstall flashinfer locally, use the following command: `pip install "flashinfer-python>=0.2.3" -i https://flashinfer.ai/whl/cu124/torch2.5 --force-reinstall --no-deps` and then delete the cache with `rm -rf ~/.cache/flashinfer`. diff --git a/examples/frontend_language/quick_start/anthropic_example_chat.py b/examples/frontend_language/quick_start/anthropic_example_chat.py new file mode 100644 index 0000000..03d699b --- /dev/null +++ b/examples/frontend_language/quick_start/anthropic_example_chat.py @@ -0,0 +1,73 @@ +""" +Usage: +export ANTHROPIC_API_KEY=sk-****** +python3 anthropic_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/anthropic_example_complete.py b/examples/frontend_language/quick_start/anthropic_example_complete.py new file mode 100644 index 0000000..bce2a61 --- /dev/null +++ b/examples/frontend_language/quick_start/anthropic_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export ANTHROPIC_API_KEY=sk-****** +python3 anthropic_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """ +\n\nHuman: What is the capital of France? +\n\nAssistant: Paris +\n\nHuman: What is the capital of Germany? +\n\nAssistant: Berlin +\n\nHuman: What is the capital of Italy? +\n\nAssistant: Rome +""" + s += "\n\nHuman: " + question + "\n" + s += "\n\nAssistant:" + sgl.gen("answer", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/azure_openai_example_chat.py b/examples/frontend_language/quick_start/azure_openai_example_chat.py new file mode 100644 index 0000000..d53f935 --- /dev/null +++ b/examples/frontend_language/quick_start/azure_openai_example_chat.py @@ -0,0 +1,83 @@ +""" +Usage: +export AZURE_OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="azure-gpt-4", + api_version="2023-07-01-preview", + azure_endpoint="https://oai-arena-sweden.openai.azure.com/", + api_key=os.environ["AZURE_OPENAI_API_KEY"], + is_azure=True, + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/gemini_example_chat.py b/examples/frontend_language/quick_start/gemini_example_chat.py new file mode 100644 index 0000000..0ae6231 --- /dev/null +++ b/examples/frontend_language/quick_start/gemini_example_chat.py @@ -0,0 +1,73 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/gemini_example_complete.py b/examples/frontend_language/quick_start/gemini_example_complete.py new file mode 100644 index 0000000..5188bf4 --- /dev/null +++ b/examples/frontend_language/quick_start/gemini_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py b/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py new file mode 100644 index 0000000..afe0c72 --- /dev/null +++ b/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py @@ -0,0 +1,30 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_multimodal_chat.py +""" + +import sglang as sgl + + +@sgl.function +def image_qa(s, image_file1, image_file2, question): + s += sgl.user(sgl.image(image_file1) + sgl.image(image_file2) + question) + s += sgl.assistant(sgl.gen("answer", max_tokens=256)) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) + + state = image_qa.run( + image_file1="./images/cat.jpeg", + image_file2="./images/dog.jpeg", + question="Describe difference of the two images in one sentence.", + stream=True, + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + print(state["answer"]) diff --git a/examples/frontend_language/quick_start/images/cat.jpeg b/examples/frontend_language/quick_start/images/cat.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..a6a8e48c972b7c30fbdb5c2e95e806a9d6411540 GIT binary patch literal 345163 zcmcG#2|Se1+dq8I*vY$=YOx~}iJ?>+q97{Fs@Y-$XE zKmY)u{Q-O9AiU{?3vSl7R>r30Mh6XT0PSEr0RT{+`+l}2hLTQKoh4bu4zAd5bNBT3 z-T(eC4qCs|{{7wopjYL;IOqSZn(f9-e^1&83$%Y(KibV{gB75a1#Tad752*>2W6f8 za?pL>`?Pbc_sf3vwijsS8?>^*?SE7D_%~%wU%&nHpVQ9KfqDn*-)sNaAMv4^J`Q%Y zZ&upBDBus+0w#dr{`_gbX$LG10JK&CfFa?ZbsiZ2fQbZv!$be9lYI>U+>ZeO-Sf}7 zf6Bzy-Orug9V6`%eCrkfEENL)=T!jU{|*4`5PBQ!68K+yo>6 zQ@{uC1XO7yb>IYW642Nq02gTL{J%T)JNdusfp+yC4&Y@4XEOX@0PzA~UJwH>Xs-#7 z*f%Fe23q^S%oYHFnORsFnAjNEIcO)?@cJ(Kd}HH?fu*x&HK z|B%+H80~f-rhj`=V82sl77!~Nt@$=DO|^_ntSn57%*;%ztN=G{Pz;Q`QcQd&&)YEb z3%J80au=j!6Dw)AW06rmy(oCZoo+Eh6YvVg_ym@Y>p{chS`lvDt;JA4&1baX^YG9p0*2KB5<)jH<**kb zu=Q7VYoYys{?ooq;_6I2waM;E00uG!J`>^%n+(NhFM2lguU1EwxN0rvOL9W8D6JVA znS{+vB9l^CFUS&mzI`VMGl8;PD0?6*aYFE(o@-q>w43w@THdk;m^*+#L3*#&@Gt~< zX=@37bfz_81IRMmt*4G{X4M<)xYE1jc;rRKvP>Ib32{vx$_~xgj-J0@wMhXV5QP|L ziYeJ%Jo>jaBAB($Qq-#RlJ)RIjZHqw?MzPc)NE8VbNLHG8!wg-Ph8slXgQ32m{~ z4)LUD8;pyA#U6lR$tFM+Hi9R<(I**uA;*%i=ociy+xl&5Xgz_sl;BybL8%zp1N>pX zDvdu?j$Jc2S{F&)5ZM?aM4;D1BBGH!IkN|oG%j@!t9z$cjG1O$6RR}8Wa`+HqfG3B zeG$gwd>;cL$e6S};CqlL1=GP-_U?gj0#jGy`!MtpR*X6{QcG@-1_udaOP02l*AH1L z;5J6VSq4x-gx8!(^&TL;MtC0V>a?IH$!D+DGlg*L-x7;&a}!i97H63R941Zl^_Y6X zwm+jDkj}tYKulNA!x6|P5ishcx7jE77}9{TxMl*{>J^R%l#hs_pCd~^rR#v|d%}8z z@S_8bXG`N>vtfmpr?_6NL?0H$*oys2!DdgHN9tiH46%b0?P#gP-Nty20PpGJg87{#AfOncxZ`o| zr!UycG^^`Fsnu&MHOuorlX9sz(!L#fy?@6`G`OxzkP8QJ4G& z`6$^FbwUkKuVFg3wz-I!XVIZd5qRmkXRqd(y|l5aG&7g!ezq38qk|$uT7Ut&(7&Gy>ONoju<%1aug#Z88pnNN!J@Iy9hCHO%kcQ zwUMVo#YdKFA4#a3;iJ7%=Xmt<}*JZ3oiIq!8m}D#--oHX;5q zHFOVf0`Gcetx1qEx%$)8iU#^%h4+AS=jSbUuT+k1EkE&hlrZ3Z6HKfiC6ZRFqa@M< zu_b6(h@6`N{~NaiU>EmLpVynN3Kt5RitQ$D*Oxc%PN3=IH7*Sf8fubAMI_R ze0H`M=thvh$4!9O;t74h8*Vvo>F6RWvORykdhYBz%CT`(e{_NO%`2^(7kn)GB-Hx0 z_V0ZNkdm@|PsWz@%k&L9_Ya3w9^com+-~+~*rf69jjXyoK$I@T%w5s#_^|Tjjj)4d zriLKIhEH$BJZf&Hw2zP0yaAtqd2Ix)V#2m|j-MU&$llivKgtHBrb4cd+ah%I2m)t9 zB+}OO0tvqQ)A)Qk{&2e4{m~Y;z~G@oKUQ+|mC_%N-NVv)u1N}yk8IQ|2{(*TtLvJd zvK%OY^%wLrm3flb7Tbn}_N;uRW0AbWyh*y>an=b65<%dl+djBVZ!Aa_pS^|=$3Brc z06GDt@~~Zq#5HZ=U+y&7LfH;+o>Am$$v2^{WDLvVCUMi_%&&Ox640kDvbh-~;`C5c zmnu3gDL3KAT8&vI^g`RPdKW61O7|5bBb!zxKOH&@x=7t-ok$!dwLzoeEo4i5_&9ex z1~zuH`6BdWIy}4x)P)8iE=y0_0WcCE*q<^Qmi{ipEHsS!x(wDBi zS4qw_85_&jW3N5MdScUs@l>s-r99tpqY4^QpM^X z@apnD)-vbrlAlUw`!`b$mavixM#o2Ub!eL^5&iXpmsey%+TXZY;JhP?XBT>cLK^K~ z(LR_k=vZLG9-t;|q8&);TwIG{E?v;lme7rz(}RS;qUl3s{JXUQTVBGlAQ)yN13Ync z;g@vQD$$Jtix~gtebYTKE4N8t9W z0dbA6U5?R?rQ2GBW4@!K2trzx=fO_!4tC-|)5p#PzIerEI{3Edykzs4ZIjP6;U{?x z0VhDFqZ3|n)cU$wAtH%D_64=N`{R+0#7zztf;V8RQA@n;Z8~Q;AUf|8v>hz4yzl3e zjOLFU1`~VYTBt-yk5&;_?=_D~Erv0+R{v(`b_H#vui||y8Xyo@c4H43e5N;^VTnrx z1D}Ppxp=DvX;QpRC3s=;C#8(Yh-Hql?5$ZTgMm^3-ao9P{o=+0_}K1&-87u>z6l$i z9dqHVnchy&4$~wihSQyv#oAbG^BSeAwz5)^OsWx^vT=f|ouGS?QY-O535#~}^`Px~ zSfr7AQyqOF7_y?blEdCWA0 z;nLWKu89&cNKNWp-LR>tt~`hR^T9e4#9D0)J`hN_Bnn1MjM65>wC}1S&&%XUBa(-j zV3P;}O&9c$6g)x%Qz|uvBbu>GM1WqE#96~UpdB;1l|34W5ndu3)E30xfj~m-ex>l_ z;;_y>2AFB7x}e8{NAG zo~V%9!BL>~=kd_+<^AQ7^FoJAoW+b;PR!D~7a-I>b#I^Z4qg%op^(Cp93XdDfsK1u z$%nuMBC&SU?O+-)A)ujgs_^I*v_fNb-undmew@dgj8sK_UaQpcEk#>W&4yj2Ns^O+=jg`4DjTfHYF-`GZvyM(pF$}DzE7tD4H*UzJhX$owM3Ity*2Awbj0aq zV={qWTM~nt5h)zmR(Qr}!~6=Ugs49dvwwcYjEEy)xQuhdJ2 z)y?s#&8=CnwYsoMZ(+k|!zw7ycL25opg^OB*JA&PL6<}% zw6mf|jQfR`bco;K@pAbC!`EDf4zOqrw)1#DGCv|r_6P1nm#0O z^4}m}53oR^N9#9OyzR~Ikx!VF8e4}uk2$Ro5ZR-T!i_d2u4^`-FpZIQ-h6y22)sD4 z3#nBxe`$Q1fIeoU1=l~p5xWPZBi4F1h}l~pUs$LOND~7Lhlucd;+E*73f>0R zOV|XAJZYH#jjW)nfW_BJ8|wRVl#F_BnmlOa8FLSC7O;UMgM#cfc$$Wz0*GBOG&8~rTxYr5gppOgO{ z57rh}i?+Wadbu;0E=9(9)A4QRv7z1DZN%vZU-K%)uXz~Sc{bocUGFqSmcv>IYZRKw zlLtm=DhDoAOb%xb5yBBeqXi*3=8hoXk*{X0&gf|7=pOK7{k7>C<1RQzcEg#ds>-}W zmCYXY{MLT3cEf`cKog#xzNqv{9jZJAJjQ(>G?69NIYEEa3Df$hjRWcM7|lalX=x93 zAkha_zw311tK3v^Vl7_@aa?1GzCPt?(@TF)Ug6PIxX;0h;3R_;@vz-iSU>L2Ift5hgt>dJV%EtGpnZ(C(hEed z2`v*NR&o2HVa14WSMP+@R7mgMpgUtmk@(cOtt6FN$xJs-AJ{Mi9!@7p9`lycTzd~v zi+L2yFni#Rq+H8Ar%-NIRvrJwJ{4tZmukmHub>H(N+LGxAa)USyKYcV?jGJ*IDj48 zr4s;-0T4(G`y!LcOZ0CO#%7fdkB>2qZZWj9c1mG`xM;h{zZ0Z^!jpmRE%5LhQ=P? zyQH%Q1lE%=hk>`PwRjh&VBU&`fsLG+N*vQM?My00Y~D)Tkuv4(PMUf1i{p%e89m`r z<%|$V5GoqB=Z+htbEopR?Cf>Cx~-5sn-^HVP8zK3h3_ zZ99k_YBL!)5o_^TTU0D_6U2RL>JzVrIv7XrG5j38vPGfdfbH_i-OOOv@*a&{#%$1{{odJb zKty~L>ub_R{);*eSk10(k^t_39)QQF2c=(KrZo{0w}u1Uu==SRKrnO-dl<;}1$L01 zcVz%t6b=w)mUq*3YXBtHEe<#g#O>6hiYNS6aEwmFA+UBp!b)#ToF#RQKGARs48{vK z9Sq*RKD=Wr4F)8Wtn__GZ~N?!TT?5Q_LEv5N3RvN5VRz;LgEF6DW;<^+kr ziX9}yY`vpv4OD_(unfEhrizbZ12=Z~;6_Qks99JVAWtS>f4Alw24Gk`3OT-p^#*jk zHH5M0#W^|YZIp19o;ap<%~}d-8B+uZ<2QVCwu0T@VGTiI6+m^p&U=3k6vlH3-3V}S z%(G2G2_8Yq%s9yI$%9$NZF6ot!uBmU47>3H)$248!h<^pA-0RG0eDooMUmPN=oWRk zf!dVxwlc*795WyKG+jd7+q7~l8`iS*4}=-oa;vAT;$!Cn>o-=K4D4B$3GtAHOOkHa z^(FpcO5qk!bK2Jk+{NW36*3*XAZe;5sdsyMc{32l$bcY!VThw~5BQx5ko?T*24!t93uedFZPwVZT`ufY0n4MRd7i0PBz0t7^CEkkDJHo3WcxnXrxafJKKCS$xc)7oUb?iUR*BJKn| z$7P$0A$VeP{It-7AhFNOQQ;4{8IWoSewYeXW0>NDKd={IS^UcILNI#wnj_1fgXEZ1 zlfgH5crH*U0ZJT*;Wp=86xy8tDak@=bU%BC!AL#DLy`)V>J9B6nh7nAj=nhP>58d_ z7;c(M?DY3VLSV>1`Vs@K(!3#Y3dcyqXh_&Zg)>Yojg9VJ=ZF}_$F=s{Jy?cJ7c&>9 zX5ww`&MivOmB|X^*ey#8ybAeA-UZ7f_-a*Sup(mAmH7vHPDSAJ!+2k-^gKJ&GkYd^iZd$)!I5aPZnpkMhv zD$pI4RLlexcKjm4jMp+Y?CIZ8Cm@(cqAhQrS`mOHAcOAT&AIGMBP((`!IpbK`3C7Z z4L%5~Gn)h&km$RW`2sjbiSuoQWbE}o5l)J)3Vb%434<-HotV@`Y*zph<@)76@j%YN z?CKH@JPPq&(*k0UVpzAQ>60KWz$y4TD4ZD(+TEgHb9x26+7+yiffoE4Q$!#Sds3-L7Nwek+pSqNeP8uBDNJF)XAiLXmJCNh zJQ5RA^YR>=)gLoXe38U4@)D8zyg9(RQN6=d=?hlzMD8NUEsZ(zR-$Adz@+byXZIR- ztr8FSgTZ9Niwb#*VXTnunyAWEnTijA;>O$5$?J;I)#XU;p4c5EX5JLX7~w;T(?)8_ z%!D&f#A(Ve;C)oWdU^ov)LP;yX%El>9&)x%OnD-FX;hI~E5P9-5~Q8IwYd?8>Rty* z0S=@dVzZmM)9~V#q^;#U&t&GF5?NVmH;dMHQo@hNzokU<3I!C_Zxm7!DlTSpZoJVTwjD zOAR9x0K=BIO6zQoc$a*6U0rPz;|@Y%29|?TD~H)R2Lug{gLbu=u(x|w!%aLVO+-OR z1OwJDsEHoGgH>uYw{b!f?s$$lnq|3Mq}`tYrNFdzS_nZJ8vrm7;}kf&OeG;u+SaSY~IV!g*MCxCiuVfEaJJ zKUwP-|IvZ!rwGDxz_y^cGlsI^R=JIs_-Rw5;T6Tk%8=F5yBRl;y=x*~LZhP>L9PvM zuz|3}eVXeZ;s%M5m-G;B+Y;$o0OTRcB)*K`qlX6|lazeS#;KakcvXk^flR>av9~STqE&u7 zwgpi77Zv|Q)B9igZm`w^z5km^`slbz^tDk2Cx9JPC;(TK^tITIPgCAcx!F%j3ceoN zidd#}D^`6*g(vAr6Nf$11)8tp7zEQ7vgemTBK~`T4pQ4qz(CJraaCB~oT=v3zs~tz zE$!dm@tpGbI{nRYhM_*(^To?Y=vgGf_z}Em` zv9-9jCvlXp7;m%hUjSL|>MhvPa?^QT1;cg18=LZ8D7L(8^wk>9jGp_vehtG1HW2kO ze<;tMQ%1%G_vsS8OUn`ni5@{l!*Qwx+3TSuF=kh2H(zKBh~XHT{<>3MGda4`syiP@ z3)pC)RqM&cjB&622ifz&W=X`HR&qGoe`wkl#7@{AaLsK1;Pu(r1M0EpgO&V05eyTD zWz!mINoZTYNH*00e_gB62?!;JbHQDwA?6z0K;480A0HtcFU^fM!#9oHtTs99dL&Kg5M)*5qRq`6 zWwRglSDj5v8_s<))@=Sgr8V)<%}rXTEg&NOR#PU&2^IWeuVE&ix!I49tA^5O#Sc8u zg|;cPrq3A|>x!Mt*khuYbI;Yuv1A~mwT5OD4LjtFIQuk#0(k%PPv!@LIvGnKr(%7to@l&h& zrY-v7yj%Ys@LP&kW)t}i0rBH5lCoY@)iXT9|CE23zdC{dZQI_W;0X0yHBF5EA?qQ} zzF>t|Y6yrOgg=QCPUnf5GKmOK_x7BLHX)e?v{9MMe#iB%p$Gtv02}7dcBRs@#^~Vi zQr`cGE&orTf+zn*PHMxJ$EEcc6g+&a`MPL?Vs$&V|LYzQ(adj)zW1@u-i+VAu%#RN zVzSA|p!r7VF>D>~q$CewGkAwPC#!r(M!<2OxC7)`NhAV#q?_8X<>Dbd%%ZtL$R-U; zVSGj+B>yvq{C~RV|IIJf8X;o#$paDPJ2I2H2UaPV9%zJ6{rgXJNXp|6o?l+tWCw}0 zi5o07^jA)A`=gYzDo;z7!wu|@5Q-bRcP{jNi7t}g^m#m;%3cjEJ{odaNmBT3U-|Ej zG=WFsr)U30GWCINR4xL3J}Zz$pO-sLRt@7>nLY|0TDy*jJ8j!(-P+pdbACx`Uu_b8 zxK5o4F8H07zFS(Bv-ZS8~t zIfz}(d31&d$1Sm8G5l6%e^})au|lqGD|fiW$(ttV;p%UKr*Dxjg*};KkPgV}97zcj zz2+uUq|h1FZKc}%L+ej7@=J5!`KMwv-zQEU)i!0jr_HI+p%SiA$f=R=7|;ModV9me zxT5>W(wl~z9KMrX*FUImG4gI2$#H;DsEq&*q^yCO^wlAOy{Hx!Ol6g?HPYzdMCyX(~D+p}TeN zW=L|g0?@CkJFPT3s$VADMU^KNl6)jJ{^g;&&C{L7Lbq~m|zmbDx z4`@%8Q$hu6Xi0o)+5-Z0udI!pxvhR*z0w#?9I0lpl;netqd6`ctBGHnRO)rc#Or5;NEn~B>4`@b z5_kIcIZq=L@BF-!fvm`L`_dn!rLpyiyn>Om#{_bgqcU|g6PLdN%CaC-hX4*7L0O^v z=YAU0ly_b`r(<(G@7^!1Mz^O0YW-mL)GzW=twixcHj{U?L>__RIL=Cnk@N4Wa6hSw znG$N{-bSXTLZY5EJAj|Zi~ktb|95Z$%=r%1TqzrBgss8(dQ`CgEamKO^wrtXZ_VDR zG!0Da;TJqY{M_Uh|BI>*trQr7T&}Y}mV;Yjh^!T-=jCnQHOvK zUJUG-YFOx^QvK1P^{WIP(HBwwi6FoaeKH>!P?|R|637XV4O`t>@J1L5uS3LY^?dNQ zTF-CN=qJlcuaC%F8^xO1eovVdO2HtZBxzfx_Ok6=a&6vCzhD>ViykWPnJ?Y-N>)@< zloQa%Nl2(tq2h0p^o^eIdk}K&&sNe`U-U8lSI>?i^P|4|wGOG8PhP0Id;8svH2lY# zVq5vA&EJ0UUN8#O@2o#+#vF1|Cxg!qO?y87n4jM{9ky~p=Ia%z=$hN&s4K$bn|DQbtyKBo z>601hgFcC~vPN80tKg7eEmyvPjMck`@sqinF;bFYPq(3A>jdG(ZH{fXNkNgAb5`tz&uhp{EVgV>Z#8<6;SGDyITt2Y2 z8cKTMB{)LpbJ!%+^vtTBXv#=e#vDIe@&V>2z^4lHGmvQ`Vsz`M@{R>ctgg-FN%Mnu zhzjT6A3tQ+dz}qjgU(0&^4>}DIV@>~%}k>?2o95B^bk9-jx-w5!;ZgKOS(S?0sM7l>@;%N6%j4Ag!DXU8kbM}h;6 znM%(%Us7}y04EfrykP<}^Ioz$WA6eb-EFkFs&xnwbgT2|(W4(e*vZ5nzF01!DAp3R z<4E*cqHV0D_pgZ{wTiHh*Mbsq>Dye=t=P_oYHPHe<0FRPm-~B)h9{+Xa7?C*^C}om z;_8mEVu`7G>toHFH(ag;A|XRtG~!2l{Kdb|U|5b^Z5T$%Q(6gI9!2}vgWxge2&~NN z&LV@1veCstIj4mu!U&AD2nCj zn;0_t`?n{*B{0pwNGr0aP$RSVY=&UzcNhOBk>NiROc0}s1R2!`h#sy1*#@s@i^Kka z7+^J<U|Eas}N|uE4l%LQhGd> zC_Vbx;g0_asq^1Ke^5BP02>7}OTklGbY;{lSdJRCi6YCRcWDoo0@K12STS`EkQcyN z{vZ4hOHc12j?u+QjEo3vKwiPI98HB!wG8iS;NutZ!%9+`*%W=X*At-T_nhdVtbkv~ z&@T;?;jID&m+u&!x9$I6Xr1!VJ})KQEWG)~hiJmhsarFFPCCvRmA8IDU*5WUQelSK zPdcT0P~hEMG|xzO?3Pk{Ns)%fUr)>CXXTuKf-RKoY)2*7zs{DU3u*-!0aT!|^RW?w z%ZW$s1Rh=PzJG&18yedA%;m&~@smr$DirLFNAXJusp*?Pf{VM!_K?by+v=K_7sg_) z{jIrJ`yr!qG3>f}8GOeNpFDb*_jjXv&5sv%g{9C}?;I;iVv{|_Dw5=FYSVI8(ZbMB zj}+Ri(PB&ZN=dm!%#p}G+MrusuTziFm71EFalX*e{*~pb&Dr_q-}o~Ek3nFQYGLn! zYKKGXFcaRFtjkg?5j{5_+eB6QbHNL4t|Y3g2(Oe}y2*6x_9e+}Aoaa-l+}}OLeqsB z4|$3q9BpT7&<|#ITj8xX(3W|Y%sd4p;}?)qK{syTp2^*Y3(3O;E6)l{NS`{o<2L)~ zrq3C%mDiEy!U^~KM$t)0MP=+Rs!Rg8rk8yzZouRiZ>lJ3GrSjSe)%cnHzs*e?A|k~ z$InA4gK1&b?}7~FSu>ohBf8H?zs+(iG-&Ng`&6px;4E#UphckMzD`w;Z#{2qmX!8w zt{BV`btZ?`$c@9|lgRlutHz&C2YD$rRgYO~Z`CoG`AcQF{Gt9Dsh{EPd@rNW**R`4 ztzJ`nI3p3~l|yEKt8E$m^KCDez*)1-fFsa%ikEEdA6+sN5IxU+C-UqzE>dP0pqjpzdTE_ufJ{eI7>-M*5zTAvc6E^)n*A^ufz34VqRzDI^8q8ZxhOb zEIW!hOeCn)NmJ*iEF_<(Hhp=LvIwp#x^^7YhX>_Cjmat%HVhShJKyIkIXdB;B@P_srE&keo%p(s3xA`>iIq^Tz@ukgG<_10scn?HWtkZb_OgrI&0}G8&OHfg8RO zS0~GytgAi>1^Bwsb7&}DmRr+Oo46;*{X8lgf5gE>qmbd zx3!O)wPGw-IVyTQtH`?IS8?PDq|b zU3jaNKqdyrW=}f|UpuUVxZ(SpVJio|t zM3w@}SlxlZDB*e7sI?%TNmG0xdW(p%n?z17hDVkE=ale2{pY`n{+p8NFO<-T#Pps3u}<+%uKwiQ6G>j5|1RomhM-C#}?E@Gp${!JVzE0oJWeZvDl5A0DzYL~WqT zZ3hQ)FF#AYQVMPSWB3o6gu6)K>@U;Ak}7=CJwdjzJRJoa9!#NGaAP57)ev)`qS>w| zPJ-`U0{rx;JB9WpcSpnr2}BunAxe4NeXmkS(pvFtc}>5Vj*c&1MurPa_*3L=Mm}YY zzqH9aH^XrZ!+&~uI^W(O5yjQw(H{T<)jzF`?xt|ny6 zc*}AcmGKG|!JC*d7<_o((AvNyo}ZJ>cOgQo){b>04t(pNTS^{ZqphR&cu+r5KpCOzl6Wf!9U)0|v*XiUBJvhuy1RjW_K zrrf7vl{MGFda^TY(DPN10;ln2d}-m*GUxm5wH1CidcsT0>FX7-0YjTsQ1{Yiw5(0n zYa?V{bgNAY>{LXGXaCcYEfY8PnFgB|te+ha+`BeXLr%L;%;zh$l^Pn%>%AHMw|=}m zOYm($Mji4;{>=G~-DFOU8d_tAoVRFO7qG%y5ca63M_a~6h zw;1J6s$jU~+4V5nrzFKQ+PsdXWhKzfFk#%b>J4w9fdWtH#^`WtL!HORGbKv>PJ|$x z?%y%S&wsgQ48D7RIOohtsoKEz8T%`QV1q$ZgP*31L;aFNYeX+k$u279eC5?fhq{zF zUXf;obo*wmAF0L-)7DS;pThAj%*m_zg8|Ws2`f%-xp8(RGPW+RK)59YtOMt+oKnpl?pbO$SL|FA4ck-X(=7RRH0>aAmx69qF7 z1Akc(|xS^OfpDZ#3op_2fzxLX$k%gOPPe#LZB>LP^po)NFDm!WSs zJK7mTfDL#X-?99t*SINE8_vULf(+Cr&Q7g1*76|bV z4hiO-_shBLP7--O5@8+Jx^uDp@^M^ot3qI|s*=PrxAs%_eF_F-q*J^<`JH;@x$!2m z@G0M6$05Jx_wW2jHYxV^{z5bhk=FWC_Ccw~M*T?wJSZT@6oUu}jy`uzH}m69*R86_ z=Uq}vr#G4I6^zNAmF853e1}Ny0ge13H>#zaQw!P+lX)KheLjc2GMsxx>rAxNq0!87 zx5!Vn5xOSJ7sGN+Kge0>ksEx|zh?Q}=XiYni;#j5wz&4K&J6kI;kPuf4C|38QO_6YFsI<^7-fX z07tXX2LS>1@5?_-E^X}X_*mLkbyFeFB6E3XAOxtv_+sTIYi#bRK{| z&$Hw%5jV-}J0}F}Usk;GkO%Snr(2o-pOwDKnJ(nkDKrUM=^!Ld1cRJemxw*p^+CHM zl}vK);AURvxjgeFj0Af0PqqKkQr>zt}$VA@!7pUU*F|Jsn9q{ zMQw2e+$&yEdl^c5n6vtX=r7qm(mH3A0M5cU%gH6{|NomeN*f7ovxS4)ps*) zf~$5b=0xA0uH~>(u!FXl*4$jWYPTkQPF(T0MnR}iv$>x%A0KPLJsX4i%a1&jVvp$F zj_MY3_%jxW_rpEUu7Z#g(B9NEMUAGbB(0QdSypD6cg*DGjhs54-S8<|3N-4A^y&S} z`~B~vLE2YmHZ?EzJ1R_^m*9Ol(M7|;c2-(^gx7@@CD+G)<+D6ddN}y%854hBSmB$f zFOkPgbV^=|C$n3Ud zUHoC@&(DS4^SSs}u6HmF;?Hl{$*1R^nzhnMOLph|@=LYpmPwX)aM4lQK0^ppHudTi zGlTj$=J(<3c>W$;ifyXWc)WUpe1*Skp@m?NyMGhu$;a|5GZ_ITD(cJ1@W&~2Gd3Rt z?>|n_`*x%1TZ7d2nPcbzP~oB6fwU*R=>hpNy}_R}1(P3JilHs7t6;Ti3HKvU|4MqP z9;{)}_D;e0w!6{5B?W<4eCM-U`O@abPXlDi)eFxadpPcC5=`7I&?1=nT!?+F$nnZk zuk)hbP)7fY9t)RADfU3~bB_wI)Zbf3zyIZtQ|2M_IG(eeJ^U%0ea~({qs{8S{I~&M>1lmU<#dKZ^tl#W_ms z)9fQka8HIff0pwyDRMwR`Hgoh{9N|Ck#$$Azh|~ouBIus@M2lzukYlVTc{4E+ydt$e-@k&RL(Weuy2an0s@QQhAi#?Nr zzq^_DRqkGN>&4*b@n)wp6kTlIzO`#Ln`6fTOhtceZ40U@bCn+e1%KT8g;<@Fq7Hqi zs?)i(c-0u`KViTp4Hkt@BEK zS(;`Iph|0Nri*Fc3pFBREh7Kg^*~b)&fR45E$w2~o`xTdVTIbob8AI0%0UyyRrLOh za12o6YEUcp;u>_=Qx$KIOU#$py{yqB{B{y0AOl-Ce=Azv$D~Ld&T9Sm_~xDO`WM?| zhVtzpIe8^N_W)&YDs|XnlzOD(&D2=DNaQgo*_`cYNggC`fcJxj$wRjk`ZHQ}1L4{2 zF9>^p>43lv#_f!j*+t(po>yA&Ho2EM3|SokuthZ-J+LgBjSLqx%GXP+_ywgs`sE|R z+n`;d^h)CbS{;)qr+xntHth2zx0)SkNxTLF(eAl%iTVwB;pmTt6%k_Sj>x31;u0R> zSbwZ!hZ#jLmh>JNI8P@&1uu#I{%L_033kbGjg6?LHlO)eN4*&DT|BAN zm0hN1aD{6N=3cAKi$V|J_W;P4&YO2y=`Usrn{QpXcRA{EwDsstd6)?zYrEkJTJ=bk zEppz`>KadU(|XJkm7^zI?;jE95%(T==Uz9``zgRC{ZibZ?ZYy5e)q6z218qcw09j& zJ%1$peB6{HvOjH|tiAArUra=9seQ58JUZWvYxcxv=p*xx*dMG)7kZ@iAC6mmM>v<_OcNxW0sD1g+A<|hq zT=}U_Anbx&)U6+kK5EY*jE)K0C3cQPeL7jOzp_p8>qMV=nZIj%`?^fIsYHUKQ{|kj ziR000cTs$%GB?8^z8q)_e?eAQzR>g3-nqCoVNE~7n|3?*Am4)~d{q)uk zhzh_ei%iJYPV`Yj4-ZZ7f~+MMRMw$iFgi|wdaetCr)ZC)L|6;0R-y+gHxgG#n`27s zrst|6YCjl=7vaJ+hZC5}Ytic^na=V24RWDYedo>0?5^5MNB&ShC0s2c)Av^etNyyc z?(UDH@C~cBuoulmcvwr&sJP$U7fv+lS?vXLBgJZqyJ;?lzmRhz?iY{1TDUmB2HuYq z@2Z}j`lfc*9xD7QFmEvI?UuruLSyX=W8M&jyAUxClUqp{hcbOrD%$S>M3916 zUgvw$z(|9Hzs8ZB4>MT(-b3yeHYzdp&wRMv^;F8+?iBZ@n#^hcdmdd$-#bn;N9-!e zI@2B;eHIcBQt{eYOZRO0RnyDpd#&UPaJldD#cx&c-N_LqZ@=m~4EUFpG@5DJg?yYy z@4vG+EhPPGg6$-D@PzsZSF8!AY+bR4A*AG#{z+!*4=xI~sv7mw`)d}5AdzO6JJ6%b zZu4cIj#DjB-}Znr3bA$?O}#eW*_peFvt%=enOU>QwEA~rp%t$4<_@Ur*b z;N`b9#UpTbv+BpK@7|<+XDU1DJm~Nw57Nh^AKK->w^}|+Pre5<-0ns0P4u(2f%ZJi3`3iBTyn3+LMI`D9y53&Ct8$U; zd*dzL+asBJj0(90g|A#otT;0=&wNI_SZ#PaV|K2dcXZ{?(yCGVvXy03A=;SX&V8R> zQ>q2s8Q<@tQuDfI`X2@aM5MgIyj72tugNemJ9lzE;%6KT{v_pr{299v__2l;l}EWX z@`NS?!_F70T^G+&R{C-C{3RQJO)*Ny{sNbNZOfUNk@gX&^%LVSRUB_WecDN*&gwVY zpsxS8t#!fdNleVIsTE5jy`NU9=4pd7Qy)1(dOvpbCtS3Sd9S_uV(L`s4*}^Le~Xh9 z#FOS`!O|ZWpWO}I13=%KO|_Sou|iwRQ<&A}6M||UDkM|};gzZ4`y*e$nU^u^T$kh2 zXm8Fa{_i+NhGpL-rNj#s7!1w*YE$?f!PDVucoWC{BPtaVyXQL4reYD^Aej z1qzfFcS&$3!QGt}cZxd{hhW7iwomrn@Bf_n-Z|gQcg{>^n91CXuIF)=fJPW(D& zN#`6s|Nq@%Kd#NcUFp4b@OyjLhRz>gP;f9ReMwP%b2gN^-cJZ{tqxW!lXHIk)c-L^ zrFq#je^4IB;|cK|j-DW!7OXXbawJJTVsg^2&`4`gI&nqeFeCo|NIk{NoG~6G4JM7iPu7R^IYwKqyqgBSc zMl7E|vlELTBejWC@OW#4Wek%N5QGe>yDYS08Wzu2>sE~d=gxqCVc?+l!8R4VH6ZUn zPt9>7>xYtw%v>u7^2ur7MGt>ppPPHC+G$j+pIx!IxaitF+Zm`u{B0bJKdFrLO>pF? zCPO{t{&OQe9sorJhkPR8y1yr6(U4pz2+G7%ZwW1upJ#V*iKo&ChpEJBdzz;y0x1$o z+C@1^o$kuvQGLi4L~@9-dS3_s<~DJ=JBXN{_FFkl`}qViKv&w0FIR$6A;B@>ax5>KdqD4|iTAIKt}nOAEoC)% z6qy?>X?<&6!f!Pwb$&OStS=k;vt1xPUgoz<=+?ZB{1IDH#fSG;AH2rVO&Pnc>53zR zc!|T0$O>S~WWe*VeIEgE=<^uKBXc`Xt{bZgWbhAS8v;l2Yu=+&3QXH5l2H^whe6*! z)p@)>x(cuRl&5vyMFP6>4KLEV$p@y*vB(UNUhdaWjisGaR~GPPY8LM}Xnn zH2|HHVV1Ka@E2R$52H+I&}#oG@Ha^Lj8(P&7hF3s2jkz~MnuHw#r{@kks72j#YsZK zjN#b;)neWK+m42Erb4Zw-ywoBn1XCy*43L9KT52$#2`G4HeV7wDk@vwGO>;n9bzpz>D zTfC>U{qLHh4(+_|@`J!`wq}}#B)b$>^UP>Ws>c3kt{q{SQJIeuD}}dG^e3Pp-mmP|O4tOZyd8uN^XV1+4}^l6Dz0aKanOY!x@vF@cH;aw?>S6MH* zy=i2>qXbXv2u5IVOz8;R9}SP-t6SE4LJ6G^0 zcxi+umanWOoOeH5&Eer-1;l)MJ6CK)OquR+%7XTMY!1 z3&;*t&egC4UTHC_H-#=Cl<+0|yWcsM=a*x)QmdE#BC zh@%~EQIGX_ph-=;bCF?_cpk*wSl13TfKyH`>(QJpJF)mY_+BE?mub)S%6x<}>J6eA zBExRwGZcM^BYghCxHzN%G`^<0M18s$(4~fC)2*;Sa!EWH{9uQd|;9GG~%!lBkqjq73kpuUVZbxB|N!K>Rw=7t#?3I z?ByQQMy8)YKB=BV0dEGv`$AvbD1XYG9{2e#1Qo3UV_WcI!kC!7SFf}EKKonC>d!0Y zYeIbj+Wg#sT=1Qb&B^IlWI~cdz7I2R=?=G_9YT84qjpAqpOBM|7ONa4Ee5x#Fh3s*`d(t&itTxEFE_ zZTfh|)0KhnGl#YK90~u0G30y6@)riML%SQ2+VVp4pMD$%lVL`C$On%lzYfGdo|``z z6O~fIGgatx7_j|QX-r_lRdVTQwHDth7KHOcVzzB$!^PsT+lpjeeCU3;U|F%V@; z0DQq*38pf3oXV$$Oi{`YQT*vCzBsOIx@wX58eeX{vAP5eCU~+66$*V=aAkjWp<`IXb+VKHHIJz#W1+qoLDVYR()Pxd@>d2YTW&*7#j zGFnj2$pWX1TdHh;>Ad^w$Um983*48^=-eH;4q`U&e)i0{FB7=`w*Ww3%gvlB$Er#= z_TQt684KcY+9^A96Wn6()3w3j>a6miG5??T>%U2d2)u`H9sQnSm-XFF_A(H#f}ra! zK;wbX>yrzixr`rm9qZHoaSO@2)ZrMEg)Ewap##`K)PaS?O4XVkjPzI<85)mcjbB48 z6-{x~3=0Qbs>wNoGKH6{ar3;!^g6A{JLL$Xoa7r@TB&M019x5yE)_I&EIO0OAV@h&ETGXSVhYCeyb@X#0+WfG(qqrx72AteH(JFDSAjdC?l#;`0&e%>!-}rFlpvYPnNu1xs|F|9(a+o|slM4S3nz zChIMp!C^;wthwT7K9$F83anUny_cu6x+Wy8F2q=IMt(%$J)npO!akw=|di z{BR`p?6(SL0ebY;ST7TvZmQ3^MG0+>fSoIH&ieB<5qyXwCoUhVpbf(=}*#XXfvQa#aH zwgz0x2cW5PN7DHBxEra(=5b$)hG+t1jVQxtrL?tpA>mTG=(hC}DHL2@JtJ32Y9{+t z@K5mawSlEA!?`{nM*$BRgU0WA{MXAkKp^$Qlss{oRF0DAO%L(U(_7bhmEOt+9n-TU z>IJ;#18)GQG~vO44HLJ#gof``gYeFKTsZlW^gT*&)f3`5S}LyBiSSSB`QbK0;zyG3 zi41o?t^M1Qsf{yrwn(b32vTK^yyLV7jxy0U1G`y$^i8AxN3qYz8@5PbJs3l))L{PR zW+8fby|@LBxad7LVrB&r_ET!IXlue&X}lDbEljm_L{n3()cTDTKvfX0|UH+8=Q|Dofa-3nv;+XM3t!5p|0=O~wzblC&C zV{K^-;u#5l{eR~t|4&$qzr&~K3m(Vg-$uU;-T3I_YDbNsL5RIG>)M3Ye)%=L{lF9x z+Hgf)>adz|C3HIH0^B*@u~U)P&g1I5No>ODbhy|Vo86g9%i8Nvdx6384|V>UxFjZL z{np;zI;C-21px~gmMFz_VDvEA(zk{U4=*u-S2UDOO!YOJjV{(*bn zrA$C!jVGxmbWyVT%sl{J#$Glq2vhy7+SBB~t~}x7I*KNUXlt8QVsQKRY8^lsu-jZZ zG82%yl3A-c9*)NRODy)uqj3)bH4%bNP=DLg?licLKF${pcMor$i5C-2kb8E=2@fyP zZpSK5zI`4oHcLzs`nGJ}T*Tuv|0nF|%WQdzJ1SXz(Z zNu{yemNH@wI7wn`dP=L+TxR5~l=&Gbx6P(P)uc^aBiIR2APR6d5H6`$u9mxfc~suf zyeRxYvWGeihVgG@L0=58NOOfpf?LSc6}7IcbmC~_^}i@;>=Kh%w#RCEq!xafpj7={ zTliW((@SA*$G9SGa&dBUdNtg82O7TcQf^h&d)dgE>h<{rQVP?bpxQ|Evb9*#0u-ei z&M27sm8G-1DRB{7tF|ovObjA>o@e$1>I}wM7?_A7VMQAG`1l8*X~fjROLkQ$x@F_c zj=?=^|BvTXNaHDffa{`&rs&ol2^8=bMlqsEEeEnW3RQSyQ4j@(Le zpPM>nfV0$1)pn0Y*^*R}j-PRJbBh5BV)qaG+#jK2D%Xo4`RQ7`0veFuJmtvZkyO9) zxxOAwcDDI+TM0Cwb-`#B@!#G@3rSH1L z=gs-H1F270uWI`{yvFr;=c)3~#PNhN6hGGZ77aK1WIdL>LR*R zN?zb5M-!^12%;4m386FpBQa!P2S>ZWgDzk)W(t3^ZU5p+g>0-}oR(Hr9dEss{B*

R7bMG}!AqIC z2`!m1@MBRbke81|xz`^$Y7h%dzHa7+Rc7=^nGxec=|eXaOv0$x*Vk2X1V>btx&oum z#2aDw6ZAc8rMDU${YfU(h4m?SNMDaI;>f}Rb77*zaTwN$3Tv)tMhE}7%z>7%uR^{? z(K-8PeFs5~S0AL&04;+o6!@kKkBPZ{QOX|140x(PI|nz6bBHd6XnPO~xRUoS%5_%6 z97DcM>h$#vI$o}DbE~R8cNR_`9+l(3iu@H1co*7Vb?J?AXJwazEhKK#P}swnvDnyf z9hmC`8y4|I$`rAnAJ|1c0pfrAa#&ax+8kW9L9c{Saf>uR8wjba6gNT4Fdzk&NQkDe z%JV$22^mcWHktkic?RKmg6#qw(L;s;`t86bq1psLGp$jd$t_5~u6t z^^Lo(QaI^Th|Nqo8k))E4+hI%-w+@ODLU#L_(5Xk((9loigqL}t)|}g8H0zn2Wx2h zgGK1d22nuK=D_A46Nm>%ktABKpYE)9qHol>S`)L+U3GJw?`14VH{^`LB` z!S(0&`nWQsF(dCrpAk(&fFonD{dDN^3+*$lP>Ily?a0WjaXux-arq2-a#^lEqRP>= zNLHPxhq@d^x7T+Zc+akNIPe1m6L`91ZC(d)3ehr1X%iyY;Ll9;#jVK#`f5w>I3r8% z1Q|U%^IYXqq?pAV*eGbjFHy2*C8cd_k>LZlQd;tj{MS+Kq}VumW*SR+3RHk8&BW6% z>1fg+IInvM4rz|LJyJU`2Y+Q1oh>_%zYkmcqY)r;km3X+gM{ceWtXuncFQ84Lm;t_ zCSdw}!*E`y#Ne;V`8i3_L2%~tuX%u}n_lXpFU3lbgy2GtlP<$gQ(@f*J?0O@UuxmV zUbxj~rm|IHas;xlA8<8jqo5#_%{h+)ciLa@SqI^@JQJu?a87Jdt+(YlWxxV+Wml zfb8B?U@6HfqUHqO*J)q1n#9Mt&<2Vv*_q+n#zoKa2gdDHJ8TU@{5xXv`BHT`St7B{oB~U-*YZ9VoMBZP%I)MKmV1e-% z_V(C0fA;aOgt>1k51ypmd{&6Tp(GjNxs5{{5v| zh);zS{3^!Nxliq8lAis^deed6;;@(JvE!qvKKM(*jZNTGuPbn8-NN~R@fSP(s>8e0 zp~1`33$jz;6V4p}EN84K{^ghQ`%|Bj|14|$s5`)*i}}+dlO#*N*zr>fY}_l zBA`u@BPI2l-yC``4;}W}#E?0zt5}1~b7F~dTVLnO;z?)I3g$o47uxu%@#6kmH7qPE z7aQgj4ys4eJCxxx69;8~jVFf91wC>0`O3wA1CG)VwF#=K>Gf~`9MI`lWJ0Pwo2iTe z6sXhm}H5tdk`Z`AbLUh$ogeQ98~fH3ac?N|eD6Ab`1~DDTarZS0{mLwX5= z!h)H=G3z!;wstlN^vd<%^daP^Rma?2NhynVl0FP)cfpx@pQA0iX>+n#_pV2DK~#YB zNRt&waABPgKp`ed@ndhBN#?he8XK_)>rVLRftnr5%Z2Q%^g2>y+7Rs&5_}e1`*x6Q z!Q;#u(tGU=0VvlSy2t~4{qrV$#k$JI6myoN5g_4!O|)gmAnJZU>$Hj*AC4#jiPpK= zUu7YRbaIirIs4!{ox53C(dN6N=5b5=i%v$>o0vm9hGZ7?YGy$WKSmUv9f<<{e9aDo zKqr6LTQl-@$YHeTni(HFJZbx}fb&gvF-*ajZ!SW1_~E&DGP*~l=yaOTb5sjXWsYB^ zyiFuOzS9{2lOl9k-; zf<;FGF8Vl>Fsp+(xyN}-Jq8lBk{md36)7G_=nx~$d`2{Jy4~hz!b|69ItG!1C zd0hh}I5j*=q?Jm|cp?uP8vxx2=fhOyRJ*dh>Gjnfs`33lMu6EtX{Mt+J*DQNH+{uu z$gF+HJF&0Oh=^PzD-(5&y5JJivRz^kzenaLABaY2B|++*lIc9o?&8Fpso3ry+MyTQ|0Qll8EUK z2PD*_)pX9s37H)WCk(WSo`boXHOE`U+r(1A&&X;#I26!_*Ei&6Q=PP;K6ZMQTPi-+j(1amiD1ONDjm63~g&oJT|drXzI;b~uduI~LK%u*-OBDb@f zFPlAY{97Bsj{gt-JPt&BRrrJUzH9&MTljY=SJ#vJFAQ9^gE;3bJAUxsSD_G(S_4cu&oaP1SRtWnXjtWy zFj1)r4RmXBWbX~C&%C)L##UU zE?=5Zl9KIL-UKHR)XIu*gg z-nuD}IBrHrYiyGy0ye7MHA~(Vs;1OXfy)-RKQ%ysPMa@OCKfb|9-ez^``bu7HUjL;N{HIpY{AB| zDJJ28VSWn0$?L%|CfV4=Xci8lcqH!`39Q};d${6A(*2HOc00p3vt6fZYkFTTKOU@H z@3XpdQ(;iIH;Y-62&e08bJr;_SqXM2W(CO05Ts;+cxgaW(n|S|Bnu7O>U3zTnI?aI z@TOGZ+xqNoNcIBLh|60Y$|x2C`-zcwxK-lW@!Z3%{)62RG_KoGL@ zgOSw*MJs3J5V%?C$RZ*qf_XqKCn!mxHBHnI1CfTNp;~XD^u== zPTeUN)v*anChjXdA)lWv5m_u>q=%+ZsJL>ziQO9|&WmC=QeIjlvii9HjF2_SJY8Xn z$-;3{C-Iet*zji$3Ej$)d&P!#JC&9)W|akLdmkTWhIn4Qw<%k4CSr=C&ek85$m4M{ zN7T_4ZS(i3d2kuzmo?a3FyXQZs^ss*LJ+W&v#;{F^zxzKU8Fi^82b|9oINxrO>WZV zH$IyeUh3g85>a<7IMvE21?G(LJwM++&;&J8uie`!>_#GX_XD~!GZkg8mG|~$XKZOG z6B40s_J|vu_oh(xRvXe0G@=xdUC`=xO>I+Bi5n%%j+|wEq;*ALF&4*)_XIUsRe=tB zD&A=7b1LC>i~a8MFARC{x9#nc;#5OJlfxrxn`^!a0pQNS+&5%8qN9Y_Z$^WcFa6w{ zwZLrpNyw*qOGskFE9A4rb$LWJrr#?fTJ?m zhqbQb83E_pquwfPX>jiTh+QlQD#u>~0OZvk$zGZS52!ajnwE98)<&>>4g6=EK_DO7 z?1Dt1B~tb~>sr+WVLw~-qiXQ?=vRdD(x#8G6f|S#U1}@0WM(oPW|%?}N1tC0-% zOmi=3txG4KwzgOKT6%@=edmq^`@Jx`HRMS#O>o9bCv$9CXkJ)TtAzS7xcu92FE1~f zN%4?{50)ozs-W_bp`Nz#$j>K5^|Q-$C1#FqOGVmg*!l!UCE~vH0hW8J5*_xYE*Jee z8E67O|C}@%Yg*Gipi^-2Q@+3e`mF49q|`s2M!s=6L`tra-CAlFGHfkKA(aNFzUqMvk_^VE9j3LW}BHT2g2ns*2Z zbrzFYCi}{prK;nRxFIX5@11Ks3C?iAbxVMzMKuMC#rtl$3ZcceAOBhiIVsgS{=%@_ zY4)Jy<9bqORlm?MgTG|kR)GRGomPtZ@{rUhICZN1{>2Fg73Qi1b-ReMS~K%prKZ_Z zXY`o~p3mJ}?f4u1(^dsy|HyS>+m9Ac!hbg6U@GO$FIJ7e~p55?@+@j5oMpgLx$K;Qf&+ju;@if5KScY(# zu%F*8)OC+qf(?W?;Ov~c)xcL?o*QRvRi7w zwFla5#Km7*qM;2h08bzQ*z^kY3TikC;IHx%MU-L!lhzJZ9P&f6ql5wXs_}aIJr|jw z7f%#jzJ~s?$mI=)i(90|=K)a5J=Ykq%lZqWc3R%7isC8KlD-zA@3`6i<$3HW`jOT6 zIg#d15u5m}tas8lhu74j3PUAEU8BtFhXq7$O#9MG&#$qr0gl$%;cs@ zo73MDFLdKlGO3V67wsTlOWI6rKA&j$NFtmdn$9q-fiHE~wStbQ#hLtR`k!A6e-r$! zB@O`E=V>PQH;o@=LhR%kEj(mDX3SmF%;M3pS6_w>umBYQ?Cg1Lk?8|wOU2oSZJB;)^!v>;^?keOEib%X;DLQ64rS+zv-4;-hZT$? z!#KOUv~jsYCrv)UAjQ`eb=-`U47b%tW2v#u8-CcIHu7&H)9MwsYqUtN@|3EvH}XB1 zKl6r2qWRr_UNvVb4K{hC+nCk%PK5k}BPy=Z{IOdS#gi^wX?EcC8p0f1)t18eR|HD= zEo-oAZ%0DTsYHQhZ%r4ryUEG2q`S!xHqHA*5Cv09y^e~#WDqLfVA;sXAyRa4au(}_ zk;>p;R?3=)LJC1`2&H?t{q^%$2;nxOhC*wWwrCK)c(Y0=FpKmD;1Q!SpxO-1b|(3| zEi343ibGIex1~;MjqpyL%r6pkVSQ{Ux7#qM$|j%Bqh2<>`C~eXcPTi215`j>FvX!5 zMG>kx7)MKNy{x*Ppk-cb>=HKpK3T)#Ce8l#VIt z@nQTZ-Q8?^NMin1f2}UXg*8-MzBZ3HMdn0A ze+?{vOtKqbV=(6-asZEjbB5-x&XGW+#PK+=V@G($?+TU5x1Jv)%ChIhs$4C4<}&82 zEc0MSIpoB~M)JcGXU^$3EHYN2Y2W(xvkoI&{?5ir3v;MS`o{RH;4?g~jFnw-@cq+} zbuAhDUH0$MOlCp$W%czOh<(x)Ar$fK-%`L=z$KID%Rl!g>m`2Ww{Kg?z* z+0K0P7sm0Y@PP2pYej#rFJ|w^8y-nj^qHG}KSrsFIh>uL)dFM41FyzEhxh*d-T%05 z&;h=d7xE1oK7^7~uy1=#e4k}x1PFXQ@BIgk!$XyKTE>QUF-UV(&~Aq56K0nBiAVjA zNA;V39)-LAjXOTlMklcTU;JJF{U(;s>ed{g(-lpJ@rS1?i$|Xi$Zd*aJ zo<}Ey!gNWA?AXj&;cw5dFbgsn(Vyk$Ivo0U<}bQ4LCq|ImwzWW-?d6RX`}n4aMz=A zBpQLyNCx2lv6{+IP84uZ)XEbOA;n%AS2Zb%e5aqGFI<1LyZejfl9yK+GsBn zo5h8By;w1|_r_O@vsG2Cj`0)256-5IBKspsTahSrQ~_)J8QEpX=K8D}AHJE4PI~_K z6km!yd;9@wrg!VYEPd9BCK0e+M~NeGqbgsTV4I9Kf56e?sPW;s@?M%nv1Zq7^!>)h zrQ?2{NopgoqQA|i%A0#|RFtFq$T<5ghwKMshT3Ke``9D|FhA3)_uIra2~r3$!~Awj z(C&o@5;SWdvHcZJN*wrExEqjXQCuLNaB~@*Khm>&(vvW_Cs#V`$W`*7wKug=*Ss9? zqDQ(!B|V>sd3cMiTZ(ilnc{0Pv}leqct_aIZDTDV$Dx%!M?pgc_6_|p!*tekd1BU^ zOC8fk$tgUS%Ou3A&D_Za+?{UA!}pON(2Fg|JI0<>ELaglvyg}Xw&?tiOJe-`s#z5% zC>e$bu;B7Du%%AnAfrs>pB{9Z!}RJ|2m=z3)VT+jA9DP>jMPslej3y)-qMAMuErgA zwn4S8);V!n4QHjCRVH0!tqX)t0Gu=5h!WTJl{<;JJOgy`O8walR6FO{@vg&JUqKR# z@T*{^7xf?8DFfqSCE4}*(saw-MR5o26OFhn1(}oLYJ0W)1I2HPY7@Vzb)pkDJJ=uD zBw6B;$LU?C7QTXw^7~_$h~aPG9IndTIN%GtV#ECEgG7frSM~HM z1A_3Nqx$m#&j`&5^`dp$5g%Q6^j{dMzyFjh8&{rJuv$|F5dkIP${+#x(cTl6Wo@76 zDF&?svg^Mvn(c5jx+;`4*_;I)H+0V{+FCmD*FuuGl%JB@|&ch zDomQ&WIoNKl?KB_%(o%E&t}d|QXc>_{kzQ_)6UW17oZFEDd}N;rD-|uJeK(VaUw1x zPSvv%Z0|Ce5|CHm=oJ90WR;(|;+zuI0Z3M!Az-1AAGLMR`x8USW0mhKsCLIUSz!>C zFiur5;*kg!Gg3TJiIMhpL7MFD>}WMhCXkJgOdn3~EjYsRszmoJQ0M!1f-nn|Kbya; zn5byaEQq8o_QSTWSx#aNji4LNb#VcltpiLfGJ1GLO5jN3 zGbZ`S^`A&)g=x3A2(6Z~l-n*qFL|$l0|$dzeOD+7!Yl&XdXmO^1Grzl2zVEbi5a5v zjWMESs4C@c^V{yxnwl{sq#NUvMI=&pOfNAl(WdM=f&C@M2I^_wH-fgUvBdYH=3Tb? zYNVQ@iRSgh`-Lj_2g++TESy`=@BJI56e@hcmV65~OGl0IdFJ<$np1H-2~7lIVkAcN zhX(^PN2o*ckP_^W*{TL38xv~6*fF+JsuY{7(|jU^EX=KE=9Xl|vZ_}toGL0fp_B?;5;L>bc18`b zn_;D2&T(<_gN%cjx$Yn>>m_;E?XIBx{-@d>VD~io>{3 zVQNhywoIow8d0J>(5Tf(h$XMEN?Jx`Uc0l()p<^l?+ERxp;oN%;3eG*A5>shEW&M? z%7^8)y(7e%K)&3*{-WkqnDMi!&@3Eg+mJ=zmgGOhc({z+XU#)Lm|rv&gf;aD2U}E% z+mf&h+W}zDW}v3a!4waPMY5uek+cWabhHd>T-V1j6|s#_3Rhm8q}(h;I5T)c;(48r zMK%*qvP&~HJ_1jEDmx8vfTetf$+{B+F+-iNCVO1=p9H)x*aE&E_m>2VR?b~GStiq! zRx{DYw=SeC|5i8l9~&TwPKf`y{AYi#HDdH(XYL zUjI%jh(}UAZVnH?T-PYa=S~*=dGuRwOz-xlq+|)$PE@biY8!JtTU&6j9rdpHr+qY& z3AC+Q20hO67#v2dx}^fLc;016*{cwm7?&gpO|j$3`y&$?_qs9{mOoT0#~vYjiH!q& z{Y53#ohZZI2Bq+=Bx16A3-=9L&LCr<#&$rR$41SYvLSMLRg3dHJd7eU^78yVxBA(P z--yDBtV|*Jlu9|_jKw`;P3UDlOq3DuxVOkCrs&BBIG_XnDeRH&L05R&fnc?Z66~&s zzUz&re2Xa-bph6b^w-IAS-B=_$W`Kb2K$Uw*i3{vZ8h`Tw4o{reIvVmbg5ynGa#$@ zk4OQkwwQmxUUbx`Q23sehq(}P>}h0{zL+5CLpn(-;$274kQI#pS#kmQcq!C1H;7=I zS(IY;vrRx!Z#|kG;a|qzn*5?5Goa?J=aJ|!H>z(wd*uP-p%C>}s@0paBm4?}t9Wli zHY8Pj0yaT9{!z(%jhJP6OH#I6`9%cBmJIKZ1CH`zCLukkpMhTv!VlGploa&T0lIkY zvG0JR&UF}Eh1NUTTo+W5%~GT-V$yfokw%I3fXIfr^prsyt0bE<&@E3sp!LpNVGge0<43zz)Az%?-D35UTFDe= zgvzrgnQ>wlCLdjyd5aDBMUONpvX2U&7gKc~-z~9Fa!7UgFg0_#;8hu=c_=54a&wgp z5iy3)uy05x1tadIC-K?&Cy0SUvNUigrO2=0X!MhQ6g&Ijz*6}1oz&j8)C^`9#q}6& zg0nkBnFn0CvL)u()>95ignON4k!LdeNRzc+d)|hy@Xl31och$^XYAtZsfd6zFXIB+>KYj-;+Md>@X!Pi$!iurRiY$g&TUlxzGC~|bP2iL2TOFhj z4~F@)o2;Y5tzH1I!6n#cTecF*68XRkhWNf*Tv3uSlEvqL}meuB}IFD9>i(C%CMSjc9eH$;>(I#a`l% zFEjGW`dGBWz2@Itoid$VtY*r-Wu`Q@YK-(Q^a!gO7@DDZ^6 zy_{R@dpex*_-waSg)zYi@g!;AJJUj6zW8~W!30+!?ccnPvS8+S{r!Vo9IO^wSp6HE zU6U#~n@=qZ(n@n?UDLWJStbeGxRmUIhiYdD`Fjc=U%!!HevRxUJzb;6gn-(@%zlig ziRJrMPj}2l-^yq6)(xguI6Am}PWx(X1mN`$%60eV9wGjqs)C%!aLXN6%072y@M1XR z^VcaM8T^P)zZ2!eDq|sX1|VNlCSm_J?=cNtvhQG4X`G<25RQ6D-*=}wL7<6VRR&b? zX!UOE(cCGiO-JfZLP`>g=fI6bm^V~GqR!jzvlEdwVox<~+@DC|6x9szM+a_8b$B!6 zhaj@gyv|!Fsg7o{6oX7|Gd|S+yz0ACTWNILJ=y3EF8C_Y7dX5pZ)Mn=N~S_mBYz+9 z@=Qm{fM4USDFq%lIX_L4IB#)-?qeV5mMbDjf875}G6IM`>GXT)9X>k*rZmTBq`;b? zt}?+l>jULAxK{|?U+hxTj|vpsnnrPTM@kiKVXuf8q|=iqlcy4=ff9ltmdKayH!_HO zMnrbHu4Zf+_wUBh?avkdy~w6SQ*GZfejUB??JyPOmAYv5rRy;BPmRE$)(#oz6Z}~I z*VO&P{d+$XjU3pN^D;-y$Fb1XhFA5}SeQ_T4*Gon$|*p!nKYIkgMq z45o@1PZ+Bwk*GvdTM5LQ2;s_mdT!a!L>klgBWSInMo|u%JfVpLVnJadQ`E*X*_>Qu zu>;-(9hC})Y#)^lQtpj(xKDJPsj6zjGK`Fj9BCvR6395^CV@Nq5jOmX5+LPywd($t z)$0*%P3q8*pLxOyv3fjPLs`mhe_`NlPkk$EtXCBD6$~m6yx#|-Q=?A$9OWY|J!ffx zG1$8{rR1>Y(!tgB%*?!!;_r4dA5hdcUyWtt|CsSb(kn}(x=ZtF?$pr$bUHUFNgxwb z^HT&HeW}ahon@Sr@K|s{?SktCnCMyzqnwZ%RnBVm!k+#YG@c{8v6_lC3lAL4E8x^k5>* zv~T{Aj&Xt;>i2w&YO{N7^JJ-!r|lViKmlQrr~bi0o6!Ai@y4Gv|2FY9Q=S{sT^Wzl zESafItd#>-aVxX7mEy+JERaO{5FY6FhKwRm?gWjDT$i>kjxujY?CZhh(|;ZS40CPgYgp@F82WoN?#l(mk2|5GPdLk790mdTcT8fIfGChM z;KxT&Ld1dWcdA14xidEHTLooZfAp*>9A>2wUo9!kbHRw|DlI58UjBJ5F79m#X#GvP zQs}y@!EpG{4~7{b($`lDx6bHTPdy3DneQZaTTW7aXJw;)^)u$TBn)b>Lfe&4ha=B_ zV%E<{SjxMI&9Xy_*e64a1agGmVG0T7{&&sfKV=mxmgR~nNhkF9KOEw+vZbhdamkJ; zv;SxgF*bVMp<$6nx>{f6sA%sN{m|pj_I>lTcs5>=ySx*5)QB0e9{Rspnjc9e_^bB) zzV%wQbN+(;6!(inS3;BE+4@TQ2ll_m+!rt3b(BBOMxSdd`W)&Tmj0NK#QjcuIC~@< z`<$&}U3|IwF;>S}g=UoDNz~`X{l$(uzm_Q}C-D_OllRKDjSYXi?md>rogVo+Pu;^_wJo;BkChit%4HuK#jN{=>Rx$Twb>Im)4@G5s?< zByIKE+;2HChab0pVMI!hm_PbQgx?*p(7ig-MZ)1`XS-o-0h0N9u7`$fU)0_8pZmt0 zfUi*s>ZT<$^}VM@abGpH`6A^W;l8FZtq%Zj5fm22Y|D5Y!QbxA5hIXT9PfR5snm_x zYt#vPa)t+&lRRySwI?cCtn@B>oTW4vBBln(7G$hsJ_1x<`KWIqc2r^}(X9VeeQO59 zJ7lf$E;M1aV0z9LqVwcfuvvezT#3C}d>gT8wVZmhJ~{V3c2dcHPg8r;YTmJZD@3ba ztj$b9_Yy9H?Bh^wrp+`-lSCJw#i?co_hc?|(Xul%*mj|tHYG{s><9I3P_gEoMe+_8 zr>70tTG;T10>=&XX9~ti^C7nYBZ1w(8A#^d%vaKIth@c9PG@xDRU#W*JDnmJ;POPv<|2b-Mm|khnN()F zPwW{&;5G4*-RmISZFc*P&xhQ{ny@z9gy{vb@uT9xvs`o-(~URjM4$6`U%j2&dVx!C zgyh;b1~T-O17(Tjej@{bxDV?8QG^KytPMPQnDaAtsLYm%BqIh!e$}gml9VMB)!$F= zgX)EMlZ`%h*4y_sXxm?FhH4q57LR6yf!3{-YrDUQ6In?r~&W&RWrVcMPWl&r@12jvLCY-dAIrIe5+1;QwV??rWJjcnp%uw>t^yi&#+i2f(IKF zC0S%_O$^r=T|M*GS?}aPk4p(+)ssyd@%kyDBCUX~$RNrQJ4THGU|Z5Kvl z#dmonB}Kaxke4^yEB+`vLS)kTdz2!M40-I5Z!>E0*A!btZv@~E_mty7f%(9jJ$L9a zFokGnH^<6G)1ah)U2%AN4zR{KtzN_6xV`eJP4xQ~V9&pE>_nvi>6Xq5SU~kF<$vz#n6%i2c@Yc531f~k1PHEdtfYEB#)xOive@RPS1gyv8>iUFJjr8Y|loU-Dhh< z^VEziDn!4fD=xgHpUIJHe~|gCF&NX|PK75Qgj?lQHYbc;PaFs-O_?hS3yn$=a0P27 z6*^irj2l6+99>;XH?5pnT#k8`B@!NR?Wdg^5p4a-63szb3VA2;V#5WW`U^DJUyp|; z-aS;P&izq%iU112I8wB_dljG4dmH}9P};JA@ZCt9YQUGI;ni4Zarf<$H_ z)WRoJWTDh4UUgT;>ojE1fP-&k&bAKvD{fK%r&>ho!3dTDGH*Q!^FqNXUFzK>0wVyj zAtvMBZjMK+(~uzghCU;z=r*W=aQ}xZm3&=c0j774l@V9{Sbcvake^7w!S9<@TEXC_{ zk~CEL;DkxM;h0@^ItbZsn9&56WV%!97iEWqGiV)*q&>m_QnHOt^l4Ys2^7uq=^dgC zk(?E?-&b#?yDnz(k(Lg{*@=lNIn`<=bYW<@&xyPQfUD@>;23jB?(&e7h~A3gKJ*+p0;YVK{F{)GPrU8k`GKYPOicWA7w6S`S~w{^jdqn_vlF2gtYO%|zd!X8#c!m?-uoGkFfZu0LP6ecE|SjE+fd05TR8vzbjd( znI|g)DR~b#+kQW{h}BX5wq#@2jcJ=T6+K!Jn|;1$77IFF?>MkHAG9y)5u@f5WQA=O(uD6G>BD&d5R(gr+mdp?z(yXPXpX!R5nRc6!y8xJGOpUmUSL zUpuhY*Fk>N$GYKN9XM~goX+-zVz$ax9(EjJCWWNF2z4Q`w<CU(n#}Gl1?_*EgA_=wFl-vy<9$ywv#!pQ~X`}wCiI3Jk1IXkhdDt6&~Ao@P$to zs5dQxrsH&a{LVTP?*(w1m}BL~$(&^U*vEVhKTKo4TU4;mO(%;}IKp95b7E;0QB}oO z=7*Tdvx;_hw&&jJRF8a;-fn4$x8sa^16xM0S*!Dv)CVx_ZwU{|d1ez`dSroQk)6&{ zgW%rLeF?)u0k)p7WhC7d!ak8lvH0;x$Lgs6JSm5x#KmlKX9k3u_&K#)BkX5tYC|_N zGa#mfGa%@{xV{7fa712vgUT`F&d-t@3UWU4`dkO8uEnU%(w(y}Zk3*;XenHO9XG3W z4B2@dnowaGy>zHWpAAc?elcQP;0(o$!-4FxMN6Cwc9DxTrL$yk5`F zQ7X5do>xB@m@JzdXXDZW-u;6v_JIiEijB)^ksGWP`1ro>O5%sCMoVQiBJ zHk6)X#2TfO)w90o*Zg~L;(I73l)}x`?`@bisc~BCsOhZrSs1(}|DbrCnKXp?4~ptE z-du7u-vf54-wgCh*;2>+U_t-y%qHK*h5%jOt%!LaAK>@QLcs(MZ4$*YE!`hE{Xkt> zM=W$A3UN{XFyA0wv$&WGg!BWgTTb7c5J)ihvw`M2^y6jPGIKz3GekXJgom*HZb#i#p#?$^_eQYubNaLjKg1w)ag>`>K$iPRCEQlOdTs$t7lSA`)f? z)Ec@-`{*`ylc=ysh&`UP$BjdD_rPy0Ql+_}$)C>{Lxck(lwjLC1<{JCR*729Ozt2` zhVlvXyD7SQLN>yGP#EhAqWv8_s^JSwQ28WpU(TWfU3^Ky#<&hE&_U8j2AP}bB5wrf zjRR9J2=|s)tI{c=9-+q{L}*~DU*JVjQx*fIJF3ntm~kR&-~E-SePbYzU0@dWu^{~Z zn0%`uYi;7PoQ*=CyS6st>Iv|DJ(I|_@8Cx{VbQj2E3_Nn2fl|JFsp2VJUXOd&W6P%Ip%#9v--rHiNLgu zyo0dlOlGl+(u4;1oI;w~e^j~UM_|aO0v;K4gTMM_O~`!#O)Np_wn``jTweDNis=wp zItg!IkDoxQs>>vQvSE(XMI5)9iM~(dMSeFA3!4K)H5*>XQa!6xc1k>{?k89m;CG=9psua#%sOm<|fQi)8Ja~6B${`CW zBgWxW%3-ijJA-n$-&Ir}?sh>g-V?UfE&hiEhh*~F+tdP8M|p7Il2W8eV2}9a&d>%1U6GKQ3HzF3OLSwQ7_$w*^>A#d@&>=Zl^4sW-*P(SO?^3pH>kg0++Vp$brH##U@m zyrcOK%7RsfLrfzjR^%H_%{*{Y1p;_8AjE4m_o23Nh0CGoZ8RFezBJg;GgSPNWI-fr z)qgC~m8__>Z0b5Q=kK$FQnObss>6I<6B{pS1Yl+>;yx8WBduW8EEEl7A*}?%(qai@ z`uMuMey)?I-?NNFTT!}D$KbwVb|JcDXO-XN9~67XcLdd`s)v)@a-Zqe3u1Gx&G*$> z`InD>B*3Q99|lw>ts9RuOur$O5kwI`4kiFnMa=}1zM{FPB``~_DGU08i3UOF??ao- zXdv|Yilm0r;M0tIE zP*2}_p2%;@BnHZ!G*+0_r!q~xj(=4D+@6=zG&W{)%AK-+g-Xe?NsSB?%g?hZpZ>pdK!m(Px7a@@e*e)RQCP|0 z+d~N}Pf{cg{-gkaO47HZU%6GH|^1LPDZeBOxPWd0WB>0H4>!t8>uE}XslA`^l&F(~SLX5(HUNuxY*?=s$ zarHg99A>8cubP?uI~-_t z_uT0e@t>a)g~NOKYX8Y86lm7}tp^mB_}S#M>Ax-l6rxW#xpDt}0eb)CW56I8QU@)m ztw5P+Le!&j`#;w*kgswE5LW>IS5WIw;L*aVT2n{Gnl9xu8*-Ec>61GO zgZ#9ZtguwC?Oul@*6L%|L^5&Alpm8(c)++4Nn|(O;1}%J7*=$)Go7HA-#u~oe0*ZV zJ4qqtvaVXTi7T!o44Mi)-5l0ORgkb5hMlYi%_14-KBis#TGt69SUsX)g=bfM-mcwa z^o`<|VdYfILCe;#+1R*9=9rmzr}h+o zoQF!&Mn;~6UNKnv$HzNZEb``FGa%Um7cf^7-QDPQ(0+ZMN;*D|rquvkvL`Pyl|A!In~ri*jOO z0~0m;HR_ACUff`_40pmGXgR5_Zr3@Jy;{i0cKO)5Q84VH_FAusad8*qJaoaWrq5nx zHg9CSs@y#wO~Zf58_H{=Rmh*VD+peXeK$csKTkg-!@rZ_C-}=3p;VIf{T zQWqu#Lr@kBKx?G!XIof(?X*nY+5ZWt`kHpi-lCpfi%cSIe|kEp(dU9U{ZvApeX=Lp zmDP3{HB#z5uUQ&H=L=_ll*Y4avRGxTSr-o$7qRW}=5xiRp90P+{Ld=2ck>n1S$!Id zU=wWCaKGxV-lu;lnFoi?=+t-eo#YM;4r$@YVwaI;&97Xtt z!{e^q&^)9yqJ4dwbJ*sog3v}j^)1j9r-&#Fg?@V;2WP?V6GU`!!Z?j5HeR1;0ji?| z?!M61F`ZxLhX_3ERCiE$>dD+yvBtf)Gr^DcEli;D<>((Tn)hQw*crA%!5LdItO4)S zW@ENn*>Rbe071$!p#$ZU-wIuA8_oYte0)AQaDCPRdcw0NIhOcrXLOvdd;uQ#g;$g{ z>F%*c8KDH9Yv}0gBbBSc+g5kS^5C4gQi6|r6jn`)smdVp+{pKWi&; zRLqmd>Y7ZtEN>^Rk6u^J&qs8>B`mMSuSbpKO5QN-wh`EA&Ao4wPSge~ohMWPuCp*- z*BJ!fGu)nZyyoA}rp^~>)eNVIfdQkPbIe-SP2x#?*LY6x>=bn@C`Zoqx2(2TWPW8Z zn@5&s>xCWQt7W9sGUa1%zNe8ZF%yNC$SB3Xa~5u$AqwTa;@MAx%_?&a>7p5|%&G0L!K!+IQ-WtCx8_PoZh`)#y>VT5q%_%aqY>fx@ zE8rea4#?gtg17lc^Ny@i{yxVXb?i5CWbV@#4T_$VpQRpp%6cl3K`v&W z$4*y`OWR+bH2OlSSU_?kS0y?|YTP>GR@IH4ZN}lLgQ)^j}37b?XjWB!b$y&V|AP(j|)0mO_y6HyIXcgZho1`$?H(qTZzK5oD|8koZ0y*n(XZd zs~p%zlMECWj1`Er`Z_9Jj!KZtAHbccoU6lw60`6EM#=j05}E?O_0 ziuwg5(qWn_XC+r{^{L_IKfu(kvrLTX5EV?|&mb4@aw!>fY zt$Mh{Dp_8XD|vLN_i)~UgDZmIY*o0hC_F)wIob=FrG)=Q!UZ-ogHT#c;i}oU*U`rvS#0=K=hWRIh zZ7OZ#sFA9nZ%ifx#6qS#KKfEeMv+07%6u;|5nqC;#7!n0H6(E(#K>M@0Uhv4L0vkf zMEjR(5%Wa36pWx|bLj6gonZ!_B0lUhQ$5q3Zx~=Zc~*<2Gb>L$hBK%q0%^F?4#6xh z=OMuW6|#bSXc6pE>YOC%fuVLFY?Fk7x2WuYi|wSOGmL|Y*_k!~w~Ss}p+o5GVN4B~ zG7R*8o0;V~n;*+YJrY_s<`-3(?Er!;ah%RvQlLi_#;;BaZQ2{fJ#+$k-)MgV+(OS_ zuIoWYVV_)`3--v;;c%C9T$_fIH#TB-&qecg!JI!{@f>_Kl_h7iRO#g zI+f7XDa_w%*=tluHkRTWmSf#hcA_%C5KDSREG9-)u-Y>yx%Uhi#{f}pMH zU87mt-o>iTi|d5PztbFP8cn&KR0P9*q3)X)8q2vU})TMopZ4c@VSd55Ahp+8BSrRzx|TkdLhf@XR=TbO!No>n&W z2vz}JEl5XzB~A-|GTmHy0&GuLNU-U|Wq;`EK68s;I=Fmp-9=nb!HyfRDy^b;^kUbB zmu^|G)_eK*OY^z7Q(a3gDY7!y{_aPR%MwJ`*W}Y+85`eFh!9QbJ*rCUt2x5k)2F%@ zc8ZOy7rUhry8`m8fabngGk@{pfFVD51>4-1reDDQN(fOVfp=5j(Wi&|J3Ke_jNLa@ zDR*v{1fcRkjff2c(-Y6+eTy69Wo{fJ8$t(W6?-49rdc#wUS}At5l-x?V__dF;sg9CAH_+h27zj-aN7EbHh0dLCnwv7 z5msrbJ}!Jz+QHY~kD0?J@|QX6bans?Upv{(H7_cOAYYrrH8UGe=|dE%YE4B?8?-&g zSigJ`IwZC?7gcX~h!!iRb|_ZBV8ygcA}i^B$(wAw&?H~lk+*UnNLoG_@AzFp>S;gH zV4>jrz>2UZN?lmhE$Hn5Q9v^rah4LZh{Mq?vz_oVNS*m2nJAwukvRF46X*P&T>^R* z>DX54%H8VB9dbw<_Xp27hAvX3@!jb%HoN}dB+bjL#$p2{Bs%V!BJUrozXb%IW3Svy zZ+E5@CqGW7KhR$RALkLby8Q@hLH#ewjOXp)p9Gc9KQ0L{Z~udG(B0^5#1<2zu$pbR z2Q}y8WifwHjwZ3C(x@3|zY-ZLy8H)4TxPq==W9%|~1Gg2+G~16z_TgTr%4x|A9mEi9R_4i}gb>k1 zeNAj=AW;%rsEX|s>Gg!Zbm16-=@Riemh%pAY*+ZmPa!C8Kt2G?ztHBq(|!s^ARRtK z8hG{2^QZ$dmIV1w6sCSxC>^mmqgghY%qE)hB0aupe#D@R{6{$BetS|#1vP>*e@BPZ zpvFY#fKuo(@H*hU%}1_2Mzl%-85O#UgsB02TmH7D{bb8?FVT%BuRYSvR!ziV`(Dy% z^f#=ac;X|=Y&x!RoLJ+N(k)d_tXftkBS#umY96risp*`$+id_A& z&sK(WNf#<-5)}V1kgP2q{n~)}(~Vk`cHVYw{WV7N$s27V2HIQIel8~6(t%v15`fni z|N4l5EF5U0NgNp|WzwvC1alW;syc0Prb<04*!O8ytLq zh4ErIlmnI^y~>!J0?HxCh%vHyKqkGmyg_WLMQq;hW8}I~*K0 z8P;Xo@CUKy%b1`?D1En@U!leiIPcs^_N#9|L)(UEI?xOO7Y|!a72&KawBfun;n46j za747WP@k=kvI=47l8&tVEzjG@mzYRwXZ6UKpmj3iRo^vV1{%x0MJ(eI;5YYH&|Jp&fqOut4$`D`Z&897!J z;?MNiYwlWf^aej>#Jo6fus)fuf8HJRkN_Ooy}OT6zmEUafhhNq(|dX#j(=#Tw45(T zV-)!TKK_svZVzcHS5aAXCpI56u(P)bv{HaJV$eAY#0IS+Yt%>j?K`@1L%MdV-D~l{ zd4y;u9=oCh;Wq2YVl>CY)&=r6Dux$4{@k!AXzzQRdwFNQSgo})2`8}dcqghH=}wP` z_i`QnqGV8wEfY5(nNSl4nty4eJ|ul5Mu)IB=9xf34-_j%tZP($J!|am#l8C2D{-aM zy}4IX(o))2-cA;eI5&?6;>)*coW>RRtRMq;BYRVdP+ma#z8wpM5m}pAHDVBv!0$Ob z121rZ;xZYTw6znmgQa1hBqTa3q(Wh3>41gQieEn&yz^plJF>si`wt?M_r^U^c9I2i z%{A!VuCRPP6z7+d#h;JzL_>v%luz$b$B>0L;S}$d)T^i0nl}EY`2T@j>Ezfj!%*0; z)teuEU;kGjG-bn)cGq17VAI{yQ&n$ngYH!mBmPdRmdG%){f@_PPFP}7lGD$=o(-AxJlZ;Hb3-`L}SBNNjVldJnjzd=A? z*1u7z7fg~=-}}qi@tAhdBT92q&7vYzbSHUGNfZ7DvV~H$Y`NV(XPGm@vTjq~ql(U7 z+y^+Aixipld<%4c5$t3UkWb?$J5$wueDzu3Gof9YiCF-ejbN5TWLMaecFP#8IaoSOT!GsgmQbLSMS^FvCaN%8^*kQa(N~|`A&o7a{B%?a$<#(!`+hdKdJO`4kDo1x_!QIW(UC6*wi|mu3m)roE&Ru#dagt?l8VeGSy= z*7=e{OByQE-+G5)ZVPHoA(bcWXvJ7r`YHJ2G(}Tk_Mf>L-~1Cvzdx?qs*N+PNTuY? zHHEH+sn5##-+}i;IKNKh9Z*{soPjm|eHR0YI``7zrxDnWYA9KtcDLXDlH{77X~r&k z|NUHA>)#S3VNqgJbvG>(sI=zOyX5C39CPX1F~TDXi#vC*N;}G=d0Cn zv}E0$lGwkiqzctB<5BOkhFnyA2ASVq*gKV^yCL`l9Yd3;24jDH!Igey_kW9m$OK`{ zBm^f3jS81#c5E@93F5}&BP6I{m#(mU5*^)*0)>qQWcw)9O=g?=a;KAaQP9iBdzwnL zIZ?M{p+H^Cw7QCXx&naZ6*8IUL z$)h&>QAfXV)0-w{{&gp>gx>u-E{3=Qr8TmWnRFAxovXr^adc5`$A3q(12feXj@*%`e?dPP_yDeM_2NGb~+{MAfUay z{YuAVv$2Z(LSKDC1X7^qmSQ(=jL@;z9<}$=*_72xn`Oh3cVrrRJC4O?+LIJcA>DTj zrUvRWhQ8}mfJ#XkGReZubQg} zY5WJJ3MzJWdr@hsCEwQZNHjKkwp!4>{$@gSU$ySpe%QeDxPcQ`vgpiKc4xNz$(hsD zY#~KAY!nQAWj~bYX4T89Vhq)@jj>JA;c5ObwMut99K0ucZC-y8q?;P zk;c`GOo@~tmq44@W<{;%MS`WZ+s$G5nnuo=3gCrXXyp9G$n;WG%gTGcnAQtVdTv|M z7n?aHrR`)xy8xXM9g_{u!n=Z}`T5x@S24MI-~qAe_+2o*+3g?hbVrdf^r$F0y&g+nP2~BlZy%nb@o-&vv7rD_Un>JRO>YSR7!m1> z;SKC{D+)4=^KUl^U9*;fbB@!;9S&IYKg0DuJ}wB$@TW|v9yzoMi?+6P^eH0jbdlfX z`SRf4aA7Sb|F$KAYk3#<>&WV>DffiAc5MyBW&6um3!dH7+}x~`grfHF!tcKPQ7hO{ zlGu?SB2#&bY=B8tB}S8GYw-aZ@8nb)Av9}aAOuGox%MlIN)Y*AILPWmiFZ#NOqARB zYHXFYt0I}yw%7<9?d$mrILYxP>y7_WoiV*m=)r+x5`x)lMB&59`aYhcyhR_@<08?L zvE9-EmP{aef78+iPPUmpvha|YWOCkE=uIqt$ZMIIPNNEO06MtDkl`s?K2zkj5MaAcq zLG~`(r?JDa8~aCV)H!Pk+j{%Q8yoqQpvE_#a@VkA2Jer-Kw$Cc5?CEX!LAdv)#8*I zm4r_GI6 zos%A3;*U;08LL-0QAum99}`hRC`_@~zu{0gjydUEf_XW$GpI}6xxxNoCJ@#hc3H83 zNBe5Y3ek?GJNu&4LJ=DZMJfK;$lo8JJN0zjahF}l7=>S93rvKExv?|KSrPaOMW|9) z&ihei+h>W&w}-xq)i2ruYiz`F<=1lv++vGJg~%KGxsO7o8!D^&Fg2-7W)R!ztYChi4^^L-Z7 z=_kkzD6Zej}b0VDP9EUNG1Nuc$>OSc~ReWvGu7sE5|m-{3nBVvu89M$E-YOhGi zr6coJG?jep54$Liw-!x*Ii|vW<3E1)Y`m~t+ORWz2(B>1!yWY!YMtY*{gt)4PS~AU zw@9J<+ak$|S=~~pyqu)V3CmPhUPChG;gljj(Q-#QZ4Atg#p_u`h) z<-71Wzp7ibwu6IH=FOJ>*+rZtPu@AaL0i*hZ6lyk^@Uh?nn|Nrsm4Y0S#HE0wAo=U zE%Nd`Kriav(J7xjryld=fP6qStglsKt^5#KTi%z*_4js}8+3_^Ir zwX;;n;Z-#%9nwTo!+E2N}n z-a<;0%CK=;5Ix$OMTV!EpIQ2MI**=2ST{eYkBWX~T=@G_tlM#vAf3$I7{kJx6~EyT zCA0Y)j7Ee49%H0fq=U1ojO_Mu2D$Sy52=31SYo+Op(?VWUhKKXC$4zKjL31<*k>W@ zui72=z>`sm;Is2ovSNA38S8=V*3Eiep>PQZ>bWXNEsQGhc2ItsP zt@ICZg1eJZS@YuS(2U9s(>ddFIhbdr4aDNLLPTwdDG)QI)iI)j!CK!k0o{FxG00TQ z%EkG`#8eG2Mh+h5l2{oHg_xjEs@O~XDx$~Xt{o=J0^|TjOo|5)z#genIwMJO+H`t3 zP@c}j6tWTS2h7M`V%W68GM3*nM<__49nz>i)tI|1pY`*}Bg>zbkKb7hEg_tK0&lX_ zqA7BOn%7!3cjI%IVWnf#&?R<4R!R_6$sf@XZuBobj7|>TA7t;B1AUfBUe7cxw&>XP z@?6n<8j#4i@=a@Cn?wueu4@YGeV{QwuNCYTL+Un?h=A$w;ysRYpFbqCPuamXRu5cB z{kt|5^fjZ4aw8I5UyQ7`z3L1N4SIQ7U7==o6BjO9QY24-(^Cxo&bwS`?*h3uNNxyT zg=--MGa}UfmM4tn&yzBV>SJco$c(YlkmIuq#(I`O7zzD28RXkKS=M=vyWN(}Ol?GT zfh*@MobKT)ngiLC&z3vx)5gBD75$D3UZ4297#8Ga5VkB$x{NR(huLg-Df9iY47LUe1D22XGypA@JzOmYn zGX%y8V#i|KmWP%fjL3f>YJ?KX42D}}YZOFs;s$Kk3^m*Q7FFStZ5U;HnUKG})ZVSp zi0pn>CmlZ9zr8B~ALf8d@ve&sy1Y*>w_6VX19h>^}V@bN>``QH9nIKXG@PZzJ-zI|On4tczlk zaGJW3qj8EW)jCK5gImunY;?>*N(u8p1=m*QWKTY^!gVRc78 zxtc$DZ}P=8LWrH%#HXDrN&i7v_3h?AxWeHO6MEaY1r7K(fE}w_HC5THxo2`(AJvzy z^eY9|T|$)~Slo|=Z&s`V0~yarhg!Bco=owE)##=9Rm10%Y&OiOWW=tWD5pPc&;6o& z*5;~Fe+JBD8zbn3AeFQ1(>btm&P0@&fAaPKzM3iYf{F(8B$FX63J$ zqF9tn0nHU^{IC5e>gq57-N-Xw3I^@aoli2g>!8$d8VM&9nV^O zPZDw@fE;S7z2?%%9SRfD!Do`3hoQZi!Z!0lcs;IazI+LZ0c=H-*PsZzzmo$vZq5i* z`I%8D@Q5Qi!O-jGV&8C1Q2Sk4Y8(*VCuChkl}-OE?#79+Q8!WY=pbX1nn7mR+lJ(* zF!cTP!T03kvQEO;u|?6`r3|II_IV!((h8*GMNTz=w?E&dPZVq^-gJvAm{;h@`zMx_ zcI6l_q6!a|8~BmnqbMl6kVBL)?Kt*+N&j#BMCSDRo8DtS8XZv{L8JgvsL*kAI9-fP z$)HZO=&vyTI{PaW{geVk8Bo?))49N_jcx4#b!9m=aG*mj%C`J)m#lA(+ny=Io-raK zj7I$t9jWQNrW^{`|FON6GlmrJb5*R@>WWmZTZV{A?hEPerw|0h3OWZVd51_ zfSSp1RMQyx#tuMJn^I{U5*ZO%bNA&hDof6?liT{w-H5^s&ZiwTX~k{_pXcTWQ+w4_ z?RykG#fu}sE~G3Ndw(eC4h2PlB|*$LS50XhUz7dwHvyOVNm?erhZNecQ#71$&uLLG zxSm+t~dg2J|+oLkMT~&*Zo==f~kbuZ{HCX`BNL5cD6nWT`c1( z0wGI&$EZV-4@qh%Yc_}fZ&E3ckug^-=NEYr__tvX2CMG5LUaF}ud~=jn8s)_Hfo1q2 zgHLDw7rJu8pIks&_A%M*Vkby$(8>vph*7{;HuXEbH5ZtTM?XiUW6Z(4H!~9c`;its zVEcF=?9pW?cj>x6P^<7_cV|Ed);qiXfSY?hQPkcniYZ? zj%ikqr7EPLE6CDBmfXQ2$LPih*yLuT2M|$#ou^}Uhm@4D31Xsa4pMXqu5b|ZrQPjD zc6tJH*38(&VjAUQswptpN_mL-=682PL9061jLp37F~@@J5GtqNk3|5V8|{5R)A^Jr zN8Q@i`{g+o;lY;%+n?@EmttSv$j#u9Nl!nhI<8Di@0NFninbR^faTQWz`nkb5)u-2 z@}!D$;k$>`i^aw&8K?L9sl>2wV|F-?UA%f3T??6&YHjdRD3;@Gep3@&ly`R%v)G5L z5B!bxD$#v2>4|%e7q6Xj_1nF!RP?!T@&iYfBo>mKo)|AL9~Y2?a`?R0G6K2Y+CP|r zlQ+GRnE#_`InP#ZtyF_(dRR3pIpkZ1Hqj|}T@VsqDu(W$S$xhmem&|@XNaxIL|4Mh zKt53@Ab6AW)bh7oBLI4*Q&Qi zhLNad<>awNT!dcJeV-(n%$O!3#zM9rRMf`q2EXD~RK{xF)&8ciSC&%`#_etNEH89v zGO3PUMp9_u(~A3Zr_(A2=teg3^V&YIOTS$N4A|DVm~3VnXM2{VZf1Y+^<22u6P4lo z>8|SfcbN4C4n;=K=YF~9_rjXZX4(xzt}|FP8nyY04Bg6@8%V+cPbubZkC25E3tSJP zHL@8jsU5|iI$FO>GN5D)OxrnWnw~NFa)HaK4b>xuoopD>t_tj7C$b}uZTyy=31D=x z`BOyGjOIYahfE#a)e8gU$n+)#r?2mV=qg1=?VcOi-Dd|$kc5D~XLMh-SG=n&Rk8kb zM&;7)=9x8~5oIDOh$~}kbs$~-8ir=*U!_(k>BL|V&ky@G%gCqbN#<}+Bl~4tprSm+ z-Z;G}W2t@Yl@W2ks?lfus+(wO6V-#G{60*lR&lTAzI4`Z&sS_O6*Zan1rFZ;iG17|h6P^~3V)YANEz+to8aKQ6BRtv74sE}h~ z7ltTVOAG5S6|nzSWa}!Qiwb6(UGoy*RPb%T0<~{&BmYL#muzPXEY>rbn`562-uGE0 zwTutV8TYvJfE<5vtdo7NxiUik2Zi|Z(Cqop=W&7I!}jYuCO{aAXe-f^qRvS>IGB=o zgw=5Igvr6@&|axb#`-5+`lzIoAOM`fhLd{Iz9ll2X8NJu+ajH9`({03Mg6G75L;&E zBvxijPdmN(D-bbl)w-wzXFVceXOkILa%nFAmP}uM$ff%Z$orG}U}jl?k@3V6oHP(G z6=(j^;C3r0VavyT@9*>7PYQ{Tyyosu-w);TguDkyo?aoe@|Zov)^5BbO_Za^S5wbz z74J~tMW+y8roINZg;cE{WYjUiq=egAgpguBF6aWa*V?tkFy|rSF-)jO=kF zGw^0P@v>ugCzE&vV$@b3s5hC@T7x@8) zz+T5mEN|F0G43pS80v&A8pI()Q+`Mr`Y)qh}k-y35-z<2czO0N;RB!)`UF%7|_i4XQr0XC3ryI zpd36VG*6=O**ReRifW{&2YEKTYkYrPTKImEv&zej>BEg`+V!tiZQx5Qh*pVWnr3tmJ6 zHP}$u@hAR5kFGi=C`b#{5V^%C^Nm~%*5q#fsw`pWQ$(|q0}?Vqqt7YLV&SQ zfxDtzv;0G!CyP_arUfE~?cuv>y69p>xsh0!_#aEvx~i?>>Mm#a zF<{EYwG?(Z(E3dS*SBvGw8%#&Et8$E5*NK#f@Tt%@yTW~mDpFIUBQtk@W!*M+#^-0VmX~WR&?24w@l6i~)gHq8h-vy*ClW;6Cfn^K@@SP%i z+U7XBe^gdLHCK(d{OCWutG_sk^jq(&r`$lKCxp)TY74S~=1^a-gCuKPNfXewS!f*(K-YP6x8FU^c_eK^%K1L3(j}fB&<;D zfxi<5hj2<47zl#8ZdN9TmvPfpX?T*nitLJ6GCo6vX$~^X4TgWaH{bRW_e$U#WUH#S z!WHQl5uAu!f;}h*rIDPj$5WI@8#@~0B&yjMQbvAIwxL~3-jrol(SqB@0q^Ki-1z=0 zX48$n_G*yP2D($o5Mo;Tg~y}XzQi#9N<8cuS!Ad4Vk1{im2psS*G3DLrexL61TUlZ zz01&|@oxE1(=&zEC7o%QooAwOSjUWC3saA4;H|<^MMgGaHi2%yL)iuXVh3RVSA;VY z6CDPMYC*-zyJbB|qsdr(m4Yzm2QI#b5Y;c)TX^e>eVMvti^Z0|kz9|EYF7O9*Jcc2 zz-`roV{%tX!@SYhAgu^O4q% zRiS**cKqR&LE??{BDvXmCMJwixTy^57aUk?Wo5g4(;hO8hy=A zyAi+P+Gd0Eu5iQbsGb!s4egar&P%F!#5TCCeva+gRy@~!sC6}eS8LNA{(o9fm{tJ32b8=hbfX`>f& z%?XVO-YW)HMwSRe6fQ#7>?=c@EIq-=&r7u-8&;`EY(xuQnN*d49>ChtjWv;k;F$Bg zC#>G)QyieAH_;*Zli1pylaH(QpWo?5qx(6iDS{hnEv%0m7tQoP;jJfgD=Nte;JLQs zmZ0W6>5ZP12FGB~T7Q#Ushai*$1LH9lS3=iA9%<@r$#(glw?OoIfL^+fSh_Z|c!OfcjbAb&_%D3P8c z9j}s&y+)C=_+Y6-=9=$beny*`A@afV8Y z)%An#Vs^8o+@b>10qOoc{|{Af9T(OAMSBmWlG5Eh2q+!W&Co-)fOJbYD2>!ObV)P7 z(2az2qx6u{($XNHchv9ix%c@CCeJw>Ugv!F-fO*=5fs=kSU2X`KrKU7B}c*Es$~C- zG~#;%nuhVdaZ}(g+$+m$a>l2W>wQ%8@r-JZ)@{8E5BKWO~XV!4^nYw zfJSv3R3fk!RDUDp&05M z-=_DY?{@V|D0BShR>0Y!vyr@bx10JhNuI9vdeD9Io!si?pPbb+{AKV4gti8JqqvN8 zZn=a`zWP-3?KJdiroyDSRio<8REk>LzHp+#963-wctRUjOiOe9{sVCnxZ|AL{dvm3 zwF#@J;Hvk_qhU*x+AT#lvEDL6=c^Tsfiz=nkTfVsYO091j8KAmBkw?8)@=N0aMG)w z5ez3ecUic4D)SIqXY#2Z9Kth$Z}x%F1Itn;)0oeJui&vM-V;Yzx=-cU-cny#_DFOT zBEto$s6H#X8}3q;k25&c|14bFjhXm5_>d3A)mc)IHGk8)GcgVH=nJsQVrrC%Q67;Y z_N~@@#mkIF`&t=TrlQfQ18&kwEx57EysOObYdSAi)r*E2<9-YP?aY*@Qr20axoCrXpBM)oZoDdyG}QdPx~8Hhq(8R;dXXK!fL%3Gu)vU(!)2#dS; zRDub`FZ)M-WHr8};8lsp5p2_$i*Q4!K_j%cfG4qecF~{S1}>ZwHjoHs>sq?JzIbV=a)S(7d=h(# zkL|uU03ZjvTe=KNT-$!AA<#9STX797=-JpzaCUTdTuT(cz1>{MWm~i)P`uR0*{A_* zI{Aue^@A5oj#!Z!AnLq%U1u zyqmVDT-1L9g53!Sq^OPsx9&gvGS73;gMWor#OEZV7=cmPT)Y9RYrDiVg}R?^Ld%MX z-bfNfhn6aAESvr6k{z$**dGk@10*Hk4&DdW;^S*#p50pN{_Rc}0GxR6@#Xt#G!*dL z1P=hHYq>XoYykM<6Xvr=gNgI=ge=coDuSfINdWXS-}@#y6I>9Yd{XN#Km65NEuEO0 z4$&WJ(4XD` zXOsV&a6Et79mqGM{5{A=sD|SvLC6PxdSQ>AG5zWBF_S0#jr*1Y=*G?&%Xi;UozkrC z>?v#8ouR{KDm-3wZS&d`>FwnnsI&3%z8A^{QTAiN?C`Oc6+h@}R%zkWMT+cC$S%>GBOrk8XOv9 z9Q+yi_*36I2-79dF0P=d8%_8{rY;*c3Ujf7hm4?xAVU1`4gU~S z%C;J-ki?PEyZO)qOv_YYb*8_|Lhb$YYdFS=R+M%&I0t|zcu*n710W_Zr~7Y72G+c^sD z5`LYM+L) z8|vfeH1fZWproNv( zDOwESI1t0eR6J*r^)I2(nT9=--LL*a(ti)It>-Fd3)-%5Qxn-Zw}mmZUzs`YPB93AGR9bcUfcG!r{KQl9e*C36Nfvvjg3@euMM3$DVE+RR z^AxYfy?x`5v%>P>2X5G&u0p~1&IE_Q$=j9KxyGAvX{|^5+*~egK1Qp?30ytHx<69a}zth$jrw-#o7QlGSSYq`wRT5!xLE-;qD?g^5f#%EDV1g z0aW-%Y$|Ffi)aJA0px=xs$Ra2%po2Z@e1br#LY2uJFl>DkNdrSOv zlP2Nn#=a&vR1m=+FNa>TZPyVEGB%{gG2+_OGw`p;DP}ySVDlHxw4GpC+AZ_U75iBj zE|^9G9+A<3vZHf`$19(D6|Jtb*Jvu$L5VlC90NBMV!tc;_$>t^JHoNx{_*$vvqM*> zJI70lQORrB4OK-yneydXtOh`+pKmA)u}xQStJn1!Uw-^ZNC_f@uJRS;lANBf=L+`L zH+>Ed98vTM+$XP_>=}AvbbL$^9D`j-Vb7o|lDy(ozac!L4|6b!U^9vxs>PDE2)U-# z1VeE{o0M0^XIH$I1R|$}(#?5Hzd8z3K7doAmaS(z<(Xw#z-^&*X(iyAwxH zd*F9oBOtw4IYko3g+dj~DU4=7zVVSIYW2dAhq%Z(=sL?q;+JcJ;NhF6Ov{~uhIb6( z9rMS%x_Qoc1uELPM_lPjEK;U}w3-X&#j5X`U4(?hES3Fz9q*hhCg)C}R?=Iet@`q$ z2b=a2@|Q{dv(Fv4i^hybGva&82#Rs{l+MYp1l2mJNT_HIl|)OY>`xn)E>rosG(V|; z1Xgs(TSRN-gU;7O2yDg=c0!VL$TDr;x=4`ZGwy73hZ8y9v@}MvZwL@Y5uKC86dUl} zP>3QiMd2d5ra9XCSJp`>+}^Lt+Cn2RSUgCxB3MT-*cn(jlvu~UelV@XTpGvBpjHXDHr>?wqj_xAqP?v9HK zy-nJ!UA%Im;gy%dj(z=`b9%Pq4~NF?k|~kL#_a@5L>g4aHl`pYGMiNL|R!Vi!_e_ z>MCh`+{SdQX#O+Gk-A^NflaTmv(z(%#t|GSQvKa!v&yolRnL z7tI2JE>-OlwqEB^LfES*N7ge7(v%0hxl>XMeAX;d4s1#)Rh?59VQW3iCQ05XFO(ZJ z)Seo1{ygC^7LnywHP)PfpNf+bM#64E>#25T%Z67Rfg+IC%CDEeI=cyt_GC)jC34pG z%5(h9>#SQ;5a{ambJxmWkJ_z<9ol{C?WMK?_U27-(?2yBTe+`4r|7d;?mD{F1TFGq zL#Mi81=7p`WYDi(7MM2j4XKs*FyN0NnG|FMWzme$rn%4(FhV6(Yfm8DVw1v)yv>5o*x z9#nyQ^-tV})8jen0*}d>Soqe7gdEHWcZ_St07!{GMCb198whBD#VUrk= zb9AL0vA1R7ef=eKKLa|pSu947{0dIV>LH|TS_+c@Jx7uJ?-SRDyr(wW0u?rl(sQ5-W%1~~&C!{lumZ`^a6rC1j}^4XjH-C>a=`W>4ukvaFZx z3vf^W)UX-tF9fly>@(jgsPl!9exR`?@_PT#n)Y5q_- zu{5k>4?bp#vL-a~`}oTE)WN1Cx;j-%Io98!T_mkhJK zGD@IW7bq~AFn&58y07z84#Am`L9wizOmRgp`K0HOH~#eDovJ9^jPJ7+25AnpOSD%~ zkCxr*!t{fTDwgQL$Mt@FO#|x}cxZ0-s!iWCcu*hgL(sfEkECm*Vj*T-jVH^lI+u^ycZKnfFw`h zv>TPQn!)vXPn~U4;_^Qa&v*@mN9G!&+ydvtVo9(u#ZS(a9rHuCrzny+KSwC(ENCVvx5p)v*wg6;C# zAVI=;V&~+$DM=UphW)r^H3=z$+g3hbA_mpUcuYYnsaDM}dHG=MaZv!7QDYZ}RjO_` z7FLiH>o1ZlEfPw8je)PJAz4h#LxtX*w+iL)cH?-9+O2;?<3YOfUxEng4s;Pefc!qp z9ju-M8~G7M!g|y`%Gb`Av}fVWT&}_!R&aX4B?pjBK5?yzwU@myg-YFMi=x3`hlHTJ zhreIoL&A;J*inWv?Og!n#ZMBH7v<_x3-1($8wU2DD;w)eSwug1F}2&4x{_Mf zFQa|nH8=Vh)1JlA)xk?e$lKq_^@D{|lLAsgC;T*bHKs`mykt9xv(B=yPWk)uFF6ZtV8{RK-0gqG1CR zdwX)J8JW_1?T>cPCbh#9Hl}iOtb$Y6M?PPAVaUI1f8nF7HIbmUkW8Db;HfkgD_2Zv zu{FXIS3k`D(MFLFxR`~H+xe@dmrI}YH{iKs*^-TdM-GH%2ujpS<>%AS^KH8u-8{hg z`tE4&!Hu9Q+m|jW8$CqH`} zc18Y3?0L(y8T_Fll$b4 zJoM?L{^fJ`^9&W8?*5Xb3K}|ZVn%(-sdUzj{t6|QL(M>)2f24Dy+}cdrNAHK3g9yk zUK7g0x!Otlk=*9;$0eXoC$lEo^8X&pXP|)LWb~td#1@=??@Nn9I|Ka7*hG1&ZEW$s zB`9)dhDz&tJn7dGvqfH%biVycl+3`YSQ5i^7Pk4nb0m3R&$=h925Dbzy=GsPN(Bl0IVLTK|C!r>0l+BDZYgU_I&j$Ro$ zJC-mF85y%l%8Vw<;U>HKN=dKyL27X}zAofsreN`lPP^Vj?xuR&uwC_wFa@o!_TE0yPcTO^qx3p|zAz%3fVir<#!RHL>6* z40nTywLBA!lc3$hwjQ(OyKEcM>~3nOrH4_0U0VAX z*o1xT8zBJ%sZ=%fssVQmqRD_?*K58g#-DI5)q-Ynqxi%N&Qg7~%ME!fk*eRm+hd^j z`;@0}=wVVK8k2U(%rmu`Mx_sg9Om9~-c(XatyoE8k+T}0jO)lX{_o&*cGPAXPO|JF z7F}&E#NJ-#JTk)KwyME>_23)-+>EyERqO6`WiQdKzIo=^;eo-?APJpATjgTkt?uHd z76%x8J5Rbu^gNR)9s{P#J@b@9L-Du_s8NW$=0HWuF1=`vSO)>*h#a123i4yehJGmr z`QsGdbI18ySE&|Ah=yq8j@j^!s0Y9vL+`+paRz0>MZmY46O7cp=ZreL+e)?|UDns- z#S*c72IDLo=>>s=4BQXXvd*1bOOCrk1))`zg^5%*q?i_1Rf%LkLp-&SV?)U+cNOg6dc10!FR{BT&5 zWjY@%v12ONy;Vk$q)lP3WG_l#=EAVxkgW^!upHd@30mcAqa0KkHfhn7n1-dbrqTvC zfk+*K9kNJ?ssTmJM6ZkRQ55e&UqXnvCO&vVlX4^|1=3<+j^v6+*84Lan*F#5{#PSn z&PV46Yz6wV-KXflrZkHl&cYhhZW1?V3%4g~v+i+T{NJ;i&a2xA(!U~Lx?7rCiI@ix z>y~3hD|fGmK!>OzC#4~v#kCjrDOShh{y!*-XF2j($NtQsqBb`B3W zytZB1g6n%-jJ@>O0M7|McdS~H-T@M7{ImKDzFFV^fm;J#O@w2~~HP3=v;A76tFHY^@a>Yi?^l>H&A7PGkM3GxY-2ec2i;7TWlZtga-a#ZyBj?6( zp4;f0p8tCl-F*zOdOdH|12jqZ-itrkMF|rqxH`{Fi1(SXTTi7;o8?Cx`N90lNRlRP zLHL(z${#xuXryy(g=zK{4jBf~DG2>p1`OK|ZC)fK&jOA$mi&N)^}_mg2jxRoH%2eu zu+pfAkmkO1TAuFHqyZ91yRix!Ag3~~jsO5GL+D()@;Xoq0(c2=4K!>BM%qencbz}X zqjn0X*)?kS%F8=v?g+{}p157*c~Dt&U_uy(ukK?#{1KGAqyQLUJUR2N1f7q_8LY+p9irYM2hN{CQyviE{dpe0VVg98*NRw z1xGchQ@Bb~^LCt{Du6aIa8KYf%m(|K$mVmobRA>Na?sr2&J^W1!;vFnw)U4|+}p+u zeUN|W3j0quGO_O{2A6LL&gacitVkN>mzadGqeD)@9mq}Q+n zLUYMd%s;DfQ4aciB?Xo;l3?nL$7Nmo*?1Dzklz?WFm$+#;t2q&$x&_c-=((x8Z9Pj zqV|$=fx0%0V60~pK)n3jo$pzZG2y}5@Pk|F^!xx@>ubo{0jG@9$DTJ0i)T(ubxS8n zs`Nzby!)gD@W_e~!3A3wR_$8#lfn2TDk4dT49bPBdA?tyDNDBoX?e5*l1*>|`CwWP znYWnObF=c4p2RFhaYsT$Z#Pgn8d9(iUEeN&=??QZm-Ub4IQXvel1r*Drz?@|1b?W; zjR{kWiBo{hbf%%R;Y4c0M-~@`Tim9SDKIGnIlg2EL9iOxvBM!!s5eBLr`ddZ8^N;l zgNuTS6~6&HqMTisKk9Kkmbri0HkpWs2I35+*_um(jA6YTX2>nG@aoiod;qAPkbKYO z4dq~eWqFs5j_lrGG~ts}3hvW0grYpcb}^xuF-;Cr>1P10>j7R0;#Mwh@$CJcLX{MMUss zfcT*oiE+pkZuA1GyXQobPsTx%Xlu6_%U)gQqgqZ*F>Zw2FtoF6o8l*$R|y3;>P(`P zl6A?BMfwTrRqZnFZ3}BbZq6*Z;I}sj)U;QXnbO;Nss%y2c&;#v9P8+nR*RCu5MXL z?#$q)W3$uNE%4*RqZXsaVM>%e-Q_H*R&%~^?7BV($smeSRphdMAdFr5f!tPS$KCvUotV<(tvd)3Z4= z^Q-2?sVE1-OsNlwS3}X#`qW4a2nrgSlLfF7ttW-LYVkuL)g233-+VOE`z*7z4B3kg zu%pKc1D*J|*_IgIJd^B)@xomgp0RD*go$=O1ETJOzkr5MC0K`Bu+W!HS(7H1JFq+T z5s7xS3Mvhr_K)}&JIO!@sf1xE%lGpZPA$bW!yKK9$SE_sg8hsDqZPrs zida({H7M~E#WO!941QnB@$!X9w4CLYi(QGYyt

%Ma(2V~&T zN$%=JV((iq`NKc-L6sm7wFOegd75R>ozvYARHR%a>r3c1Rtmn7l15LW&6cSZHghee^OE13>!DeM%OH z`V|T5R>wBO&)mNercbAqLi7q5Q~n%6w~r!6={}6BM=fXVGtZ8{nQIhVi=4SQ%Vc+I6Ny(gcOE!-R8jIaZST51jR{io5#)2z(C)ZZ(p5r1!sSPGiOgNV;R@G6D zr*g*QVXyxl_n<8`n#dxhiD`vw)E)E!DF1-jT3U!T>SzjyL}x+PG8PjoG?{yg)^Tvd z^zyk9m>^V8;N6VQp$$oQtF$K!4h`iHgwWab5ZlZrc^2_EZ?Ddr5-6rU%>~K zxvy;YH6mkl_jyI0DJCmrizXMxTSw094C)1&!z?j|Z_bK9bi5D)en2Bv`{U0FpEDnk z$<%k&vh4`FKJgT<%V`SsK74BIr`SOcm!ZgvNm8q&#b5l$KEfEa`RH9K%GuFYnX75I z)*m?~(L(h}-Kje`I6VcZnNdV~&5d>PGi3|g#ax~Gd(z}v%r|uU*cSbteK(YW{Rh9! zLzTf4%NN0ADNj^~eDl~{)>k;2?-5>uLS9i5DSXb^Muqidtk>nW#^K(AyY;tkR6&>H z3c;BSZ#+KvIqoEL@r>$g%<7XMvrUq< z28`B@M=paS(xiMSB%-bVbgFn*im_gMaLTi>1UPQn7!Z=e{X(NiDsM2QDdQ@8rutZB zOg(ZQTkZS!Zcb!FOD~$k z#2d~T>)Y*$%mm;ru9vow`d3PQ2J^9tJj{?+=^!ypcm?l7knJy-HOf49KTtB!9o_PN2V3-1=#-J=wdrRgI>A0%#qK$bd9~i--cJ2y6q& z_^6R)-KY967bzbn6sDBP)%U(R4=R0esR!~D#)ww+Cm611+sGKs=63a$qXW^s8+#0r zkst2Q$rB>YlI7?_8&~>4Wt2&&BZ^CVB%N%fPA+~pvl#bS_&H&eR>5-tV3_=>Xt=|b zMya3)E>;-L0aC0nN;`193UBww#^^x2&5xe=@f}VZwx>@&?M49qcq_b^KzENi{VD4L z#x8hiypm~%92||is`KkMJaTjxcS!8cYq;_jgf9wl9@S>h^r7Di!wW z;@s%>8HoRja;NzI+bUH1w%(-3Q_iPGL06T(0F+BW5bz-wdm;cDF#j8UUJE_GEOb6? zvDI)(_&4_x;_DN0PfnT^TQ&do_H66b%l5CF7u~5n?*xpjaGre{xHt;CPMmcndw&Gb z_W)W5Q8|7p(b627=W8_21Q_k;#W-m^pGZtZ$=fP%j?@%5EwHk56hEZOG^%@;3Kvbb zZwdL;HoT;!IB{}8_7`QR2`w`Z=lN}cF@_>yJU))z=;LUiWM^#Tyt=cHJW6=P+x=p# zwFuVL!DCL_C34h=?Ln2;OGu|@n4x#+=?u>TZ+LOw5MkOe6zat%n1J1NDbvk%Insoo`*F}b{0nB> zxTLq0o1ac{#0pT+@g$~gsbjztkg51OqE5U-p4cEl9DTyQ4^mL1Ue?L;*ZJef@K{d7T>1Zy-Lc?<)&e~&QN zYvo~R=B>fVun#8gJUR<|y`t1ymbY2vtIl2xz5dZCO3;_`s6B$Av8NWFyKXcJ$02X1 z%!LD0&=Enp+Vds>u-acHM!PUBaw#AC6AnR_kb;kQJ4}U#LnVN+xb7X2A6=}++uTGk zN&r!&cQW#T9fG`~DUl6BFhwJ?Bp$q}7NE$ox&q0D3oSaq1h$6N^9~SScBOChx4&w= z4&pq{Sje(FI%;;HW;2`pn$-7hPfc-StvW;xqK8TIB3w!>PgXxvE-(q&{w3gkd<7D| z^}$&g(+t);!^jvATx7W5RLzCpONI9JklIXwVPqYnAo@Mhi#$C>M}~8|@KVg}w_t;Q zn1|cf_mba^X|wmIRuD6b(edO!00%Ru5vNZNZy^J0p@0MOJ5m8TY5z|ah)V4l0Mz6dbR2x z3to_bWSM%50)Tq3l>z3c`q@1!u30ow8+x0ZMIZx~bPLf~jG>*mz+WG*3(#@t@U%lT z+%deEGEB-(kA{nj@nn6m&jeQVdfyNGqW6|t8S@#It{D$@m=xK{GXR8x44G8c2%eMM zHv%;y!iAD@3Q+{PS+8+j8otzND*RZVEb?f*-IzI485E83r*A#7eQf|>uwLvWaw&gC zF|L_dlf`~U^ut-f5G_B>Hl+e3VOw8o@o0|WD@lYy1b5J{vy&DL4-GO?tFh2?kaYG0 zn`Q&coPFH@U)_kGCW3#{#EGjsDs(#iefi{{7~xB-&(O0>J^emc1dNBE#NWIG(%ODlqN4i$O_GfnxLgl*y8f#3VPCi%gYcy`zo@XGX9o-!F)z8BjA^dGg*pg~5J zn&#&oGvqGt!Wg{deOdl`$`cqs7IP5AemI=<{gT2`k4?An5K z2@FY2n`Wl``SK#UK&QptXK3@B$}WYbLyp4ta>}g5@c_N2GgW$9mghviH*KCF62!n&#tl zPsE!ZQQO4Krn0E-6IAg}LrRkx4(#fp%HQRpcNI%4)*2yxbrXb?)qV-K#@)UwD0B0l zAUNFz2^H9{XKpT2+V~x_zCH{2{c`V3JXBpa+7(lb*MK6>e&7=}=xine^t}M6|h@0F;6AlOapagf1EzBOSap;@uFlq-ug| zp)MwN1Zxm(MRgMDO9FyzO)vb&oE;pKz>p}=&)fBamFxL2b31&objAEyjmqF(v6ztf z=OItDOx}gqSE~GE@wu|;e1jfl$0uEG?K;ExI+`wy^5Q(KRzM1~Mb<|L5^?_RhN^(> z&oO>+ZlBZ1lhHKFUeQAn_vf|9-dA?CwE=Yq>mBY6rp$^|d0s_^Ir5LymMu$eR^+$>S#)Cs z;JT)>ER|yBdx$MOkCo1qX4B07B!s;Z`uQ#5f98(b864+y<`%9NeXlFgs&!ZN>5xW| zKSU?k37zCIKP)Il##P|03FkdH`P4e0sX$9_6JxBYYd`7Ua0iDvnKO-%WAfNw156f4 zXQv*PQTNrp)L|F9I`f}T-@lnI~F}=5C)3!FUDor*;Hk6K|b!O*IWm4-(`1Z`_6lwv~1AdjUwF?V5T< z(7~ntuOXNYHXXS6_VxeDivNH2rA(W_(!<&Rk3(r_vW?cT%j%J4wTlO&I}&PMmSN^7l*?L4f$|RTU&UXzG6Bk0seF8$fJT@6aa8?<+=l%;8}yn$@V*fCIu}(VsSw~XN=L;Vt`&6-W|d*ri!Uso6prrwI)wi!ODPjS z{zNI2x(#k3D@-l>67DlToNK7@!w~)~;^)VASx3ZIzFrDBg$`rXj$`X3qbbi7uadydbn{Ew1K>ylCpf9}zkPN5t-xktKOH1*}JwKPu z_afaLsxs)qhFj#8Ua!QDq|y>)xJeXvm_&$7Pn2mp;kOS>eSS}_6YK407P{^B?KH-< zv!hvY*omxD9Z^;PQjOf;R^LJ9klou)mW4L$gFAOj2 zjt@{6>nIG6W^*BmOoF1pNb^5;8cR{T+T4#60^zKX51lZ)cmm8iDF4RVKD}Ah(ngaZ zv|_IF&~q7=P#x#+PH+-hwPJ3tPMcQEukT{19y?UI15~hqkc?!c6gc-~~i^ylYzmd3e zynlb-L>`94Lwi{}U{5EN^8vAdVv$H#POji-CKu7{{B}d+U0_3PQE#yF_w)NS?9?Qm z;m}-Z>CbfZbRZyP-S#3}{oh12xm$XGO#@v~LpdtWuv z%R0&k=N`O4;x8!`i4)ARd6M@auN-pFxLzEHWm+k^RUHJ18Riuaj2vG1@D8k&pP);} zH5K_gov!xGD^3+W1y058@!Y{%&?2hKsr@oK68|TaS^X?6^M1j1ny!K)`Q>toi=o#i zXnC{1oV#7ix8ahh{WdZ#qY$YyXgu#PCF|xE*f!+u?jEr@LsfWb^jUHN zG`Pg#s285D4m{OWWG8|orCBIi1eWC|;o`kE7o=@6!&L(~w#IyV8voombhB8!t81My zPx`Ao*e*)R{W_zRo?D=3hA?tSQ7wI(;9am|iF66xnWv(sMDnW{2^P9*R3pGx_j}1q z$O0;=ilS4)ZJoN_>=(y2dV$?O{zZR7F`_R^ysIE7Ynje?{>Fw?tr;;Z=h4oGNS*Ti z^qb8FBOathcZMoY8h74Cwer;bi+N`0;#z_4RdY_ceN|7hRB1Zf)>x&<9>YCCnY~)H z&G+iHbQq8YZS4;GDI=qBy>Ob^3n9T$F+b=s;eQ}E)T;F<+KbFJ-@C6!LwpTbo zr@ZbXhSHG_P20x5XK?s3w=h>{o>sNJ#3x~q?jy1xB{=bX$m}CW*2zx2 zt?vzu>Epl1IH2_}%1LCO)H5k>+-ml1{)N9NN*1^#N2j#GhVYv1o<~CcXO(krl6M!k za76zJ#=EHf5Gf}cg>VIakqGMTx>CoH`l^-3a`u4;#rm7q5m@a@q|G$bN(!Z=C7so+ z>pekbL{hfzzr@DkB~2|Tcu0!wOSsC}c7N+KSzz8}(ZRhT)%8(Q$aRUlaj0!xcX(dv zRWp9r9YSh>5qDY#_ajwjFRSZikqZvdmDH79>j5Nx$HOJfCk97Dy$>$Kdij3J)G9~# z8+oh>O703><_Wx?Uf85XN*kkWCdEF3qI5`s3}6AWk0I9b142CTvzvG-vmEn+xZe*i2H@i$1?neAUmTX-2=e$w46-I~^q>>AqS$$=v(x zUMDf1M|{zi=GC@BrdA9kaiIQLmRu(^08v_R#6kTW)`$Abuex3*G(j_P7@Hy^nMu=$mZm?3my zKpag}t!MtQH{DqWXCcqmxH{>3O}5fbWtm3TW4@G$KDN%>VAu5v{iGrh2Q>Bc!jvCf@y=1!6b<5vz zbB-iV-nFDpt>kJ`%Wl>2UqQXO+kz(oM+Bn)CU}@%I&)@B0Jez$)WLv&!4#z}abO7Y zij&h_mQ1elIl?XU{sUJ`hV3f%SbqCjz;4_bRNpX8()wc<}Y0NBm=1 zQX2IF7sJy=MD9vOmY$cJS7Lj3hT^0Z&Q8bvPTuWl?$+Td|@q&Dw)Z(Iv(ORE> z+9WaCTGzkd*x|hjbrP+YxN;o|=utB1J<92Ff2bF3WIlJi~#lxZg#k%nGgwsKUqVLB*49c)5(5Lh ziq_WR1Nciy<<}e<`(A->4pd)uMR+`=s)t=|^Svm4SzI?wq*x)NwfU(G1q%(pyo`LO z78AX^9`i^4-26Fd;5ijgrf$9eJJIf$W3Vp~GOIs_UuBOE&4KTS(cz(w92<@ciD)tu zk{Eul7Qm|`GWrOk7q91Auo`_CTqLl!8Q3p2x>K2<{dNNyw^-jX23ZtB>YGW%197YpHeG9~U`~%7# zoow!7Ipn5UD{41+A6`ami;uXdTdG9K-=B=xEQju?%Qpj8K~T1Nh|%txS$WGVP9{!o z`Tgibs8p)}=!3%#9GmoT$s?bE-xFGy)gu*$KG*tSF1C3MUYcdQS?EsOPxL-RP0h$SO`;=`827ln~WHg zqN8HdpVlqu&8vU06uRxBwQNj6`ub!0IrYQWzeq+P+CINrXn0$AW`^=?o~Ng!Z+kGch<`n8nR@y%)CEuwmLpTT#~*yFYRzUcx7%udqm zEdtVODWe3Hnl6G{;2QiUeZ`J1*c3pR5}(kwJ!4Ke*o5|f4b4+rmvYVjG8T?Ap8i_x zJU4M6SfnG~Jpmn#2_(R>^oflR!9Lx3WALphGc?{PX1)>2%P(3Mv>ndHqg_hF(6X#7 z2f^859rxRy@{SHN_vNJ5BO?yuZs1R?UVH}$Bg%FpV}cg?{jdjRc~3*XoN+Mmd@3?^ z!BbYrxkmgip8{&W-j>gL+i)~M5^TI%t%&0knRY|)HV`edI9;P~BxTHw>Z$@M*QgzI+UO(~W=?gvlLaRgOqBNh5 zG~=;a(VxP9!aT=8s;Nzo(P2BHx*`L`?)zv$NNFP7<*vzK#jwqbl0szzc6>UAH<^sb z4h_q?;B09&2ejT7XhOv>(fxRQ_B|5_#(aYFSSA^3f&^!pwWm_(Hk|}Eg(MmKD{~!8ABV#5jDpV_80zY4+r@uy4j0n>?kDO; zSb#jbCt_=&*|aZ>_K5{EU;>|YS(Ah0f&8h#Lin~v-f@u*=D~ZJDJ2?2LQGe}I5@zU zAS&6}mvUT^QBiX0qNS!{Oy@+n4hpv4%8HUA&_ON7&XDnYxl5o}cKpjx$Qo3c+DgEi zCp|QQg+Z%#jE#vfSx-Q^?&@NxnMg&xrCv1YGX7<~U<4SPBsD2q{OhY)LvOt}lpl^W z(dPp)=FGP?M5w|ZLLCMJr0f0^vTw)P`jFt;P;7)%LR=r5=qUsk7m|}nR4OP-tEsAa z0RCZlL=61et+i9FuRhKI+WJ0v*|F%P*HF3B5~9tY1=;4{)49SXC~bn|B;k5u9<`a)67)vzpg|h7Ij{OTAp8BU{-Dpu`ZIxGq=qi)t|lxh z9JiPuHDyW&W`%?cw0o)s8o@+>dA$AI+vyGQ;x8w})cJNMzB5Md{~?(AEA+|!BJsQ} zuD|532NiugA*KEaV2OVbAC71xyt&VA-NNfYU6!qF(J0FP6#Q!Oed3*EoeHK+Ja!d6-o|No6|JL!sWi|I<|I+v{-p4U#kJS7 zGr3qe!bMvj-}_`Gp)FjaVs5VlS9{qDN`t$w=?s2|TH^cbsV6sWW_3uwH=iEXJd z7_eC&PvoPjpUz3&F?#Ve0T7x2DJ-dZ4JmvCe;-bi1{Q@CZx1^0`lZx+jE<-y?Q;Q9)^&U38q-xnEK ziDcfAb)-YhE+2oHedS zQ0R<3PzME~-bG3h3Q#6P=`$b3t0*LivtUDyk@c_=dmm7}jY{ptOil7o^L*!A=AuUibd3&qWh-b8CE1fx1 z4t9*d%@DRpM{LsLYrc;Qd$ z=qnRlq5g1vdY!9mx%Fd=#dm%~B5#q`MnkaWBDG!`pgRj-ufG)`U(Aiz{wUfDKK8}G zlk1I)@wZKBpr3($MS78j{&@BcpgWPBh_X;08A#;jH#R2~K9)+un>^tovF+H5Mw47Il{Lfz-48$D(J@*&dV&-0Jgtu7U7%85@60ZP~Oy?UbKk_n@2u z!IZj&aSeY0D0oYy$|eSNg7{D0f`gXcrpWrPjecif&f})p$NNSRiXBe%T45EbrmVu? zKX15Q`5a3(5Y8K9O=WW7CzlQ>1$b@mC;j~OPW88h<1GK>Bk_W zG#ESVles(-B**_Ixar5A*6Co<<~i&Dg|cYcTw%Dm1>R%Qy>OXsoJ<2>{^y1L%-uQaelI~xc@}1&}NiO{Oyz5_@65ESoufObWj;x1RMA#-pvZRj@m*&4hxnw!R(vGL>3x=2AyVo#ejNQT?a5kw z|ED~JQ^BFF@&5ivLH^%cVNLr}B}%`Q{7+#cD~q(gvM~}G(y;QU`X#ZOMC^mdZb;5! zfO#B}9--C#C2f`3K;>d_xJh7JTgN9c!R#A}0NUf2(EmOeMNEnjao7v+zrrd-J|iMM ziGxV*ba~G`N3_ zQU8w;K=?l#Dd2?DT};SxU<%c*w|P ze{+WO4SA1~bYt?R@78>;?oD84|3aq6y8bO@-a}RpqGr;AX|?3KInz0(nn(6vJ75TS zv($UD|3R6&zT)uM!?n~X-tM>D!q<~JGrj@`csOYYO(1-rFt7q59L?FboDy8Q#bZ(3 z*~|4FqX%JB?<2KAE&WJaS)65OKg9RVoD_S{$1km__{jS4U4LBePkZ)`-g1xC^;^jk z^nq#yvI_pBPOap-%ccDs-=SIfwi8ipoJNCWH^Y`v-AXx+S185{k`$-q-I- zk@#&ylQe&p8xcK%>vtv2 z(OvDXz#m{adA7#txhR0UPWcz9{_L85%jZkG%%=y9aIr?T4r&f%B)2~qYJFz_i6^YJ z93XU*t}eXW4D7SkcIVe?c&)ldXz{)yen32uR?f<$tLgc!XsT!?`sN`(8Q?b5@)$cE z+<}U|-!kfdwRFB>q_-W%r5gL$F7ugz`KL)DDJxP+FuRuU_ddZgvB?cgNRyI5b&#Km zolk?3Hyn|26Zka$;A3g=NbS)4s=~vfQNlknl;9$B^#h;V@8V>fiqnTXT&mU?;i2-- z9-2T2-X{HY%7pO|W#OHbOyZBeZ0*0=2j0t|%U(*3JW8HXzZ~7XM>#{a>gyX?ykT4k zDQ=u+oUUoEtB$C&WzDpPrtH^7?LBR^%zeOBQ&8=D2mZZS=x}FDqXg>f+PK%4uSuvQ zGq|xBJ)^h*j!WurpPD59raTM%6jmp?h`-{D@k*I$(X$!jv$KxPL+c-fOZj)A8#Oe^ zl(ygy@VdkPXp&@8(~0`Ry%l|b_hRri_V2G7ZqlfHn(>o~{Ai604=%f%i#QKB+sq?B zxz>7lPhWb&t~v)kbz0(l5?c*A!=MUQ%3b)M@Nhm$s`pJv-2e;4*=LUJI82z(hC#ZuG|l|NdKH9ek+otk9oi?%!mvuh#T zldv_o=e#xul=~EKHpdE^jwp$w1n9$Ac3eQ#7%=gCl^2a zzz=k&6h~~w(iL%+ity(g=PF)5))IN8@Q}Z3 znE&+CKVr$-V*6)9nob&(f0v<0Zv(?@X3=9))DuxGJ%1V%{n$^imI>dTa*Xu{4j^Md zf@X%g!gvX8j-!)f7@u}4D3CuB?bQQ^Dv83p4giF^1!H4|TZ(`SC!xe>p8K}j`?%Fw zpMHfSc0X}}y>MO1qD&Ada{dRPRC>`EE?22|TVE+FORwxQm+w40Y(G{jp2;Jsx{qV6 zpLMM|XfU~REi;`kQ4-4}ky4UQ$Xo-}WkI8yqxa<(OeV}=^h}oT6ry$<9=E~W)`OP- zj3~gzY85nm$_{W-)%V+pbu)Q=b(30huF{2!5jm;0O5kO&o*vwhem6T;se&u?#KMk+ z{8cud6%p^)DV4o?1XC{OM3E_6NR4y#b0G)lj8ZLXy513UWs?CkJyT|ZjT1GI*qi)2 z_4$(MKYW56XZqJR(!!FEn=cM{6$9zGMVk&bxV$jYX=7QK2sRZUD)q}yvS#qoUL0)3 z#Yk5~k!rik(wOOO)rwmX!+spP*^OBRcH$}Njky*QsZK4s;Fl8Gr|7J@u%?$P@g#zf z2AKGDCKgaXkAV_M`gZCnF9fHld>iHKpXVoDalxB+J8&jYYn!`YMKNs(`B{Cxj z5vkb`qtF*8n85ns3X)NerNXbK@Ap<*azzCz)E?ZB>fN>7)~Vo=C;3G{b__?OQR{$^@i#%5-h&+>~#rZAe-RJ*3CxKjeZ4AY(*(1 zO0TD86xRUD_@l`$caN|_UEQII)&}g{f=3~a6($q|DNydNnQF7BU*fA4LJ6_jjn)2AU6>Q!3^2zqyDKy-Lc1oLlrtDlmr}TIvkW+jY|1! z%W=+Xl)sAS%fSVJLn9r~ovWRQzVRB(ng(@izVxRjjzgML8XHan$1~L5JDEKj z!g@OBa5+5E&`@tU)INdkzVCmIh|668ExF!3_kOj@E@rWOj`8~T#&x_dFSA5^$`xW< zA|XLovb3<*QdYN~AHY%B{c-r#EvXh&@ka0fy%z^5!&%<^{@sdhTlj5t{hxdg*!KEu zAIlwXCR< zTUV3@)(qOv;TXTb#AW{m8@4@6R5<*l2W^A@6iC#CCa1rw3z-zetuy|MR3PYDe{afR zT!DaS@=|e~+`&mtC6Wu-G%9#NWqB(6@=dWx&0b5OEJxYw;Cp_uAO@xd>ZG2^6#8sB zuOZD9CwoipH3#|k$BDn&_@ovHpieEea+1)3`Doc3Z!QH4BW;_hx6q?b)({RDfvdnE z&X8EMgu!mc9l69WVA|m7uRwPK*SmSsj_ZhDgcq(F>=hDnJlIBY62Ag{QkxWCE9TnF zIgDJ@Wh($b{otTKLP&H|EY-pzyOCUwxjyBU-movf;yhM4_C+xL(CEZ<4-h~9go(UZ zy$&eXWVSdYU$mK#y4*FShRK^^Wu?mH&n7P3g80i4ufSsWY^Pq8 zbj#u`kis+T`5aqq(=LZSNzO3Jf^e@nC9Jk!hIY*i?w=b}Z{^bSaZp@~+usHJ>+Q`o z`3g))uftNV=upQ#@u~nWKm}BpbIsXF2XPi`k#FvbD-algbWoH@ik7fnIDMFZ@rQkY zPB+_BPu^q7)S|5PptjpxbjxQE0S%j6+|4nRV7A^(6+GAdELQ=F1iqdi3J`^q$M1=j zeqvUNcrC+J__C$~DZB+cu&#^^fJ{RP>iO&nTVQ13QEkIqTrQ!>mWVw^g=ugVtcTxnVd!(O*7_MtGRxGU@#Xj zlHgnCcM#IE7EPzWrcZx;&bK3MfpObbmM5Qr$YTFO~>YZ&LJBAA@xcf>`zL$l~Z&!~0sn66_Lm!;C-5ac#=f9!FF z-_NeLmcaH|Tk5&XLYPDqd4CFV!hy3p_I6l=>FUJv{TMKkiqA`z+zqrPhSH@I&AeOk z?h9ezbO#kg%S)OTZ}Jws?!tHS8jWl~?+1&1OwXJ+ z%u4X2hia?7@z8pC&z)6(=dw&h=(Wd67wOBw@g>+txosl4j%e?&I%sQ8j1T>(Mw- zWH24AN@Uhoo3Lmqs9UYDA+y@Lo8xPdba6$rM5&`$*JOjYG!ukgHqo~#GUmF|NE^-A zD7`6roC=MajEh+cKtQS_UE7l)Fy0yF5*8@Ybo6N{C|#8phvRDvk+YoDeYx*lGyT@j zw|kVT5Mg7R)KNI&e3(z_ z0cmQPZmKMwXi(=VSF5G%5?eHL2}FZP*IpI?RITt0Gu;7d|7cVV<{X~G4iYVg-7b-1 z&YTM;)!RSo!0I_24bdF1xRFS*ag20^Jnez$^I-*LJ>Z1u^Kjw4hu>Fo_TCajHnzg4 zqU(zKuetE^yc~U%9)f1e}OS6rMV@c*BObT zN5FTk>X>+iLDE{B!|~nP8k2S_xUt)cVz^xLTs%kt@0Uv?djHPh!-`X;TaYznOM`y* zuz14jv!6mna%`^+lI>_3J#A}tTG;rr%|?tg(f5wFZcl@w7!uxcnh1Eh7{lpTxC4do zxE?vv;b(?+_xr)seACLoNgvtBMNYnnNM0U>2sAaVkPms;<|d99TJRNh*a*G(t?yYX zVXYa#C~E8-&OW56JJy#_cBatkwtl-3xK(4iRQ7cw=e)ENG$Egp>+64nSUc(+hBFX$TBBuvlFlJ6qjQj;3|#b`oBVB1UB#RjTt~>fw+beiaBzpaG0}<#k^ULSJ1|o|zUtd4@UUs3$1caFRG9VG5HFRN2dbo29 z&?K5m+#o8VS6v*sSr*>OdtFoGe~}hKOe2p+Nc8_9>xuruQ~G`EjJ)ra{qN)}*tS+L z9}rURsL8iV&%DUu7#6FZmm9xVkVui;2F|$yhw=Mt9iuhRRrwKJd(De`%tHr1io4y*S6_w6Qt4GPg9;y##dsm zpx3?cQ)nE2i`xEdl?bIK#pY>w_?3tp=?5#(XtPA{8(`5+(yEpjotomUa_ls?^@UVO z+aN408X27In})sl@9ouEf=;A?eKq|-C+VAQ{WaInd+USgf=pA9>VX?AX9`&e-G5RHve z4o9`>)I56>959v*6byrj-ia{jc8+3*c|Dhz&1}0x_0<)Y^mD{IqCl)d zbdSw<&F(+_bgH~&b^L8K$`m=tbyhojf<-Q2e;W{L5UjrqG;ZFo^aGo&Fqz^%ePAYLN0rCHCi_=^M-Zx{cIl(9xXR~I~d6Fxe|==ZB+))-PEpPAsQ>nhor-Au;D z4s7wgIo?CVLA&@cBbsqy)GA^%)_VUvoZ}{jn?<$0g#8U|6c}r}V{f2Z5^t=^f+_E6 zDtZ8~!PCB0&s?HLS8+@u(?8O$`oL5tT)I}80YsE!t`o7c;PzA5(8vqTJv9row3v;| z<@ly0$wrr5Es*ClL4uDi0-;43JG_7TL7?gU;~41uOnTh73*;^W^;#PmM&D*&yR7Le zzxgsY4Ro;@r26f$h^suEl1sFQ>Y>Izm-h~%V}c{nkd}AIqPL{`yATm3hKI*{w5q=N z$9s<$c56Ug>)jV|K+Q){$8^u}X4`dW%dXZCR3G_SqFaJ7*0lIggM4vqt_gnyl6}rI z`xhy&%X#({JV^wy;&j*7G<-Trd`i)g9-pQ&}>%b>V z{$R!p4d_h&iX-j$#;CJ8kHNCBMw6u(G1cjQm#rgd9sU0I>%9Sw^|fcaoO{K$$S!>0 z|KmCH+_fSFo!%2(KAhkGjMcD<#*TaNIgXR$>WvJfpz)onZ@fanYd{~~dUROcyS>g^ zYa6|N=rt8k73yibmqxkCrvrK;g(U%M2)ao$p=@l}GUp~9Ay_~oF7YKm<~Dy+z;Y2c zNk`mSqxnLcYbOx2A5A@Ydc+eLH9H!a?$OIOq{`ZtI>VPnb(^U!otkPA!l`ovsYg_t zeu$^YegjN`fg3i}46XiIjqyII{LF+pDf7pyYu^C$Q>wGBb2(mM?ds}z&?dNlQ_1Si!lG{49DDeMcQlr zfdZsl{?Kn*?yNLK(FP?M*z&e^pmH+6=bnTPSc(m6e1dnacR56T&_6)vRhs6I2J>uP zOe~AOw9Zk_6P{tmo-5nJT!ROUtW7M^j*W!5T_?eOf01$|=GAJOioRg~! zC@*|P^HQUHS-`$eLk$bPen0h`j8#PhTAbRq+q!+p!UFaS9;3Pkzs_tkg>>XMQ3E5} z=^FQuhUBy)9l$w>Ov)b_t}_a7Qn*~`(%qR<5>Bw#wCe^h=yaCCNkwpTF-J2fvuxc% zl}AU*@c!zBVxq(S@LJQ_WfpN|Je^_P&BR9#HplXG>3~dkk zphP}mHplR=5I{+VUvCb*E*!OkPb|xdDv^jG>dX+8L{t8#E=B4!*m`hmtJXSQCVGtr z{RTX(11*dnI=9!=4lFc~kdy;nHoK8!H!+eFqqc;8CdYae3dXtvBZFK+vEJZ8t%bc= zhm1x0*S+OUW=ng@nK$*=)P4XzzJ!V{N8urTD`Tr;TjQeZV=YnOaA6cDJH>wLSL&Q+ z$b_#?&GOi7II4rab9ouKhV9-Amg^EKj0VdNO9?u*KX#|kgDF0nkJtVLjZEhH4SPqd zrg8MhQmKYuIR>{LaT!UF*AX+gAvlpZ`RAQ(kwedGQt4CAlO-oRr6-&EDJRV$Ied1|jR?)fJ0$wza;B6{q`^8S`N@vV_4( z5x5yY4@JVn2^;Oj=oIaJB-@=PkAiE{S$GPyZzImPN3J<=j+dsc)A~=12$|}Qdu0)k z^+zjnSjz16{i2Dg$s^Liqj{cQNCu4VY3@j58^4FK{4)+5y;vi{yliy?MP95gda)C5dszfZ;MMqrD5 zC2T%|N1i;<+%R=vcy=1MTtbh^ms`&RZ9D-Yj^ri9$+H4&mM8VrQ}0HD`;e7s?dQlQ z9mEPvL(Bd75NJW~^`}agr7WioUfCekKfXXIAdGCOZdi#!6_z=@TQuB=wBEn)U1Y~f z&$6_F5;%U5ol!u!F1HhaC+{^!PALm1z%BUPE@9ZhEQHGQ7wJ(#c{V@aAsK`{4nrYy zFGJ!4`vASZ1u-f1LaD2^)Cn(XWkG00;DER5;}YtGTU_@QZ1*KV#UC{8l#N=jHHhn8`81J%+qe^U4doF!GMg|S}RZQs2m?-13iFpEa(&KItqoxw3{Q>ir&qzJvrl)wx|Ka_4 zv#Yr7md_Z$BR1Z>>cF&!R{+*3i_c~2!`dMH{Zl;n=S%t?1RsvL5$*S0a1p{QNmDx>`6#JDBtY*h^|Q zA{4|sv09St$M@6Of9dp$a&~BLk#!G4sm$Pv;yq)H+p2O{Y&@;6#bCdX^XLPoP1DcZ zCY4$#G)tK_v0@hZLtRIJ2`O;Z+ew3v4YK(vi1`Xpc zHCm{WNBKbW9~t^IWlBE$I&e_av78#OF}8mdp^J3UwiB21%ZJEl6b?MV7p?`p_heLZ zhxs>%9^KL=(sFSU0{~Z1hXWkiE(?UR;dz_`&iQX}7tv zl982pdf9ZF)>cez%?8gzaPvx7Hj3)=h2-)!nHzSD`-Vrb!g1MKzY%|@c{9C5N$N50 zx%SzhJ+l2CDuH>6Zzf3)x#B_F`8TmpCKUuk=EK5%oMU*g{GeXKSEK zq<5qj`lNN+$X3O7D3b6C=Z`cynX4O4aF5_>L`-R_RjiY#=xKi3m?z8yvLy`XoCm1L z>05MEoP@MbJSKEKA!6=MKw)#+!7YI}^_usfI6sXJrq|q*$2{{d<$Mi9kZ@ey6I3LU zVeD`Pneldpe6qvhQ<^{Zh>#HaI*~5NUrl(EV;uH^jRm9w0wI0ZYsZ;Zc&3Mx^f9JR z7CLfJzIeyMv=ZWc|J`zf#L0G!c)@O-8G!-k@V-Gqiy%mdpyH4*j!)CXgM=2%UMw3R zqt&%Y?GF7Kc(o%LS(;2TrXqf*ytg!4swqLoYM>c+llr+FWFPSuY;6!f|VAOT8PMzjhIxZ>l}x7r4tDw`6DNic<3NXwleDUs7Y^L;!W&6 z;WI)q%J+M<{wr99#uD!MNZdY#Xk=Hv{O^%6nT^i0BGkbgmSsz9$*K_TXI>iYIvnGU z1%`>^PuM|k&C`NAabJ}EOQ*8PT#8~QNKmafJyT$(BOFE;ED@69C_?{AlO%U`Gw08X zSj4I-wYh|7i~*<34T~tG9%mO9513et#0_gz9mxOUrARDs|0P1fvG@wDK7Z~Wf4-=B zvUiDC3C2VLT(&>wN^D$UAd|8;bQ6EOPaOEa$#O!ZFNYIVn+^1g|I9}(S>F(QG#+2? zfBG*Ft7REzei-y3URTHOHN)cnIpBH!%c!Cw|ChS$Qu-G{ec*Fvi$MBc02GPuHNwR7 z4RJ@0I8NrD#^-;yP2fNA(|d}T0L}38Moo0a|Be0wu-b0@?}HJlpZ~a?mNNg}LDZBF z#vi?dCH@lu{da_yE>1ZJ7JTG(1Cr3jawJ_@2{rB!je*VgtG`IA33`XhlP2!t-%>P< zRo|e%I{8sOzwsxxq;aveC-7gXOq*^{3YrMK<>GSh+6>y|KnrK>S`im2!M((xrnA>)w6=d)@?pxSGuIP z^I14Qx3O1qroux zbi$_v(t7YaCS_n%`$UiH{MBiYVabvBearMdVUyOClVUFQdBmR}rA1bqNw7TT5CfD9 zq`o%DF!6n(Glz5eZHa&!eNB+J^92=@D*55VBg3AaBlVuqp{`T05OeXp9Vc_qVEY;Fj0k$I#TWHItdVHM97 zy;MkzJ5aR2s9iFC#4x-8*= zUle^Kq0gG`?$Y$`MB5@bkC_XcG!^}kC>!tX^S+;7jD)V~rzYEI18(GBq|ow)Vc{0L zy7IQdj1<4WNJ9_ECu?h!{N6%dCWB`nz?i%~>#$9c*k7cZjO&;d+QhUV+)NEk5LIL< zo%VrR!_pGlJ6K=B0Wy7$_6SIEd92cdKnKQpK50C}YvmJVbY6EVYV@I7?nB-M#+w>X zL02&rMac$B1;e9Mt@gGoY+9VDwr>TP=}c@4V(}c~IrHtNm2A7SmwNA^ z+2c4TVT+si@gr}WWk`K75qD!Y^V2;`^=f!X47+XVN)YeBr@37I&zl)dN{utrnq*ee zlTbJfR=JF21vRi{>FY4HDSgPi1~xC=(P&Q;)moM$W-P_beEb^X~a0L#cVjUrQDl|3ko{bwSctE&vo&emq^ zbi9rTRFX^UB#cZ7X_ie!G6|@Nn+vDK!>HS>Lh{i%S^%Q(9*#-cYeflh$wU=c&4LLepF}ij1@sH#JwUux2X}=skg>G$TFnR3yI3L^E`V=SdJJ?J!Tve%(_#up4A4 zAZaw}0C-V?P4+r7xxVij|7NR~AiLVNxwMFeJT;#yyQfBM2LBh@loPAD&r_58r&auv zAMyc=s731>mNE?k&n3Vlyn!Uoj$0%ek$5wxX%N?GB;g#ZkAI_xnVs9hr^OaW7kGlm zgTV)`hTan;(exn4r5*klB}Lzrvcq+LHAb+eST$A3&|;2@TiQpoaeh@7F8C^+8y0&F zB#5P7PC#>_Yw@{Lz*YEkBgX^sZ@?o0EuX%eXNA@vE}ZBF)#dcit!n0(@7Jl=bMb%e zHHjPa!|JBR_+Nnrczt~no_~=>eSbx<9@CVGIFWO9pddK<25ga*8`n_@~UbQaur<0`36ry@xC`YBzoyrfCU$mlI{J#iIS0Fe|&IcEiYvoa8XK+jiO#T))w zr4+4?P93OYr#F41{Y^)o=}tQXj>w76=Zm(K0EBT<72^`;8s1sQ}Si)XDDKBh(_>=1Pv1MbFSAzFQ;vEndi$0bHPLM?B1fY z1#!W2S{d(SExd5!2DfoA^RNeu;TD84aho&zIz!i+U*u_m<=sm?T-~*$7{{59`j*1< zlo=bm@@M5woZiD5607hvl)?)1_qt13Mx>86;*NhYeVt-YTK11L2AUA%A3iBf0p_FxqBv>WHbtnDj<-wkz>EdMLjvBXxkO9 zBH}xc=A%R`9Bt&~DFSCfBTAJ&(BM8htE^a0U20_sYh!Y#7T;jG9oGUfg|+PR zcq$nyd;p9XRLkw0#l{)VD$&jc)+fr!!ECH4Ar>8zYKa=B!q#AN(<*Ss3ShzHtq+#~U z%_J^;IT4XPlztDlqqJ9zN#U;BB7-9cnzlqDXR#C$=V&g9o;SA{A@@g3vvlCkV1s+q| zO8<1k*ND6%6%lgSBN(#e^~?)>ac6_W3HdIHpSfb~cw<=}s?-(3@`i>*v8j30S$f^z zhc?!iSlw>FyOUP?_8at=j(OJ3zewW7Zo^WSHa&v`HbxQJ!!9i1d!^af7!SV1{@C6` z!Zt^o>S@y)1bs!e_uiOQwnNnHM`8U&wVL$0B|4-#qf;g$Q1M=NSBrF|zeu1-c0X}? zHtVY;Ex|)gQL14L3#!_JRo?I05*T(*+{JINvi~B{v0iF-lXb_!%>b&DlP<=|hhEx+ zQ;@oT_3!+mxx>+QTP{U1vKqRQq@>4S`C(VZRhA+u@Dk7R*F$HU zXobo?g-Kc#Y{~CtLo9IeHYocaz}cP6Z0YpE3Jb+rZ+(xghC%&{mQibNO_=Za>0!pK zVou%)#X0*yp)gq_Ys2-?@%$GRhgR)E#QTiTav`JL?+`3- zDY|(MiJFuWumQ`Tir4B=XdDqKuD?2ZUKo;x4O(2TGB}XKBL};s+8y%&Oo#Z*w0E^| z?XB6!DSq}OHlbEhTz>-UEZ5b(pzJ2=XK${`IArI-pFf(leu}>(c0vy=Y_HYOskb&S z-R#vCWJ`(k+Ec#_(K?cmgEZbhuvIUQNb#3KZbCRazvj-=Td<0MXkD%R#c4GXGaD>j zs1VaMtR~s&+A1FD?r^S%MFmtk++T8&t!jk1?v)I#J^*e<5{u{!a{X*l%|JfDypqL~ z{Olyq6mC)2^=YBMeU>tHK=OhVocYIFOsLGgQ{siJ-A5Xc1l!8BaiR{uTw6n%S&bPq zqz=Uvz|lHcHQ%Ad->KsZce9GTBI4&C-wp(6(-O;JP}lrK z>L|35z~0pyNFS^J%v*##4x%2+P*Y&YEUBZUXwM)xuq=IAMg4pK}Zt$&%eY($IlsGXP5CY~jjRUAAAGt@zEm4nc8^0Y!j zKK{}Za)*@8aMN2{_Ev?6bm{N^Fs(Mdj9bb2$R1GV`RI^Mg@^8E* zW#-2cX|OM)8jaLKj9Nk>G^f1sa_4I9%^1=YiM+gf2GiyzuR+zd9QRn2v&lUCSMI%f{rguiM}2**N>uqsw}rhxD<;xdI6)3N=N}56!}TVO@oqm-$uZ z5y|80^6C<-@$G%tJAYa7P=5ckmB%*0_M{OOm#HL5w=9X9B`ORXVIO`MN|TNAM$>_wb9b*hXD=}5<3^@ zu35)Ulcoi}Q_LpKO$vnmoMb%paXS7(OF4Dyb=$!S^hC2q| zLVU2+92goZpmpIB(&hq#VGp3p;j4eNsM+c}pW|Mn%(*E-382 zk(h=T>8qI?+IP8TUVwuPWg&xn@JncyiIn35@AZY;esW|XX)R2fqu13%l0Gm*p$;)w z0;R5r#9q@0Ikb>immIvK$V=Ue~sseF#oLk!#6dvL+qtV&a@pGc|8GK-(F-G?b2BDwEY zV`o#Nh=1{dTd_}1L{2O?9JD&}OXCpxc;mLmU+LD=BbxFGpvAxC>MF{EmdJ$^PK#LK z^ZXHQ$ofS2dx89j?9PYsIZXI;NIVQ4kK$TKW5`Sfy9H3r#Kpx+J8l}@z zoWY~LvD~vBS|6AJBpr`c;So+eIIif@pY{3n?7HdDZLqdl({jS0py+Mw zdl8GERBubVnYUbDjxMF1Bh($jS$cD;cvgdz+@^V{S8b?yh`G21goE=8jE zpS^4T#o(#C=F)_PUvxdLT8iOA^_x?~;Ez9~ngvs|fB=@fQdZ4~6U+rRLhHGwjG=7pj~LQMP*0_Ap3c#*Pd zp&kWHc8k#m?x*?#If~}x_Ur!l6B|}6(`3`HllCQHX;%D}9z+PsU>m4T4DNUE3iheQ z&n3FRE$%$ArqhCKWoDbfsQrk3@_ujQCW1BoRc)S@85KFhS~g;&0x=!^QKF+)ob?Tf zVi4@s!(`Pu-Y)XuW^>kF{l9LWv?k1U6LPEXv^&Ljf@#wI;=Xzq{4d9~pF;;9|FZuJ zTz5|NzZs}NfeGAh{(lou6E%KkQT7ks=CanvHJ9Y}UjIGD36|RGd+z4gK4Iy6kBIzs zEqZ|BKa)JpfA87N{-0_7J>I_#dcZIFa+HD3pER^c6vp0QA-sJvL8p4bDk4TDcItfW z2hV9%3;F!ZWqlhz8K4yNnzCfd^PjtZAol5?IIr&eXNB%` z`7b4?>ze>diklIiBi)6rHUl_n`s%&!y#vef03w?K7YDGTC4kp!m*MLYAoAfOWXTJ% z=l9&X(G180$UH$-{q99{A`93ukZ&s|-2OcvV%*dB~_BLf0@vHs(=zI#4(%rTizOJb}K|FlXnE%BH@rF35Z&I6gm|zcIa}nYy3F_aW`m ztw!b@6l7VC#pgdY-N3U4}K=6ws2yUFyrx?ssDvckb-^u|zON8~p?L z^wnHeI&Zl&Uq%ernkSt%mRDT4<6r%vU3DNC+k zshIw!+i`H(5nJV|0R62{PHwsow|W1E`qONIaJ<(aW{>hGn_MM$C4V>FP^|jw9EwrLXV#(SosY%?1Eg|Tp26`oIYz%mh<8A`uS%-dEfFzuDwk# zzT}jrl%`Oau*XifZvu;>=&B876ndFNr(7W{W-ZM`$7lN59;5hH(3n@B3sR6kYz#^NTz@P`Cq9pN!o42!fbpS# zePQ^o#kge&QNM@U+6Kh4h^9`GC-pr&BF(S$Dfj8LG1+zz)LgbOfWd-iC#4dt08CyJeG{b3iMOfv$WOY%SeSth9# z@5hUM6w6a7$3TP0ffaTv-B_!(r8CSwRXsT_JW3^;l`7#Jw%?+bL^kPS5#vK|yso1^ zAAm=~e%ktJ)W6Q^Wce^h>hN@)X4=I;1v>ghVN5!fjgx_Hheg03*;0G+ZKAn+sYp<~ z`60sRgGpLagVgjxx)NT>vRXnJ~H*|s8;SAS71=Zih`1IPV10o~*EsXsk7uV0z_xh2+NcOxeT->G;6enVRUi;hMIuCcN9FbS=uv(P(SJFWVKi%l$Q~`;OK?Ub~FwqwhowB-= zsd^iIU!SpJ1h+I$+Ih&ceOKqln$WD*Iiulq;bSEXDc{*-v(b`)m|UeXEFuS5v%<;_ zi|X|oSKi`YmL$Bg9C-T~lli{7+j+OCLBsU#-;;d%Q+)KQ;KJca!y*Bdh8&A!rc=sr zJDEM24@ZyYGRm>Rn2lLvQKOYWVM5~B${t>Lk`8C{*!&jfwyz9Lje#2nV?5&O#ZIU(ih8st;R6=AL{dYm2yoV6T(>g5#_x-v z%>wo-%`BoIttGf==~eYf>eeU8sQa=p(#n0s5AoS}+BFv!i6Zb`K$Hs z>mz4j!c{tRgZqeHj$YPs^$9+brb1r`u<@<6Uce=Tno~^Fflw$VXyR`u{AAd(U)j?IAMx z6$$Zn%M!Z|g6N&*#OSZ4tGlV?`NoHctg8b&TkOh$RiZz~U2&edAueK%2ITQw4lFkV zY7r3?y9PP22-OWrnvjE0Um1xo2b{*>Z*&=_Uwp%O&mpg#&DfTyN3qT>zVEsR7M2_x zYtH^-~pJFEHH{9iay8-X0FUtSV+`xPg;U{Zu4U17|053wjFY zs5>b12u(^R_;pw5J3ZJbSNLJL~;%eSKuEa*l= zJdznzzWD+wt|r9PJegHyQnWFat$Jz7%B3yVRk3VVZ!;F^%Sv#02>)b(iI~kAHM&-4 zrs%pu)xPqnG9syT$d>Lg?nVK1r8WbzQ343ibR*+*+)ADA=~lzz@FU$^XE^D_bxr4;H!_BV?fnt72G~10@~0RXL$ZT~zuhSW({2LCS*~5TpPNY6IYRoEbXk$IR%tGdhg*6E?OsU~jfa zS6Nt+;P&qtug#tPO0*j)l9Hy3$j6~%fL8NMh0UcGBpf{OK$S6F+x9;77kM2{Tc%|8 ztmmB-huD1Maj?vrVJytA{S#AhoiC9}8|McaPlP2C)+=ys&WMpQ&X-c-#>ut3`;2}P zyQiEKz^U=mF;dExrmKBmS+1yp=g})BIgw30oT#l>ts{&&@=PeUh~}}eYR6ngAU#?> z{0cdRZ_FM;(m6;uYWoC=om49I2gL%x;)Xu15*ZnFp(ePZsW2B=>Dh8eKVM&)Sdfys zNnq|u6#dCD@OE_BsF1XzHWfYYXUJTmAv4r7T%PPRDiEilyw{#-P5%pTgT`J-)hhaj zX@*GtbHPeDOOjGbmM{&2_*8z{)-{qhDN?#|I5)yglh z?U(~>nijp{Jb&-|D5uK^SS2RX)cO-2o3v8T&4eCznUt(-5jbxQp~JqDv@I1Ki@4QL zoi#0orP&1O(C&}Sb`w>|^dUM&7Achf@_wjs3oz+VzYUk(d?8Rdd;RVs{QM4fTK8LI zv~lm$w4XvMTA$oELS{iZ-)xeX_{~>{e=^t9N`5AL`)J_k6N=S8fD7A@SB4`j`0(?a zNAlKgas7K6AJ*N>*|U~xSQ2?w2B#TISIXmJMk|+E*|-w1P+69|@2-EU8>lPmOKZ>^ z4P){+XTmy>0q+KvWrv2T;3w}73DXO4?qCfIBvZz6FY(jJI)XA9#7+;4n@0`7M4;qn z8)OcB@_L2?%|ve;Y6dz!*DMCHYk2`mwC4nTTVoOi^nK_fnlh zC22g@c|(5;lW~nyP1H`CNr7zFCGKWj{A}Xn1>eJS<^3rVo964swi?bWfnp|wO6{u; ze+06Q;>&+Iaz4uOx$EX}M%SuO_dQmqRAPzc3k>iLU=f}FaUV}O9IN(n*XO}OvSnDV z<#PI&Y0LKt^`b66p{*pvwyY6Frc2voAbZR2!L}O2pwmp1rCk~1Nz|0A!Xf*-h`CMr}F8A>?#@p ziTOE$r5y|kHuzE*lit3Vu!9wE!$Pxvx=y+Opnb?$11=Zz(Swxla~y-ASuS{0X_AIkG(@a3A;Z1DHJy&%j%jDPEr>5R zY)!Et^xcFCc-~cJ!qxI;;tW}Kv{-9=)8i=AMNNr~8aVS`) z?Th;LrVHT)OK`z{Ja;R2r4VO1I>;$-*qKaDRj^nal}e6s7su2M4G*D$MjAAmuhtsf z;}35DnU=d-$_rv(b0+=eHzCv|K0sJ+K27|B^42(7>-}zm^sl$rtf(+h$Sw<|oaSKL z=;$AguzxSH(U`X}e?$}}Z>Ul&W(;D}GGfoh_JQ0h(L{~EKQ}k+R29l5q8gS?Sc zQ=c5PaRC0dE-95Ic|JS2PCwQ4a<_w0p#9dli?t-wmRB9S*0#DU`aq0jqdeg37LQAL z%4fzuV1>ikFy?id>X&(}w!Wx5(MX7b23B3LDVVOem_l{+R4-6ps&trn{n)Rf7oql> z;fA@Cs9xJ(Q#?b*HYUkr@WPJu{XrVAi{lmVaQct}Rd>qQXsT zQ$+ByVTLWCe@meCq|Ra_!-+96MLLW6v-G!g`kRVewYXx0nOWSidFA{$;~CkI51x?{iNzL$7)bOD zZuaAa(sg5e*~cxJ%2wb31^bsey_2UVqoNz=eFlDn#*O}<5ske=V6l?G zfx02b-BXc{+^F_QM*tc}Q=A=|Dc8Fn47=QO{WMMKmQ?aVCnb6BjDVUnLl-)7@d|zc zH`NwIQ6@YiF%B_%Km$dK)Eu#)sF4cAn&1!%29mHzDIP57##ouu*W0t3xy z40WD*1Ns)m{*j*kjD`|kL5Eu3@&#MSpOjEWf#p)6sCV;(Hh%R5f+Vt#!Ui3d!M~;= zM;d`4r9)9`sP@5eMSI}Vk2r?l3~=HHUZ|k?eO0)|z%vE$p4B$2hnCM9Jwm6Bo{+S_ z()feiql!8gUn+@$JdZGuVuNHoh$sULhZ=lD#3?zH00&VM-ywr%62#x)p^}PV@rt`nt-kTsSLp046CAiimOoy^C8Ktv zj+JJTdISlg>^^+8NE`+Shf^;FVoolSO)K$bcQmO4nUr`i=Zqq8*PDlkq_>=>eK?gp zs)q#qU~*pnp<b#2+oXJB zAq^6=2XwBXZP)X%RpL!qFg4;8dH{d3yM{K7#pV0EWV`i+FE!-X-+?oW8(hP0lgjs1 zi(GG5UUdmKDN0_&c|qpds_eM&E%gvS7M$g}mJkJ=OgRW1Q#wI_T%@YnhX` zKijylo3^^h_9l97Qi}IodNQ{~xQ3;c(s~M+r1$&5ZEx$a5zI7t&B|s$y{%}ZOl^r_ zNsxC8pKRwD(bFIdRrf;kQ)Awm8D`S4imZ){uNx#=_NR}K8xscuQmF>LU-^xNQ*SbEwiS_W?VL!Wv^OR#S)l#;c2L0M@ zqIV8qH6%PC8pWT7Q0QrtSTN8a&9!{IROeBNK=kmt)3X%Iav8N~`rFC#G-0o;)q@g8 ze1%*5ync%uY$jW^H(#bYZ?8$vyd3==5HGT@Cn{@ptWtQH+fxc$Y^es%r#jVev-_{C!tuAT@#5-WB$#g z|0F>45H^aSs0oJ+5Inqe!N?8%%hO)G1VH{G5Nxb^JGMNFRv!euzdLml{0E>AU;t1C z{QqE8`0JjX0NrmL58>ocq-XEnh)#g6>?MGIm$5HEB=mUC$Ay(00jU}0Xgmn10lbC= zOwNvxUj?GdHAYVy_&~Y{5C?s{0=nOP(eT=T`pYU#&gn^>LX9Z4>cr+Ecl!126n1nE z_)*rUpKZ_f1&%1_63H=(u&r#BnGZuRqW&>~GEjZ~}^YzdpEc z?!f^NZykN&FS~w2b}=XI9s}h&*JBpXq#s~g0ZM-nWXI5S*3c4bYM;e6wikA<0#S%v zSeiG#u-SR7M!)c#+XZTPinw>QJnAn-0r1&TIGmK7$+%dv)y_UaZ09J-FBG{cG}zPk-2zhaQVL#M-ojCD1d{1b@+o_jOFXFXShQA%VlUuY^Z4~|vzsLeC0vB#XBjm9ydEc}@&9(+ z*HWXuaH3oRYX0ON_5UupKJ9VQ0kJ^6aZ1@eq!!g(I)RQ`+<>{(aQ zBxj{x(k|K=y_*TYX^qe`!KQ8%q21|?eV>S=p3`G5S=NJxfh$Y>{y_O zONb?8$VlYZFp$)h=?8UZ*Gsvg5Qz;4q+F1OMP#sE7T*TM>RZoB!r>u2lu^Mm!r8R2 zzkk)X_dk)44Cy`s$1U-`nqpTf@B&=2?K994?F$(gtBEWB?jPz>)bi%9ZN6G02h6q& zIES4vu4_&36x`cYh?3TE-#bmVB_zW7EyeMb-oz*3Oa)`8%gw)2OD1V8ClA$g{4yEKF8D<+45_5JNPfSM=`U+ z>Rl%fefpSSMon4Vupw24M~$aor2*^6K-{f4R9I)wk<-SoMED!rxe2$M9I_|353_tl ztS^{d5W&b$;UDa-#mhf9Z$!%a7|D!t(v8#uMV`Jbaety}-R=6D@-guOxB1u>?2w4% z4I5Uw)}P&uV6OjQKa<_kw9Rrs^2Fv>v0Gh#r}(Nm3l!9mUWv>H&eLD&$GmG8Uy3+J zs_vk_{V*henuSq1*HUW>-f2ov!beq8$&+Ltg09CAyDR^wn}4QmcF0?%J@9hQqhEAv zbDPab>&rHZp!G1u5l;W9p7zX?3^xJ|=vyR?+Hz6rzJy9QCcYCS6q<2-=T&+nrAEaN zkGnFTG%IebWJu?Q& zScGyYWfPXbDg%h<1mBDBq&2ZxVkINQ_i%z?7?iXK;B z(Q(S5#V0}Fqr;r=`y-`8E=mv6-)AR|Y%&%m8Ev&`@wlis@SHYsm|cv|w*=1ZMWwuE zuD1&VXcFQ_#Vk|AtPdak)#i1vHm| z3Cq5SW)k4l_`3}rwFb>phxcH|p@_hF@=pNIn?BrMp!3Wat^ulkA+EF(i_E`deoq=t zC>F1sdtMFUH`cC1g?$kaQ)W!xc;ynFO0_+jBtbm3Vo+8zO_I=LG=@}2+?6sMfyH`c zyMH@=`U5;!%QQ5p3rv%W8v2Ryfi6YDTsaV0GR ze9m93mCDl+B$SFioIgCBifSnJ_W_okI4+!9j65>VW)xYG6eH?$7B6xp&KOI6T=JA- zou{A8?=AoJSI@5B+fL0AY4ek|1eep}gwnf*{hcl|FiCoJ^I!Cu#&}r{U&`guV6GT_fUk|4sk=UZjf4 z`fT2#f)itZ#NTR2}p)p4sJW2I;Dz;CFfW$DcK{4lB>&)ux# zsebe-xhB1i{mz&tM$cO(Xhf{XQ9iL3hV59Kg*u=MVlPc1K;lBjLa`5}STk4Dc zbGt-H``xoC+s7mbGkXl3htuQZH^Q(NMoeH9sA@qQ1ZKwFl`qmVPD*BEA*C)%55~`! zKO#OtMUY})M3Un#@goq)tbc5pd|qf%QTuXK`oy$u7!zQj=B<5KRd#(`Mx=><6B=8# zF|h2yD|u^t^U z>C&X{yOGQY<-QH7Zt5uV5g7(YT7DplfO@;V%w-pUl34^X92@NMOS*eAO+QB_w;Iou zLDZR5_a@3D1v+gql%kF|9*pQSr=8kI=8SDWl*A>L?|{C1F7TyXCB@!P#!#;4O6$5% zJ?oxCmkfK7D%!9)sUM}=N2mNXkO)Ls+9Sn}JSvUP3eC${SWH`yw4`%Qq|`VppkkVW z3ej=DQ>S2hm`W;vmL@>IgBpz=s+dv0P!}b_dQerDHs5k52Z_x_rvF!O##0q-`vTwF zX!;{^8BS8(3Tol_OKA1GIcA>;{rn-YeOIux`cx*^pF6Vb?)~k3gmbVbD6N6sdxwOr z&Y9xcL2l+BfK4+e^Lda!+722U;l7n}$-46)&a#BmJosStOVUi1nZ!2d(I@byyE5B| zT>h1h3LG2y@)=6@rAOu^b)S$3R6eM#uW||Xdb;<31-FFbIs3k9KU(|)Sl%#f-9xLd z6$LhVu`tXZ$s$8twM+V^e~;4;q*(WDCtbO@Pxol(el(9~c5$MAKZ?%aA6@5sg%b{kEZxSkiapp$u0HR3mkj3UG* z<+^jhKaZ9U3Giod3yQTNAh0jfeN^-6kYqRmZ~Rq7@Dnkb$z7~rN6M8%59P;gJ&P8U zbzYFX^Uu1EhLS3w%q!X8ibUZW7`3&5c|G<+e_6r;78fg5*o@0%`>nEs*m7+H(=Y$V|=CH(~)1|9Wt{NkpH zThXTlE>v*>4IpXarZ@C$98ZI2Y6WKC8y9Kl_j0?r1UpW9!yVxh;dS}jM(KY5cp$R} z?T#nk7$Y?w8ZGb~Nw1=d1tI%UqkYAmL<;yQKnG~dSglelJ6Q#?P2tOX<+U2C{^cN_ z!^b@Njg74+h0NcqEZx<hq)4dhm^_ySEQh@YU&JMp~8YJUhd* z#lR@{A{YN?L}K>tL9$U1Tq-X6eO-8Y4rpJ|p%@DKp=cQ`QMB;2NX^KQ8+Acrq9vV1 zOWM>@JW=zHy}V-609I6W`6m)o*-wTD;V@Po*uVU4MF zgm0;is`qX;z+m-0`aP<*#aT3aC$VxYbWHvuu$gu2hGo${_cUeX_0I))iJx+w6A-Ae zgQyRDxMiML&qtU6gxaNn(ex`ETnm$ymY7ymZe} zK}U{xe0;Lc37vczVUppedx@vMrAtA}cdua!Js3%s> z8ruG*9aDAuI0_3obM$@$mwfkJVv5vJdSX8HtL{m%L#&LU+VWBa6|F1Vib~l-&Jr8T z6Hx2(ly1?GGuQTH+_*Sl0yA&^1;;b55xnwWnKjznNPn)KX&p}s_+-FIy8weh+d8~3HW8->p;N*4N z;CGmx@o;TtYSg!FB~ZW5HR8wNRwn(B#>uB;die&jR2Ny6{De<+=9@D-5PPje0Ah|B zzKwGA_V^Y*d^kV%+bIe|nqF#Xk1O0V0^MU1RZe#R&`pF>45q*iFw@V3#=#^hI9D=N=KqJt`+`468rVVw-WWK_8a7kwVHGPizs0{0b zeM(%xAuEkM69Y&KhK*Orpu-F&Kg%1=Y2@Gr=Q`5RsI)o!&DR#2I4>@|jh>~|vp%ZM zqF*Pb3XW%@@JpcLC>KaaxgZt)z@RTA(ewRd5P?K8tOs|6ibaI;!#sDBk_Pnj9I9YY(O%R3 zNcBULkJlxzeh1%Wii6Dnisca}SLE%k!&GFPV(!75AJ?>A_J2SMIJAk+Dm+%$)r`B+Pb>>EmE zE#uTOeJAtI5D5|oy(NPb%EWCNgeKZwW!%ZoB-~uYmCQP2i4=KJwTW21(^$-kMs(w4 zfpVITTnl%3ZkzW?+1DvL8%HYAd1+!NeA>lwX}w=AWAm1i4!smu7JW6-;;7|ElU!U` zde!pUSCMs`yBpegRwNs2%L?Qqrdpx!kM@zdpjgQp1fEVDRuH9syeLvEXtZF@Sn~cI?I3RK` zmzuY8^{N2&Ma%=El7E{%{4fh*<_w|;mwE3LJYa=Q2dk@{o(mf&x!;#}@ajtso9St^ zDo7*Jk(8kfC%n<5z{`-X2l6KuwoTh>$bK-H3&q`$)|~zyM57IOh>?Z$V8dNX}`rHb@DWYauGysUU_!MRoHtdm7l`oR2j3XAr7qw+b!4`RhLWjOo7 zH7vd1c+1#pP2Er#oBjfWgrA7g<1xY@P1~@?ul5%VXl33|x1WE2;0(V9AY-I=AzAfN zRaL8&pqc-#hxh`Em?Aebo`td&WBPr?NzDo^Mjr3))~@06)b+H*yi$YEr|C3(`{I5t zN&Qg?nFG!5VUs+b?(Sx4G67iBNAZ~$?BC|8f~fF_3KH(4!u(}by?9cV1C-il8Z_U4 zx|#xfy*<6XCDT)ZG#7`*pznt=hwiT8xR$owl=OXLVGg{qx45^ZV`L{smoEg2b&ueJ zuGD&A4n{G-@4G}y^ASI6-?;JaasvUM3$X0XxDUQYkzNfLFL26})p4hfMcT@pkSm&R z{y9H&J4M!CU4JLeN%alqlrbeVkqc@j1?+5{7;eWpYKJK6zoZ&|Euj(Z0v0COlzmSC?G(~)MC<7bx#f)!eY}0mExoS|oxX{ghi%p=ckqZlRcjsN;nA<3Ra5%I`>fO-S$0>Yu@4oCMpFLhIbyN~ z$ym^fue8LI*2Zv=UXw4QfJ|)E=a@1x=X3HwQisBqojXRlCx(oi2AuWiC3 zanZ(_ir8;)%cL(MPl9HD)f@^L?3iwr*nl22JnvJ*@95?WF!!0%mNs4Tq}vBYv*|1I zEf`x@u58@DdM>TKB<|{$DK1S-)LhJund@H~5-bCgl#tQY!%!MW;_{FEs>i!no!PuF zJNQlfnw^OSn`EXOy1q)c_MD*QaHQ7VNhiwc2`?8f{T+d9l`e$M$hv04yo-6(w{uhi zS*Na-6mE)aY^5q8ZbP4|@|t6u1+F^HzPR);_lxU0NR(@Am)0R8$6W>rSK&gI0QTq= zn1b?bN@?Xk7qJ&_=DdHS{Gdw5A)E+*`=|%Z4|Q&bO?oc3m!Ap40o7@gwS}eDcP?Po6l^cbCj%Gpj&qX(U(3|l2?%nuHD}ZptFlvds=r}- z(7`(90w)@lP#wye8H*|~*D}Kq0iO6+M(VQqltrJK6LH?5(ismLaU90&y2MNM1Y25z zt7ig{F$@h&!(q{S(QwTrB*fa<;pDHj;%O6my?rF)hu-6l*;EI+Bs_XmJ~}b{^A5O- zc0}ei`8*1b@;F1&)%G-#s^{0di+iT^-$~B%_j3f{8A5aBaY>!PEvK-$u$_<%wnB~lztO4_qb6-9o`kR#(#GJrlQra=V8dO z_|INw(czmJ=X+m7*5uwAe_e`((XOujfPWKIHIJD9r4zBHwTtfk=hLoK@({|{AlO|h z3VC(f0XBeg=UL~kc;jT%;y(a>0C7gFFpRWWBy`H-6CkEL`!^zsPtMrgsxJV?&in3~ z8ouRlejJF>d@$J&w)*6q)p-w(ugmW9k=r>ot>wTb;S^}(&C#wLN70K30GY=bA9?6C zY!-KUv#sxK&K3M3FMHF;5E$uRTeIDFCe%Xwd?xJdgy6vc9rW*iRSzW?Bc0kHrhdrqMp)=qk5H0|Cyd}F^c{S4Nn*8xyQ`;m^Z&7e%!`JKa zmUObP23&6c8U0oJOptWa`LJojKQ%!^Dl5~>3YdrJziI!U)=!Lb!^m$~ROD4^TM_P8 zM{qW?gWT)y?d^rjlqon%8W(i)*E~V?_5*f~|5Fzth|c<|Bj6(~+yfkUd5IlVHXX$e zBL2nF09YI^-*0{qghYHH^LdO_KqH3N2gn4v0jobY9bN`kVY2|Nn%`gj1-763ns@Nj zOiurPCNaor>Vn*g=wntFe=6o7>)gJChmK;+iBweoc;&EJ%2!TbMn!dK5_ZBfJweGO z4K%%8EnjDLekm5&DPf&p)x>*l5w;kv&WW4#}5}h&86EfyNAFaZ0xDQo8 z@{mPRX|!3Am9`_#rVZX|Fu zDp{-k>H?r%yDBj_?_B-Ps?40Ee*@F~Ey!tl3gX`i%@?z!W_5LLf8t_e8tTHYzTx98%dtowOe!*$3#j z+AA2~gP*o9g2JSUx@;$hbJ^KbM}HrvXeZgHKjtybS=#Ex_YI8=Rq_*(T)u)%;0##4v`hy8lLuIv%>W;`3?RXdsgT1sQ77euvm7Q(O+`QseQ?|=IZ zx-rG@9!e&M)z&gRy`3p9bMGd>6hSRp^(k&Trt9YB@oCcHh-mePl!JnKd& z%*&g~)3XwJ!OOvqk~0(C(#k$U5=1TUt{$N`C$C1GlXl)QL{@y*#6uey;nZgDM=26! zUQmR!c*9hYv=IQ*Cm`ZX7s27-xGO zz-Z;b?%DjZ*lq0w!~c`2qKUMy^yQi$u0kBC#=q6>euLp(ehNzm2W@sgh#bo_Tcxkv z+l$mB;ZiWQ5a1KXvbz(sH1*c5{kESg(No{ox-^ZnEsYo%Pxi1~uklhkl}t-f&()YJ z$MaoAE@V!7NDH^N6x4Im7 zI;N+U!@dpUCSS+gus81VXKv z_0hXsiv9-x-wadsZyx0kY)Xl`k9KS0qEU#YD`}Gv4-x}JdPi5nk~hOh?>brEQS12l zA{*ESrYJP%IeV z@t{g<%ZwA6xkZKf9@y;t4I<{DG9C2wv2$SQ0VuOA63kEz;n5zEgMN+t=Q%Q(y9 zx7k`s6LxDLQxuYlBB(Cw(J=p6L>0Z7G}3R5;gXq=_q@$LWn)=9U%Mpx=I!Q6uXB9_ zZxdJ*0%y`&r#H+T%iNGp_T?JxN0FoD!tt|vm7wqA!hS>Rih1b7q~qpiwvoZA8V(8X zC;AnK-LRaKKelqg+@?tMqMhw^h35BQo1((PLLE{u99~>?s!(O5;Dof37{mA~Vx`2g zKN8Ay83An6)T^4EEEO5>j=(Vbb*t zkKErA3!WK);Yh9}Mn@>k$eIl$-{eMKgWl&pf+qNC76fYEL@!}4}-Q6Js4N{6* za41e&+zG*5g1ZEl;!bgg0xc9TUbJtn`}tks-^66eP^jQ5{(ShUgIXeENZDI7!f|N>E=A}W0 zT4Y!iI`Sf)N^QBNcmt5wB)n~nzkZas9fOg8(bnMkcT_$_{*FG%ZHYc~;Gji8a%R$~ z2;qI0K}K0@Z4Iwnn#jnKOWGMCD(c==6J52xgqyLjpKGssX4D8GzJHm5HDW2tVl#Yba@ zfdXwBf#kMpe4xGKG%rG2>k~&~lkR96CLDqgq-cMjayWENaiuWUvzQCG;l(bO>LeH; z_!7V3$7T+1`;<=9M4?W&;K00%8_SSR?9_SYjf@f}nTJBP{3e!AQ`uZPAB(6&2I0ee z0h5hBc{HQ-020|=BW>M%9?yV>DZz-#Ky)oAD9s!;*v24z!gs<=$Zd}`j%sA;o+!sK z?Xv1yQXf_NCD*rx2Rc_F7l#^7?TI+3Svjv%TDB>5yjvXNMv;D0%hsV|CN^5#w_(}L zx}@e+scg|t(Z=h$N&SqcK}E0Ul94FrLDE)DhX%Bi9hoP$@gRPznm6vx_{`5%QucUS z66+y`bIv1VcwP3oa<2Bz_dDH3{r0?4`hKmeH?*V)%hnna@L;5Ee>2BIf{l+dXy%Oh z2_v1MVS$!#kIp?kSuS$^Q)7m}wHUrhsV-9+i=MTv>1W~=Z(PG8G7>LNCAtGKpWCccROY3NiC-g0LV}#`_pM`Ap zDDU~nmcgf{{}%Ph9eUf;w0h}b3=`8#W@NYs6Qve2vL)VjR>!e=i=EMl)wY76Y2Y04 zG^PX+t#Pk*ll$Wl=%4uqXAxP`P+r`(3>Rn&am9_q;OakKQ5~Y4*JT-Ay27ugVlJj? zgB@tb2A8y@?2=Q;+LXBV-ZXfD1#~SJ*-+i>;+Tqm@s=|`NdSwNY-fM@7y0I*m0*<& zWYNDL$N;lNWE3yFecIKE(3yg39?I$ZWgvY>0h&G&*E>Fcr0m+>o~QDY>3|#;<7YMw z7JmKu9*utSJBK4`il*W!t&omCPhV|D;&r`v2Vrt)wkh9Xi+Q8TvZr_>rqFEvg)UX< zo0$r=NNaWvRW&H%^(DV1OgxV#iY;oh z414t)Ls<#Ca;4gf_?nuYz2+ZMDn=h`b;f;h80D+PNpwqojVVZaoK!JFKyy4;hxteSXo1u!EwCGx-Tc50c8lWQvH#~>nJs-yc zSI!`XjoT2qjR)%r-xC;Te`#MZeiFH=e1^uttdw4}WLzm9_KqpaTlMC9Ou$G3Ar_Ho z#>Nz?vQ=F&(gj+0?}%HQk66xMHPJ4#A5DjJ2vOX9Z7>EE=V+B%xHz}Cv=?sjIjh*B zK{a3Xo+nWrf3beKjb}FrJHwT%B;n{nG(#KySes9zCDt*BZ-#u?q2w51vtDKa$`*gv z%xF+qsW6b;_|q+9^x*`AvznRAPf|ypK@zUrIx!Kw_RfvD;wH&4*H-DzhPDy5oSGV& z!h%a^`B(QEHBuTwb0FH$(Fap zf2E_1Rp#Wh`#1%Ef34_a&<3C92YVtdoo;N>hJjqR?ruNkTS{3UE@SCGldbqpfpP7q zm+S`NebJ?+kxgSvFmGS_&@rZDUix-!d*&kgrcs^@>6*?S43Yhnw2CM0GEQA~D)F)f z;(kn3y@Cv`b_QB#*oc?HqQfNo2NX5U2ZZMM$;!kdj%6uLYD7ZOZ)~;&*P3%xQ(7@S5_A+s>3Lbc;2t6*O8W34`8Y*{f;Rnv0)a4c|I=uD|lvotz~G z?+~oFJjeK@?X}O<-kH9+D>oA|FIgBIvFgQ_+9j?cE>UxELs5srSW7AWcvpj*JXxE@ zeMiqb-~ZGlZKHKPs3z*pEb@e{(@GZ{FwMF=CAT$9k*` z85jyI@hzYdUo@9)v%O*dDD&=X%Tnup?q8oU61(g!+dV;MK>=NxBg+mfKNGVcuYUk3 zKAke@gGX%lONK)@T+ePoUQfMELoU@pJd(wE=TGLnY78TsR;NgOZ8Ei(f$a-0nS~3J zg;JF#uGlTMW~t2n&Qh((p~Wf3t>}5t;g90Q)o-NI2_%)m+^}@1EoP%+Hz7fJ@a&U< zjYH4hUqS>pWm`l0;cpE;GV+Ms15j>#_$Flu|a0>{qm%3v>vyPph7z;$n z4D^k;;);ipxh^hZ@7bfjWt5Ac#=K?d5^ZFZHai7@z8Rp$z0ZnX-W>YI+A1Zkl4U2f>S!+Sx`#Q5f7mMbO%6OxCai_mD3xC`twkaL8h~a!ZV72w< zM&;UeDlIM3sF||f`W(Si&$+_Zp~nL#UV0(SZgudKY7kj|FL!j^4SqF;*dU! zD9$5$#Pj#1|JnR%($}(hp1e)Yu0qXPZB~+%alobYV+642l~C90r!|O6`bX-C+9uE@D0^z?1#~Z!*U( zi;LeO*89w|koimr;*?*ACA6;~B7N0Pt3EM>eyt=9oi!}iJQ@9^AO>Bqk=IY31^a7^ zr4Bjsvac3>U14MgmUUJZnc6K5d9~xN0H>TtzbRT{PmbIqq(F(Ok*bWr}o6bH^l(4rk->nIf?gu02s*>}e=6)BtV))BQfqFKE zkJc=B^=MS@0X0-ux_IxlY?q1!KKW%OZSWvo*T&xg5M^5fjScO?h(3^tbdm z>brTCKWB-hg~!e9Nm5q^rz{&^a@IO$*qOq_al>ZtN<{u2?j0|)3RNwn3w`c^K!NBJ zb?4)NUNQ`adq~_IYht9R0)s;UHWBNDD^ct3;2Wl%TCi_Lxv5+Nd^7_0L>^8ni{h#2 z%*-El449`A0+!)P_2;rGm$EJ`}?6_nfSL;u><8wdDtI|#$#yP zc&r-gA`E>~!kR8sX~vpZPzh?cNK6$ge%ca!PD)wUJSETP1G$ z6GLx_!J~kJ*BZhZtMH>zQ(ax`4cWF^@AYF^nlI%Nqdlr1rg22Toj^tsMVtJ6(fqCv zBbtv048GJX@xCiLsU$o;wrlHw5Rx!ti;B)RZ?clLzxlhDx~0bp=Np*(A)PVMdw zlH5!OI0uQW{b4@YzZh^VrN;aTaBkgiZn>aL6x=76NAAb>0uoYAxsN{byu&%WKim2v z*$Oapse8P=-rc{xPsDm~4KCf@zv@K?pA1%CAlGvwWUv|FBmPWv8W<=|_XGR^bG&W$ zGlYBiF+{WfXqXbe|N942k&CX|`ki4&z?k&)NeD*R-?Oc$fvwiZ7EBKW{r^PcB;dz; zQUV}D<9ybJeEXMUuo1vF2prh6I}X4h$eRwH4nP7B^vFahWu8QYFo1yF zN$?Il9n22y$prwuUfokYe**Y|-5^_)0j>8p0FPxK%o^@Zt7|C9Zls&8z_i#0q`owU_WpCkh{l8 z?!TfDhj?Z&!eXw3Hm~e*$&7&5q?KWJo}Acy2y))4%hvJA`RVa!aD8CjbN(B#-Wo z@(ua;?k)7cYPS#X{sD9jo}R&%!vShH0sjDom^2+_vgpnU%8Uxm^EAZoYQdWMoX>)2B+>So~`@p?pt`lv+%Q^ zaHeKj@$gnPdmAp%MX?yIo!ngFmKLskgyU=NV17wu(7L6A*!!79^yv-5`dPGqpMaNJ zO9TA@rq3_t8@Nl&?}@M(@{&Xp2@ zB6Jx8qZOn$?COMfgKh}WI9DhH)qZ(Rm+gJ;8<(;0cQsF=wr$UF%l7qWysiV;hD9y= z_Nsy5WTOTX)ofjAb__?_)Kwdwj?mV+kYgpH(&b!e*+g|YW3yOV(d_X~=843E(pq&U2yHe^VLse@sVyz08r=3^2nC;4wYh^1A-n1L-^t~tIhi}Au zGvUX1EPL!v81_qe!|${{yVZcJSkvAlKNv1Bkkp6n2gKVbdc@udpIq-V=Pt?Zqb zJk}0jc1+>X4>!8d-A>PTef8gy6q}UpV%JvnA9lXTs8(U&31)?w>a*#Mr3I?GslK=i zquq$;Q?%0MV|U>VAE8v5IMIVW&y~x{>AP@Usbmm}vP@A-6rj?@?fF$x2l>nU%k`Lg z{naU^fR$r8<6}pRw^utS#DHN1}X*F zWUmmCbdKXpXpf!wC%G4V@ZZ`Cdj2}+sP{j>@<>1lg*Zm{9i0j_I8)QuM~gY zt*=3(2dtnA1Jl!47^@&Chw!kkAFhYV7_xY|NjOCI8zyAH5O3Ndu-jydB=P@vC>8ZN zYxve0VL@QPja;zUTxmy&17W!UbEe00rUSJp6L93OR|TJoS_qsYjPr&pd+A*tdiezL z4C`;xu(_1RrMJEIZ7NB}2NB-we5M$ekF1a-qH}8Uu9@xze{L`AN*XEaDAx;{k7Z#p z6IV-_71oGDAaN~LhQLc}*~}*y=@wE>>k6j>wgj5~*lHyW3<1>)UYpu(3j{eQG^1Z3 zzAXC#kcL*LjV5Jtv;mP~%A=GO04t?#hqEeVbcuFI*z3;r5FcupvngI3Y7Y4cMqXUM zjH?(oYHkiEPE#5na&dD$ah_Vwlej|`0~S2xqIJz&eHDzf1`6gsX-l@WXj;0B(rgqspPGdjU_?@!3~0~aIhP5zeA?cQ z{ntcKS+}cqqT$Od$ArHXcfd8I$EI>p&ktnWAE@fVP~(oX0Pbu`%l~}e15}fb@Z$b8Jlb>A*1K_O?zYNU*IG-a^4isdaN}S zG-!Ws5qu1i)o3(X;CPi2hk%J|Mju~$-v(t4 z{Xk@&Ui(+)5E92P`9ZI}(8|#qlRltE0TMMHMFnTOqy)YsMbP@ITc z+sE$<12Ht?53v>IzbIT2abRhDu3!C`m8i^+mDG6s53qUucJ)P~(3=~zKOGT_*1OZV zoAa@a8RR)s_yRwa*?{42K!7X=jvl%mV*2-c4mlBG9$poOYdL7cO)^8th?n!|l74^7 z1b$z_VpCIXynp!%^{jX7?J>KBc+ygK0p{xKmCW{WfJ(LXU#3PfHpOy`5!~9%tAss+ z7~D}#yrmu9@AQVMZW`xXs$LF0iWp&GblDF|T%$sJn-hW6t*<>MW{B5TbX=4;$q96r zPOs7|k!Z;yPsn4E7b*1Fe8PQKayAo8Wa4Z;s{xAuL4^RxZuRd-JmTI525N~d z*iE)P_}3!VMe0o2MB&wl6aN(<^0+fM{}3OEkoxZ3`AKLcbJipG=6eoMQ^m9ERl`C0 z`eu?1`nU1{c8`9gXcBjYo;20$MX*nwu11a8XrQuoba51FFPOSoszzy|O7dGBk4}@V zZkENK{t0!BmH5rL8izPs=SXD1;dRT8O)W@X{R>O z8JAEM$rhyXt)(E43#08Xikr`IE(W1(h!~ImKRuaX!g) zr*KN3mU_y9;#c6;5`QPl;`)N%u%132>F%O#8^@met$81EcVM=ee--TQwCMPe$7bS{ z;u?6&UFnQOij*fdhf56A($jd|{o#9d9GCp}*{g+Dadqsw6xPB?D^X*lF$fN|CqVuW zD`evK&_U;bK!qV`VANQI+<{zq_S^hb4lC2m@dfg6b#V&T!esq5vA0SKI=IS=CCeLP zF_E4W1TjAvPH4!I&cW5EgG}JG_Z}&23r5)GGuz5hX+8dYsR$;H%2anu>{2v3mfqNC zw(MxOFv}?N{4dYb)Ww^tB5<2Y1tUipw)(0mHo5jF+!&RL67*EfnlYz8_FgvQ#{we zj%f*Yi78j)hlyQ!IuWwm_q-9l^7+;;6_xT+BvhoNI_f?y9M~Jdh@nV9mSlxmY?KP9 zqMXMU2uejiJYJb!CQb9&6%oQNiF7qIQKWI3{oe4Vy08n|nWL#sKIEH0D4arq8SL0P1rI^3gn)C91l`nK#j=v(Rxhi`RG|!gp-D> zv2bFG^R=%=X2azbtx-irb`%x0fgr_{-kmT;7bY#PQJ+t`V<59s&*ZZDBf?c3*lvX} z&?^Na!mHY%E+WHSCZ%Sw!HnpTH8Fo(hXg*%bKcZ0&S;~eUW6K@s?E!y`97?Ug)l`{ zMmb`_$O9Tj1#D)_b&7I8hAvbTbd=nI>E(J*b<+()o~Qz4USK-uWcQl;E&*L=@oD(t z>KD}66%@_CY8V=8Qk_BEOv*8O5#td!);!;vcS~r)nlUx zL)A;^8d3M7jlY=b*pI`jX>MkLy6&JgH_&QxXlrpW zy0y4`35RjfNj-cELR{EJjV3xbtswCCRPycE0YSlUKBmxdqR)hi{X(+BCQ5YlbbPGR z^X8cg0Y~r3hlwE^_t~n^S;}q}W`E+LY48y2Y*C_7+*-Si5=3TdQkUSUnpF<6s|s-fJ0lX&$^u6-Ooayrhv zW*helb&c@SRCshJ{l;9WCs`P1M5UTso_CtZ`jU{csF2w2{fzTx=|`5M9Kxv-f6z>^ zy5teY@eCiwVzkaS1CZ}insSizB4K$e)LI=D5siwSVQ&v$+BNv3YI?x2`VZi;V0&j! zDZxyZQ25(6Om_uo;~lJ`u&Ze*DlS;u@SCe%yfzVztxScBsMbhe{fyI)+~sO$2yLlm zH&*zL$g%dDt(%dZohk*c#kPiF?5sqCCgEy-QcuNR7#*ktcv)S1dAJm|Opxfin<652 zXMwK1B02r0VHdlCcL0tt{_?5;#u8efB}LV$!zjY3pxJKjM(6S-_~`9WTVsWn$cm(z zVK3iUb6Fl|2p-@inzBKdD=`!!`7uBECuhw6jn)4bb;scu#T|50UE4NdSRMUY>5n&B z2O#vvcXg^LPZ0mx`?eVNz3pUyXMf48y&RKX=jWJl;^UpxOV3s;SH{uCpBRzHU!aAl zv#{33G)cxrl2ZLONNehf8MUgO*UglBsMtdoZt>U)G9kqSsi41G747YLJGu3F+v?qp zp63HO_;E(rEeweB=2HlhHhstjI98!rap@>;fs-ax~vLsDi=uz-HMtgLloY=(n z2KC5GD~gyt!)E^g$Gm_bzuDZ}B9Fy2&NJ#{s|pKwZ;qtEcbolf>sgz*xKdH}`q}62Q#wYQu$9X6#0%T7HFU=(lme1FpWSUP9 z?6d~dXWZQRjk=^Mn))k}TRNKqaSViTcFft`TyE#6JA%CGezDt6F@o_YL`C#mb76nT z#MtI5n(pvO$9u4$8qHwDAOo_Z7#>#dIjAb942yrNXB`f(UgdF#KVIV#{Y6yw=ks{0 zAKJ`jp`MX98!45>7p{CPjVioI!Jq3PitA>N&M?Put4HsRDc+Q?`#pAKoJ!(#PTwo0 zTzNC71q#|R-=<&XaoDhs_&Ksv&aEWQMXIWmJF~rc2PuAqH+jn{w0A2gyp~MDSIhDz zk8vPC2Sz4TC@HA>)npvVCt8a1X|Os)X%Ug?`~0TSYBX}Mm1E6cB3|ueoiEM){I1MF zKScs>$b^-xA$4`WWXVx~N(Y4=UcUkA5DvgJ>p69J_S1|TFv0F(MCOG{)Dvz+1%8$qhB2)30FKlO z+)tjj`gM}TY9BLaeYnQmR2!OA&Ef*BGLMZP(uc;+gpI+G(ZUJU7dV_3QY=_dY8G4D zH$y)a7D^XCD4==!XV%AbthGl@+}B;=@IPoSm6I63IK~BwWGjYs7 zI)pHo$69Q$ZNa^i&Y&|8d~iGHAoND#vtC@nI|zj1<)9T@%GX+-GLn4}*hOt%|Csice*hclmWPX!iHasCNr#bdOz>64BwHDx+wUxm`NNUv zv>tmN;cdrRH5l1o^&^2cCXhtIQ$W5WpZm3@NIA1k&D6>fi#Il5E`!5-IZoRp5P@0W z2KeGKY}qydJx$)|2z|(ZH=C_=C?K(ZEOPWYepb@M6^8UZCkJ_fgL4pf=j&i1xlEsw zU*R1u$PP2BpqD4Z-t`SnfFty(yj+%MV)5o-P->w@d`j|}t2Ha0YBW$Y<&E)5Vr4j- zPCQ}n198+StNnM*Ma91km+MOT)*eUX`d@-Qd|`AqiUtzx14RheMJ2SP-ly@{Bc!`! zqBYIP>!n*M5hRV=jn=u9;F(QEHPQcT(1TGqn`#Akzp=%NNzz9ntga(MDi}-=fLI%O z8g|L0lXxA1ISn9mIlI_RC~Ld!MbA6A`v))qyzCwZ-~tpPuAh;!SZ2U0j2;4W$;Z4J z0NUBno@B@elv^M zpBcS4PVAHHlvghrQCYTdUOMK!<&)G9$^V9|10nALmv_OzEy92gC^z>p31!{?0BLhh z0ELi8%(T2Y|GN*U!Q9+SsQ{74`w&hD_cc1+vwkl)n*Z+0C(JLH6E1Ep@Q$+&uWQQ+ zrbgFzR7eH)n;*>910Fs|AWt9bm+LZ?lb!zn_^w3%(~c){5m3-*C2oyRx1{_btVLF0 z5+`a7+<}iQ|N4C(>+g4w(x(4It-tK9MRjv2<%Ahp(GcphD-nRJnbh*mr)L{7+$Nc} z3#jo)@_^8J(x^RC=fpjI>h(7_pefBSReEQbqcmL9r zEA_sB+Z7)32r_YeBe=JIfjnAM4m%i3{{Tq8t<2V`zu7XXo?Gt7E!pKd12SyOZb&VB z*|zJ#o%(;Bq+rxEV#Pak*&15fOm8A0r_EhJIUaht^M(%a2Im|?J0-d z->mQAgR+CO*(wmN$T4*pq)xTn?MIL-{*T-L4b&6=+U%(YsgxM-Rh#08X^p~+yX}0_ zM$~vs3%@0c=3(q;KqnQ8C)mKz+N@1|#4nlOTsiP1yl9BGj(WY&B07a6Bzf4G1UlhN z)xT$p3cgBu@ifjsoxt0sdy$2K5E zA}hS4YS1@K;9B83@@Sbr2i^7;BTJ>BF+b;wbc~&@+Xi~ch9at-JH_fs;7-91R;pTF^E~SssL|dKH#MY~oMh2&pT*B1N z?c@W!_r}ioH9j5_x9_r&KX5mH6Qca$rlHc5kYgpt#Mv&%$Gd;vd7;OYZ^YRehEDvd z+UFhGo~&M3fy68}c61~c1Ky~uuTxa+sM?5iMz@}ptlvWmZhe}qdf?DJu?lc8>p;8S zmyrv`0jGXxXmzAH0#?RJAZMRZn^xlk=TGg>OFo}f_qFG~%RK5?JW%pvvh=yvNM*#S%Rvl(#$=K1O(cci6gEoM>NA(~{lU=~@+ON-!bKS||PVAZk5`ABOoN;Ix4 z3kn0MLSgx@Qc&7c!)O(3!R5f{$Z>ICeEjbU8>1bT3{Aczd>$5gI6dXT_zDRw}$5alx16sf58vB>$tKKlI>FfP#Za~yXY zCT?f|6{@f?Dj2k_KD-`gAo?_fr=g;s7fkpa2y4`mvvS?oUZFh~baAfH-2Z|XwJqk; z{P~dw!cNdts73$zx5CZYR>JfWihDxr4N?w}_^>uFjJ^)xbP;nBmgovvE-YSH{`i#n zBO9_ppp5^mtdSux{d~jMVVe^HFIVgZ0RFbTxkSJFsBUU3s3*@m{Um7QSzsR>x0SQL zvgGUM^CW88iMyP32_2BI%81^-}kz z(^4(t?+vrWE^dLCBj?@yj$gZeD)fK2K`O97Bw@KR8}K3d*oD# zxMSje*(y5b@sIX8VCY0RwTj$?ip6|&%Ze961i2TEvxPjX7+Q>XN=l8PqPJ05Gmf9k z%)Bph+Vh#OuK$zsWTui)j9hq zLl?itQNm6{NK-11h^tgC+eNj_W8yx26H)E$-i!R%u;EYm@vyRnP2Y+D<7RAzNVoLMt9l+WvUFu0;FWOfNU>ty7r$Ysdc< z0L$Js21BYE@pEJ!kz<70MF7fyfL4M;i6xhA9yVG~f3l0($1wvIBtcS?jLNlNQsu z5|hdTfr88=`f3k56HgC|F}SoVVF;xr_bRxDE}QO3iL_Y5t&x~?ikPab`D}tMuIjBR zOSP~Bq5AO=FUQi)`6R|Zm2*&YxteB?*DG>Edlk*%hQt-D`K*GQnS#_D=ZfP@kdBrM z?`KNd9Rx25$8*OEgMK@h>Tb4sHk#71u^%9RsghN=vLc?66Mb5sPw5nvPni+ozxfgeM{>EVr@d7|W z*uZN0Q-Et3F{4V!ionWzYe#i%d7#qw%?bM*T}7^O$G0m)$LyWcJ(LgP`=L6QXe*i} zvOgApKS78)$G(p8cZ@2o@O_y&95&SvXHB>TZ8M79@q&s>quDam5_JRYod?3&7t+1t zu6^L}vfRhU)NQ2cYf{FhwUu(@j(3uRyaHJgP53`RQ$+r$XG?|N(uY1>e$lG11y&qY zj^!v;Gco5cK8*Q=vk6{JU8D0nrNCHzpCJSV>5T_pe^mVN@m{Vb*k7!0YjJ*UR+Fp8 zdNyxROmSx+x?F&q{-pyN?29OCj{{}lK^V9ET$<`zdM5e$Nug4tW# z+&DHuF|@Y&S#cypq{|SxKBcH_;W-s>B*ga1(Vw5XMw(6E9IDnd*tfo4LT37mqK}Qc zD7Sg(LCjXpp&5*DK7u{5}&E9_M=}>B@#-?XkHuXnWuSfnS-OodA z-GSU~HAMnV`=Etn+NRg5mBAVPu|qErt;`9x2|*?@^f8aMK#XBsq1dr%WmK<|1;+mGDA0)0t zu{c+KCDF70&LM0;?HtzM>aXdeW^2qRLX+}&Wu<{Sv*wnadVe9lP{Wl`U1@?uB|eSf zcDY8@zJYkHxFCIIcUrA1<^fJbH(aAp(O1&L*|Y#o()y|x?xJB(HpOhk;GHcQF0UIh z+RAH<%yN|pu0&6rMsYPjrXeM^%+(ChTs3gaMxmYzI!^&HnIAb5E#q|45X%-u%~7|5 zaM?#+h_aq6vMQ%UYIoQ>yJ-&_W&nmhnMk0%a0ruRb1zDp+%a}}AO8XhU{#ZIvk6+oWr3N$lDFk~@}eqUJ-U|4A@SSW$oYD0UWni;## z(Q4%#-yvr%1j__r#PeH` zJ`#)Wx`!Ca)gV>_1Xv@l?gLcV!}&3XVjM2BO_vohtBUwyr(6-+*QZ)y$z7y_SXDs2 zFBOA4e7rLI)5^aU%gM+_qSpWrPXrD1Nf`)Lw?1a2#4kjN2Zxr z83l%rQ_Y*d^{983T)MEU;SCPlM3hV_gX?br48Tk zC8Cnft$eZO6|rUltCLT^W)w~|pK2Af2j}Y`&d{2gX(E5!xyUizO*Rz07*aGREMde- zvC3d&{@Nz9^5chR&6RQJw9D|w+oqb3N3B+=8cU%73_W2leI4~l-Imv)Suu(J_B-G3 zeVxci0!~g5FE6=No*~1Cs&?%wR%LD0#!AzwroB3VLsFNJYBtAi!S018}r$t2$K1rg%PQ&AguAzZ51);Z*WuRg;zl z{@{f79!(;~AU%Xyh{IfnsMlBuEkXCA=V$VF3!Bj5iuc(kdk?O0P^#_-{%8h%wF{6KzkuIxJFh!8-h_J`MI65dhm5SgHx+DqIOH;!{D!2E zfjX>Sw(|)O_GyCXzC?3YOk=u4^Sgf7HKel{LaKTJ>>mXLrxi8s_?D<$sK_R>1R>Sg z{{VE?h5e56wl?EK3TeXXaMWfBdRY*03K3EDs;GW*0Xv)X;%6@ll$W0dtu53nmTOBg zcC2@loWIj1Ov1B*)F^zA`aL2oR(!5iZ{SFs{*u`1KcQb+w=|BuARHR}B~`X>qmMK^ zPZM0kU*H_pSR}|6Or+2b@TX=PN2kgHRDm*VI@;2$?XaSD>3JnfnO}Ok&Rb?9gr}=) z?Zrp7X_Vt=h^J30q2o>w;UY0Y%11Q1Y5MrBon1=MpKLg63)`I`JGhx8Dd3&lUFW|q zdLy0W+2mZ(`s(q>J)Ocke0wM&NSu|~Ie2R|l;W`|Oh2FYiGi03DB%#N7%N3%bO{XG$VNh|daz1r8kU(F)j|L zCu%tS+y}Tf5?%ZcfQS8tbUuDd?d#Bu)XsIX9(l*e9S3ut<*O&9c5$W`#*rht+gm{e z4NpST^#zjs?iT1F#Q`dHR2@*xtG(O$T?#7d`)~(-ez70GK1SXsuqW;|S+AQ@<0wgI zy&Zx8-Hje*qvX@?JQFM6cvV|sbof;_v>!*?>OdJw2ijaTIa%24aWy6T=nsuvM)m&zqBs)2kTk5&mCV@wbTCz&Eb$4vDn4@)bK`xG zU|e=#u_p?4J#-elD=}b&Bdc~J7F8b|Ol9zd%MQV*Zp$u0yeXUmO%9-yNd);&j!Iic zr>#E1Sj-aHBJ!wJJ|sA#b8pFH&%E#Lw2|q5%lpV*5cR`2UV$uOGNCC?U3Bd@&B(pL z&>LcTFc1})pg!F+awbH?UMoyUo|E~Gb`(2*p2|p0#~ggYa;+e|`c4b9%aw(r=I7(@ zWp9#sS+P0s!Vl&&IhYGYm}*_>-0AW?8dz+a?hK7mFM3FQSsyL;EtG|ru%qL9Gmkr7 zNI1P<6PeRUuPD+gKN{|_>upk?P+>?7Z;(9oI->xmw4bG3VpFJ1AxP^IiHW&e*AM6) z(1vRSU+_0(xx`j&BOd)BJYhivTU28mjaw1#H`-mt&VS7v#zp*4m5mOeeHRlGbJqN@ zdLzYIpQP5&Vf9evpU$d`S#5B1p$db&+=!0Ok7aaZNbki1 zF39H0sEEsQ!g`30yjnoLNiA#s{bkD6t25Zgt~cz~>C=a|sk|qI<)Wt~XLz$a1wT4s z!Xk+B3%XLu3}3lbbb@Bd=Z9ST{+c5s`em(Z-6t0KLTe@Jgyl?lnoIz`1PGSs=D7s4 z3lic)?qBy$ew3)~zFfYXU#c;`D6~lWpIkiQqB|b$9#rY(pWL?uKGqS{&HNXkFMx!yBj~rwE}C0{hxZu-(r!^wPZcfk!v~9 zh!M6FT(1F;bLhAB65mc*@)jS0T6^)V-OfIbdot1QKYf!lc{W6;7QVfEn+VlFJUmN* zV^P}!X-C}%UCeWbUAUIEARB+sD}=PLw5;S|XBItz%kdZwv6!I^-6rzrxxGlbt^Zf^ zb2<3 znlt34-RS|S>~~hdPqu+>ko2sV7O-r+2(`Rx9vURipuG- z^dY$R@q;*{g7zwSypK};K;q`^>o5f6dLQMN`(nURNDKwq^6djsT8H`H&rJFMvB|U+ zKY5ffFz;M8`-2`<{{m{ZYR~pVUe>KagztA){;CE0hGZ+FA@?6dkU{SMF-m9zNCk|~ z02<*_snh~sP#PET9))UpB-s4_Zwc*1BVIM_>BYeLfeclfr?O$u?{Gd13D;}gAkTOp z(393}z>BsA)9$Na8?Kht{up~(=2EAw-O`)Wlh?U0;#c(|n`n9{}qg zfas8?*8Hdni8=HjuMyxt%Zn(De}JRw;H%-C>hnZ)!{c+F8N+2TL8s{@DBvA;9bVFb zd*#zHcg$iB%GoK;r`_ANn9c}YEnw$I2!vRg*&ol+B1o9><7Dj-~s4)1Iq^r4VlI?3{8%*`R1WPpI=iZ4zgSSgnli*5qN0agkYM|<@ zK+Vp`ERm`-7TX&^9Sm^Jt?-~whfm}tr8ex{Y$#OsWp6ZwFj)*4S8GQKN7)Ld$Cj+d z^TXrGuNcNvu(-N8TmN!ke@;(~e)X|3&9n90UGnW9!K%lT&Wez5cP_(V?UaE|k{&8L4U*Mb;=M{sbwWl_jw znlruXgf7r16excmmvE>r=T8T9S!Vy@*mH<6tI1j}74bSH0{B+P`&3MmS zO%jE{bNUa^hB#U)K>r6I{Ihe*kn6we@BU$sk;$I*R>`c82J_c5&>C%n&S}%WLZg9} zyfVNA2L~f@F3;)sJoCWoJmnyz9JGpE#A~2g3Xjxi^CXgsAF?jTXGp{k>nwJDY+Y=K zJNd}TI4pU+*t4Oz>?zA5 zq3{!xLgiE|ZMx6W(M0SqD8q~;*CKxy42yAaoD_E*eQ}Msw`I%aw8!(>{$BvE4p8yxRko>(#H`Wc zy-}rA>F>&>)aldWwCnMvLZ1Qn(&I>B#@bz<~iW<7oKYyK8Dj|h50SE|3&+NqIIn<;e(vnmsvmnG?M%%37+-H|d(i7R#I zq{ww^b@$wHPCV+=M^fyC{%-wtk%HGc#HTz-h(z>*1d!N+ow}|#5BPm8S zd#Ib}y~*H9I-Za+?X$x~$kx$*J^RV>O+{pQ4nD$cx8qkoCVAVN`51whAc!~j9dP-oGk&ixe5 zH`rrE_SS^0_u1nmB$UlrWa>&mlv1KS5R8nkBPFPej@X<==sHY4XKtS&HJVJPEAr$7 zfP85`y(&uaT+|d)tL?tpKZk7qCmBaS^=q+QT1*SDYA*d_+Xh%PZ|@dP^jiW_u4GQm zux3P1D!Yfpkd&ns?r(UDS>pPGiW?q4J{k&=ER`teIQ*;mqec_W!ZJ+LveaLFvT4*vkdN#D$+H$J4?{mvr4i1&98;?2JI{qG@!I?&ob za!Ou=>kO>w(k@78XMMNa1HJEId_}L^p8oS4E7^&e(@e_fEBw|rk2Rq!adVtEBdtv> z6?AD@RCOm;g0o^tAZk%nxC%EGIK~+klemw3jwONj53*{d=O^CM+=;4mI~OwfgOQcx zt8OFf{j5}4RL+ZKD&&|fJyuOi+=TT72mRH^0z#Q3DhVm^ikwZ-RDz`h6on`p9#s9) zhNFA$b{x0#U~JT~wiauFHu@jzbEtZPT2=nB5f$*f$|LgqL>eoy5|4!`d<2fb z6{l_Sh-2{ciH^@nB#}uy5=sRFy!8#P%8s~%0+m?!5-)HFIvP`eDI^rBB}p36pi<&) zg@quLo3x~=l#7$#GA=&`;_w?~kyGN;qB5nt5~>|aVR5J4W=N45oZFGqiEOysrqIHq zI+mGj}H=tvDY8MQKpQ5DwL884S)27~WBSrk!^}NQn%V>`c-`MOF!QhO<7h*rIVbDJ2*)WmD$rJjbN;5q=^fwjRmU`m&Q2 z3>ZjmdCn0N;zmR`>>?vcxLP7bOEBAYExv{l^KGp%OxU(R-pugA;*i&9_?}R`B+W`E zdR67!jb$lDh9vAoQYtewO4TgXqnVkRnU6oRq~{@jr;1}z<_Xa&6nD{BqS5QkA{9?E zJ-WIW_rH8r@@ z_~jJ3OE9Ual4VX2MCsF{NVRw~s^E{*IP`A$YAm~kKlB})50`qLsHkd<08#Hg(VBgT zU0BPCZs}4?XleM+g{TK)QP^-LYf6@lZ2tfaT2w9xzT}f4`2PTzSBUyI+8T{4Tp)1j zzGE#*X%6-4H^CuDjC0izDpJDN8P=MuRp=DTf}_cfrliJ@xK&xK#+Mz|rBl?VAE&(* zL?}+jpp!GCTO{U_dUludR_Z-vc)FsSG*7d~XQEk?W>iHk)fQ%w(Z$W__0?KG%bsQH z-z6$!X*C&=iAggdw>=|ubqa(S@`x<10v|1~ltf4_%*%kVk912O946IKsl6Va-k1?& z)hd#sMrut;bFwx@3NvECp_$j|lyD@I(ujzxA|)A|2#BI_fajTB$(yr{Akl&n%}kq9 zOwf`dHbjaUn{%V9FjQErOdx_<*3Cj2~hP1%-5zawxK88T>p_W_)`t9~S#YV8z~$ZJB14a8k>of?66# zre|SGjiUC<=t`o#f{~=lVUjqzZW>|~4XjGENO=ix-NYq&P^5&UCbL^lcYB$5jY@-2 zlgeObQDHrO7Kh%FjyKwI(N3YE4a!?+LINBMLXc9GC&Ln=fY1sG31oP#5XyA7sD)_M zl##AIkrOk4Va4iv_7AEuN@-_oqj}u!uG}>WjOJbi;$v$%cJr@$ zICT-1R;9NBtw@y(Dm%xKCD^_@t*BXHPIzdzDk&EDK)Jr@M;V<4^%FmtF~#zg2`_2+ zJYX$N z(Qk*5tNDp0rcm^#$~?|yZlxHA`z%Q(Epo-0N(lQ=>!MbC(VXwJhnqbsTGi2RoIF%Z zt)ntBh2P>!QS+sBmJkpAEi{Co1F}*vtrCpg93p>({{V}HmRLHfKh~7@@lL<}G0bM< zlA0JPBwkc>SP}w5jUj6&P3+o=t~H<~VFgWv2GpdPr|!Ey`$dDn`}6)A&&Y;5sBhvE zxzG*IH&7(l@&))DCw(AX;*#twz6K<_3vYpsHhsP-bqF0StEL>cqlr>!P#^HsVz_f^ zuwqMNR_a0|MRnF4C@E5!acC;Ch&boJyJOGm5Y)2$yH|XYeWIXlgJuVEn(A#~DNmGp_K`y+}DN=4l*>SZCo%SQs7tCE<)Xa@cl4fzo zR=I8(B?3g~Zlvr)#@~ZsV-fBrN%}d{i8OfAq-FOWk{CmpoUP^+AgkdwQwFWzC|sMR zO%$Hs;?Yj*QkgSJPhP!a>71H&U}$D$%_joBK4WFF?o-Q99SuXGQ=_353Gm|}ELkce z_=Fb;1P#i`+nigG5m-ZfB$8dqsX|9);gq3MU?dGOSsV6GkpK^urac&iwrf=SXTu$W z;9?|Xi-ubtA?kZ@a1|71ROe-yqGPaav?OeViyIAx7_OA2u}ahL?%X1k9b=u5hiLAY zN=m{=8b*_AdFsEZ;B0oqL|S`1NKjwGd%MVY(A#XdkWi8?b|7{b$TLYA3MI@4`^;iq z4B;5v9{&I$>>B}+DXx%F7QJ4ns_lE#0+pB|wf5B0L7e1Okt5!iY5xG>uXhj$ z_(t1+2(`}#10ph==6oPEsuAJ4tt`t&Xr@+tFvRgA9%c0*31V!NGLYI5vI0}E1LN)V z8}qg`3Qy(_YP<*B`5b1aS5r~M$~NxvF{ph~q*8MZUaZk8)W{W@Jx!PCwJKdsBNJ)! z>GH{qDje978*zy-q%h*zd6$s1I@*d>2*w5KYzVdg0QU}wRVrZx6icJtWAPW(*sWwA zXwI)>yrN#C<&3Y@9PvS+QjsQ1#_9B$R%WQdr^8H&gR4^Bb7j*6cRc;)>K#o2c@3{{T4O5%8A&^BjHq zqR`Q}s}Mq6X=*}>NKgtkAn%V8 zV_lvj2TPGD-L+}J$hUa!AE*RwkWVCBpEHdJV1-AK@+ayoZ_fY|fCm@=U;&8!zx%!* z7?23!!?zzx8~`~c{f^$71NqWZA3^^B%73Z-0f6x(6C&H(Vx?>e?l&KCfF@@Pjz`#Hs{zOB z>DvJjNjv`l`M-%E{ulr!a0%q|^gD5Y2<2oR0RG-zGxxwt^?%;*5TG~oCg1*L{{W!C zNhWgEMIZgRc$g>vQ(@owkMDpa%*jzZybs7!K-*!t-{=jn5Gh2<4-z574?iLJ5swpL zY%bwg%4M|pk&5gteF|;qdtdL5MLUaGDm$VoSM(>I;@10MBAZKWJ7i*mwe7Wtx24i= J_qGGU|Jk?Iys-cP literal 0 HcmV?d00001 diff --git a/examples/frontend_language/quick_start/images/dog.jpeg b/examples/frontend_language/quick_start/images/dog.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..dc4b40e2107bf4fa949352ad31d6cab5a2ea18b9 GIT binary patch literal 416580 zcmbrmc|4Tu|2BSM;!>6wd)daAER~S5Gc(pvmS|HdTenoUkey-d$rcq^My1qU!hKr{ zV-SicgpjdMNQ1#x#`s-oK&-#^dg#hf$OIp=w;@AvUOj^naDussUEPnubn z0UR6vzybXL+v6Mq7RJVor|oRaEG$j`tY`zE1*Qf7o?iaGcIHM%`-_*5JfnZUv9r$E z)z4?=_5ZLyCzUi-_>73#16{i1!L$)SxEf!ZS1|Fo5M+Ae?E z`aA7Fe;3!3gXhiyYbtE8aq@`iq{10KL% z01~hOyZ~3=DAYOzr~&GL=JpU^44uyZyJcsP|GN>OZ*MmM2$1s=*H11E1i*>l;6iY0 zHv_WJ@#fkCt^Qv-3vh98!hk*8JRtAheNYEWIMjg?#>EZW!v(#aaaHTOsG$+)?QZ(3HoX zlu^EdB4hFzMt0DIXyQM|ja_{2cZ->vdD?i;Rb1i3*=yr|{t}9&F&X9Cg8&~F2eivv z2mk{lL~=SHaUdY$<8|%#wd@Q9H)d<4{CKU1r36nVW5c{d)|LnsWDYG$WKEIFvsRI9 zO?Bk8dUe4y0=tXdW>m41TJ($HVbPFRuSgf!2GsS|w}I87d0xF>Prtc#frP8W#iB&; zYGh6rb`G)WJiK>k+;HJN=Z$ghUsSwL*6^i%TUk)sAa$)CK01H<|)G=9Wn) zsObGf(rFblzGfVqzp>dI$gMy{sqB-f-3DGcQmOp0UhlR6A@bEV`;|2k4R-j>A|~i% zO8q41Ex5mmj%JN@TTh4EFo}bsLI*GOq2Irkt>2saw znjnJi_(Qc0u4w$wEOyx-&~;Jl^{rEV59&gTVZ!ej>x3}H*R+K#r+C=RK(v9SWY~4W zd&+OWh`As-56Lj1bRNv1Nb4JCddqABG@f6qMdN4#yuA0Vc|rbGjGSxL(a;sj>^ci$ z9|#Q-|I@}Mo9*)(HV^G(AZNfDsaE-{bX3|iUjQ%}juuBPmS>mm7s^+BecW8rJyZDC zqNw2;35{b_Cb5LGHup2l8|K@ySDSNrhsGloljAezQ33kQG!q6pWMJ%zaf4n^ zjh~u)$Ag>senc=JW71%6O=!w#gHoJe(k0in903OtFDH7pPFlrvNxECr$&7xpZ_WF; z&XUf!Xi2eKk6zOzj^5UpbvJFPH+x*Cj9S%9{#^fW`-`tMqEu%3CE;5+;r@ePMo|rZ zr7J6|OXTKwpm&4ZDZ&}Pntz)p24<$YUGs~U485~CMFfNJXnU^JkYIlb%(f~99`MaG zc3iM;P@)6LeuLfIMBk zng`stJaIrag*~|qcpI9)OLYY!DKI6IU+LA7{2dLwAfc_<1jcx!UEqJoF6mVi? zn({FC!a(gCP7bugCr~$14rj!xT=xt*mQwaIg{k<|JevF9WE%{gg^bVQRgQXji1+?A z{xf2wVk^{#Ad1MV#xYm+j__NU(*S=TyNrzCRWO}2!78+pw!Vs|WmFHEq?y7^qFwml zR`hZK;!(Gw@-PJMkV3;!jxg}clG8<%)(%5lvdqk~5Z<^KYTZ9+0jyA93J(Z_0eLAu z!jGyFK|GfeHzp?j++&=Vg3noQyB?Ge}z75#bFHBO(s*H|= zE6%XhOmp?=%$s^JVfLeiF}mRI1P>uzYDojBKpWLu^Xl=WUi8(&{SIkYQc6z#{Pjgg zFuN+RDAy?w`4`N8Z7au*0@E5^TDKvo)|KM;?A+ud3P~eA#v+d@m}qwC2Q&j6shciq zc-?cs_-w}e$P=3E7heVvYgvY;xxs64JktT{KUH1n63Dn_EJOWM-w3#jUKXndQ3;w$ z++imEv`>LkOHDz-Rn1$YQ6h9b{x)0gzGFC8QgYWZpWaQ?59XDXKQ}2NXT8u%)25RR zJP$VZYuqlcu*?0v+2#e^s#+yEtZe|c^#mQ$j2C~GX;YZy+4UuuBd+*T%t}ei2&V5~ zx^`5<)74SJGI_%>i{NiGnpFCOcZPxKl-v!>{G--XVP;(MF!lD@gM4ZPWh*U-C!ez~!-9=gJhW4&Jhzo$H9jm zgcp!zNsdcCA@u~oK9Iw5geNy+=^pZoZI=Sq_=0iU$!E9x(Fpx}*)n18#xD>I!<^#& z1TUQE69dER>l7HjMT(o?%eB^#RSN8w0bw(hK?IGmrwj|X0m2Xrer&HBJ3#RkE1n*| z8=2$AN{tW(5$JF=BRF$Ced%t>zuUlYN!&INu7>?Vn7u%qScr*c1w9}y%IaBzW*F}q z7dEw8Qn}=9lnii~!(DH>9!mcrd{{_m8~d6vN{V%SO;+-fZP3C9mTMluB)3xQ2X4vx z(cW=yUJj<0E3c5Kuj;l~wWYbc$*Sn}R^W*e!cp4`GFS6k*sfD?6kD3ZT541*Y=ky` zJ$tUAqz9&?P+@p*MQ`xI;NeT}Iz|FFCA@cow8eU2>)PhpnblYe1=z;r`R?XWn(= zx^cGe-FtrjnAD3?o~){}x%zEDeCI^SNxq&rwNxj-BqvqMxT$b;-R{UvM_PJ&Xi7VG zR7m($_dfC&)w`ND) z_p^rF<6xQ)H&u_J=T8+NN~A*XJn1onZb7(i7kD|$Zk4(7saCcCtd!|Kjv zc<2z`*X-4l%w|_3YI9?hxvmw?(v0OD50|WG2dbGA*}Q7HpD*(QeKIH{D?nNxPPDdNX6;dgU1cX0GD4A<4NV3>lYB&M0DM^PnS@|y%T~yj3C^9j%K&5p-NhZ=D{B3wx zM>N_(mQzOP#Kzg5Iyd(>rCXmAzIEAuz(y}R;zHeYgCk5yWY}Z=d@=5APCxcA3<$Bf zAMRt^aDCchF?M7#-*29P$rQ}qC&;VcP&P*^zdH2$r~s5a<(+tn9KEhjheL_~as2wlaNSPk2-W5?B01;yXXVu(BgYFzFU~eOn~l6z0OO=GXDgoNu8$@n zl+Of3jyV~;Jq*9Bp&lssOzmyTJL^8loZmJ!2KC&*qX}b1s^KMFqnfheYS1hpg48a* z8E2&ab(h^UzO#OR+f99J_Ys+^y?K%LwgdordhRYZRQ2AwoZrMU-y{z|ZIK)-HF zXOt)nObx>$r1!408a4c&|yj7Rr)wu$Z3CO zLMcLK4SAOz3qS!&6vn0g((36a7@RMvEF0Q_n8N+MKw>rE&0LxF!R-yk@;sOa=wb$Y ziVb{lARM^|_BYQ~f#Z#7O~9cD4-gL&TAWNEit4}RZqEA2A@k1j7$|oJHdCP>*OR7EXaq?tAdi{cDr$v2U)rDjbwG7*GkymFn&QH+Y)i*7aEkE&vHyn@jiL#wT^}D1Py0>v$8v z{T7m`Wiv$35^m}Q?S9vrVCV*<@Y^}Q_GOH@@5jkR|?m2Sa`%$K6_sA9g1stC;?bC3}S9Eews1s2Fn}HNkS)hOo!(4p_ z3W-{rOE%feO7P}=uv*kNIDZ4XOm%hj(XnhR)0nhrf~o)Q?rvOhY5Cj6TEY#fr{-ns z7A?aZr?|OEmdWuKE(<`pEPgFylnB52#mNGR71p!QTB;rz#qo7}3%e(oY*>0i=@Fzt z$B5sm5CPoa)j(-RBV|ZTAxqAH=3Ca!$aE&V%ui5nYcPcZ-51YU+M!LGPo*VWmGzQ1 z`Zq&ev}<^Y(B_G5t#6F|Z6j{WZx{3~;lN|XCy_3b5d+sbhQlJhbaJ(K`P-Haj^xo?-e_qKmM|;G9+Jq@67YO9%hmKBBFl?Gu94FmU%;uUGIAdRZf)@UUHBuA;dO?svGe0+G;7{4nTl%DlG$W!g4y{s1b(3_^6SH7V68M~%O2cJ%~YWW(BQJ2&OBZ`)sb|AF}o#sXFpT-Z( z^|q?M0HHVHz3JZhQ|0T%aZQ6uFJ+G@7yuLjG3jr2)Yj7GzknAL$(ZCQO|q^2hqh@o zqxS}P77G*}3>!WzyeZc->0RzklrrS5ywx}wdv`5#i!|{HDv1)%-c@J3M;&uNOiICV z@~%?vLc-(=2Yqn+h#*S1W0QbsU8BH+VF+w=C+khH?-0x3W#es*5)$UhJ$TtZ{YaUx zm@bUAUSfh~6Ctr85Go>i`>%_RJRpjK>3$0p8;n#hU!O$t{m*^@lH;*Ru!zkZ5#y5`7V`9Lw7xSC9*qr zuD$4rb@>6~?!3g+&CLxd(7Nb7eU+WI{Gyo|%ve&J--W}{yKq=w7Y@hopTtB>hj4U^ zoeeVm`&Vm(Phsg&Q}(1>(YfT>)zXU9T9a0Xt_#ogFDKB+aX%JQ^0Yf{9y6Y+f6*9u zsZi$?car#Egpk4!m9pWH>%~tmp?%jfm<;zgP)Jpwfl`(zi{NCalk*;e!5LzR{@dR1 zq(|~lCDRa+fBjL5K2&N5;In4*5=g0;o-_kBYHmD=+VI5dh;A-NW1}f}L!* z@Y+K+JoMK>HS)G~K!|bVx30e7j`el#J~g9t#@HT!4{pjS=pa#wX$Bb-INRj{gD`(# zP{!5kHn-D!d)CS}AP+PCQ!|q{FU!Lmw}0ebw;$LHAy?l5w~1^R!t`a-%a7c! zpp)S_AikIr_VpI|svtu?@^$*eY!> zft#@dr#cOS9ovO=>lQf{6}obudSe^-m;aDFN#(8^k3med{$yC3cx1#oR@+)&GbSq! zx=T0DYy%$WpP^N=zo8|LSxR!rc1ko<395DOiQKqSk5D7sPLy4|ts=`1dX9;d)2n}R zVDE3i&Y=+g5_;Q^fZP%A-o%&;#Q%VU^4p1~~!4RjqUQ z90p;F4Ao)m4mHvJ89Z(LscFKjvDUbIQkUKy?xTY73}G^98@NfBk9RjGU5ywex~8$$ ztPgrG$hzmS6iE)EGlmNWRoWgHhTS&sAcA)rc&B8Z8{#;IsRuD9RM#(zk~W{b;uSye zA`t02%(Yi?FmTIOBIIrrU*qt|#+C|SQ?^_VGm|;c|D7rOomoens88Fpw0cJw>|dXs z^l8gs2oZKRfIJ^_(7^2nSeq`sY`B^7LY2)X%kb zszikh?N4a<%#pEKfr(R>Z3oUCM_Qu20;k%fnWT%H`U|Etd!EZ}1LF{3k-L1Ln!i>K zpwk%SzQr;cQ86x4z?}X?@Bx$@qnVp`Rw*DWb*M`b9%EDW8{8a%?MbO9NeWLImN*bQ zF0k3EBqlj@D{5?zAt4uYo)~}rYi!A>vx08l0^-u=Lg(8jR>5itzkKm6PyI0BA-VLP zzCfA*Ze&Z%*d<|*&6-AChrt`uteLeNIf><01FF*1iJ(vhawtm(n;%epI_ppw!WV5h zN)(Z(OqcXc;~w-zv_(rsqd=7%u<+P&LMhGeq=ksjMAqxGm&FKf1zh|lF^cC3IMiGV zWXgnKc^%m1Lv}&g^jmX3Y8^7v_G4dG0m{dZk5&FYY2OpNkH;x-ycEZQx=^Jv;9V5j zurjNXunk2MwtYhUMOjbMr8^@MXV1I~7-3R#qhW})fzg%RdQ9PoA+woq=^t|^HPqv3 z;8{~zbLHW@srEBOHwf_I_npvTekT{j81mTIknNk((xtp`!fg($d2e~N{aB)0WCTey zg;?JA#i;eoozzqZK0^pg8lTO(`V0n_f>6aFF_`eAOMfEFl6RmD;VSerzBRl?oyciI z5EAU!Bq+lacCZT=R-Yjg@h`X52zoR|zutOyI;e`Rtp0s!+!1y60YyRi#g??Kk3jJ z?oH%cdj4+y^Tx;_6~@4NhnIXu3wVGeDdW}8-%+jr;(Y^4^O#}-^qN&+EwSlun?>)Z zQ63n2MpXs-LC__sG*D_3ZeE+>T2k~v?p`M9L7VqK&t?pCR;GGgOcc-jU~Urd|2xsj zm$>1AFWOgAN>jr)8A6O{Z1Tri>sG(^^x@1Yn-erAb&N)1V^bk= z`lDekRD>CK^U0joOWp{*GZ`Xd0$-5t&(@lwc|tQ2pXhg0F<1nsSSVL4>e#`Q@wgF{ zeu&gGX`9AImz#V}rc53&*F(hE0{b$mQYqP^aWPRe>Ai%|Nm=kWi|t6|+2Qz|M@mRdEAj6CACBIRVs{pRrE`XxZkZp!rv?SG?wKZRecA@v z*@eroT3c8@1~tO?1*|A>_Or@FJ*Lelsx7Bwv1WBHQG16t99Kfw>Cn4Z zVV-SUVecTeJ`y!jCQF5m$WR+mM##E5;`XN=)(T}WhbEnYN8%(S&GkrmB{NRJT__o$ zUZnJFNMD|^`M2c=@V*p>)V-y$wM~bY6mWt>ByD&7uMljLa zfKMEsP`;8n?b&NZC!$CB27iBx1q5limZR|i5A}RtEM=bns5zxE*9E~%QI$3Yjvl{Y z^^TdJDtbx3yP~zy=N1yec4Vc+jC;{k&@kV5>^<>H+YBwAp?y-0h2aH~*4V&TMH(Kub zTpUuj@3`W3hJnxR*M!JTqhR`7T2-Ya59!F4EaY7Z>^z97udOW&tDwQq5hGta{2R&r zb<27b5r8`w3cxL#7W@TOD?}wc;$_~rQ%h^nX_d0-X&>!@u&|>p-W^sDH*dBUN(eB4-?5dhnYr)&V%7qiRzl<8isl;`trITN~K-U`1`Qu zzw4zTKb}fi&M4H+{ylrJnrN07cE`H`i{uQQf_AHHozsQy!Y0U`%Q(vFCe6j zmUT=VrgzE8AL{kG9{oK4pKjb zSh}1*2Yn$@4-BmNC8j5RGtRQpdmOWJRJ7tmd_!4fH(%5(zbVbKnW?xBq>HNCfUWF` zBArh~0YFlSd^u6-5l&oOxu(Aaz%ag2RanO*SAw>r>_aBRfup(`@)QS%5kLkJym2z} z{sWF)A|%~ZBjZE_&a)$+T>ec#Gym#8;VML^zLo=dGM8&)E#cpIz8I!Jd2v5(Msp5R zN__#|dtyw&Bk(^tc#7KY!3XwR$hbmRx)u!1Uc6$y11Rsw3q&Z=g(F-7cfhjER_QoY zt@;LU11Fzl5^NqF9!{5#`V}D?R51t9^V$%*VEjDjMzPma(;Rsa7b14dGxpMcw!K(GY%5!y+MzTb}0XWRh{qyE==q=f;*}G1nchCxZtH z%!gbLkAE+;!MsfC*I66s&JuE&G*9xC6`J#FFwOiy>^qrfwB>0dQ~M$J za(_c?immG$gd{WcEb`j&SP5)Pvo&N~4(DDsr;)ExOrE#Yj+R$9&ba@GSVY&na1<|V z78(2({_D!BBK^t5Z&C@=9`;XR&OpYHU&8pW>_#VKoSkuA!my`9vefx<8!`iebAMEh zkzI-Q5Bskt5O3o0oWDd3d5I@Ctk@eb5ufb9;u%ExT^d>F*5`3vN)?~)@vF{Tpfe)~ zkA=HPs>M$pi}p-oa4a zN}HU^&eyZA{-dzE0dc+Lrsi;&!xJ=ja%+E>j?4mmT|$n}mqd6UwlE-P+9qv{IBy{M zNmiA4AJW8aHe!_Z8n{B)uQy1kDy)U(=VT zp_azSa{=o{kxqo!5t9K=X7{)}?68`ZyH&uGqGh$UT9&H%q~cr+jfm!`EdA|CvPm8h z|1}y9k&#lWK!{0FqGFapPk$#Gf5$GE)jITflo$y_#Zd&!XQ2(TgyW7E{Xr+OdoA=> z$*f9@o>MvEq-5ywP>HrYN&U;`G$z&h9K<^DkH*K{izv}fRSQU$aTyP&D2-KJ6dOj6 zwb-S&(PeFazJRz_s9{Q$de6|9rv7hvL`M&KX0Ne7=G6K|_1>6d*FwpwkeV-Be_$76 z95PPRW!?MZLeTrhSc@p7U+5a!ps1rQL~kV4#XIDm&!Ul$J|nwn$6D_C)I{f36E3*s zAaiA=qTb-Pv+~%itF*3nv0N1IsftsNdZSM5JD1yAtD~H;RkWINxlwZ)k;9Q4e9U=h7C$u-Nfn{#8BCE|`vT=$DApRn31X0N0BpomP~gV<(sf zXueN&Ffx=43W5#2pf<0wIzb1G@QBbAa*Gt)BjT9m8hs&(DD?Z+@Q28`@Z-*}XRR<7 z>sAMN2Lx$v!Sn00&E3@A8nS(mC3{n1h1?zXVgXCTEi8tlz8zaO!X+TM-VFZ^i;D8P zVuMou*Dbr&uuE{??5X=*|L)~Ja2}E??n=StduTnI8V98aP+q6xEH&M+cOBuOa^ zC-gCFRk_e!NTi%9dqy1_(_6_ZUP=f`OO8{ZDvGw5-K@M?UbRWj` zDFNckG?+K=+k|v)pX7=nFKymNLgBN9!}l@X@GBoZ?LD!B2eM8NOh-uksnyLJitFX+ zqJ^9wELB_9p0QFXu*Y*zHi*-@b!^2jbXXfLU_6GoNZF-Q!7Y7`^`=t!5ow5Y+Di%w z?mlFg-a}uVifeMDi@@xiKOCCtgrsKe!elAmrA6DeBu_-oqyLPkEn3waKE}ThI@b0d z=I)indjXpjnLt;_NU)^84OEqlY>wm8uir<0dp)sLx3L~PhGNyc%xN|k zj=ri8^4`F{?k3%6{3*QQ2iA&J@XOqmAzpS5zIHQB{4Y+Y91G5&&mUSz`NM6U`DYSL^{IY8azJUwOF1)el_{Ld!s}N z+n0Drf2LN_FWw~@5tPYjgKQW#s4)02+(;xE<7jxb3<3h;>x`b)*t?)OQHQZrKTkv{ zqsXd;SGny3ZR}Kgf@O6m{85=LL)oPnoK)!@c=Ou_CrB6VC9D>$HLB5eJQULk+W>~$ z3*p9n5E(PNKPS?y)i?jPesV_Nnu@Qam0sn#VQ-S1+eYg%C14bOS=w&wr~gqQ@Xhgp zksrFkU0HQSkKT$Y%nF`|vG%PAg8T6X=GFwqqa=YCeiy-H$B8BP{csn-{x<2@#7a=s zZ``oqjO(j~xOoqF0huHV-VjsoD*TAv*ani;iN^S+Ikrx}Cs(QhSj~J%AEtv@;TG%b z7Lmr_>vL2`nL<{uj#HDXVQU)>(*k;+ZB958CU4 z`Yo?_J*r4JzjWB@=Qbdl68YsVNu4z=vl|)ko-GX~iE|ocs~YL8(pP6r*9={a6d<_E z;S(5SDz%h$1TsKH{=8t51$jSsaPz1Lp0SQQ6xa!DNAhn%gwaTjtlc%@o*mI=NAojp z@*<2hg?frnVQNKAx_*jNB?^4sV&rdef?AUE#}vH3_`NH72;S1LC-Pr>BU}CTePWGT zw}FS3hgXCcLys}w6i0fbw>&kOV~P-NLkM}6&eU~CTKY|^P3|%-rt9#b91oqcxnjQK zBMdYh)kCzYMCydh zPx^XR4q_AXM?Kn7j$tgzwzfHceG~KYw)z=S3^P2DFv#>=`Rc3~fs1zzSThOZgAh*b%JMuI%b7=v`&y z4`(qp;IAw33dMnh`0R5}<=tuM0fE4#v@^aTr`Lv>HcXFL(pKSt6DnWt9}=K@CRQKi zfZidj%7p>rv~)3>^uCdXSgVGc^+CJSLW38jz;zaARe4jwrN|@`%6l z36*pwk(lIhM3w7}my6&cQ8ZH0u!|uk%dwZ!N!p0@1WOn8pYNLFMy$)mkI1o_$uwz- z<5}kY;T(IDmqji*mXQG~+y)qYqV zj(NWYh&@rt$ykWqs;`!tG46dF7PbZNlXf^H8K1%0lWx4Sh01h(Be(@%i}vo|pna;OQ@! zNUu9}vC_%LrWSLbbcd)x44&z~g3dt+e4I3`NcaTJX=p|CCpSxy_I2KVRke7 zQ1VSi?kr|}%vGkWpgWXD2et7*h^38CLYVH~`fTI*>4gFzdxACj%o~h_erjEv7PIO3 z+>e#`0*&`cm+!=W39BcxpcCaNfG@Wu4^4Bxx5~iXLgE9acPB0t8gA3*$uO7iTuq8|`0(h%LM589>lM*oPA zH`LWxtcfY@o7JJ~fz(q}dsDPH^1`F!u-el`i3u3NAyTE__x16}gXq#dtFPBgySC!> zufRJTK5A$@%)GB5xVo`uf=%59l9iQd)r#2)J$J7k|MXRV>|09vHlWF-(Vj=#9=R9i zb=2nLNP5nL@Aci|ON&{~p(k>3N%5(hpD$Yb99vSt>0~<=^Xl8QHvD}8x97Ic*vQ^- zwtD!iwa8cmdi=tGhwhf5;8exRH%m>=;;FBZ5A7^Y-g#Gu!XGFJaNFN!mC-KDxICoN z{udm76(btmR1is6UbMKBrP*w73!fwR>-w!wXI(eJf3_aYo4ZdJ2B;!z_~T=rh}k0h*;gDN_5HnY@|;8At79Go zAX-SY1Ec!+Snjwj>49r$aA6U;gd^!uqRl?8i$Q0f8&&oiU{Pt;w=(A)|GEuQ0y5C8 z0nS^QQ+Y-049lZ#HFsUGA0~=AGabHPxBJ`R0hgHN<=s^0Y_OB|L}e;hHrK|q1tum| zW5(oWxmPd;{O)2}+r|htXmR4J`9aR_%sZr+sKB++r{CHH5nDEgi~y0hj>20uCrxk2 z+N5u-M@e9=)9wnohF38hXGEW1u2K8{BRVfAempV9uk_9xFCTg3?We!z(( z*<6`$txjCg;vDea1%^nA0j6wUPTkEt2huiW;)mR)x^AqG?oy30V~V(8v%G~EP_qi> z61cX))tjjKAa(R;(Q_~iMP9Lhc8#N;l z={0LxPrY}Q%pF=!3_EGv)2-9IU$L-ZE@t#XhPQtH5%x{Tr4kB;X(}>IR7|fNAGJxZ z^`-JBseO<;1lR+AsUbbfk||p1!d@;uIKaVK-p(@VDlRSf7ivL8L&Kc{v(R$Rbo;Aou{FlL=XTd8YE(QT>1H=P3Nh0K}5;$xN!`G2d{_ot^plFC?+LNE0 zg~=uESvB&0nzK9uD@DUuRdZ83tF2pyH`RvI!}BLrNp+=V8qwyW&zYMIz4_5acm!a= zNU~Z_2|^yfMcRAA_rqpqX{T*xgv$26pP{{b+pW01MJAv)>Oqh|?wM;R@0_n-^VM~VlQlGMx|XP^-?2`;9MQy&84Q#-p*#fge*LN zT7}liG)jMIOFg5daQ%isNOGl`DUJKXST!P0AHYohPx-8F?rqP%$`6LY}OK}&@ENdg<5D+3CUY9YE@tcB> z|A8r!`>HiuPDgZv5zd;Vqc=Iwpj?jg4hN04-Ud(vcM6C-GC>3PjC))&$(U;=C|9)Q zo2UT-_ht{2zM!2Mt4wR;TM~~#Vhtb%M(v1)2`^EiB}dO!dUbk(8uX-7Q!)*oswbiT z#kq|cPiSj=vBlxL!gYI#@Az_gN(MQX-3M`1wv|m@WwSPmv~nk+kU>#)F5LLhMk-Y< z?%8hi7Iwjek(VG6R4qE!UvI-PeX5(a?)(*8Ah*xCzpNHtUYQY|s~mw&%h0E81@lHj zX~^Hm6B>@PirdV0bNYMcTyebWhyhhjNc_$jZ=nXg_3GnlYtfj)1fj6PEHSYr=P{(X z%mGLUi&vOtE-{vA;+v6pWUR25f)hVotLkWx3!bLc`0n+Y&Z6fLr;SU~aOoT7kaC_P9aoCT)|T!g=lnMi%4 z^@kRKyMQZ@u}S4M>&BoGr^+t_9bc-zZt!(|vf;Yo@VsRk(7RKKCdwcI6zx?KgOn-Y z3vt{iI~2#D77ZiW*_02Smc3Fk+3)NpXqoTl^^ZBJ=*ft#ef;yz606~qa9zpDUL{0* zi#Hn_y0zg;5nOy<&*pymqvQC~n9uCX?7046n_eR4M%dhv;fdeXNW3&JQ?^FtA%*Ma zEzgDpUtPO}1$5LB`4b29We%0;x?a_>PN^qFjr?7+YUn$_QqJs5`ERvHGew5sgF(Ms z@=@Lk2W=vbHIf#h!`EW{1?ixh*de1xYbv1%a_gRbq1vg++JC@ePD|{;A5d)xBj8D_9k`I@T*|P zGV!1~8J}{MAOW%d9x@RNqdZ81$Xp%e@t<-ZZNzbcpaQGLs%};Lu|j-IR9^&}u6^=9 z4k#?ZCpRiws`y2Cxv{ayf{XB^Dd^0Oi1}X<3(OR?8rMwoy51&TV(4QauuDTe7B{(^ zh8HQV`;iVT|znLf|0Et(!c$t6kq33XM!@c2{SJ86~QY;z8xgb3ef7)B2+ zI`Ou|4|-c%N&EFT*y2X?Qc=j_wNN}=3j&J?NJ;<-|JG>&E7oa1x++6l30;Fa_MEqveY=BfOi z>dx%?h4a__-^j-;(Kv;kACO88pE&1*Ow0V{HeG!9g8!_N(q)ZQ^oAJ=OUo{-Sqpn* zEZ=THI2Uy(?ywl|k#kADK9uCra#{9+H{tkY#G2?Y%r)9y>!e%o&^v?2(W#wo>tW9x zINi;C{cThz?H<11?Qvz()HR0>BUN_=0zZ`dUs-fw~zKSde@^=-iOX<^|xPrF+;b8bE@ zx2qh8cp@@i-XF1kX7;4;_@&bvL$rYR(#fCx`93h*Dh`?9jtG8+EIG%U7;z7Ll|R1> zRPS8!Hvg`9hD1|5^ifU6jbp$se9KcUxjn4}TzvAC`@>>{Svz z`|1$P8P|KFp&-S2UUtE%aoFtY=l2K4MHcF>m5%o`=Hll~y-c3Z+S>WD#yEnL6`nm; zsmLx;?=9KL&QSqnYJXwI@vn~gTzS(JaYukAP1F}|c<)j=x&PX^YM)n)H&+7IZ_hm2 z)H~R5RbFS=Hg6m7v*AcjqjhfuSlS>Cofn2bIMrd4X`h|-v3jWiHXdKjl5O2eUCsC& zYjeAWCGVJF_u!}Lttip#qt{$!+W!r4S-o(%3}D^}t;MR9#^Pljw}oE$KGM^+@?FL= zuC+sPph!UXiu;9_w<^f^Hx){H9PaCNLsS!XTKEB%PRYu;jOJ@WSkhqbH45g?XA2Ae zsdZwb&wk=sli+8I`zJ5oZ?3F}4&5)VIdI27C+zk=<~rXhhU{NzEm5JWIqQ$)rWK_Z zW;1e_=TGss`>3P+|ERa_Ha+=LV57cOvQ;nd0XLe_AN3`MY*?=$W1#2H)MW*i{GCEh z6YQNW0#DV@&h)IPrJ0SZjqEa8f4JK7+rVV%lP2f4eR1y6HtmohL9H-#UQ<@^6@`aN zqY>Vyr6r|Hku6yTU2f-`;A?OTq?0Ogn7ho09m`o3N z8Y*jbWOTzHbJQLgvlr!>l|qn2+-~pZA$6l4soQ;2z8Qyn?|TG%`IV{Hlz#gK9-`}? z%<5k4OVHw+22c5~N05%D^o&h!1Mls~lc9bV;Xj*R!nF5@+~=oog(p zdPwg}nfPJ1e}j6OALxZi7Y^FY>u-2YdlI?c^U%nLb&62#lXfr@87IqNHnmEb!o}#O zwNag4j18K6uA=NUu2bbZoyb*^Hdl_DwrWVwqr%jUosR4BuezFRxw&%5vr;kdqi;|U zjpjYCFs+{yNk`zuHU_5^0Ald|8Jb^Q$qxgnIt+{wPYGK$V@&fdH-0sK)GSNR!S&;f zt2N38IwbvwnpDx=`vT2Dh}HEwiGNug`}i!{XO*^bv#~sWe^w39K$}@?Y_Xt}$*6r` zXUwli+}!HGu|Tb+L)Q4UlsxXJUj zPZ2wK*6@KFDLxY(-qILWOgs>#qs~)nk2!?J_s}p^ErDy1-VkK-{^2b>CWnXc^uC)p zcT%&n5qjABAYw}qvbd^LQedcUz+!dd9vtg~iVfOKt31ZPTRAPS-?uoP`isF?AXT4lGp|(0dItqwVm+^Sn~>m`_L!00j)l+ zw{&JmkP(4UcFVam-BBG9b|QmQA{ut?VB}qL*?NoAu7U|4)gm)xU0fqv#?=P2Ajn;k zm)C6p`&c~)4p>I=MQhcqCOO5vLO$;6KO#B3H7$clG>9;!r%{9j=6H1PD>-&9RLeP= zlgl`VS)X5o_~pWJLMSi&Ny&XH35t`dkY*|SKTYiVSxGk3+K}cu4R~?lUY_)!V3{5b z*k${ZArQUDDH)tc^P6M&4#8jcerw4I41NQ5I=sknI zF{~DA^xSYj@n+a9i1aiUKJn4W>g7LFz`NMtio1!nK!JlhWLOb;u5Vy@c6QH93Bf){ zy)7Hj;V{;f>HMX1G1N=zK5yMqH}1YF3rNl6>WPT8d!skHY)~QdriSIo zCvgnRSuRB{vm@4>60TdXH~}1XQCbCJ5qBzoIrkKbBzJClw&4{+yA01%H#&BbU*zja z#3Qu(F4-Kc+Q7jW<6agD{6j?S)37EU&4xDaQ|{k6ED6r zQ2x_MrCw^U@C{`2LEhRzZy9tH+Up59_IxA~EQLf|5D+*GY(PrF1IxAn>4cg|1(cw} ze5Gu|TU&?!TBW7D+b+3`q#v^=_m$s^=z1My%1*!JKXE*g3$4fVN-s~v`MjUqLV*P* zliEHQaeb_JY-}}e=+xC*W>#L zy`8JFv=2S&nuH9X0?J3Z`X5HI7!ht#nN}ZHY4jdplTvJ&rsPJh*1$FpKL$3~bL;`j z^4hADG|3ozs zy?3J2&-oNWw zYkgN1NUcxmaRs*qpO?&C#y_Av>G`UbGP<}pbQK!7#5S3(nqcG`iunn(CSCq0Cc0yNizd$A7Ki=*3S<3Oe$dUBN2M zuQ%b*^XZ@@YVt4w_n6L`AE!rV|39xZJ&e^3od+S8=x#iY%zpA4c(!7ly-SA6>C9ji zBj)>N*RL4^4M=qucKOVt0|ZYnP`F~cX4Fguqr&Txtjp!6L0$e8F_oGR3`#Cr?-qf< z=^v04`>RXztL&$`YZ)e{x|vq@Ln0mC+LxtqGnRV4;*`Jts-~=$sur3$lALJOHfX_h zTy>$Skr&zZj-3wjoaGI@6i&D+W2NqktsDCCVWBRwM?0;2cgMX}$lu0;@r>a3=WRKTmx91Li7LT9n2`>-$dg%Md8IxR}N(~))s!tk5 zJoG;7tVFkeJ~+(9WXi{LXGp`^{K&>qvKKHRMQDF$gNo$l+4v37m+lH^GY*~TUBFM0 zW>5Ni&&FY&x?VO0a+}clt*zg_m!EtBjT(NyOWMWm*987z@~~JJ!KCm_`bO-GWM;)E z0KTkde;r9fAafD-jy;YnXk;LeudyAg!6OFH;5yRJ>sNckUpD;YgH?M6$vRnwx zd>yT@ew4o_TE>_VO!8^I#jQguS_S?_wO0tO4N4KizBX_cA z7HgX$=^H~dmFSN#UKLJXODL35*S^PNI!F{whs6B|4Uh)Kugh$45rwEo)iLiWJeRDv z!D5YURnRkY)bz}Y`q}0H>5lK6XHLwI5TJ4`!TT=xH~Dy-mvJhRc@LFsOE|j)41;al z)WVx=tJQZ2Q=r)4{eDSUdIU&#AgQC4m|BN=XCfWhI}w??4+5|1m{$7i!prSGw6Tfi znc&=6cfP4ouPD67Miqj8FPH!3GaPYdr|h^%E+3 zKHSf6iO`Ux<|Q3s-07OH@@XjtiDZ&;qEl?ta>Y^Z30L}wLR(mN54bgg^OgGbYN90Y za3nG~Zcn-03s?stNKJi~1#YtvBDMFF7-HXth2S+u>j(KNj}C}-zrEscZJPWlKHpM= zn>qMJpxE~7+p^ZJS68;Vw{OE&2)S6er9?C8ADOcaa z%XGJjm6#n?_I7j$?J_&Ov?6*ma415vN^F^=Aio@onQ<+5I|uG7A+i0wiV8pTK!5;$ zY1d5}T&>*k%(O!+=>w?f9%b}{!AI_*D}np2^=a}QuXxtjL5vt+b|Dh#@X2QM!*dGt zxat2+GXbTWk={e3eH|;}Cy6FMHXT=4qW+vQ=5oK+tc0}3nj0sYEIjp{qf&+ecqlbr zUt z@Z$DS=$KPLB>svMp1nW+-rt>xsWh~J!>une*kf~OI`(OvryMEcg6?=K>))LkwBfI! z0Y><`)>b3Pb$ChI=jm?I=}xqs7fdzKv7mvH+`9k|DW$3+W=*sv1Vy=sw?WLwY8Tg{ z7nz((^|47=N9iL8Yw1zLr9c0SCAkX$Xb%gZHt-jw$7#0Ec+%Eikk#NdeUO-qsR{Oop6yM;1SdSk(ufCprLf4>;Fvq~Wi&;*&I6;2S`Z6a zgs;Q)+6Is{Q))iP^O%)v0p1>QFxg^WJP;hVp*UWXxIOweGW^tf+{;`g`)Gy;eRsD3 z=Fk6Qx_lbcs|$E7p?$C1u1}U^JZ`o*3yG zv-YV6!%_qT4Pq@-?0y1iA5@~=6!D(8pX6=TCz42k1jYbEtnr?G#1OTiH)SKc3Q z%#=igS@@<#w0Ea~y)b(H+pTYI<>Vm39e=y-oCc0JEj4$7c^oo+CX%u0-F`|j4rV5E z+$DK&^2gKpN|vnuE*EXU!X(*v%M9#885Nd1|HZ{HzopBQen}R@^1FW&kK=y~Ab2XQ z`@_@NNnMjx!L(L#P}Pi?L2L^Pxf(@BOK8Oy>B&B+G@}y?DN~Ci(c(MFd^6rcpHEV( zGO`d5?;Rq5PoP9<3Ps;J+~?-%hFmoJ#at;oyV9LT0F*$dYeB5|gYxcG?Kd3^oxAhpX)n`LxBNH|T_Rr^Cfom+-9u3#H zJ=RVw0QRUItos21z_IbaD0tbz-Bg^k^^*9HcV>}OLgQ=WCjPx%_9T6q{d35`hs3UH zmLmUSNDUNhq$N8^w0RB4K=pZP>Ro_iKHdlW2}j&jNXfEFY7w-BSGX}9{aIlG=kW2otH-W27+Lr@OJAIxZc#|q{det}z{oalkQ z8J$5loPFi(%C4%@HM{EM;29yu?r{ooZ!%_R3Z0j0fq)R15&qjn3dYCP{>v9Ex>pxT z8$blpz|}0P%P9c3Dy9;E$?c642@RRxC18Ex!dF9j1g1^`a)%>8}FF6ep|;w2v#IX7tWC&Js+IfGq(Jbf0bDeziHQ>UUE@7F-!?ypnzza7edJJcO<21-Kj zQj&F*UsuuO!^sJN$nO}iDIhOf9s9K2R7P+Qxe+C$-@113ai21Dv+&{u#02m%R{Q3b zUTR*d9QP9^&_h88l9i21ChY)Mh%7>$qVR~I5wl=IsMz9K zFdKw+dYP=9EF#~;>>m(b@0sB3fgCPQJh^lS`i>r0t0bo8&qKfl$JmM?>{uCGf(VkJ zKsms8AzQV;-r?^-(I?vuW|n{|z6nO>qP=t&7q!bGi+{iZ3sWTY1KV@+8M^6wy5%D& zmf_g&tXwrIsjn-FMIsHtE+yKV=OvHaR}8MfBA*$AtVwM@gxDE;X~)*E_nvbo+ygVF zzkx|M%IK|+Qb6fLj%II7Cbr4cglCQxL>TK69NHKV%enu$8p1Ty`Dd@c<+vBOWj!UP z1A_6}cP{jt%9s`aL_+{Fz&dY%_+lwzIJM=@x1*#mCF~wNug4ve3}STLYnx)@ zFl2W>Y~?A2{EN}=FR>TVfk*VeOq@)r3gGwSb~n#&%n4I%y68%_i8|yww0r24#|P|A zPrBYf9A@8LH5Wk6utcc#WnQZ?;CJHJWx@PRH%_NULb*ghv2UyaPo}PF2vka@Mthu) zaH1>o#iwi9);t=T4{cbZdD|z&2MKftqwKC5e9eu#UM%(ICodSYI{xl!DC7`{j@rxz z*R7uzy5ie7G*edE3+~fpX6jrJQU}-cXQPZ92wl24*&u3fD%5mAY+R}nTI+2)KF_}p zII=wm6M6&tPSu!?v#n{q^4u^n%7*)SzM32=S)Csw9{ZJMU#p`Pbo>77?JsJmN9fhH z?L25YrSd^4*F)hawfST0q(D)t#5n#v^(V&2bnv>dy5VCC z53f}mf;%RSQtkzn!CT^*=(jqWH)(TBNdNp@%8v3yA=Da`OmZhvzgg}Oj0UD8dZ<-w zp|5#Y=EHg;;ekI|M?uE=`8FpdqPyr}`35X+=yxBQB`Hq#L3{vjQhAV7ix#@h1LDtSMoUy;r&R%H$2Ku-#jodR2|EgPsBjDK*QUI{QIr%?qQT+QCP232@tqY z>^da(D3)LMp?Z3OQlq%YR!{KPqj#BJ+Eo!SG0=HFThJHkPq+N|)z;gYtw@1n8ji+0JaZ{-Hlp=GW5G-V-2t&r6XgzKl0`-6s$lh4<@fC&Ph;JM z3&=~n1b7PHiT>cOc8%jOOe})kBN!afhm}1|tfwqOgh7y1Au;L?xSl?Ev~ejokN=40 z$D-FJLn$i4`uZ{OzegA2`4PX}-@wu5=hLn9p$D%S0@Gdy@q}?08gr>n>?LD3nZ{_M zxfGf}^DsvCPbMtxVbx@p%?tZqszud|>&l-;rCP5Yxu_PB^>g)s?bU-PT~BLkqX+hc z|AY@hKfi}61}B$^qmXQ}V6B2elPYGw(iYfO4pCGu@5QsL^$t)If3__D?*!&!f-StNuP#v96s#dG_ zSLVB}zFY4)@%S4VeEpKiRaDoA%TMpj22V~&B)<2YAI!Kb+eV0R8|WKkSJO2zOs>>n z+LAUuO!~YjSd`(Q@a=}?@}WV`n`9S-UC)=|Mo$8Nn&v|>@lRNklsLLc^A3VfZ(v8%DH2m zASvyvKrQL<*VZO1zeg1suX&yKZgXlep#CGTLE)fZ>01*ZE8wqU0?gv&v4_FpXl6z+DF%9@MO`s z3i0HI@km4c6lNQ>F*z_c+=mpkrk%p;NN!0TmT4HP5)A;rhIViZH*y8>W7>ZU5=&bL ziCM{TgNtUUnCG$Mjd$jeNbZOJ33No{PW?AuJKCexcBiCVICdou+Ge}STN7dm7cVy* z8$ACGE|eo7PfPNqXoDKqS#7SgwIXt-x(5CSgQHC*@;?|HOdcuqI@2%crmAEg^hy5+ zL0tNUdDFj0H^2>!_}2|S>Hl8NM%?WFBVIBV#lG~a+;n?QY@T>kj@Kb{!`hlvVRAfZ zP4Htt-#Iz$a4880x(0knwcT;UC1>q_K_Xam(ML@p@;;_%q8GRG%Ps$=fWr?sobsmd zP%5n3^etZh`MCJ=H%$(&+%9DM<+6-#FK_w(_w*@;8+mZ0peFK2ET@if{N%qJ1;NIE z>_kp6nZTffC6IXwx{+j*xiE4tZLh%36TgvBhRX8mYXm!u`R|YUxOY$nDtw_{xDpG9 zGgMss|A-ZEcprb(cPL)jJXCHm12FJ1XMQ4nks>nX;>*VNaCsqYPOiGp;YCgXzby87 zFtA<__c)ErZLn7-pBe?Xfi>&1cEko14bPJtBH@TPOytu*U{;7Dov|~({2Vx{N&+eR z463L_yF4y{*GYqEuGAYmKN`aUJF@QYkK2>qD+r?)VN!=7IWiWU<#6C3_+V&$&?Bsz zl7!2-(#v;@ewTZIa&qct()SEXT?pMRC{mA2p)KWlNl%(rL%6JnM6(rH%SFr0^gXK|-?D;mbVB_1H!L>}v!UI1Le1wT3;V<@-db&Lk`+TQgWUFO`&8xQ6+;*_ho=X8 z=j^&8;X2CyfaGNYHh>9wdUYgaZ78oontx194vwuc)af4tr)Z?ZaVF#JB1D3pMDBx` zHdnjG)-x32vHS__B%Z`PI1*RDFx1sovv{O^l`+iyGH!$X;DJ-*b(h~?P9aNzfB2|F@pR1UjkKt}FjF*C(Sl>R-^^z&Ks%gS zEo)=q56R|vFz3hbEEclV>>_m=!|4Z+tgv9MWLujNyW!OL2lD?W*gM*BP?p)lzwQ7I zD2?Ln{;K5;JusKSlow3{6LxKC6z7AXb36V%3}+O;xmN=!PcCDLZYc!D){Wm2ug!}% zCS=Lo_C#0*P8Cl8T`|FS#?e8~6!WBKN*HGfDJY3K;^>=c$Mt|WfR4-fd|*CQYs61~ zXSYiRU1>{P&hjg0#|c|spSB##L*rX%fHcVQ;=sQw-sL^w1(su%=7^$32!{ow&B{{U z4*fvjVxy@Jjd-sm7mXSjkC_5?$(I0fNE$q@&Ab3@+(FWKkO~?z25y^ zFvC2OPcqm(E8}}3er*;ENoa0|@1OTYhitgmL*;a@A{JSLA;J*m53Lbi(hs!e>o2<7 zvBf%Iv(>x{OgtwG5Cd{@JHM}|=Uu#x=K(RkAHkGfk{cyyJ)WKaDRE07SdVar``i|+ zZPqAS5Mfx0SHi7u<&k{5KZM9xN76BlWTOC{YNpoj|BAz+r0^+=v&Bv)9x|0p#(zM1 z5v+fmW@LYjRSXdd`98%6x5(xVM-zbbm_gN1Ao=F8opI*3-Fnf2$n2faW5Q3nJ0UH< z8xnNMg3NpZA8MX4XtDi^B&^W%OLyM5p*W095ZSD|aEI#e_V*xwsPH9VDH8Q>Tg-3A zPpB{}({O{Ikk=1cEHXvGiV#8uTlk5)r-uNjk(_JRBV$zsh|BiIi!_pyNyfoV+Y(H& zgWSMsEM|1^WNG``p$HPcn~x=%gg;sX(hh@hFhtmzy(M`GLC3TF{349#-wH$F#s4#h zMQ_WB@HAf1wjET*hJ~E30k;%=3`BCt*MAtepR^@0)i37#0xVLiBufQw2dByu04Z}O zxf&&k-+g&h$!g*=%ZMdm7SRrieUFS*kYVzlRl`% z3ug^Rhhd8`yi4>>0=`x3CDrqqc#=Ia@1^CHqU*UW_-RYY>>uLn8tZE=6O~n#3CkA9HX`+6ZZ@m;KL{j}bptxYB*cAUCVaKH)W36Sqt zzSS!C_^1p83?Iab{kvHV6F3%;{JO&;ZsK26rSzJ^%*-IECg!$kqB)YsOgrAlnm)IK zHm%k`cW7mI!12P_e%8v~;@aQ;A5c^W=cb9f@Wz0v_|tNX6xO=o?#Xt)aKtNySeJ7L zdZc!zA5Q1?5R)oVd4)UPk$p-QPR4rUhnxHU^f1`$8!4Tt`y~$3*$zeK5^}Q|m0xIB zV~1QgNO+#g^2GLFU`Gd~rKqs}WLbv+>D@p7fJQfLZ`n5tjPpL4F%_6#bn2 z!4o92&@ecdB#0H5nX1~kJXTaH_V5AYj0e$$+|F=SIH%j-udiBSyKKdodh@y&dPUwQ z+9-&w3dvgBmIC3cJ`$_RGEbrB@+M|$S7}<3K>!m%y|MDO#tkD?z`2RWY6+iw#JY4girnA;0;z-(k9VySL(aczo4>%i$};9x(VdWZ(+IvcF^wwjnL+c6 z^o0E%or21D!xyzY5zsV=$qH^R#pqIQ+~Sc6HaS;tDbsaa_Az=yJ#AYjXCpTln zu4h%bZyxK;=eV-0M+$w|-I;TOo4$<|Hned9gWC_j>%~$Qs0D{|W4mKEXLder5$aM0 zEfYL_OmAWt)qQv5vX9?gv#E0$n-$n*?s}!3h&3`-A-QmBZl)<_Ah1k})cerXuQRhZ zhDhPzjxcXMBY;e@nz&aQrAi<^2EvvU zXEoGk<1fVhtd#$tNhg>ZwF7W0W==tZ1gb_HnkM5(?l@}~3j&_9OElsgQG7UZ@w%UVKYKOc zO4WQY96o(VyD3lXc=@iz;u~ET{_f{_vmQbKIYDI*+$D1HB~%Ej90!Hd^Ebpi1N&PGSmOHz?n59ZR0ASZGcim>Rbg`EXf<-$ z2|~a^Re`eDfccxAcvx9q%uc>0c*FJRJp&tpb5jsQ8JL92EQ>s_ zL5?oVC9%tX!X^n~4iZmfa(U{TX0gYRto}e#d+o$!sJk+NNWiNBHCewxyUdM>#Cehr zM(2NiUpo19SxtwHdyHkBnlcb1s5$~8(KJn2xSr0@czRFip{QvB!jHzGVn!$JUHyZ> z9Qj!nzI&AMunHA5jVLua3Hk8k5}k1nGG5rPl_vk?f#$r|jh#}F0-6*Xma_51xgYY}_mr00 zZM2-bM@;3lFI!kou_vgswCJ>yde!EzD696aT|)FmVQpw?B3#tEM!9SxyY*e^Y_nUb zle&JBwrUL553YQ?QPX^@WXjK(O`D3~%Mv`ANN*W^OFUg7j)i_t;&C3Ur+1V%%%$;N@6uFH>M}x|yS(!|iuh zx;KTt&v{mRq8@+8lKv9#sz0aqrq*Fwe)7wxn$Cm~Dl(mA{?Z*=Y<{1{1YGg7M9*m} zz#z1RQRT)~q=At2Pr~r*8YrfzLQ0C~XpZc!prspz(?{|Ts7?5&j$S`uXWCK2yNT0H zliRl~0+T!#C`aMzFmiQY)Z&7wz1`v$Q>Sg$nALqh>rhVUdbzP-xjTp2z5?EU?`_En zzcWK`+$PokqVe*TfVHmdMQ-Su4UrQXZB))(fR{C3SGx|l-UWg#T+M=nO&0jwo#$2!P3Jvh7F^~uqEDlv$&gAg9 ze{2XAdlKnG$3Rc}J|2_EB5$Ee)k1{`+N2Aj^8?n+L1onS^S6FiM$pOs#0eUdocnlc zu3xn5L`HQ2ebr1%blo@EeqxI8A%{J@&fifmOi+FIb?+wDw2QLPmCfN8o}w!iFhqCl zkZA~Px(|JEoYv+Cj7?0Ht!0cJSUO18tn6n%P3m|>@E6iST)3@BUJ|w@7q^7MmFjp0c zYi;qpTnY_Lz~h$#z_hNGCwYoVjYmg;JR1_n^1_#?7p6r_FIW~B?R<$ImtX;G^rT}t zh!VokX>38^ck2L1N+S9eY;5Lt>FiM*R4U6p;#YZ?^*t_ThrjCGWIm&#sc8(uEI$5O zTuE;#H=LSWkzdir>oDxsJVnaYc4>;-Cu1*=doi5MW8;)cVF)K6&*a#^0&6q z0N*zrS<^F4vF747-M_*s4+PPNw=4p#n(r7_p)4W^69KFw?;|p4Sb`%tR4ImSj%Mh* znZj96UbfsP6+FM23_4e&kdGhpLLA|K_R2S#1`34u>02`P!f)wY;I}uke4`33L}6h- zvPDHZkJT|@T;k3Be3$6$tk}bOdx-=N3hM{U)jI3 z15`hlM}JT12T{X~FW+1uv!M#kasXpM;f^nWlVeJ_DG;bTXnW12wL<$9y3tbu?a=%O zgqd+_S?@}iA{n|ShIXiv+o4X%n(Yr}68?}0I6ar431s(Pwk7L}RKAqyuDy~sO|Pn5 z$Jz`HeG4|dxpuvX22dK3M8@;6#m8%wE*30j!Nf}u{B{$8gKP6Qf0QSw!Zd8pVC;O7 z7(?9mJOa%J@%4NXnL`(s`m%uL(gMZktBe`m>R4WI4Zmj{&J(zMWvS-%9gG8{ElmKEMy|kJemtMH{wqa%jP3K!nlT zd&jJwlB>(fh-m)kaO~_(|0{GQcVBkj-L|wCz=OewPBq%q7fTV~L={=zmIE`;Tq-0P z#Dn1kr_(T-wJCZ-@>6NBg5D1F_e;wjN6CoWQVVr3HugZ!8ppza@RaTlKs7 zO@lLHLXWRHH$HoK)LELV<(+wB=9n75JGlvlAp&zyYO6idB^ozhdDFV3!Tmz_F5$Q}?Ysr|k-b{<9}=+S#Ir9sG-quBuxmtI^Zr zjd6fe$F+G3sEW!XQar-|%AqQ^Sav>023F4I$%wxX)@O{cncG|NGGNIE=v!xitoGl} zb-|9(!{duYU{3i~jKg@^ZmVYO1$?r$)WKLkcG+utxKm_V@SOzuhtc4c`n=%J7UR8l zzu(TkJWo+qPIqqk%jlYf4T^+)00b~a8u^fon;jX8SYPuc6=DS?LzKDT*^YwrtJ!E7dTq*|(R|$kuWSvY5Q%ngH@tso z+I?clcmUB=B@M)me2@T2AH$alHp3*b(tAE05}@gJRu!HYF`lqdZ~oD4P(~*{ZSGfKzf5bTl5S z$jZCKb`1lA#x_T7G=8PrX={j5BSBttRy#sBH~*QTgoKd#iS;LvY!?As$V>;l>= z|0B57_;{1Km(WOD{~j*)#M9)>MCNDggpsIBZd&>NTH7QSHx4}GF>Y=J0hfj~-CE5D zbo%9#WrVjG?7j&Fi> ziWX`e-{iYkUCne)aOp$n{Iqo*6mHn_tCS`?C z?HLtiN)d@wEPcgVIYwCC$g!$I1r@~8#!Nt{>FtIRXn2S;Qx=lKhu#z*^wg%3eIGks z%HNCme!Np%Cd1KVF+h|JzgePRJPOMAAPY`!JWEZbDJp+Mgf**J+{i^J`{1!7+W{7# z%gfw9kK`i>llX(!&%9uvL!81)S!T+~MY0DR9t(LEX^&oBLelST34oWnA(rp_4W4hA z?5P`TVLn*Y^A(rBW=8sl|5E#oG|WmT#C>${4U*Yb>%fer&uqQlgi)i%wS8v)jTAjC zsUcCf{ZmFX9kmP@f84=-BQJswS`biB!O_mE_TRfZoA#qA+uKKoS>(R)BA@3G1qM!kb8*UpPABf7Q z%!9$+*CLWbW4&@;;0_h!Byuj#%qdinn-ikAzY1|G&5VXQSxh+0!o?YI)sYXzj2}T^ zY(x$^wPCP(7*1Gm$D1~tTd&ANuutzE@g50IP{Asb?bymao|kHHF$eXxs8c(gH$IB9 z;u|`9DGlJ#c3q2`(ey7>+o}a3sy!-r?_c3l3!9z1O&>%wM$yA$ zNlH+>A|tPF57Wl1w(QTV9qrkklh`Qs^ckm>78gbxe`R*udlUxCIA`^$F`G zD$5?7XO4|p8z|Ks7N0fUEOMlhT5{vq9GAptaXl*z9ACw0~G0N zwajXm5B8x}jSQ~wsk4Je+`DZ3;yqFFk+d^@qaE(LhitF4-hA=e_CbgbR0&-V=9xo5OjHN1!PEJ5;n*Dg*Hgf%8; z3tT5sFrO3i3mAp(m}N-mO=p;(Y?**nvc=iaex>!`AJF}pV9p1XEQyT--2Z^=hGvy> z{{cOGz(b=~6YZ1t&9HfRre%KZMa{tACPnLmjAc2|a%}wC2x>M;7Aq*W*V`xqe8Yr; zl5x}gg~=<*VPy(B__`RWB%7$oxV9-3K80v<+#CWC@Mc{dao>PI^`@rVg))>Y-g{bU zhhUQ&UD)qHcYU-VI^12+oRx0rdp!QuxoFP!>8iv!dlqh_0I1to3+ON12aXAXr;}eJnlWbO>;NwqadLlpz#S_AVR*!j=othXi?? zFBIRX0W{4}xceGa6nWjM1zTbCstJ2I<({H!7BY)2YdQO`5P4r=0!X(^=WT*eEUFj& zx0lP`Uv%8;@tlMUY3@5{^ENM*E-~CROV{0=iMo*K7))Pz*C2>vBxt~LA_5!RM@O|k z_em`}QvAPqvu!@kXerHm2`%|rPX=H zP+2ukRx``X3x9x0YEE$$@EV|W2EO+60wy~ShsN3KlRGU)WYLEcA3vvM92R@@@R7_A z#2y@p@RRyG>3K0f)jg5Xz0p1l&fQl#Dxr2Ru*5ks7BOv*Ht0XkOBa1E= z3lFQc2fwyivX41bE;a%aM&n^a*AUW)~d=Yj+GRA0&HeDu1Q?=F{|V&;DQnCyrn zg|0~|^jX*oVqgDvJokQ0kRJ&E2OW!FN@xd?|7$t1_}^l+|Lgh!tE7bgvhE|NiDiAO z8WlvRd~I4O)?_GoiR3fcFNgMu`QFX2+bo_0W9OqRofftv-0&Q+GMY52R>PZnrRBd( zmendkk06Wt*WNqMf8grc0{g(%D6;{;VmbCL`yZkuf%dwnqXjvt6Op=W*G4Khpx^o3 zi6n~A9RXx}0A4xCOTF6M!DlG@?p|DrelBRuh3~d z{1JVUA2e}euoZu_yheZy*tIFdZJH!}s<6xbTEX)O(5c-1@ULC>ep_WG(acKLpLdiM z#?QZQZV|O7Gi~;LC#ew{n`7;p6}d$c+X)sGgX0u>O&m?^IxH`#oL_GY|881}l9#Q| zlL4wksOYh3i?#VJWCoE(06`--$mx+}?@@sN(CSh!;i8Na0E1IMx&}TRB^HEfDY~sy zD{y84=$vW{!fJu=L0{Msuqhs0wUYue+W83wVVt|?%^);N17_VX%w+9V zM*s1%M%@cu5+M67+e{_5%EYg*YcoE@s-DR4Q(g+N} zKrG(NCZZ;@hUa5}Nd5L(IyB#gBZH~KdjQx}g>6vUQiCJ!yCX(pMI|t@TI^YM znWq7j3-%1E(JnD{irbML{$=2!_eg7oc~iYnICv>a2}CC!nlu3TjNxsvfC|jU{5eo9 zSPqmRLF#Ho4vzu@c+V<1d>~j@u1J~Wy{_CNwOcc^Vo8Y8+$qv^>af2s#KQH(-<0f2 zm;Tik(2%_+rK67$BFy-H9GRX7Kaa%Ryb&2hm_2Q(Ri&iYpyz7w#jcBlFY{P03dgx< z-ve-Vl8#C4sO+e8o_9`osYrMp2Y#XZRU=~|DP5&sm=9zIyRT;a2-3TZ|D_d-tm4YR z_owY(kG&r0m1Gb7rK!WfL9e}ZHQQNJDw_d%HnJm2L;`S6nEQK{om~K7T#uFJ=jpp1 zV}!YX$S6xsLvSz_TKAq;4%#dNd8cQ;>ukQq;!fhc&*QX@X>8LC8dpiMs>Qoh zD5ukaJ|7uz=RIQL!sjW*Ilt``4W(}nmfrI?#GoI)J=WoHarp;CA*LQJD40~`+dh`- zakb;LZ%4OWb>WOez%(K^H-Lr@USFWS46hA`scI8YLT~ZLhtss3A#>xv1z#Z9gt==A z*3R9$+>s+46I+}iI<30A&)_UaNl~-IM~}(4bZ`NPa+C5)L?gVP)vmYFqyN+F7pEb*zBQgnkpD5?a+Xfm!T1+#!RQjOF z1Mo9S>`(P?TzcIne^S5le&g~)Jo$5vUC*Z~{k*&hlTiSZA^^Ygd)AS z(8x`uF)g~!pE+J#+JE;N8nJj4Is|8Wl4cZrPiltC;(G_%h}20UWqh9g9en_)*l>}b zBU8N3CXG6ule|h6lh#Zb7g9-C(9vkQH4g~BJAK)OfE-$P3IVLFxy;s@9KJ`U2 z*P_8T4cO&wn1C&O?W4JFaUFY4@MwhoG%mAw=v%{=>8Od)=UCtE%T&5R0+^pjL5^IZ zCM_OG(iCFJte}SZkZ2KA#ATXFf~&DeTH98?)I zw_c{IEGS1#k-SuudJK0cC*Ar8!SsyfeaW$6Yl!f-Xw=zkl^9}kQ&2#rQKa3#u3)6$ z=gERlxMgl5>UZF`1zavU8)5kUGk#$)BC5cg8&qTuAMYR(yN07^0(=-maj@y2K)VNf@Rsl+v@?r$g427gHpx zMh74g-nFTRZW8J^t#ea-w&^gidj@}trHOMqnH^>KL6<%Dtmu^ZX3uzX8%MRQl<0Bt zu;>Wt>D5QUQPk@pAWG?tP!^en65TyL zuUxMdV%|5KjE$JtY$zvv74g(Ms_lDXNT`IZln>6MmL z*7XOa#O&PdTWtnU=Rzn2?)&YC`&)6~v8l4B4^m&0^YGju#6ZFqsTS%e-l#!>=jN-) zJhjQ0o+COh)8dPSBdAFquZ%#)GI@n-!Kv{}1bWYrUU7DzwL3#ZN#?cUI8({@LU5(O zk**`1Cksj-wFJ)gKP2^On~gI!Fs67#XLH@KC`ZqG=ZJb_2hJ8EbMSB^~{VOCvAM{6mU?7b{ZIR+7I zLRFurJh&85I2Jx-={mt{u|lX{?2GDNR7-TEQ#+?h&`#&h4kQk;z%#;A6Gcw!~s=@V@;Ze=IJUMYi@v8QDBO>{cz+*pb}vj20_4kDlV6<+|-G%$=! z&QShs%^xaUTSi0_cc5NO98M<2CqD#jl`__rVAbeR{d(e9BfWC^YGB@-c#vkX>1a%N zgUp4QVZVNiL=IdD;MCuOrzM*4!-3X-@Gqn!zaM_ilUdIGRj8e=^YlBHuJ+9v%6?U~%?CEA z`D;>oxMYvcZ=!TSAwPY&W}8fmq7&i<4kL&nJE_$|f=M7&xLc;y(*gWGs1x%fk5;bU z)#vM|caB7~kPKckTyD8Deae##oI)}2pTn_CQ30J~P!IAPjz@(?KNqcje*5ZW`Ia&Y z9oF67qtM0W)Yjt|zFIJXsm-dR+{Jf4vb8C@ott4?T)X|;epMU7DiM$DT~cr!($wCl zdh&8P5hr=X+$hFamj6Owb03(;^L%KBo(QSaeCioi)B8Cz@*r;7%!-jR08qte1*_ds z_I{w69WAWxi}LC5hw-Hn5^LRMIf7cVBYSQc(N`wa@|Bh_#og*yFBRe|cYb*r3ID5; zlSE;_vPdq6IL4WG;%7etplGTRa(Tcy{%y!e)ZF0p?Txb>BmhvffNsk6&EwO#&Us)QkD8hgLM_VIO67_NB4{sp=f;+-X;RDY@W+bN<4%7sh&VxEFgZ^N>+pbc{m zs(0XcFh65V+gUV}%yzI6UCn}NejayUt(FZQm^k>OLUcwpp4Fi-=`}!;%GnpSm0Q4k zn<34PcxP~icpn%jT&8=OLUk|$A!ktO1-c|N%SRC;$fj6pr>G9!;U0Vr=Y8MIO^B?> zWjcY<(26R;5B=p_)Z}kq(IVKm(c02-`0KZ+N*tDP#MM=vuK)RVbCgtyo8x;LpH`ES zul4s9&cn$O-u^z<=!>Ws#sHpgck%IES;^@k!ZoCYLp~;c8)T?Aq@EL}*WL4yi%Yt5 zA$Mb@>C65vcz2lF*eP*4FdvSa<0Onq>R6+l*@P^?M`HvV>`FJSn(I~?X2GmbVUg?W z*%^3XnZTw~E8jO@Qj(SUmvqsopV2yzx0oL&?Zy07sc*#ABVYtV4s?XE20~Rz_?C|H zwA_Fp5r_!l2(#noRq}>@Z>`s0rW3q89LsxMACbBEMYrT2sf1&rBs^c?i46mX!$GwN z9ww&k>|{YH@>tN)avChMY7(@#3Xw>B?PSE6?da^c< zKg+Y+2TMHvm~;DnkUmq1?t$Ky!|0vG{2OehO7w3NIDA$Flq_ zZWIvN?H5)}SU3i+=dYaj&CJ4OY8XFt3E|pusgyNtl=Ei@(l<+%_}=c*xcdJKC7wn! z4~TRWXTQ(yeit=2Z3D{7S)ZZaXWWkujIO?}UR?F=HgM`A-y@zYkg{0#LK1rY<}PQ^ zi$zgx1Lrr|xc+L!ED9A*v(*XEqQ)CKs}z6+Gg7T3pr1K!-q#dX9&Y_{uAv=9eihL= zvcd_UfxnGwZ3Upupt$hst0SgG{|#uF)-O$I6HkA5a7m}2&H0>v$tU`}R~MycQ@@zJ zn7^YcB=$dh+xr{`s-z23JN?bmlKR1a9G~4DZ2Fx0TOffKv-Cs+pcccik)WuMmLpKT zfSEXBJ&o&qhbsIgY;;M7K+?dE=Zg`yT*s~blw^A2PRUqN|50Zn^UJ94RQP1fS;hxL znydaW%Pw@jt`}YCbO6X3`Y~5=2@K+k0geD-n0nq1GUv1S&5M)#Cg(sgP-K3AK7nI= zZhuwsK-Qzx*W3T0SbMNH)<)3H+na*t6=b*4*72*5b)Wc`WN0t7T!oFZb&;Y<@Mebr z))#=pK5`%tH(yheg+^C_I z+dGvUz?^%aSW>yWCzAzCX{-vZY!BpQn`%L&@=EFlv?T|Iu{TQBD5u{~s_?Vsyjk?ruho(IZ4U zq#Klw?(P;z=@5{XZV&{d1*9aTq&xi0_vicjXMgRSot<-U_kCTj=kxK9in+Q=v@QDY zXc2C_G+%*3>qmLp*DrpxGJAJ;m8*=0LGkmU@+MSvzxy@?bM7YKd3}bKC^f^yH_^CG zrgem%VFyt#Ag#-K9C?oTN1N}VvuJDn^01|ku~yo-oTBJSEEcnAj!7j9ZdO#`Q8!GV zUrW#T^S!vDMR^<>AS!(IX+m?wWGHZ&iBaOmDqeRkMZm@wB*YHiRi?Xjko8?!b%41n z-MYSOJcwCrs$!R!{(4~ZRH5YC({|ub@qpJF1mU`7@(4M_);sn4WW_GAoV|!ptioLT zW$^RL`pF`WCb3UP?PVw-ZYXv6{5AdqVQEXN8&@2S8XbrE?XM|`BQZIAQM8--6&FP|c zr7TN%Z2zfi^{0)h#hc!+O7%B*XRQSoQTQS;$)NynL{=0=*dj{gqulu5kM~Rf=)+ z=o6WBjKjzNQ6@3T}X!Ji(zK*YpKb^`Oc%PTs5-M~@L zf1y%6ys=xvF>w6}jv%k$k{E`_oy%eBTf`W=tbL_#7N*wz8)N)w+$A=QZG8XpQFmq< z%jkKPlARr^J%!8r#K@S)OThu`LLgJwGFTXt9t_9{uJP#+7||Tk$cr1+WZY5_?bPPh zbx?Wt^2-;G2aX?)1*oa!^~?{(d*-p)($Iby02DX!Nmpd>Y;&rNcgILv;V> zOXe$L?%*Oz-y$SfAtC87@k_X*kZEr4Qx%L`>_0O`GCF#;zgve%I&@ydtyRMp zN|?%wxS7hdpwmJ{5SSd~gov?9p#&Lmy0##8ZM&?vrTa%AkEh*nE_2rBH@TTD(`MRH zu}2+#gx@HEeU&`5vtZZ+ol=f()@rF644JMSXas0%K+OV9~NwHcCJX2U$q|t$OS$kAM(lF1Rs|{|fGgVQ^^?8)Ly-1^|>`@k# ztpOGGCs~fLRyLWKs4d^M1@9A{7Vb`~d!0mbLk)#?PjPw)Dh}bMk zrNX!KQ_kSws{qB3Q28Q`6J)3}2CK>HTdNKZ)P>Zh2;r-**)By4|N{OKI|E*H)9fIvsi$)&qY5W!9lCOP%wX zjhdVN7TGA4v=?+EiPo9dwgc@B1h<9OtxdJf5{4VqHCnp0b|(if5-c!|$jFi}XAj6yJRsUd`5!YMxUWe?w><_qb9F}LpePEfbbd!eVI@~NgQ)c#MbD#Co=Vr9#N@1qOaxtF}WIR*`S+=g`M`7KV z+gE*PpZOBm2SxiyT_S(w?}~6w+g{Y)wP1RhJt}E7kPiRi&9ZzEPsS8?`kvIgY3~sw zOP{ql8!iCXn*I6xEd`a;^QR2n1EfC-=|P5XxI~a-w-Ol`<+2`RZ{|n?Z9ZL{3dWb| zEU(%S)3m;j+&rnYT=eRJwb^#LiQKY8d$hnlS6%6AAHs(9@|g7AJY6_&K&^P$Ju1Un ze^x4VJ8Z(hOsEjHcyqob?FWX#676BO=;@!8>y2?b4q3||3TVnE+$@bxUu>QCx@Gqt z`}4B(r|tv~JipArxAN%HjZ>7iP|-`3&M3l}U0~BAe=8pfUn-wSTeIgd4li|SdvFd# zWzdf-vqPVsLdP+C9A#IbtG+8V_NWYS+lu7>d>S(d-r_Zg=XPbvy&!X2+BE#2K@uFh ziZ>o~|E<_6gATQMPM88VT*3+}BO&6}x^VatV_c~s_tgZ)2nO75JcOgyP=r{LckhH$ z8|1rVSN_8f&x#USsp%Z;G*xmb3xm@%&|;fgK5~H>)8%r-Gb}66)r+97ps4&L#W0mrh_!TIdFCG@wZotT^#1x}neyjzUW=rb1;7f*~;hD~Q1a za-Do^e4a6-lW<*5)aJ#aARc`?VkQ9-C(i!eC}#(;sJLgNyzZ~L($=n^GY2>#y`GuS(Lm)l%^X>TbqHrfAnWv z#0}j+68*0=!7#izZ~$R2cKzF7i+q>%jMFY`@RHHx-ky_hTnz_ShM(ucfUN zWXKFjAU~WD)O(Z~ew4^|L*&()5X>(e+cyQN6X#WRlsxqt5Ljw@|5By31^29O6a0ze zxBNHph(b=UwQ;YH#36&5XSdiE1D0-sL=~c1;7GPz+zDBs+Q*!If$y2>%l_qjV*;z) zrV<<@bSkE1d~_ZQ8OT^f8ORuHoDfxzSBcNrZQ&#DUO>d2$|S$MPTsY3o)kHWR|#rQ zR%g)~OVq`meLAk)`10qlO;#&0Yw8C>YDIM+kuM!cD@UYG^)|X^cmC({AM|O0IUVEs z+Fx%*%%M<$C7~Wxo3>k|r%(C!G?S4WJhZt-fneZf0~ln@%Bx|Po$@NXlpr6)e>8>r zT?`;!6O(-@;QTfhPr|COf*^IZ+kH>7T113XQXOqKd4}72U&49YS-O?XfW97rAuI8W zpn+ndxMA*U${-9ytRY|n&uqZTGTv8_ZUiBb2(mWoNMJRIhR|<=R>ghm;zh#UPVYJf zpx|8EDDE~gCrRM{02)J#vMY>5max0`a|F09;vl~Sh37; zXf38z6UyOZ?h*QNq@MJ~;$)A(=<>Q)cYCc3z{EfuF z>Ls$*^=}!MY-a*7f0c*3CKV*Ot()(8oU`Qp!`lI=|B#Sd{F&3axs3pc)y3l)V43(M z`o%TR=;kIw6R}?`UCmn!>~?q)nT&hu5JEFuc6{;h7hnJ-PbZ}GO;UY)=BHoM^hf7( zJPQyLi86SvyRNI*I7oyNWr^WsWyr^)nPJ8zg$>y@U&E_YQzmCcNf_Bj8C0yH-+(Sb z7&Up-Ycw#xsEh`oEENnu5J4!;Cz)^EC_vBzNS-JsRph#7)5hD>9uB?8%hgg2NV9@nKr4$NYC6&-75>A%idrB)Qsht>X1QP+IluEy>+k$2T zP8*~)*pa0F0yMl612gZFL$0JX-O&5hJcr=W^IEUh1!0gY;gZ7aO61ujpR)Rdw z`tWOnI#y2{Zwkop2_IY-?%K%cQe<5x$a>6zqt^yMfMgT*<~Do!p3O=S#%$M})&jG= zpiH_@HbCDw;~L`_V>Y_E8e#L$O>pI$H_kf5V03fpC{=jtAfS{|zFy%JH3|PR(FEP> zY!OuVkM;h)S8V9&w_{{6OytKwC(GM6U4Gs5ORnC30o6NCr$yK%2r(I`R}KFU<5HsK zaIaynQL=!wiW~Km@cmzasm-(^!rAovT#P93{f{UL4n}kr(TY6GC)6&JB3CAsy%4~h z$E{ye*HTLTh@l7(tF~ZqBml>`9xwX&MaSb4#1Of?iWa}QpamJv7d7%*r5w7cspb2b z^0>Ll8_&&z)Rv+z+=SzoYPTnaC)I3zy-CWWlLr*lEycP#T*lIH8<_&%8BHT0BlJ8= zuzY8;jBad6AhCoWbDi>i)!Sn3pl zCg)q=x_A2*Kn9LtxJ(MFL>Okm5p0U_a{31hs&Ey=XBJi@`{a2fe z0=#w+H+b^Jli@@38tWmbP#R@7-;37f?q5J?u=sB;q4Mz>8Gn36a<;g^YKklsaY*M(5hTR00Y1eS0VP$2yf;xr>=%y`rrpPj_2h51q2o@FJ`bg zW(ofmrtvE#5pjMovq-D@Td`&v<|VKnXk!x;BsKZJ?Gc;G$J{?}#aBMAF*5N5MRZ$T zugu~+%UlFl$|hqnmWiOU`YaNS58#}MSRR0}GK+ocm~(3gL#2ME_V zYAd<8T@tShz6qhKqpGjMBtk~4*!_ldzkP7fQ*VPKNS3iM^W#6>(!rFI9;z!f6CXKk1gHy`B@?-L+0fuTx!`c|t{47@g zOZL##rD@t5_S9GZ?(6mrMg2^VIO2U1lgyAz)7`p{{-522b#Qa!jP_oy(RqELp8IU- zA@P6utXP?x0>qQJKjC#~@&3aw1jeed`ke() zGN4+7C1@de?tbh-b>2l~HkhdoNcy^LIC1xrx}U&l*salogx}V{dv_E)L6e-*i{F~~ zFH1R8bB5p=C2r3e8^GdEoiyj|*I(1pcf1f@$7|DgPfh60j)7|(9{&2hMO&Zx4DASN-h`c2+cNlY5}a;8|CduLx*VAa3PLB*YhI%_< zrF^fbb=ZzR%-3CzXQ;~TpejPasbcnq=sOg}jeLQP@Kw`d^O}rm6)kt${$hv$78+hi z(-J!@{I@_uV*^8i&Ef%O0JP(ZXF=pq7}q!g)1!i;|JB&LdWs6^h%iSIMYiQEl9iyl zp_V*VR*Z1L(g+5_RQNJcNy%@GX5`@F>6gEU`BdSVAiO#PRHEYE_Y@LXjan;dN?Peq zwTuawEHH&J1^P!(K(l=<`*vw~p}1pJ>HsSbajV9!_j2;l`Ok%btyCDI`x-+%Tgk*G zpGYqxsrt(Ln?0FVA$8D&Xq3l0l=EodOphK99;4<5GR|g1RDHgFsNdV;%|bne=9G;% z#pZ`vx+qlfjGXzxMzs>Gh^DO{AnIy17)jYGJld_Wuu}-YIIwx}YeNN|;cMHpK&CHO9NrfRSZb!g*pUfbPlA0qxuO&1KXdImRVqEOzLQu0}3e zSB%-R-ppDjB%f+CM|{>gohhq@@4z1BA~2#QXwVx8GJA}&3%+^2dTb#RtG0mH@aQ~) z+veAVj$9qfi_kRmSgo@?(A$=3UKMq7>2*cYoB|-AF#`Idxe}Y zyzN>eNB6d%rqb?LoU3%794IbJ!-VWB7Vf4@J$eysD4Qg??(1(eT^w6_d%LS z&D(P$otd=apzXts4(W)r-#O49HP?jmd)rPs8l63BzxnNJ%IQYHlTdv{g~N!jSJ4U{ zuC-vCM{!rI5>`LK;SLOS_$9kJ+89?Typ*n7PGX4Zn=BxMibjBz|J{4zRx6EQfROV$ zlN#F3*DT+!?US;xbI|;ED+iE+SJ$q%`-#oCeT6DiA2K_uX@uy`P(ybuZb;Cle^$*Ki&3itp6C)dRG5*|0>r|KLyy`XLHeE z|3M`s>{`P!9^BBnJBE4$aa>fVuAzttk%7fwtWsOUuP`u5TqwD|n?kL}FCU3NpQ^+X!|lW0aWBbG7@xn4RL zm3R^lxyu&iyM4H1?Lj%*eIFg>y~@6_le~}0_8lTk-e#o+h@ShK@4Lwfreetv5O1OP z;|yL0chtl~ntijRg}e{`aFlO8;D{M`9xy%|%3{mfN@hE5dQ5yH7k)Y7i(?4S5G^b| zEG&kDls&c1QwchNq9038nQRC;n7)@-pSCZMTP!$gl)o&6HgHqXxPP*eV9B+Z3ox!i z^FYV4`Yy0WKLU<`2Hj8Nb z&eE9a#1WfwFKTXP;>0h*>0xY3fr(GP*-AWkS|*IiFOJXHe^gv1_~m*T{$evx#-0~Z z!>zBWJgM$tsv3}maywj?r+As>*V{{pOq`UNtvFK5Cu zI4G2Uh8$m3mT6FmrrZ@G(dG`ckdjB>JTe0KxK%hIgCqz$eUroc16WOQt(pzd)HbmL zyESWtb`VRI;e627T}*bOYIZ2A-Cw}h6$qQ+iOakno|+Msk`C%xDc))+p9w_=iOdRF#4#rUr|UB?msH_kb@~GhYm<<(7=1 z=YG6}Kn)@XB6#XwsIs|bq!*2(6KG*+HB-qfRvE_H8b(|;244}iQ?&Us@}R{l^q55C z9?n0u1z9NYN4q)df2=p3bb+#`XA~*2d@&Sq1hx@b6J#(W77Bu_MNLPw;=LdnwH>B_ zi+9-(aO5yPwD-cDQR{Zy)<61`GK#*rK}aZ?l2a;5LJ{Uh(q$k~l_wdMeMZ^# zC1Pu~^qcobB0Pd=0ja2AKBr|pV|d#|8voZzJCY?_4wx>eiOV4j6-A2x%EwC}*!TIX z6*VFAF;@N8G(&)}up5>0mDeT=*114YafqqFuxi+Gm1I_vkjI}NJ=>O!}jAdE8; zCT`z>A3Nm56a4RcF?^$5Sa6FJM(}BkhqPu_T0^}ui<%^tr%OU5MqYPVh~Px0#P$Bd zC8dqz&M0ZXo_9C8V6i*LD|J;|!oFae;8%NK%}V@w76X6DW%oiZ^aH3Q3kxV=I%0R$ zc*b+Y@8|kIx`7DSPVe3S&Nd;NLCs4G9y3OzP2Ti z8|}B$_0i7UEfLF&9S`0bP}9EF8Oh;%cS)RY^}Q=$Q~*8e6(gmem9JGR_tVGlr-F{a z51YuAno}N<5lK8UP9S+XY}bX=#1(^)8V`hHm`&{n_rZ|igTjc>BQA00-T9j)&`kGoo}e z{de+Cq}AN+aegk-I#5Q@bRhBk7Z8MSiCpyjtT+N`-kS|Xb;^99gXU7n|2Gwpf?*oh zt=+3-fmT`429XT#w>ZiY@B-UlE|HWImXpI6f{uv0Rik}uA*uWN@=-F{wIz?Vp*cC< zChS{b0!*}1uY!_Ep(lP0rwl^cYObMxpegQ~H^z%mXlI|(3E&2y&` z{19<3O@S6^y3w z_hgpg3EgQI526O?kBYTdw{x=T7%>RTLfP(m=SepP%HjC8srPyzTry-Uu4flFedv?V zj#GiR1ikxKuH{@`mS;ORqUQkByn^N4UPmZWYv}mWf#m_u*O%>#{1iIiIOlqlN#>wd zNumG1xnKpe*^17mN8V2&H1v2^P|^~PwhN{ z(Nk1neGD+ilk&=MUI>bK2&%-9a16hQ1m)6$XMgDjsVaYzJ`^qbVgEWSYEJRvuUi@f z|5h|r1h&5PLHP>z>Jx9`K|@T3?X?vmPeOkpDW?PwI$p+X%89&Sa36lMW3`!cx8V10 zrqhw&d$lx)WNZUh=Re@YHZX}4?G~!E$#?lpk#=Fi`Y`67-aB*cre_x=K;14OmoL69 zzgg>13dx+*SOH5I#-RFBsR>$0)BQOCH_L!1_+qagL;Ct=rNGpL7lQsjh*BtqUSL|M z?8;ogJ)&egQYgIF(6%fby(jQpc94WiUFqT~z&!NhWm2H2?TJ6T6GCB!q17--=s30u zKBhz=Ks~YVru^>=_$7))bJ6Xq%dn4^iLLjWqyIwVga4xAlL$0ZQIS34i`=zA*-NmC z70G~A415c~opH=%yb8yS{mEZ|cC60IB3Cd;OCb~8CN~fVn~JhWiF(q-$+uIq}FIOesbwg|Q6)(3n7MJ!D0q`J9u6dpDRDYv0tdFV?WV z)4mt2a}zuS%Bh_%DYmCIJ#Sz3Im|b2QygDXE)bb4$-vGjdc=>RbmErge`HDb15=uu zgI~(8kL~x+<4FHoMbw5GRD`-(6iL1L84u5>;=+$WaD;M7L;*}oSU(H4=~EeFve%ve z0vZrTWAiu5(nkL}#Y+>b_mlzoZS)Fd8Se_v&Mh{OEJA4jM4;CtNGzUcznnnzuZ4BR zUkue~oC^PbJ%Nk(H%LtnM~0`-3AlT4mc=>qJK_wuGnlKQd z`(9wn{&5U46fN=bSW~b_eTiSSUZzOZ! zz#ywy?RVZi9MG2b>{frvt|LscKRY{) zg9)u0yt8AgM5&Q-lf>v=j(1UbVj*PWI;PpXd+I6C9aLwzwv&iRdCu}$kM@=t1S^|~ z-77DyVhqS-18B!sjx`GRiD!~0eV=l7FYh4cmonH4(m%KHK1frg(EHxS;qP=kKGnCZ zdZKmXvMpBc1Buxrqk1@AAZwb`@^us3KLRROGzr*dF!

Pl1c`y^!TK$s4p^S;NOb zNz7GQTvK6Na!s~(%SI*-j#7_ysn5LNN8x=Zcrkpqywa?2b;f1 zjQQRA{n~Y3qD9Qx`tt{R9pTsMERvH*$X-U7wTw2|6N7w&e4=BYXTlki>`+-rVc`Lc zZf59iBdLDR1mSAk9K2^y?|> zMi!^YZi7$L83%QuVdYTb1W|Ei8YY%udMPnQN>$rKlVmHC&M0?r4=+FIuNN*KD%rsx zPDyh>8o&A+k?aWIILPVL*Y#5-bG#C7f!&$Ti?uzI#DtFf+|Sf;L%Z8sh+{CnJ z*36r>h=biMF7uI$+pq)ys^hCK7bU_gRdfOXy6!XC((u0kg^2@qUSv#Ee)qkay=Jn< znINjMp}Za!~VOgwFy`1dF`fc?@e`ffQbhP$$I72T=LL&{s z>tvm+KkJr6=E%)JV43pP@3izIQd7cU>GC>@-m!XZ3H9p?69h`V)??|wv7UNNp;kGx zwp6m%rOTR#Mos98oLV_8qAfW#P;*J;%uHgcoN8SLCmjSQ1WVjvkf4S=HNB{O@(PP{ zOf~hYo_5P!=;yjk?0$~0=SNhQ2%yaFIAbzyNqW4H!NtKAw_tkCPaoH(axNt^rQmdm ziN0cEIf9IqbGGyC%*(GWk7F(`g9xagq`XVm2o4DAfO!yCRN>;neGt)`IENt9-4^K=(}raENFGHR>O8Ol44dKwzc2VCye zJ&YW_jw6>@ROze63?A5e5fX-eK~`@2xW?(XEaJzXV~d|Oz?4H*&Q5pAGi zN?9aDMPM9A7tBZwmy6#|RbcNH5XcBud8%MU-t2SG*_GxYG(Y#(5Hr8eOn#Mc+q5Ic zI`u6&;pO0mHv^b&4_3!olE32$kGrHtpc5^xXrFT30Kmjd7_V6?wVCMOss(!35VWU- z1Z@J-9yuwW3gfi59ZW_1b3ZqhRmt%#35Zq>B>&l1YIxU7*3hv({PgLW(}$yRnWFTi z%@N(3ukOq_LTZf_$QZc1p;sCip&3OZLEKRUKXB__4dcSyU&RkZ^+AWW1T8#Jf-yY1 zj#g5r2O6bg8$_Fed~7)St$j#;sSZSC12iPAwijHo{Rvwyw^W}j+hjtA{Kz#t zm9l1Nh)QEKq;z$&OvEV^#n=P2nwXqLG&bj$TsR;)c`qBKqa>8`xYZ`qdj9+%#*5ii zYH8%1>qs4ZPhEA{{#)mJcu|xoqgS~QQV&n0RRnX=5z<|hM_|#%{=k)>`M758Y4(G^ zfTf&D>!@{>3!cvRpg*A{QK_@e^OA3oVe(};_Hz}OOtLol9ZiMik2sW*9?&8Ong`d8v4c=d>0C}IOR3ia;lZBpj zEb%Tnw=0)1A^mP!^|ww+5+X~xPND8T~_rak8>WKlBl9%RPeW~id-sU>n39VgtX(EoLW=mx>FAA zzhq1?(sBE5!U=cHo-#ch#7|yFDfYn=|itYmH81C%E~ZfI@L4p z7v|Gcq_TzOoyVUAf8T=P#AXU)!+HO%lndqF*jF|^0ggFB^(_F^D~<=rgBJ8Erb3Es zZ2YRbJMXAxOJ;h+e)jIInueKlE0{nC*PtF;aQ!Y0OWEn#_x5;`bEjp-v+Q961qut> zKM_1I%v9ANB4YB$0IUZ>Q%fs7G{q()XqWwq(>@=mIL8PTI=HM>!l$UcaT9y5D^Yz( z0@5dW)F{pvaAGX12DwoP&xGub1JNFptzV_7b*SgnA0{&Ctag_D`~yoTDs9=#g6fwN z=tD6LBG`h;@pvfX`zkOCHxsgvhgQw918v({(8VsDQlkjuO6*YZM`sdu-$&7lSBH1F z$(->%sp?g7ae6yw;mPUBnL{8L4B88NKmbvx%J5XCSAQUNm;2Q@?#{B+ z8LiFR90w{-@Y`0q^zO^oIkB04jJIS}(B4HJAfD;nc zm@2vm31KXgf&9DRCxE5H&hisF9F(a9WNm9FhJG>Mm457pYg09KVU(S2}?%e3ZL zrp*+xh^<)O99}-8rhAbUm+THN4|GtC8nUpfH$pVW!n?_0SC;2CqiJ&l*KDgm`Ymhb z8KN_T5^HVYz3voRatkG2IhoioXW)gfn6JuRa;nt+2XpVp-1|`;Agu3VLwWqLw4^_A zkfRwDkig`^O4+AXAZjkbkYv0F%8l7`abHX^CNtV_coXj^D!3M=HP7jqgFrhlcF-(bzfh5fvri=_t@232@6atRLqLP zYG(oE57v6@utoKpY3Y)-@zGR&-4o$2w$#pn2}e9GXF2-fN+Rfzn+*nrg<)#&x3C$L znL?YMw!S1-0lM5l4>a#bX=#|eS=B6&9HwT5w zr9uEBu}_`X?FNs0$PO|q^$hF%zqpxI8B@K|X_s{(5Q{scC+>UJW4vPSIrY&- zC%&A}Jhf^OL**MA0d6Q@^NjyD87`Ae^|cZvDzlalin6^{7`o>n!?^oNLQ&#y)(%x= z?YG|5=KXO4S(fXyy~0EvD1BjGi1bg9Tk}xXP6WvgU`}O&}B& zg8UVqwBc(1!3b0-DTV#|^&}w#2&@DX&_rb~+@OVwG?}77c151{Zbhw$#QY(M90zRU z0B;zrEJ~pBN+EzBn@l*a{ z2bn%ldA#Qbba<#2|eb9Kj*Xbg>4z7BQJFgYU75e?HZx2^GAg9bA@3s?ZKK?)us zK!3%5^BoXXC2VKB^6=)g4=4g;;@Z<}3t{7HS=a1tqZeYTbvrE zyg;>AAuXSYa@JCN9sV+bRVbiuTQKL`uTR=mZX=$GaVkcx4qeX3vBi+#D6wa} zE6XB>?l@-q((Bs)(<0^kx8uPCoZN^;(nSJ;i_WQ?ye=PhxRpC4o|Q0^|8@I45O&9& zeIO27jASp+@9lc+7o*Hcl>o{1I|y+U1cd^Ro*P}-GhplcoUdmn25qOmOw#cmAV@Yo zCuPEytdpE!!+YCi>3_urdG+q+jXR5aIoHq^0~ooAH>)|(4KXW)*UTz?_@D;~QOgIk zdr#p7(2+8)@E%|}fTv+uvgoV!%QwJChK#%`923LZ(L)0j$*tF+Dk8f|3-p}ac=GO% z{d#dTC4}82l=HnVx7---&kRz4=pLwb>p*;fD=Fc^nu&CegffsCSy#pbondYxlqP9L z>KOs&{?)MOx_-FrlIzig>&37$ zv66$}AV{Ln`zvP@bH0A;LaY@qmwGQq!ny_tTMI_~!RM+y;=Ha|T=cTJBveCWYeg}w zGL(J1_2<4J3m73&3s1t!wtnaRzCDN@3d@L(!ZVqr9h7Kn-@!ohC7 zMDGXz2}{|o%-%etVu))&fzVXhU>DhxZQw;N1hE4ln1(=`l*41P8PeB*pax#s@zGEn z2~Pm@*0T>Nd>8`+h_B|iZlgxs4Ro#WNnIW~bcZVm|I?TvVdD1v@fMdb(5r@@YB2+S z^X^i5R6VI>;}O;8o88l1b`%QTV>qi{)JvfzB1VsWLgsRC?5WkCcEr>;GYv|dsIwjOdzv)_Q3Cy@h#2uZ$*|dxlFw19I5_O*DXwH zP0Yupw*6;`xq{!ofL?@4b@vSsNwt3IMhL>Y(2#GnGXwPZ8o+96YO|s57I~K|oD~jm z3;yZy;_!MO50I~OLU^;&0uX9+`3s<+O?b4JezT0)`3Sy0U&3a=dPDSE@{iH48c?MQ z^97o(GqrS9zUko{{;yJ@z0HPDMwD_#&Ex?X{0;RaICdEmII&{k%*Rrc)~bGbSsrun z4fO2;liO@`s<8wwj;CR&(=tUY#As0%89Z?-i|!X4W^U_egbRnGt&?8-#2?L-tgR@j zeA&={=_b$DV2&v%xIne_0cdumk;iOv=4V`Bqya2tbUk(1yzh%Hh=%<@h>6(fjP-%3 z5RWSxpprdNu-y$Q>euu5y2JerHkj5>E#REi0+4u5?Jv z`)4_I$JUu`ijX|46RlsSa4tb*bK$b>fDvFV6sHHP-~;!gGq`kiX+UJDujtq+87q$E zR5;nMi^sEM2xB!cS&ip;uI=P}GD~@)VHU zX=p_sO|BBwGa-{HBKWD{^6ki3&TfHAEQfyFwh76Pv9;?-&+z z#x4=d*U`5BQpiLjb`=Cn&%E?F7;AiZ4;3i55vvt#m@}?U* zuyoA$1}=vLBhM6&BdO?AG^v3IOA_RIe>3(&b1&GWe|dO7s*#T-aEy_>1S_Lc!nIW@ zs!8TdA81MM`aOkItz9}d{>Ty6YEW}A7{(eRfkOenLZsKhMHQ$Tbh$~`Hfrx@vPGLs zswU(!aZR%%43tct2S2v0O_{Bq!Hk<{uLgTRkqgsLJ%c^4NP3>l-*?)2tbKcSFgMU4`u zUXFn{LtohY(();u&8Ve2r;G5Oy8nYPv3%3dj{~#km->W=SuF(Wk=qrBe_^pw{wuA8 z<_AZ$u0ZTW7H7a7ro9lBN5fpLEY54z+1XL{ISSQSth1JL6&9MZqmhqm->w}La(s+w z2=N6Au8T7r7A?2m+>L|63{M+R1Olhqw*_VUYRuxNz$pOAW&6wM&POlRDn$GYjRU;(q zk0rur>|eu`-zHnVt>ula z;0f1HN-xzFm$_|Nawo2P;z0c~;ay`b5~s14g_6tWnp7ep1vo|K)}_cLLL6JR;;i0M zbuU6yQI13>p7}+Grzxc*n1a}WfP{oundg|cn0v}*|LH@)_!Os|lPpgm5FKe%t1GAL z9^-{2E6e1`df)7L1^#qAEb7G)sVsS6eOtU&;7A4{ zK@Ya+9{t&A8l>apGy&`PPMZrq?YHaM4zSsp4`6UAcXlYu1buKz4y#iEzZ{nMlycoW zNt-oNd9#f{a)AP0<2s=<-)&NP$k)TT zTv5o=5$cQW@Jfe$CR+zju<#ddI0*C$28;4Vg0ROslXa31@@5?JZfmABPPfWy{E25! z{OURJtFf&P@g-73wOC*1`~~o;`~?tuzVp$^Z_5hj_*__8N&G5;ke&J`1XFCmG(A~2 zqXdUMLb{o;RK0oQ1g@vy*Q{ryJ#1Z_{wgMUUWIlm!hp}=w`(mbjRdapvOl+&MyZ7x zS-dDBALdYJ?_@(m`5B#}!^ zj>srgWcXq@%4q)d`Ug_@jFT~nVd}iseIv-qQq&8YH?j_&g0k2c-O<#*{WzG;p$RU79^h6m@{f0Mr^r&TT!uXZM_2rlKM&C=lo|!a! zUm(m)mbEv~VY0;LlOwPTO{Q8K8ZkLu^YS@}-(%RA8bk-oo#K(U5zQ`F`2E}A^+JH^@0A&54u+A<3r_6aOrqNwCWPC}pvX|a z1~VYy`b}x5yZI!%d0O26!6vm@KZw?-gEwg+fQMgE?sLT2l*b_@#?qxq@ zlH&f!v5(43gY80`pHv+__uhCoM$7)9rnEnqYTT2EjeV7?R8^xCk8XJO620-qwb`<9 znUZssn!xrqBQ9X5VnB|LI<+0K$|fLGo#EneWo_U(>U)-B!&CY(3wC@Tf57XUF(|6i zE2yb^24gCd_1*iqbj#NnsS!+I(N}yS>Eu9rIAFf^46 zv?n`XIwGM@ChHWzaYX>dXnL{yj+_+0;UkI+%U@li&*kG6g%!joB;(6b?5y%uEFD{6 z!{6u3g8u-ccQ!Fm=BCD9^wxGWoM1mrM0w8?rIQ_z`J9x9se_0VD1Xqa#0^s4jZqq^ z@I?;pa2_od&R4De1(dU5(?o5Ck|Qbx6e$z_GVxq@w1B{MuJ?$_L0cH$kzJ?#!n@>yftAeT*n8N%hy3m@AZ~Lbxo`ic z$v`HJi#-yQM|Rs3v3A*&z8RP)^raP!p3AIg9Y4Xq0FLJ1djup^q@5vFE(9EZ9?h=K zEzZkwj7`TQ1fm>DmutMaDIF5@RA!*7Dq(Y$1nKV|l{4eBq;L{XiRl>=zF2hsQ=x_n zq8lKmO(QL=yYKDnlUZtF?zsHXenDy#7&M=qXZ{-GdUJCl`u_iT?I;0G+&2#ExxLYB zoygdlDYOXOplCdQ#gTk70KI?pgaG#5+TD-NSk_@ezU8vgedZ%vV=X|6W%H?ibZpR@ z0s!5{lt@*JOBmf%-|$1A6BCu`r4li~Feo9_$MuIn=q?YURY0E2%W`s3{s}`-3^ljs z+Xf>ZQXVLm<`Rn|DyclEJ_sxE?Oulw=K%Nn|lj<+6? zRCtRYFoFa5Wt?~lxKLgleA@yu(i${J9NWo^{Wiz%Fx}fyv01fW#e;La=`c= z^5T%W`;%4p+S+akjKwK2z?f)G^xhK5(q8zX7?^uBOW<(ieR1!c14d15PuI+m|Mg!8 zHdhutJ}&+RWEk}qoJsL6jrFeXoIF^HAprOE|6ai8*))OvWjq1`tr9t;2ufloKCRzf z5~o2Z1As9vF;z1gAtCnDSUsb_{6iyjaUM!-Cz%8d^I3H!Dc}(sOqN%}IX-j($-cZ# zTIU4GjxrXgx=z^_$U7&AwqSr2lz9XGV^CUvoDWBpP3{mK@fjBpL@_1;ieoF;Ig<~{ zWNS?~R%xf^^u9mgm(xxXrtciBOXk8tBH%1yS;N*C_Ugk0G@yh>Emq6;$)9dN3{0l5 z7#L`8YkU`OSZ3Mz>O6%L++9G%?u4FqL(5Z_B@l{1FQavHPxY^f zaH$5une`s-u&nF8f5b>>)2?*`X2ux07Hz!=09ZW+3k}ijYY!1#!GR^!px-)(87#Wa zaA@*yo4?~%HglGmuX`hkG`H^IdP7)06wBya8H=#+9pWZ@0)M+%sfZWHV021&T%u#{`~^VH5IVJ@1e4`;8e4PoLfO5Ahll4|?adh`Y_yKW!>dqW z$|N&3X1-T0z|wb<=VRjT$>~z>UfD_k2q0xvx-kZAdF&3*WaE===~aCd@xn+wN_m^2 zzJ7<;>XtjGXLg%{`l1OgCNaVQBPOHq5P85{Eg<>n6i%cjMpne_GhTSXbe*MXxf z@#VK|Ik#Qae6556sYa@bxQHlO>+^6>H%C#K_$Lqw24s+G@kKZ8$)X z;O_434#g!%kszhG6faO5io3hJ2B$!AD=q~J6n87`?(lqBYyI!WzmvU;jFF7YIq&N_ z&*Px1MN&3j!0S=Jl#nRh{>xo{1QyjII=Fs`Rg|wzq@;V!*A6wks3AegsT%OwFY^0E z?Dzj`?XinknKcvOxV}0dVEVHoQ49RENtYhj11XyiS(AZ^ zet3>-{doWOR)qh9M?-1}_N$oCQY*{qKLki%-{S$!UbBuz0$gjwpCDAI> z>!0s?0?Rsu!u~HP1ttmI^~DhyZiRFeZ*aS+*iwvw?NPf-wvs-0#IgsScG;eNd$$pV zg(RBvIDe%=X8lUC`1ZoKbT@GxBExBA=jFoRbxZ~(r=!4Yqu`}8@mh0^Nv1M-NSl8A zRlKq>boF3{k(I#0G?|^A!Q{AV z_f$S+Pz&WZ>R^WUmfhPyyU(5smA?uMUd^QF0}++X79G3lDANPV%T42)CGm9b=ZD-t zCTN#NuCbSm0x0P>YTU{Imu8OY{u2@eJA#gyf(PpgL*=oG0yW`;GRh@7)^V-G?ec_) zecTyW-0k{I+HtuClhqDe)#eJu_`u)sak!XFbsMf#)1aYV*x9n&ILCznvP;q z<52PWu{Sm_}Ve>JQUr2p!` zPfd9>SFHm5EyJkcZ-LDbW=;j2aKzq@`F@G&ac!IOqilsEmS6JO`n-mxKCQ2npbG;N`;X2kS(O!O| zx4?MU=%tOyc0(=6{I}6~no+JOq%R->!0V-#`$@~h-dCiYcklXmrOaiwvP!Nc8Ggo0 zKUz18agb_S3L+(qTfqXDtg&_`j`HFMxeVtl>F2CJt^5t?H!C-vV^pXA8zqQe@6fzY z`8Qm9|Ez+Du<#SGO*tJ{ZO{Xiic?m?0R<;Zb%oTJ^8xEtvmji4zZ-9?tTyxi(s4in ztb$||=c4DZIez!@#LgcC^6SzRuHZJY46(qqhowG7f>bNv5^p6)mvk@;5& zZ9F7rf4E|$eX9+0jVv-f-EYCC+1QHc2FEds2NU93O~B7 ztX+=8L5w2AAQMFd=i76u!r zN^zEqW3s4hNSOZzz}-BiTXC{ayK8DEwEU!$A=-?P$!5SX*snqKQT9t=Ej9L;zB+%j z0|U;n+Q>84=!<9;n_nTbQ_X&s&BmDIbihFZ$|xy#IzAG78gOtuvl7iu@BVJG_10DvvYxG?}^UDZn7 z+2g;b!OZ9UK(hG?r*W%mf7UOrsuK!y1X02@(hc+%(%(A|^;}|PmT=K~uW*Id^s(ME zd#LGt5G?=%zVnsE@;A_9tlyP2fk^KzUrfB>n>fw;H-dGVKf^#3f08NIc z0(Ka4xcyZ0lWN8R&JA^te~<~ET3xHa)V_Lc00I4EFLsG-ut-!g`i@juRR`5Rsn;9@ zvlGtMbK~)H@7$?<$;H$W0%R0SkEkRn^YIm?i4;^FC0ol`u~f(NF;RSk(CDwvHO}M% zu*lpuFpzxdt3JjKdf~!02DmvnV4%VQd|S@)v!Cs*Yt-~FkbC704K>d+6|zt@F02K z3>|;SuAdK|f|%{)cesk_X3tu;9}umDMDqQhsI%rDb!DCS(cR2o_N<7%OoN&l0l|#W z!2HvZRwm8B3peCF!!V_kArO~S- z`XLbckOqOcA9!dVo3}R z%+R8-7o)>Pw#mzzGH_q1x|s)XR@MS09AlPB`0}0dRqG%xJBNw&jQl35kNZCJU86xj zgX%Q=7C2kzRJhR}K=k<&PJ;|h)5}D#b+e-+W&QvyUNF?1`H|M3_hNLSES*AdEP^rsN@g;c;afwgN6vdD{ z5_Ei6U!|(DI5?tj#nWu%bXYz;`1v@c6Pc<; zkxM9=$jScEO%lBDYOx%g_LMuo;812qj;1xijF(|0E2CYcJL7KSC&$jR3$>?_>ss9v zoF3X2LCs*FO8V|hk5=C%`ZrAeX+el`@_vsQx4!LO*kuGGik-*H$Nw@7eV^toi5^Ob4TD#Jr!__tOZ0M|vLVYs*q!1ICTqF9XzZcNh zfMvH3bPV@nwOi@zkz5yxX8`E{HOPr5OV>>b(4>g z=66g*o#JPL)$9 zcx%1;6hF3K`ymAlGc&GHJp8Gp!hbS61fV)ul zUOKCmMJwKtq-q*BxS3f)mxbVKV*V563)5dFC`otTh*GB2y9MiD`0!R-YBsDnNeYUZ z4&E>9P(t2K*x9^n54{abnlj8n0Ii7ETg1X6HhjBl209ykKIfZLnrEl#x`ZLjSwa{KDGs?F);Pk7t+e4Kuz&0)VnYB zP6aJOk`j6+g>xC1tE`aBs>)axQqzp0!If>)F_DiXIlJu{){S3{6|=XKf+p*kGg9}L z&5eY8f5;>&>(I#2v%oELsNv#E;m63V%zbs#>1apcrFW?h^x*T^S%*FHs+!NGO&P5# z3-N20VQwOvB`DJR{!;vchbb^C`Zwe%S8KlhJ3`S|K@j(_GCPPdea4N-|D(4c#Gfq%b z%+nZmy_A|SY#(679lFe30#q1*-}Nfs*nA=i%E4*ofUOcRqjI}lw!Iw69ZB!T6Rs(n zF-X;uBj1WNv}LavwYGY3@bD-3?)QYdvbl3$j_t>X9m-4pFF%f1(OznKm1v|YXU}cf0JV@D1~K5%q$pZ&72bV zVB2^;AhsMCer9`9LPq)A3%Y=ke@Q2-^$Py~7ekPcg^Z4DrqzY@dh^bg5_3eFL;2{kZ%oO=pN3e~HholHgona-u@zI&N zqlh=qdfE1TIC3hhmr^x!63mGS@QbHw!k%aT6L^AjiL}V$@K)^1trgL1@cqe2h_;m7 z+u=dzC4pTO#e8)cD_YXnHpLbe?~a8Cmoj!Q3O+Ba+&6 zWkJ{t`xXr=cI~2dSgP+}yg%^W1lm4>nPlcyUM~4d5%I(%^3iWcI~sNQesMCK@`J@x z(c>A9FOzYc=i}E~tWC5HDrp1PUyJXdhiVIyrY?Y~I*HD+kYBa3ot^rr3+Y{imE~Fs z2R$AuAnq)HAxX}rDXo0O%#uRwvtZy!r6=H3x()mJV(EEI%;24AyK(u%(}t3Q_>fl% zp!RV8Gy?mnE}3Q@c0W9Ch-`%cVBFJxNbtyytBI?NKve{%&Fdpfcz+Xx{ZLJt($sCsZBfDSq#r{@qLOPAOy#PQwj zSG|z@*}{$(vE%+cw%&zG3^_e*h}BiT|Bk6jB&AQdCW;W6`3g_7e7_ zz5CC?FS)5A0K~ud%_NFXZ7d~h?yf&vuD~dO6S(wvWI>~ED1(VeqK}_WfUsxuc;u77 zgb#ED{j&>?iO|%8^`0Y#+Z9&O;AnZzxhwv|kN5MQsS=YFI1Y2P#i2nRfu40@UHy=unf({Mj?S|G+O&Yuf|XWAN&+^`yeh|8xDxv&)@~ z!!|Fu3Aa57L?JKz=;^(rX|5m|TGQitq#36+Dv?+vf?z$7N>izs7XLSiZ(mwrtrM~( zxjXU^ktH`GVLPpXxxu(LbPVLUSBI+*C_)`C_<*T)YAUTZLZ;Dot6tPJn9Y}l+VH_1 zD9)%Zd_XHfwjE~qgG%AMS05rm;gWB})8IGRU?t!gr2(vz$&Rfse45dy`qF$ZvHb&X zyP=$K2fa5tJA|RDI5>~hRb(>te>D){2Uvq#PVyN=jSrF z^UN1GR{vydC^tMXnq;Qfs9|^3nUJoP^5ssU`x`{{tEHljaW zSR5PR4@;=>9-<5wEYqwBaC&B7cmyb=Q@3jeW{^9p3m?yKunZ~9HP;kj{OJNafWETG za}>fcmF|5=PYG5hKccNsnrp9anWz5TJbiuoui5pg`p(OHA`h^T8zo>m(HF;Rev^NI zpXJAU^jRMQA#v)>x4XI^brME@ z)-{M6{_{H6Z-=JQ+FjAyOfdvAzA@HB^NH`4FM0o;dU;p*}3*l zdP^9BhMcu?C3l3YX|?^ZR-QJL1rE%rL>dv93x>dtG24K7vzWV&))K zv4MbHyHPRnQfNalYS`=(gw1{^kUKIYl*RYWRCZnvvBh)HZZ_6IOQo; zt)>_QR|>F$;q7#T_6WO2XiajSp*Hf2wW;$~)LE)Hr;4&tD121FEFH%v`r~d-ngX&^ zvI8ORloF1Vz0e)TAZ9%0>e?B9>w!J_H?wvB8>dW_BuH__-QP6|?DzkK=@iodUjs>0 zV8ZKkD%((SHX*sSU=f4GbCzOHX}z}3qmuBiPoH}@rY$a zQouFYyZos@d51K|r3wA~uqgeHeN*(0YZrLL%4@>dERxg$Eto|6dv;5BJjVDiP36#| z#!4=IH8{AiFo^-jM=w$c31tQGKLCVl_|D$s-&HM}UN5UX2JVHG>$n2q6LN4v>f$^z zL)9nc3^?vE<*B#AAV3zd>r7Tvmy1fI0?cf+i887n@iG zt7M9|zO)-XRH_olz=q(ur5T($6GQ z*kfChzphexh6tl-1H()%ZU!O^z7FyM;n59L!oTP~=h^KyD7)-keHcKVjoDqdFKpn z@DM6!tNIo#v_Ros+Oe+d9=m^|*|BcH#1nXA)`|B{F9&Kpvf@L z*V+26>omU=n>dMS=pM^?go>>L7J^pD^z=$=~I+8Cx?v2j9j)Q z_A-L3y&Z~fcmBO6Lhfh|=alM5J=GN6_~ydG-Mol^0)Q;Iqk4u_FQo_w!<)X#UK91@=pzadysG@SC`37q^^B%|ys{Jojt?z`sl@o*=pVzDp6s*#|1xMtSF_caV# zJ<`O6weZL}2rvWQp{Yw1gFZhb<-VQoL)P^+79*>Bpa7ZUzGvX5HXCwAr(M4mJCqyAxSn9 zlcqNR21J)`mOU4lYdEOfR2daB%DLwv<$c}I9Nc?M_ooJbH#$?K>zx}Rz9Pb~%A)X@ zbJs5C__OK`Rt=ss$zo`(6ixnD&5+$pgXZkO_AY>{OSpyR>l$liA_U?e&z0o*P_Yxf z-&#rkN>VZ-A`uWWQ9o0@UG5(MbPQJ9zZ{XtyWG4yJR^MV5%rFkEZ!5a5S=l=9S>T# zBblksVa9*#BTY7Afw<+$M-2W+;)`33?-W?|&%LcIpvq7hRN7I0jFq6>f#zKl?qhJ2 zC~Cs85%y2=Mq~w<-o?rBm%@f11{^=7IAvGwZ~QW^bp6T+7N-R;OYU!(17V?B#7a#C zvgqqV%5S$3OWGE@>Yr0IlOIkCTLWMoqU{^c}Jf6PtP7ZDF!*U`mhH3llroR8USe$eA0#kVH0;%4= z_3F5m0Xr@YT?TS_0oRQ_Z+Fi)31;sFWL`&9l>QleQufU#0@EQi8kxUd|Jn8->)Fi^ zQctL^NPhb!gNAX7_AJnN+xR3MT2Bk}YfM(KC;!Tvy!T+Lw$Q;pPax zWO9mn8H~V*B+7&V;X)0vceHRCkgzWH!k>QyU@b5T&s2}&!vYkrOIk!*R2!|OE-=UN z&pmx)kU2O(cxNY&Q<_LnOmb+z!77lLk9J++;$>Sg-*1vi!CXX_3<07#iuZs+Y0{I9 zn(s3|71B+~=Mcse}WaoAl!!NCP@JhZ%Twq_*t4)3JfUHplNYhQ** zBYcakzS8*^zB%dkJ`Ma`E;^u>r~3>SsRc>+xN?YmRB{3t>s7`P#a zCzdCNR$y+CBp3ucb?KlHQ}$w&Ivup_O#!jV|A$^mh_KzWmJ4HD43U+f{s&O*joeIX zyY0i#hc6;pQEJ_krF(+K^VhJY;gXvBGO-z9;+D%9(K&2KFhR5DH5&;DU$;dtK%-V* zEok9Q4yDXD!PsWAdGn$tFBA|rvN;8)D`+AjxY5FTS(}o){OT}_(#>(9y}wToXIDgL{O*9{jSD(x!nujXmPLuHv#0E_24V*) zQOa#BH1MJWEH!**06`}t_fRi=sBxz8lIj*&=htoXGj}neU#qM=7()^F$HeNv719^K~JC3iwT6nW>r+Y~Vj{4i!>O0jP ziX#Q_l8YyyQ0p%kDu_~#IwXjZTPAQ9W2+Y+`0}KNl8FSrfCW=pX1)u){0Be=vJ;~q zOk;(k&}#Y94@QA#1!0tjG5+6)Un-t)QJ6`k0837Mb4tl-{ifiEj^b#3E1arGEoUZbQ!5hC-)xzPolw!Jgb}^ax{jL zIyCX@ySuU&LOB}}v$E&3aOUR`21LX_tiIXP6^DVkiFm}$S~t@%8LAdHpEI*NHuYFF zz$vCy;-hcnSgauXYa4P5gYH68DTLD3i5)lCH1vQt=D`;^1sNPDe$i_1mY*56xg)`J z;FTtEcvFMJlJ{1g^*;O`U~5}mLSKWF)a#h6OSbS3EGT%0d>!z+Pe$dz_5lIf9v+Mu zI@0}%@Oi@9)T8xCO_i$q&@&h%^_Uu-e)6?YHOFCg7KDm$EYQQ+&m5q3%?Vt}tmmV@ zyQxo#!p>9o+=M}A(`lVvnppmKG7$`$$V6wDK48nyPi#P>;V~|V4sio?5+n_k2q0_| zJP6%ga6P^%@E^lo!~UJVUdl~tmoZwbzJl-jfD*JoMO8rXS&`!-jTBSTv!DNK{X8@3 zhwn}nMz9o7U<|sPk^r}CKbRJI=C6$+w zj4E~Eb?>g8I5F>jdnYAox1#trbr;J96sF&S6_r>s$cy5 z$ImN^IQ*-ICMyJ}hGQxND$A0HP#c#hjAKNH_I!zg2c5)Zos1F|aL`5xCI73;ceymy zFQ;`FlCP9nr#r|*FOI@Jq>}~Pdh|L`N?M7cMh-gu&Q~-lb(Mbp+^#dtu`9bO2}J zGF)#ublk7tbb*aGzmoV9Xy>aVy9uUzy)F8%7uHkqhkt92YV#gFh#Axt4H5TWB-DdG zJqAqFOa3EhAsGl%#M2Ja5DnqlRt*W^-Q`&mESz2lZzf=I#MNyj*Q$ba4yA0c$badm zj>9drs5f&lY^5H+wnSSZJE?USC7ct9CzJdz>7x>hroDrxmpI9~wj{WI733^Q3WUxH zwydBily%mpqG+-zX=t)0!%M?~P}wxPQ}Yxh&PYJnQA~3JnZf@G8Ot&_TH@MwKV;EB z((6_=t3;zoNe|)J0o5qe?Nhi!sdw_S(K2KdJH0djJCIgMf2Rz7VsWz)X;UFuyi??$ z`q0F`&x~ak69R0~$`UQ{wHaUXYk4iVcWBH2uBe$f)OY`J zwJ_J3#E9E$W9k`XM>B%!M$a4R%nG?JQpQXP9W>v6u3?9z1uzFiip+b@PwxpBlNNl0 zmJlSs6RzF^oW0v2=HcA$Ch_kl8cVA;7!{ibL{}N_RTGw*=OR>C)|lDl2(>)28R8kP zL<)gpghrV7do^`(HI;oEX|R(Jw1sC)4DZkrC&4h+!=CM4^a`Gs2Olw#JhM4FspMJl zRra3m?`zhw`WL*X(~0ocC3BSi|h4SK@feJ8T=0-DIT8ezv+Kx1SP@&qr(xbp~l_BhZi>w zEx(dF)mLkL*NojW|Dnf%5w6XQC*wgo3pHm`nxy{u@zvbkEiC%Z>g=I3%+k8eY^F4P ze`Vx`d$WeW>tl66MBwk|8r>{(JNbwYZgg}T#R|mqHYRozZnp{s4hXM8&KNp`rd!D! zSk07JIPEk+(oLwyl?Wy`mjOU zsJQ^oT3&%a+>L0J2l>Pyo2~ZI-0CExt;#LY09#vYo0DU%C6Y;+G+#x#H^r9tOHu8_ zvD2vUKisa-bp#o-nA4A&dB*cy?o6O}cp}Bh+$k0*o67#|Iev26<|YFOK`l#zH=7ij zl0I2yXta@vqZ`=ASK?rJElgEcSJdq2KLBA+>-mya=~u(LORkLrOwDR0^p5_GO3T~& zk8HO`D9WA1P30=lveDk!KS-2p9eJM0`es?Ve>AoZCX>67rl>tub@g6yw*l>&e~T6D zS=2GNi#@WIjoY(T*oPpB`XV!SXERe7&U*5fQgWTRU%HC&0kPy%Pt93x%sA z8LY6u+#B%pk#Lygd*3;yRSH?!;cgGuV!Dyt)VmK7pCOQC{XCOz<@kxg%5x{#7-DP~{bTN(8a-66wlWYzopG7= zIn9<SqQh;(8 z^vv&th?d)Z4=nqI7vp*VIG9MaNqRp}6&Q4iEM~?k20cDR`>;)E8Kr6e3Qul&58(Kg zsx@EerAj|4+s8t&+6&fD{!}SQ96Q3E6rXm?^c#pfm@R;&85iPY#2;7E*j@EUj4eJX zX8a0>zep59dCt#}d*n%{=ojW!@+$DoMKV!YL0LXw=uN}y_#d~3V4;B?NDP)&b2{^)^JNb%c+E;MXl}zMQuGf2iq^|ha)_%3+U!oBnCe5>U->pC@u@g zP^H3n?K8AS@|%$AHIinrNlosGk|}rsjT?;MX2D6T>sL&TS{7wIjTO`^rL3;g?# zoL%9rpPs;JunQcBfiCA}^V4!kr-w@GXA|4YWP+evyqH0_rwOR_b)U)QcJ@+>)6z>x zsj0WFlV<7yP1UW9T54qkMCP+Q8y7X9QAlNdx2ORes)HN`EOYwNu#>Qc{T*?sXJ9L! z^y~5uorsCcKD~f1&^6X0YLVCJdHqvqF1>Tg!58(4(UZgJ*3=Gbe4L{BG~V{q_6;I! zGFL4%8$Eqg-8A;V(lA8N$q|q6FHt|WPW(R!RU78j+9$pX-}W57U}bGN7mn@@jeoEi z8gPR2o$2i-3~_zo&&#_i@s%CG|E&n;yh8rx(Jwqo%UkzLn3_)_0-=I&EAAV4rmmKz z!V+Y?-0R;laFP$Klo zIO2`YV&C%d3+9;hEAeSE1OiDl|2teWiy+k}{F!cnT9~ajm%fy~Oe)@B1|ulNM{n|q z=q1$DB&BY;+XVJnLrB3{+Xoy3CA&bui&qa+9)}d6tBz@lWrT`R%XJ4Pp zqVZ{}8xYd(ANz^ykhvMC2m1hnD>G8!!;A4gPC=}Ur95P9o(o%#ep|qt@Te^w;_RWf*M2P8IAKm=}wlnPOr~Q%!s3hmjMuUk|m?yA$l*L{gUBGu343ew)Z8#+izuq(!ozZ4yz6JHNi?oIT7^{ zc(nBJT70;fIAVh_Hv_c0vJmBugW#1-w)_H*J|LVq`W8jB4(l~v za-HDxLZ}VK&zJO6w5FE)?9n%YTpU|KEHN@aQH=5h^Cjh1w(%XOTY*T@hoYOpmTq}! zJHl2LdbdQ?=i3|;G!TOLeRGXpt=QE6jD7}esYfKeXkVUug2MmLSW+NxbhW_(kCaKv zR5qAQEAiwqNS~|{*FkDaLN4-_tWXluv!Z$O(bv3*seIx;fbjoOfK~n%*^5C3rt>rY zOGVk{)c)tC_ForwJhuE}XI7VI5WUl$Ex?T_eci%_&w>tK;QV=jTIGKUuJbP=ox9ZG zz?6zCov`16DJ?H6SV>Ffa``k9uE9i@!SFqEe!oJ1*h8QMle&$UAntF{@U%>@U zWHAW~zrc*RU+g|eo!~iJT$tUX#7?tiV!C-++<5s;4Zg;jMnpdr1kvIO+mU9LduL`r6$^E@V%I=ueYKfh42&E4YH0NQ=vqp^lGnHEE$5Q&`3ra5kvRU? zWHRxf?$F;2TO(h}A>ijXyv6(t08&_8(sQ4T!k{sy`s}OkH%Xd@7f*g{BOXFt>H6L8 zI5!d{pnw?!mDyUdVf;u@Tz6d4;b9M0kE}y}Uyey@{oy45TppEUq8ofi^Toa#Mf7B7 zjdF*ce+i5#M32z2dNyU@tAtI%P8ASJ1e8Ee4LVtT>;q!c-;#m5|E)g6yu|V-n{y&G z{C%kcAFS74Uk34Ei;}&1J{PQ_@6Ze7C(8B)nVCKK3Zl?zPI@c$j3nK*!QKxi*dRIe z(BB{ke30Vg52qqZeCVm;;wU9K@`mJ7!lGSX<-FEp+@@3pH=+jp&0J)gm#fP(XDQ7a zB-^NoJ@< zu^Ki+^ko2IRku&D>z<;evy;icseqfYAh$$^GoDa&rM&#pj_#Vymq{z1%u;~j-cL(9 zjLag%6|j1(VNJ0eLBEwnqoLexF!4xbK;)0iB{N7?s+5@#<#GKephfFO{L3jR6KT(WOb;Fg+wavH|LQk{?;`zQ+Kn-t z_Hel$okNJ_rh%b+ZGw*il+khW%dPNM-fs@1EZJ`R-XW}BziyOYTNG1?a49AzZ7rds z9ieQU#b1RpATVo)nfmBnciQO&m(Ykx|MnjLsjPGQ{Wm>%_xYphM(-Y&BUv68Rsjm+fW5k;eV+Lye%Z_g~1WEc#gEJi!? z8`?b@O#^9897D{NW5H%a6E&>xG63)`$-rKB%D5yKGaphXUL<2%$)`t~6rNDwkwIDPBhx2Sjgvf>Q;?x%uy;j8(MW0oj`5s2GZ`U=s84KoOY3YjBx z2e<>WI+Qeb{GewFv|neU=-bdeS8mOHZRg@ImVf;IEmz6t^F7>(pB}(wZeZ0dkAWrT ztWNcJ_jX<<3#u$b;;@MJB{VFxXy-%0`3i~-WLlxG4}zyu z#ppHreib9Jx;6pJKYteL**z>AXtngPZAMXzY}&3j?t2IGcFnYMjoDFr^LH2_Cs1qA zu0ZE+W@|@Pf`v8T%Zr@Xe-`3@__WNW@R`e+Va)Id0WZE?6rh4k?PgK$S;4dIq}&rl zxp05vTAyW1zemE`vvlbtgNzp8a*GrkFKcvrv+?6cRSWbuyVbC&%6sNy9!s>9!g&Z+ zdChr5DCFB5ie(z|4Sbd+>(CKaaU7jz;80}!AAqg) zk5n0+MQl^7V0gyfRKbU}M+Ib(>{>U<2LsX+MX@;9m8**k3#y|BC8p1aBQaU3CYJ`+tk&vCOiRMmu|ciz%0O{gK7g{F(fR zEw-ph^q<=nwGJ)RM#*iaDr=E2L}TEiXH=z@R=>2Jkiqfo_oK2TmMm_=SLD^f&lh^C zH;v*2Y!yWwdAmBo%(%bHO(hh4yU28&!fB*(Z1(Uygnp#j$jbzdTGzJ-L zH@8f8@Qi?9g#r@i}H)CSMQTxpb=yek6uK$hF(2(XRc^T8c2# z5ia}ZqQx`VbS&i=Ei;=HPWD=us|f$aShe!{Nih#NvGJffGdR|&*9QLsnaz`6*Mhp6 zu8N$kNCo&qHING9j8~6(ri{$3oR=anohkK(wciBzw2P`fxuT{*Wmz#=wF6HQ+Mx7v z9pNF`6HwbSkE*PF%i$$&2F04ih#lYWPu*sk9B)bA2HQ5y1IqdZun0WJ2H3v;NMJ43 zVIZL3Jv=(H$x;t6(3qWg_&D8`Mw_1=_pwFR78A-TXygzMDa+8ies0S9_#a>!F%uTw zTQoM!>Ni9ZEL$4^{ep()y-KeZY`>?KRcY5$C6%R9;A^8e97l&nyBpsbXlC8T&_iXR2i0)Vzm9Z8p$m$ zGd5AxNW?GG(LL0Uz7sC&20PdQJSW?_HRjExFgW=M><~8V42)vGa4(04&MK~K>T-UJ zIiiGkR4-NEI%?Ne2{3SYR;ttjN?8U~qTx$VYzIm+Lr|5!7Em7C;l28IP-X&Wbs4e;V;0UWX0ZmQc6w{a z6xOPB2<_F6#rcaykiXtqf?T=52InUNc8CW`%c7@Av~4}+I@+!ktZ1y zJG?rZk`0LT_0Ve*XgC@V25lbz`ROPq0toGNn9yaY1Dw!9)Rl|f9;?I1g zf2cA^JB%l!0l%OXTY633Ete1M_ROQpx%_&|3CFKdKWkzV-j#Gpg;m%=V9WXQ+qAKC z1%b6ro`S&q>oq$={pZ=jg)1x@F@~@S1~s|eWQ@a|@@j$M9}fv~!f4Ht6#u#!$VG-~ z8+Csx?xkong=zDarU@3$6%x=F!&Lr>zlDS1H(EN-uzAuy?>0PUT z^&_|bkOtc%`gu6i&yu!joGF;<9h(e%ic?xyG`sWV4zm{L58HpGK3=$TO!RCfv5zMZ zH7QaJtv}>0XGu4(97Uwca8~Rd^oaObJ)Z2`Ml`sm_4c%d$cE}o4{zrrHD6Kcm=|} zhvBD)!U8Onw3be*as?TW#O&8u$85#yPwRwF_nSYq=hT)S7*gmR7is4zrWL7K{>jj8 zW z;%LEec_F!EX?(x+XSXPT)gvdmhcr(@Lyl$iU8XN@cCj(19bQOw2DyuNX*TmnTAEv$ zcC$`Xns!xD-T0b&M3%lX;c87=%bMjNPAzBC4*qJS#$_d|5-uGT82T|yLc8*(w$)f} zqT-cCpjS{Mwlq4J%dbaJ_>__l6gN@dR)gy@F~zE!EG+A+9Z!Je#mtPKnp7O3K^zyA z>&WKl4AjObUWef$Mqk^%8&d78!cNDfEB!ZPuuUwSL)eA6$jz;!$K!7dh)g@h4sCb} zr3u!)C#9Te6Jq;#Q#>QFAj>^5irU4OLB?F@FznFTtA4G z8f?*fPw5R<0a6_|i8`PD7Ee@;@%`rCDUs~#1*uT#G{S4E^^;27Q3W7Gj>%-dW^-yx zV;iS(jsKg3Z@M_3JKuB`Q=t7nD!^Ruxj=xx<+-1^XtHAI+vH-xhRBXaVR?^Gcs?}0 zb8+a9Yq85t^<^-=XUFey8Z3@VfKNqrNLI=}^X~gb6REMZMuutrlHf}l-gAeMP|#&K zpcb%P(Q{1(1BJi@kQl= z^@ES(2;G2QvLmhIXYz~S4^ghIfsDUlFGiB#Tr<1RB9v#lkI=I#0%s8UC>$>V7evAP zPLB4MbH4rEduE(^Bz7*mH&mOLcYgm>3JZkrZWb0lId<2K5K9~0UJIYWYfaLJu)t#@ zlz+s!7p@4T4ME0@I%;LD3n=~0WhMY^KfaW}pc#4%PvjEvJsT^yC#H@;NAU;L+8O`q z-k=Df!czOrX+HIEasniSf=vLyIA>zr&~-ooRc_u?k*Sw}5g%sQ$<*Wz4?84?IQioP z#Rncze~$sCrGuGZ`946?ifmH5`{7l1y0R>RizcuEw-L4PU+&;hh67J7Kpys4b!!BIG$ zeDWuLzg?MeUTW#1aB{+W7DzkNI6|?jMX=uDv95)0#$Vpn^IxsBbHAv~`}-cLW4Xu`>eqqlcKb)Q6|(R+HxZ^}E}|FF(qyRSL&8b( zZM6@PdrcdYcnEG&IWd#df`ds5^e};>jeFR9*d0CzH zlhG;&h*8dpror=ZYNmA{j8Z5c|t3b_>Mwh%BO z-G?3b(p=cS+F!>sUQt4_J2z-lqyzvQX6J$_ec=9}U@E_d$L(XyquJ!rrF$iUv%GuPR3CkgyAn^@xqq1=Lei^7b4+%jP?=FcS&v^XbOI>~%P<+~c;YSfBpIpD~R0f|G0IZrM5|_VQ=|WS?3<~`1%Abp?foO7?e3Lxw zth!H18rcaYt;bL?dILF4Wpb#;J=p6-$kgsawwtZK7X*O(2QvW*0Bpn|uze!duT)O? zVqfzkuRhq4l*-#i3CY&Dvi=_JVCU%aMVDJo8lv2@l?q-XCKQ)ke5`reAtO? z{o7k<(Plk7zyBAC3+dt`Ha6$t?=uyb$C5E)NCg?3w%$8D<1vpq8?}wsZ&^~*j(6%% zxc>v(7#pK^aDoz{Li|q5LHeJ4(;wR!OsuhAxk)l9ok7(YT=QpngXkI{V42dyO{AeS z_pLnWS5pQVs*H@!`aeK?hBg}0g|uHU@#5;sadMGjU zQ|W1?m)yEx01LwM$6!4P4%3u8;z4s~Riu561VkENSP@7KUZ{K>waik`?sOOVcHH+y zOcI0nv9GV^%_xY`jT@ zmtdhn=K#Z7SCca0l+Sc;yZ&0lB8epLufjsjDG9WIwQdb#_u?2k$f|XD@RdU zlf(FPWhIbB-dX897mba-#n6r)3{h#Rmkf(0p5w^&VCN{HA5C+{yMj&gZ0Sf?aZi-q z>rBx4{ed8g7AxHM> z(QkRT{wyf*6G38m6Fper1o+w#2;E_wNAgeSy8+W))rqb)c{j1HRpXN_W%F&Iml#d^ z6AsXr?Po=K)Hk^SaXv*;+F6nS-Ow?o8K*XK8jfp(N1K)o^o&2N0xAjU7(CqN4f`oJOyx>D_hze-ZAvaWqKZHH27+( z>yR_U6U@&T2`@q(1Gj#2Rphn&ine@iJ8*Z(Jx{G^<4iw{nb8!l(2z=1xrTst6Q{(E zcH)z;#q@?^pv{=8%O=lD=Z~G4Yqik@!7#(-&vr_EcmwRs;t(#DsYw zg0-mUp4U(B*j{20{JLzAx{(zEE3hc@ijO-#g>$8ZC0LWcCB_$A46g;*hloFfvHHww z8uv8X$ebAXw=~LNi>WlV@7+BOJMMMBPdR-`SJ(PeR`#QO44qXyVm^n6GvC@8m zn*HaGR(TEPO6$JxD8Hv0d2b9O$fksD^F|S1eBO%+&nX}xXLM0@wuYYSYM~frbF{0u zTC>{DQ}Iu&yt`@b%~XC<@kYgd>fbFH)bfhA5VY{7RmUt;yB1_K1lze4v=COJ1pzo`RfD$m+lh1$hm5SYiuX z!}*14;f9L*V&-wKk84cjzO57l$mraCMoOPb2Dx}JXX0W2t~7^vPV7iV2F*ztBGL@hc-)t#_aDQT)+Qi`68?x(d2kg4ZVWcRbQFpKe7# zVKR!hwaiS`M<5^Gh@?Y;A|BQ+0wM_2E_W@ZCOVh*?h=`EO&uGl%=T4+%y$0(!uC1p zcZkMse6Q`YdVTm6l~q+S%Lbz`D@{#H6?Uw{7@q@kH`S8&NHK9$5 z%ol%bX^O0z$rcm z%+p-PHr>Z4d}f^PU{C4lQM-I@qN`+-eHgE!Wo>F(JX!_8$@=ENzgr8|5p34VAStha zmC`ybJIr;*6nu2XgskS#zAXdVJm#!0JCR~nn8D**tSUS^wGA7+4oUPIvM{AmY&Ar(=kBn_CNY@L$nQ3u(W-LjWmAa`MHf|iuxGZb2i}OiC zhzNa!qN#g>SDB5Wu&MS)90lP>AF&w9SnBZpoT=*s836soPrVwRdP7xasXo;3wkUCVC_dJd4|s*lMRLq^m2c`RD7XFPo5S<0>=plee< z!_oHCznIf*$-W`DanE_uZD~}id8jq#?(JYPlLudGSKkbNsG7kpD`t+$cCrGps56Iv zcaxOUqkDEx8H6LILB)e6vX;$m9UkaoOnsKP6_8JwQAzV2Or}bI68h3+L}*SuTe<7f zNkqL`s;^guavjRE-`M!n`yuB#pf&8wjP__|RJtVmBFUeL7QYO!G0`|gyRn2n8_aLi z@ioJ&AzinEr(}dRF|E@Q$9DIKdaIF1@?sub_&lrpjg zu&vis-MFz6+)D{>MP%o5_-Wdo635z5=ao;mpeoTWyU9bBR z-mCQQ!wmlGq>C@>iuv36N?iM!@~P6hE?=KsxPX}*e&z>Eu>sWR_YR}jtHxV`xu!b; zc@-m;v?=aq$ygrzNh?iNj=C%A$tz}fR&jcfm6=L?puI@!_&kGSHJ94=qydqrqI-o- zFDR3q>n0AXtZqVS@XI$f>Bv;EWsrj`XQey=Mg>moMhWNl!ESiYl&eXT6+{E&w7q<= z26N{eZ}NJ>Ainy7U%tX+#f_=kQy;!Ciz}C3i#$$ZWmlN5p0Ct0-PguLyu^{;Nu-Cc z&Aiw!m1tt=ABjJpMpB0b=!=nc#iaf{oSA$7W9n42mk;HjN64XAdk-|#>dn_I1GZD2 z2An^|joTemMJK2Ljm=(Oyb}*80!ot;NyO)}4jpvskJ%T~hv*bzjZ0lUPg*7^ku3rf z{ND2}->KVd=AYeaeT%~SdeK1^lk3m~4FXW1J{i_WdU!68GlUd`)xUl4hr~X}&P*Z5 z!A=}$I=h>Rn1%=OaaM;E8dZ{(z~iL!jh|AW^A5uLDBE<5c=2$f*Bp_z+4y;h`ozGg zb108y&OVGIq4fJ@KwAQvxDIgYb1yvi+e*N&$to&l`qSO{n%+s9Bu~XyMZnOmN)$dY zd31N_?JW-oy!~Z=V{S##OWH^gu-cWxL+2Q1dkY9e+8x6U7hT;P+~_d?351}_f>zHQ z5p7fbVfG}jfk`kTK%Wb|Q>-zJtlw}pcMOG8ys`YUZ>9eLJ?s9Jl5}#L7g1>iY$a^i zw;ZqnI9L5eLD+I*JJLhA{~v-&O%oQ&zcA+_DA2tDS0Q)Q89$Ea>N@Q}!`n}sVrWz& zGswfSEH!s|8Hm=3XLEi>Rv0D>e@cVIf(=Xsy0qU1;}9o&I_JaYOWj5cR5Kb+Vhe0R z){6jPJl$ozB|=Z=c#an#$TmTtj!x|FG@ks_vG^Vr(9ff6U>N-7%O<=GfCk+$1R4BZ zWIw6aaD8*k5mPDZ*Q7%GPzmk(de%k-4`*=&5e1k~nos#z6tUoV^CkK3z1?m450SSX zygW6-(bZ&M+0}bXID=oqP z$@*Pfm3|FrNW#k-DNZ2x#b1`Z+wH+`fcbx!DjHPPMR*^amju6-IJy6SW?aI&7WYx0ecq8p}3c-`+N{=RXvV`*&>M5;Le2sT_QG{^^}OiJWejr^ zMWKw{->|yr=toE0Q0?C8$M1XFy8gy`5<*DHCh@QW?e3n-gOWIHD*|ROtHf;?|P}j(^8E?4nFcVet=?r%v$o-jPpgX*onj z77h!gyE{5SCKN6=FCb%`Q%%@Jy1KC{j%M#wJNW+gk+rP22X9n5vGKvyY)&UKKDhN5 zoAv>AZ*_F>CEzk+3;=iqH% z8()F!`+m0Hy6WZ04)GUsb5fhcAVd_$g%P^hKsZItgKw~;TXQ+~ZU}FpOQHS3O9cvS zZolb+<`+B6Eho9;OX-9#HZ>Yq&>-w&ZkqxiQL+WQZa_nf9v z@FD=B03*j|T|Guz2XlBOq^Oq{Xg?V}Z%|fBiWUKmzxa^|LTv+*cm5DEa5wPK(GO`% z0>1<`8Xg+}h~1bGbbH#GHOb8P%Nm-hnO}SgoXJxE;3)Fs)^>8%LiZrh?cU)TfvDnt za7?D^>F)uiV9J7JeYkv=GgQFsg-S5rrp}$hsdZ=2dJ%#S6yPzFIs6Vh4yQ86)1pH& z2j9cIoyrb5gNcr_wXyqO@BrO>+bsD^B)M*B-7tLbz9v5 z>p^Ayx*fkiAh0@>8A{JNdhl*OHtl!$y(01M7Y4IY0peo+^XJJX^fN?x4v}vwSmyy= zP(jDGeE~o=m1D;p5;pFV!aj;Z<8v;078ZV@`t1-^HCul3E4H0ot}k`~|NkOkI|VfE z`$gvrIJKRb$5?pA!RNp;zKVe_1;z%U574&kpLpLRN9pJV6W%-h>!p+2{xd}F0dV;G zwThU&k8OR4gwF<0f-GGAO20Zrr28@qfo;egCjS8zkQu2kIyuR)#k26bMNXw}TGmE7 zdb$u9dbc7&0C!!hPq}`oOMx(JegS`|YBOIq5&TP{vaUq}Ta})04*7jF`!^3S2o98Y zzrX>WH?wCJR+`z-0J+b2`!Cyn$c5>$Xm^`Pp3e5$pm3|N}N3+SfexbdAeq!Y~AC)_u3M+o}0EY)Q0e`&OH( z3<>>(@7OGQg|S_^zf5Az5Pjn$aZclO`3#?!U(WwMVLnAN{j@jM z@6*An9n_mcaOt{|`i_L|yB_hp zNRtQn=KI|ApEleu`W_p?B4T?~)Ou99$LJYQG6#e?6i)K&!S=&$0~dINdpcabov5vP zPEPgKd`SQoET1nzSDE>m8yL;dBGR+5rx^$}ZQ)bJ7~`L2Mm8F5^6nr)22n6gHqx8C!dL8)qAqv?H7;7rRo9DbDM9FBJYf=~Jnnd5fS^JKIVh!>GG^ zmDI2Kg}*EALz_}B+Z}X+pe|;-81zF49W#vZ^&rB;xXq#A=ZLGt9i*rm^xW`9=8nct zNWMolfqjQTKqr{>3v1V3I_}wMEd_-j?yx2N7ux!O_)q5(Bk4jU^!#ztYZ-R*RcP^p zNX$CId`I7}XtVG8EZ(O)RDFH~OqdQgiXoitT)Ef%+)PD8?|BN<(5b*LYikzVNz*DL zU{Z3n5v7OdvVfwQLD4A=ma2j~bV11Yh|ihmD}Sn$hLaC=$`P){VXE$2bpXYLO@g!ab{HdqX0yKWx- z%M0cM1j@zuN9S`z_=jF;>4x@|qrr1xih&w@xPba>Vf|d(XH6yI+D|Unv4q?h4V@Aq zKEppu!oK&_%3&WYYkzp*N}zV&4qsf1$;h&|2>B5fgV1sdPa%j~>8ya}-af~^J*}bL z$uiQmFXtpPOk)lcH~8GI7`iro>mRyNcU#s4H9JEaa>5NWL;3a7(?FU;%)}K=TJ4fA zwVvw>rNumUGxV&yJNp0(2V`bPzEGc`F!TTa{3x~fTtdW@ zkq7+}1$*hRsAZm9E{j0Tm@Bicb*^`-5!e~wX9W2HJ!I=^ zdCH129it1m#tK*{vrOg0g&9pFG~f4qYsfQ8IOe_wDs213xU-^39);NCDADf=u<_O# zd1JzAl`@0kft3byPy}M8C4Df_5{?Q6D=|Cv;{F{;I;EV6($fm_teQqThDG5+0GTBe z?Hjzv@*^guHR=0kc)bdjyG+R+pB#uS8+_S9(xX)i{%%^j4Lar1k)qJ*R39nh=iXKh z8{VsPs|||0{RyqF(L5uz>{Hf_y2SPzDSnYeVl1lk6B<^ zGf;i}^G*D$mixx{FD2~fo(55I2!?&a)m&%Mtk&GucF?B<9?>&K>$D$XjI&G&WVI}c z3yNQNVsfze;9E7+H;>?P@abQl@XueY1GxVI2A+7#+x^KDAmcZF)7qdez@PrlG-z(N zM$AT_>BGZ?Nn68S4F!9(R>g|d2>f=-mwuFS`bKUy(CNq;q$@cIXP;0*k| z?fngg2q%dzj6P3xVQ%hs*uNGsZ;u|2S1^{JfBwc?Wb9OTC$b&zqxi5uTLO!&s^*yW zXZZ*LGQ_$nr^5PJ$;P{2MnFvR-OtaonJkjbPwaa{Lt3j!h9#O01plv+JF+9o7-R-+ zzCC0VR;gLF82lq`j&NZo1jJ;ZS@Uiv8WUWlw)GFdK0}{!@C_!QTCr(<31TxD06rlnQ@Zq=?)&IVV;w+aUj}0zRbpe?hI3d!@ojm-GgmA_D1E;j!5@$NH&kaDXuC;8nHUbLX_5^4Axy=>(`8Rw+Sf7_{F1{?OVe-O#a#ZmdMRh!MTa9Lc3iF z0=tM&5wwH|xv(;uJzlSri4w|iHj-hnSsE5~Zbf%GB;0Gp_|BbPZ_#okfyX|dFIB-aDIN( zy^6~E9GCREratbYLT#1xPq*fBwRa76Ee1m-5Y*o+fh4bTrD)NG+IILQz&vJ7a0sMI zyxVNlx`q*r1lJ zIWZBB3dRxV>cF5E*|F39mbe=z_z-ez45=K2izaoi9sC0e4r*zZ%#(zb3;L~q8;XK5 zgvU8bXZ}Y3kDiVQ9u`3!L#>hQ*U-~4=7yzF&X7V5gE=O70{x50sX)1HxoWc{w(ab8|DP_2syp>xOn=R5CS#&tZU!)w&v|fwt1^ zf$NXgoWx8}Tbi@APEk$Am2fGbj5UkE-vIey9{nw70l ziLRA@VQDabChOFXoRTiAsRABvN{f{wgR%80&HMu+^}p&;AKR{PfvoTihR12(TAtGg z*V!Mbspk7;99+t$r8tIc^udf8RHs8t3YL|Q^U;9nuek0x!N%OYF#68DAiXOZ< ziF+A$jbHiu^pq6ti}~N#CjQrMOiUqKNng42AFye3`-*pM*nO4{ z-or_^(b-{m2()L}*qT?=2y5e^7g?N+LN4RR3O0A%$xj{Kjx9hOqdk=?gqF!C!UG}5 z_RJDvvqNr8`ixF(AaQql7N=G)c=6(}=5znVKY&MRy@c3N;BB`=Q4Gz#(1w`(6}jQf zei5o_>I<}o1Vc)v?=|SYSDaYLy2#(>NSAzDSQKpCXxpuaf@8)}5@{l&{&>2phap?w zS$VmhkTd~g`j7#Bmfo<~42+;dbuvB;(D(Oi1Z8Bi)($1!p`F#g^y|bkEK zMU{Sl(L3rqbY!nEmhzK3cMUl4x(%HC*Qujbdh(Ps+Hp0?dwNIu!TRs`*(?b z2*w{(E^v~Q7YU#uXGeb9oxSaBdF;av%7kw#exjfnmd85Mo;=*MJe>9F$9Ev+n12%a z$q#3|i6I2vJ-E>pa zU&5e-7;?-a3m7u#o+LGXBwJ{Z$CeY_(X)CE?csLc>$gJgiDVw9lFfGPTO5X@>=oH_ zzzb%U^Uaq1)LpdF`D$m!Ln0UoGvPFM$oFx7ToDm4+=^OYZFzCrl|3p`hL2CoLK??w z-JH(DAxD>c>ydz0Gg!@Rr&@x7bT+(K1Z8gy4_I-=$+0DKQ%byH_N`Yd^n^CBsI{5Q z^yfsNol2iTI+jSHSJbTSWM~b}DjRs1(M2q8(r>3?agyC$4}M|)H*!1Fc~IET-9{ZP zJi7hY_!FG##sVa`p5a7og_hnX20{i#{-gQ+-`JPD&F}|E>Dkbh{IF;zNHc(QoIx(usN%oZ z9}T~1MR;UtwND{nrl6IPd3eJKWtvsl`idPXV-u5?F8bn$8pt)&ul`@oizXbFUFdpG zL{!|rzbJ&^fZ@Yo_o*tfi_PcdcfMylBtlx(vAox)F6^cELiGR*Cd+S+i|JXJ_UwrA z0Dbec0+ZIYO~88g>tdkD7fRbeR7_|NZbZEy1csAB`1QU(atWsE`NYB3OZ)Ey{$cdI z3qNx&J;cOIy^Uz5f z0DVU~W>%CV1J6=Wf*Gb+2?gU@85Uv_^?KnIqc10Jhu^;w1Q+rP*ZFfuV5RVq^>)_! z#=ph4Z=y&Y4!)O^0P}_Bg8=SC-|EQv!CUF?qunGD{kr<^P()-+{9p%b>3e@ZDnS?7Sfyd;^S zZ5VHTX|Qvi^fgy59K9gK%krT7q9aoAh7^W3bhYNrQa^l}b&2k9Q7~5rP^i;hE&pWO zS0pMezc-5WfcM>jL$VhD#ER^Hk=_bHRtu8`6vN}uVM9a*teXbbl+C2U;ckz=1{G#g zAb=QxKN7=@JrbJN51*HN{Yl9qMk7jCM}6)@#E+(MWyDi#HVH2d;FV$JDWH$rNdU`^ z8i>Rp(G!HEEUA>@8A7-u4}>rUFPdBDKX6V3o^rgywLQL-i0LODJhGhOmsSCA+VQxY z_XMzgd_$aWDDq&Yy+&-RssxHD$zCBjyJlN)L~KFub_y24sX^YEz?6^G(AKG!MBz73JX!A4o9hJ4sL)do0gMo>} zkLwp6WE#8Lrcb%US@uuAbT_f?@VR%)#U;lF%n!DAd)zPvX$hE!YGFy*R;Ijl-XsQhxKSe+}Tv3^`kk1fDvsx$At&k z`SMK0vVF@;1XIeY48^R1x;!-uqoRa-kSyP(0!4Xd#x|x=Xl~ztCoW6mUT#+!oYfv> z)C)D2z_?fkr&=^KH#5>oL;3!1!Oe{tug{N=SNX5x2uX*Z%RTd&702-BQJpES)hm=t z?>5LUuqFF&lU+)VV-zKw56KDfC<+e5Qj(NUWog3K7PAP<7~yoqDMjP&Km71Q&t(^w zG-ot*dF>$A8#Ie*gIVrD$PP6M;xoN`gEIZBuc&s2;<=z1hq}tLt(u=@L&>VE)gYEN z$#i7(Ij!;}g`_e9Y&RxpdC|FB$0c&Uuh&PKdXTP2QGUVoLhwhffY55 zVDybZ%TFZ3Xv@NF+)as|d_#66+<{*0q{uhLE8t*}EnJ5(;-!gU4v&X@J9i7<3k0cs z3>U-jjZ`?RL*b~%=dP$CP*oj~{B^WXKOLh32=fIcKN|7M^zS#q=|~GOrDa9OAiXLc z4?d5puH|Az9)zqg=ux(+3zAOinWq$M#ao`U=#Jwe9}d4FkDZMmON*zZw2?TLOgpTq zY(_AS3={h6_8Q7;Yx3ECv_B;|?a|PB|6B$cR$>hbR)2KCPeEW6ayU_NVmx*#!eCU4 zQVbR#k(Wcl?~w-tc_R}133kFza*Go_8Vl5cy6!u(v~CWmRH#BP`tJgCHuT@W7+9y| zcpl|gMidVKTA4DYS*qDIH6U!P75L#y7#PNgV-`o@HWCIviuKq(V~?Pm$TG9)NHZIS z<&jbwB)&?gm+qhWIhXE)>ZHz@{nkn-7gFh%_+4A#cC{J!;MpHDJ#!(WX-l7yvg35s z)g9Q6kc~6=T5_iiThKY9ZR_b3AoYa-b%@+R|MmYhZd34-Qm8gPDU-B^`F~ zU+tN2zU5hLPxRmTGx_RRs7+JN;gZNAHLs${>Q?D~uOc)i8@fS%f0rSZia>3xkB?t8 znv%*@M2W4}9AHUVF0~k4gE#e!VU7psSRgX0EtadaW{gW_|E`J0z19{|TnB)leEBhN zL`xvSHZ$KA#=Gw;@|^TkW(ug4z*3t12%!KtSwAml1=mP2v`RRE2q6s2hj_i++kEUZlbeWMt< zd>1p1nyaVS%HXSK^(iD8XvBAF#Cdhs_DsP0h{#`Gb zWONB(A)GAYvVpC7AW)sR40}k&pQ35KwpMWcSz7tsJ)BOkRs76R%u!Hkz3_bFH@7WC z4j91QWOq)gIZrMm0YxN(#MI-Uy}&gy*{ns{o&9$x7K^nXnf&LKxFKKFX6RL06%5PD z!sxhBSDKwalo79K@ox?fY>?`A=^c~Qj>#TS{kVxK+anQG((9*wD6IH>-4c_L_F$(a zqWrUB%Bzg7DkpoWP;m}Lqf8$q6J6KI{#uOJ;nA@$bA`u|=-aLP#9?64mjfRAxblqb zrZN?+9G|NCS@gQfvACLf&;j?E2u8DN1z|Hkn70-8_J8JMP3KFSR(HPzZkYgcZ3S)3 zd}qez$~@@Ma@eOUcsZ3TTuPEzsw z$XY&fnQ6U~qZ;xZ@JX3=@`DotGV3a6o!^+Dp54;omJ#~q z&8|;!kRK1WnHf6sq#d(7i<;0ym9aFItpxExb=lIi;VwHv(JaQ_Ds5A3)w!y4a;rAq z`3LZPI)lwHtjsx{M|Yd!wi2E5B^&iBhym|GDs94qDe&ho=Xo>NREw3H=DAf2rhwq4 zZO}0;nL{?Slp=r(Imdtcr2%m;JiqxWOqdQO&W=TvHu>Fgn}B;m*yZf7Vem3_{w|R( zs{xdj9-!WFXY^?%Uq4)%izEbsy{zVu;{yIvA;qld+bPbVrN%~@qfuxt=bGP69G1=^ zr4n8m&+Njmuwb71a5p-ZQ|{cTq8L}XZ>J-?>9Vipw(_~gijL0O*`0|QMME=29Mxq7 zHyudWIkAzuG(FeyGJuw+UiuMMzjMAG&|JA+sA8^kKtk~WcmHAz&&{FPA_R6EjOL+tYn{-YCNrwU21 z4UZKYF~;$q!@wQdhUeFql5$9utn0_-p*&2%jF~f`MBY_LriJ4qP+5_!77^Kb+O!rw zETOEZ8QBVX!Cl$#_+Lsce&KIqS^}g=aPcV9k>uyE9aX#U#2?#k;z3L;ed zDO$m0z)E<;zF-AFZ!(WgzWB+*%i{$Ze4dS%=LTFO%$%!pvte`lb;FNHd{jj(RGD&# z<{e9<_tjtybNq5X+}h$!d)czTQevT5BlKolE#VK}%n!l##>2lpkNb*F=~Z~t{z;t^ zd*0)$nON5hkg+GofQk8FK{DM08q(G}F1@8@*3TK|Q$D(v{R zk%a5(B#p#FcsHY=2Jmcje3kg2^>TA{%pEqk!H8bL3W@ND6nh7RLW**kl9uGQ#4=Ew zeZA=@M2Ou!3x;JL3h!_1WLfOif77`YII_jc+uGZ9cKxn#hS9X%@5f3x;bF8w!CauFk7<0ONeBeSpd1B#*d_2 z=pUD}Wb>bLQ|4WEO}|(*8fsZo=<_`4^|N+bv*~xD<0`pn2PKl}FLna-bXqL%6X64Jx>~K< zgPg9>!a6kkp5U5lxRs4DGtO%;Tj>+<4&opVl(frF#Jf5$I)}Frva~ zQub%9Q^CepREaO}+5Al2IutDMU}i~8r$0e(BmV%m`2z`I1HRC^8#)dZiHVG$neCGqi)EbLtCMZ!!~#$sbq=W0-_m*ktUJ> zfdCW8x4n7$X1M(;>Yqpsj^fE;8hbw5xO#ULqUTOkn1Z^Qfza4lNPh1NJk@uj25J~> z#FE0|Id3p-1&MUTj>0!A2h&(B^$YRsV&o9nt0=4Du^p_M?h~l9w zm=UkZo~WSFadX-fUk|)-d(*_-^>cBGJxpD_V};?+oUHIHY6a@91Y6Ke-X$QzNcdKWZvCcVgoC>23HD zgR!?mfUmIK$d2dcs&y#-wyy_^!$&&M`|*A2OO(*&@G(3`aB~SzIk&RVI13n@+)k#9 z6?yi~A%P6c;tf*1Cgf|_Kg5d}a|>;A%u}L$fc$Bz_P8g|E3uSsO2s=4(9`?f11*|v zoi-!*kF-PczmYn!LI~&Py1w}p?3U60<-@t?;Vv}P3QuQzzo(|Q`iOUaoAJ@d+Sl8P z8gY{f0)+{OY*F<0b&~l$y>xI|ABg+|6m}oN(_z75)1QBOW2t`bLsBiw6Wj4 zea0TA!``-+d_(trNu+l<^20Vp70jQS0^f=nd^m0yj^* zZ=kGORbqS9d#qar6%-`-8OI&?-hHy?_r>5T%>VO_grF|bIMA$>h+3{byR@J{MF}$n z=DT17Ha=)#4Az=?g$YDy52pZBW6iQ=2EeSE zfXx{uxa6GxO7#T4#73mVbJ0J_AjvmB2u+>)bv{86Ixw)J{{8`z$a#nuSDW8`AEY97 z;Ge#Phk_LVG9wi5Q1!71EFXQDFTQjHc5#(R%K|hmr?el=pO{Ff~cQkg6NQsP67?Kd)ZVzx9j#Xjc0x1`~nM8nsT9e zsq!-u3P!hxMGl?YJi1Ht{4cJDLxz^yAOu%tKa!B-R-r4Fd2#si<^P_3G^l`Rerrns z)_lBh?fo%@;&5u#i#hHQ+YT^KMtpPFJ)1!%54Xa-{2t(Q6O$DUexHEPy=u0W=zAM4 zC!;^776$@6DK`Uscley^YW@KnoMEM4)I!PrY(*)ry?Ae;l|CBm-Y@z{9&fPy17LOK z>R<+c@QQtV4Ioy%DBK|G-oFn4i{H?tPKxzFr_ZILoKUWPz^~AIzxut3=qK-&0j~1r zq8tIu4rPg#TxElE zze~x1uyUkAlP(jZryT?$_QR>gH05N17Z5*^C`QXrKFU|Vz1?PXipOgpk)Jmv@@nhA97hm;{yIL2Yk!8x0Jqf8tDP=N<~OET=1BN6Mu1f6qcyx>Syyf?OK*<=-(GzSC$e`uyXA}am)hNKt z8c+VP!5n)LA$UrW(!SH$6{xeeV$~`hoQ2=dR*>A67;j199;+!+f^FAM0QV1IaxLdo z=~}#SapATr=Eme!fD&eSG?L27DE|+D7t5Rl${aG=Fw2YVt41%}uE?cmg`x-ijEr*UE*H_hbx7## zP#Am!O$)y$IO@;9CTXRG5SIT&MgyhHqyZnSbx8;A$!3TgxR`FPC&vy2k_+8SSqs_G znQgpsC6Lr;OJMR< zVj=sG<1;jfRynoPxlq|AN4u?xw5>dUIT7GEWN4do`I3fjIq3cww<^L$5!<((>2CpO zCR^Pkk}~gB;nDSs7ZljRb&83^K7_5On&;e>vO+Z)eGK1v1+yUB{8;`z&)$FhOR#SE znQgN1{m=Geh;!E3FSfz2dJJM0c=1!SnP09TL_7UvM0C^G;uVyaj{g8qC>=#)QrB`I z$y2i zZgVYaT^=Ge=tRK;v}2G%3&7u}(Iw~VC}&IAfhmfeR0sTS(~IGHr@vz>>&+^BbV7Zc zhM$$88;EY?pZ#gtZw*>GUTW?}`CRI&!N8D6+#xmjAL)-u&{`F7T=dCLE^mZp>J!Od zxq`prj}Mc`Rnp!^lXBg#HrdRIQRuaUS^YUhc{FtK_E>-x3IML}b)RmYWy*2Js z$nSXFvyCAbG;M*ryME0z&DI|It=URVXM!0ii+%>0$O}g^gcbPBjCIQSKm^@V@{wRx zk-|}>Y4`*FbUc@{lRKdZLLMHw?xA-SW>5oVijYMZkdj{Iv_(A(6n3RSe_OdmAMDV! ztzX#hA#FXIa-a2Cn;FioH0qI|`cs89wY)Rn$ArBIHyv1}3O5Y%k`?U`kmUX82K z)chY{?j|W=`zafW=6DXA(20+PdtB*ZbAdVjn1j2>u!C;Y%NJgQLatiy-N zLN*f_f33J6MM7nppT5wN6f{qus;S$baHJO$t$*@a`q7ST2A#zTljrF@o`6+NjP8t* z+>tqej6Avh!2(Zxv4a07TX$DRQvvsv=igzTB5S?zLZQ z^;M^c#H^`#Q@unFu5sN&k%ECoagxIF9y5XtNLTis9H}T9%7JAuvOmxrTySQPNK=>5xFbx!&dV;>jy6Me{AJ1PkEXOJ# zV@f~u6tQ0tS)f=<8KIwU6`q!T6jB0U((|$2FJqp!57?K(I9iaO6-O>>6ZL=VVCd6$ zo^s^K%c~98c|%S@*-Y=5O{`~NN~+7X4T3|S5eT%Byz9H-FP)?!*|xIk6Y!I34FNWO zMrQ|qJk(&O-;E}5U4p_gjt#8J6g;xpHZ+w_tRz9JcC5BG5LQ}AJ%{$LE7&}Sct!iL zimw*LSL-9iLR4rI~dKHnm(0Zd)lEDS3C6N?+dX;}Z&=|3}qZMzz(oQMY#frNZic|bKdB5+BamLBtUq-?L9efBy>k)et`TAh1m-MKqh4baxS=XF0rVclHJ zu0nS>!)$`q6_y!=`h*hL!8l9p@FA>Nq94$7_j=N5jfbVaBfpCZ^1EOx+43)SyNrtD z83$do+)vYUmH#C;|8Pk9Rd;D&K`@U~_g~rvpO{k<0x9Wm(3&VDH|3kLRE})p+Wpge zqvvky?%MD5Jzhd%RVTk(Fi9)rWuPz1%$%V^u_ zWLIA6poxXBOT^7{*iZiGM)^0L5yKExHvCL|;1oMuC70+znoHMBr6$QouyMiP59&*c zTpb?Va(}TE`$%Ll;HpS>hFYlXCK5&-NLepsiv)U|4mBFe z*T}j*btc$flbD~IyR$Q#n^cC!;%BBorqJq|*Zp+T#q7@Q9>OPA_A|sp$3YnAaw@Zr z?Oap0WHg1qQo0xxXq}pNN?BPgC(Mi(l|Qnib1`wu`7P?Va&hn4MV@ulf~OKbIg!N%JRPV&K- zIKvEggcTt&oTl+B;ENP)Ab}L}_YXKa*VGg?&Wp)a5kD^k^@if$RqUYl4Qrr8YZPOa z<>E3%h?ZM_%{%!o8*N=hE-v_oQNjvb!vuJIEp3I7X?NA%*N3N^7Vu5}`|Jk080Zpt z8Q*0SwNbV8<45;&1}+Mbbb-Z*2mB6y8D9C`2|PABeFXF@$5e{{=GucRX4bVpj7zm1 z+S$Q+Ht6H5M)1bTNhPFG{{hfmxP#B8m{=Fp`GD1lC|!R4h+7eiI26%8X|f1bhuV7ggueg884e+644?56 z$Z0~8@07?IGG7|$*M<2BsAQwFWCv(UuW^$x?cQl&fx^pyG*`nZ;q{i>T4yAVc8WcU zK3*q=KZcs0tx}`rl>yjyiMVFH= zHH_w9-F~31vpCLsrbLyeb!AONn**(!2Cjz#&_P1sq~OK8>F5x)Xg#y|LWGwk_#zI9*D9&wY?ulE@taD2$-}N z0WEG~xsI_xwR^xSpQgY~+N;m@)x}Uf+S5bSVNxhDRT8#0)XY6@3bET_JVAALwc0x_nE;ic~}RAnH4o!bTYq<=Q5Db*Qr2 zb>=-ddbXtaEKY=r7j5P)T~HEiULbm2ZE2}qhxXSm@zX~u)1JoPhxe@2(z2{dDP(PE z!JoKL;YeCiYv8@V(PnqVit4h=P4T@Zuur_m#7ta0b~CXx_f#VR8$0~vD_2yBK0|_7 z8qd-T0iQs;RNE;9u}F`xmIYBY$pr#a-o!1rJ;Pli)=2v3M*?bu>>THfkeW%;ubZ@- zBKj%Il4N=n#)ghY=t(Hl;2`XPlC;bN;nyraNLX?15F*-tfGLZBWTfg*#vi(vkP98^ z+=L6BJ9EBf$;76>wPNLgTy=CCwJcRM`d2;TXbVUr9BmUU-JWg;r!oKmTg;6^J}0W; z!WFQse8SnzYdjf$${opzabCP`K~r8E%EhI7!_4;#m0Y0(1+=b#n68}0{XO$~^&=0E zNm_ao?vIHWrr(jMpG*&6hGdk6@B)HFdVBIqp=V=E7!kiUig9}1mrt-h2rHI5*mmyn zff|#as3Ho6VmIA)^z$)wmd9fTmdOsE?v_X{+ivD&(m1GYiA0f8OZ94s?*4ta9ls4h zER6|)L-{#=2-A;UYUI8mu`;_BYjk#!&@m!QnDqBE?RdmJtvh=axJX>1hf+tzrlw_8 zBK4H-(tp+7h>1_Hoq|}A{JG@~aJf7UA5KL0PAPpxePWIy>r3<;iJn3g7=+>b{_8k6 z{PwtlQ-)|6U^q9% z8UlF;$Fl7)jlXL%|K+nnEc448Z1jH{o)aPpdq`p`VlpDn)b_w9N*aCnl!$x zujaisocdMNbFsj9d_?Q9zPFXquIG(F?>k?lM`mwz!ttI7S0 zXS?XH$C0?R!uL@|Obh}8q7J8lX4Cn>=1Iwse^fQ1UFdes7$9+Rb%0Ply8%Qc1D+GB zoe-oDjDB5Im>UVD4)vRK=*^oM2D0*@q{-9Q=QmebIhbdvtH1S7ZW*hW>DTz|5SDAQ z$!2z*X+59cCti0wn?j=Mt3(0*cKx<1UM3C(s>WODzz(f+jVRs-Ui3G%zt?LppT8%wiSzK zifzQ}AJs(qiT^*@n6_b{A-~jpyR5s@eRy!7wv@8~|N7VSsoKgQ*Ij{2#Q+-I03v4G z^V*J!5D%p%;-#A~tST>f5w9CpnF$qFmGbCij1QN&D3s3vl?E4ZLXU`RRjyGvfGB)( z2fK0fju*UG@ncZ3G61UkYnRIwhksw8{LXe@h-6>@Qg4_D?VK#^Th3#&36yt%$=3cl5x@x?5+43JKzbVV8-Ya~Ll@HuoAwn{-3U*O2IfHn-u z2|V_9M4`b%Or=I!4w7`xX}Wx_x9z9C$D#gYz+F3S`w;%G|2r>IFcu>w@UZbU*oqh1 z+4({Z7w$W2N&%?qA zn_RDbXR53^_(2gF+uIi9&>T)hslpRT1-*hQEl;PL#MQq6{Gmddwy7kc3_e=bf#70^wAWews^f_WM*Ky~~ z?EpFko|ci`mh%O)JzTL&R`CVxxuA|2Y_`P{lig?3BwgYf?BMMfzV+JX@`^_fdF>>=Iu~E0@bFYM@2wK{04T^sa!xG&-e_s%gUzry zqgGI}a!`{FKZ5~8(Njk0G91!!DBD>QFAbsJBshG;duMvC8dm7E7+l=>FS=K#GsX~! zL&}ikzjYuc+Bi+p2WHYrGpEsutb!Zvcs#sl#5B3MSHUmg*UYCX5@I{2 z$}MpC^XqpM^+7@6xXdowt8yG!_HSypHHqPxz% zsVHs|>8^(~mE3Z1w5@RP*WG1F^YVNju{B^PTTJ6{*g;GKX!rBI7{r5g86?Px`dgex zH7q?~q?M&CyA&1asVE2WS)O_HKB>}k3E6aGVdLf9pYx2B9j>*JuI zf}XLqv5zG?;os#CA8eLdS~$}4rjb5bW_D#mB8ZGUbGidYRWL3IC(7R`U&Pj_7We%; z*6lNMKTrQs7N)4j?b8Yw*ll=}3m=#of#E&!Vv@cotMfvpqkwwzWzYB3VP9jnUzB?2 ze3|b8lAe+7FoLIUP6G;3vQ#dZ9h+WzL<;O0A1780Ech@L)iNxWIDl6MuM4YU=euM70gj!@|A;#^8XU@<^QNB308cAQmQXZ*rUEVK zAQ5H*o+wQQaLhNIdO+1mg$!PG9CY1c#T0b>7$-Cu=ViQoisOZPDCaK^uk4SsjocZ*V z5f1)|P8sVo5F7vwD(l&T5SOiInlUBKdovB}=hQ9XzdA?Vhl^rrkt8@uxn=+B0K=UX zUwk5{Gi~IQ<+j(I-vL>^iXWEJ23vR={QQ}@xS_N*!1$sIH#o>)xGo zU^Op-uo>5Plglc3V!~QfRMTYexJ)*ffJV0PxssGBXcV;AcEY(8!KRU$(j7#<++*v7 zXB%&urYUk2gYtK|&xX}=By{An9Q26+yXgd4hITqkhHue}#l^b9-Fp`~Z$^FSn@NjY zoTm22G0o@QT?>oi%n%rkYY{#Dkk$HmO+kUL%$^qF-U9-Y7o5)XX+U~1CJ6;~Cnl$v zruh{=)rpt^$)X`;ECABjatoEqvrl91C=vpm3pi;OI$>$6Q6%;b`_p2Ocy^N!T^5jX z7MEapD~o8f+c09fwN7a8W8a(aO>13oPIKUdaC~EbQ?$`v=Gn@MECXb6H}|W|8Zi>5R?8e`%m1 z1c0b{20{V>0JgeDZV~!W>;(?9iJ?RAE&x6Y|H}r8vMr7Nm<**c+FL={-MEhhf~IO_~sX3)L^bE&F1EMok@-?pmMv@LXN6uO=#Fs~wau^*Z({ zUlxgtN+HcHps^`dMqp|`clEb0(=y55xJI5IhuIa;?Hmf;pKz9DI7oFKDeur_w5B{P z(2!&dQ`5Lt1PKyRy&O|~k)O48xCRA+h>#cb*NNPewklbX{6#YK`5T(jXlGA-(u6^R z+A?1<>70ZkpLH*+QihqqknB@s6wQHh?HqqgB(|%MCsIVA;UiX#c^r^E-lM$(H6#-g z6A1bcE>u8ezMuaqGgc6k$}6+{_RDru*Jut6sEW~3Ej^6GieV@%&TZe3ELz&QAYCii zX`KJr<-&pJ*2)a3^)2%|XJ`gK@*XrS!`r>D`SNG#T~esJS6XSZF-Fwj#8+Slq=Q=q z!iK7W$B+yMXfeKp88T*HRW)n(hbYS)^oL13uqsunMXUGDL&d5Mx^SaSelShDx2a@* zDes+`4+H6J0XQ-Td;_vL6bxP?)epmW!Oc3$g#wO6g8a5W{gmFa^D?GgE7MXu7X$|@ z`R_YUZ5+}r5H2wLNJCuZzUd4SkQIx-+Ee^+UL8Hblhw`flO50!F8lXGfz7;YDy^~3 zR)a;2qqWHKYQhN+kG&J%+lm%+b+YeWk4b?-$NRoY+1=@*n#2M^B z2dW5dgCAp<7t9a-zFOyJ@IQFgOav|2zA0yNKdz3?9uvEwxA;j^*InfXz zKBmRh!zM#v%@Yy$=^B%74!K1n0@bLRc&{G#jH2IOot60+2Q6pb>dqseeTv?e8J>L@ z>mYA5dzn!Sk7;D`mWh$++yP$Rkv$vuT?mCGBIA^-ws7#nGG-m?jyn6-=QWeAk}wai zY!=V*2=tsHR##$}QAohK2u*H8MJ~P-Mo~=sQvnGt%M}tjSJ%j*1x>s1@;ETGG1;Sz za>TgCeom}oME7GQC~_G2H0b;FE@{QE`NPWjKaojb7<~fAV8X0*(_`85;+zPlYo|&E zhcu6F84@Avyb#3@W68zLvgaubZD%fn%_I}1bdE*Z{UMmcWSX4u)#{)pO-(#%o^QTxo3&DneokR`{Wz@K~s~7Zl^}%#i%l`$<}yQ(|~G!3#yR*c-5s$w*$E_EYlM&Jwnt}15?3LD+cvwp|aUv{V z0oUTNLnoO-*=%~JqZ(#saB0Zmo-&{}H~4`5lt^U_v$@4Ekeqj-@hQ+UWQ2$09RBAA z*}+N%;Iy&v+4b0KP}-vH7m?pYu3bs_2dmE=W&5&WVs$Ceq;vQ=KN*8VYhyP_57pgN zw8$o_DsukPrvHp!y>dk%bi&|a${=(->NjeO?P5-pvGGuR_*UWUhC6uZn0rfO0hE>`2E9p0w}b)sIuPf%xK#^h1=@#N1^Bt{ltg#zoX9c&s*K~!A<*Ke zw$wD1e_?_I$p;q5V$=vgo|Q+%jA;+DI&w?;9g5z>ppIjT&xZ;D7Wwu!c2sWS3L_#R zYu=0*BvsI=$s23IRHVi%T|L_H#V|xpU7F>D_P9TDpXAp_2=Z|Z6fnLzbU3`7?2QUS zWy6+5Qa&^tvXaoAU$B*X-Ljb-Wqr416kD7qSMuDCo=D62AhJcc&~yi4LH+#C96cQj zSCHjuqS;u^ChU+*Mu*>#Y!DCJ9icFJb8J2(Ng)4qyrK6lGt>>rAQZ|f@AGLpUGs$& zaLrUK84Z12;Dynra-0jeCQmA1qBmV}8NG}XE!JlL4o_$7_J`24Kbjb|4sg6>RlUnx zKro39!V{Al`5!XaKErEuYjj6SB;R5&vB5RV6R3Z|C+^xYuye)YuTJN*WmOo)muzWu zutm$E)-7u3euPzD z&Ymit=H=W4h*3212!F5v{8S%i^j6KH9lBCAkC_f8w6~B|c=JghdNz19Hg7USPvR2D zSanbB=(k>I@R@UVJAl{DRV0-@*v!@P7<$@JXCqLiF@v)^cIJC^adkmv{IZ$v`bk6P zgKiq}Z5?l=Z6x3CIANppz{ayQ3E9E5G#C>nZGH_qEOE`2G6il0lcCbW*)fy!(Gcr} zq7{X0>a6Ic(;cxBhINt39#*j-^q?Yw3C_5iO&0{OMAS`B1U1->c}T%x8kyJ&o>|C7 z1Z@S81$=}VW|?KG?yt=xBnf6^iy}SS!v%Z#?Y{p`go#pG@_*$|sZd{%YQR~Mwgzy} z3k=hx*A$1Ob@`e!fZU!IMYF96K`I^)uu3E8Wl7*9)GD#2#e!f|xxf6@yV88Zz8l?S zi(XoG!l~luLFYv2TIAFr&{&9j*j8og5I%NU&-OpSysPj;8SwE#T97wna7wk5Ef(sG zuyiegWUZ?{J1&I~6fY@NVdU|bY!`d(^o&GJ@neBo0h!fPgBxdI{#jc}IQ3pU!_*mI zK0E!aV*Lx7HpEI%z{Cssnl3x`&si%rpTv#n%YJz39e)M`lOfuk7N9sD1bN_u-5gtL zd$;5?zSfUbOCF`SW^&vYUSU6lMGnbSQwxrxpqeuN5XHqal{!!NbFsz@)jo1PA8ogaq{O62C%a zKdR~5ul=!IzMoZj#rFkHmFAK)h|EeVYoMyL3CM4eS`^XLV}f^SSrs11S9amW&0eJ+ z#?~)w{N=kd==Q~9gsl0aDgvIjh_#6}Rq7Uj5u}R#H!VzA9J&x)h+JW+?7S-5UD8@5 z$JsRyUy(xC(9`Wdz$PZg^}OKck}K;ILH|%*FB=cEO6}-1XG>9Y?e(pPvdRsS-S+N& zLM5bY;^(k;CU3)kr+<-4T0(AXY{~;sBoOzlNR?QoU8NIC&iF{R0}8e8)54Y2Lh;;Gv^kz! zo?AWpr2V?*z6m?DRxaPHPP55It428y68vpThNKo^1S6T`O*GOksQU~fb8+-f|4I^{ zG{e3ba66ACYo(eAAAX0IASI=$2F>!B)Bwxy;|DR%s+1A0_NK!^5}P*IGg!3{Q^f^^_qFY!!c!GMhFqD%pkfRi7C0eC4cV(ZC%DJYjbWAtpm_yJ>34 zSSgIW6F>-;f`1*2wnyu9=?w+HWN;(2vbHn3d*0%j8AT zK2L3Zt3aSK%A`m;*<&(7XA-hkMQdJzQOaC4?5h4d5HEn;Q(gKKJ=@f6H*vLUXpVKq zXFc=mNhG-t0}@6ii(pL|d_tbTWjrV`4L7q=fj#MGyxmi8LmZWI&OBObCJy$wG9Y6| zm8h@EM*M{=VFd&jRpw%<-P$(v%f;@(-(0MiisIk^341hD&az*3R^G?_9JD}+re>u|#F=5&IU#Rv+Wnm6)!rt6zJ!faPO%TZy| z);x22E4cGp#h$ha+WMFe3;uf6--M!P8K}xncoU5Cv*twPcgs%DtZ5vcZJicw|L}QC zygNQsY7TnTpLelKTI*z{u$>V;N4I&1w)qNzs$^vByLwYl0YA6YB4%}j9rZJEy!cM* zZSBO~vg`-Wu)OHF2u^qhMSR-BoC`f$+oSl$G38+AUC*q>mz54aa-KPkV1*qvWY>ubAIT~W$9J+ z8&1l`E;>I%DV>IsBghR9%fPINpOuqhTRO+{<3ZB!A`6xLznv<{QeQDQ5HF|pl(il{ zrR^|A9!sCgh3x&M+b9j?6YobH_7}MyD#jR6Hb^=}M!60cE4m-#UhzgvS)P7(2?_kU zP@1wA?}1WT0B+ydll(u4)Umq%Lkue_{Uj1!fYUid2PPXG{)`wzl7w8_6I6psmna_~ z-pGJQIE)033RAKJ`-Ee0mry81x^E+85y!J`6Z*o*O0R`jlyy{#tK5*_lpG=~Ej!Cc z2|_Rp3fSdHg~^gRU3ezxT_jD#ait+&1MUhE2U#oW%5DD4_fR-?(_-(<1(pBl2u@Up zC(I_?LQW0dfz8P_`DkMBeI7#RfS@$?1(}IBadMZUD)8G)(2UTNrF8N@fW9wGhbLVtm!%rHg+ zSHO$H75kzj&K>UTvN2Npnzhw{D+l6m#Oq@{ot8t#B!vLYHMNDV5@i@rH}L!h2Ia2_ zN@MUk3N(E}xMlFhL#Z|TYOiE4H4%gCVR%RZGXZ-SDSW)eEGqxrcUW)%4-++n-Bkvi z5bCtcq})NXwG(fpI?*wR`yTk0&Yn-TNzM(4FV4Q=Xp6s=7Bw=ImwCHfocz77d}x$c zlQ5FUo{0WD#|Jfve21d7Hm7&+?s2sU2}8KGTK$;+WX7R?{P2i*KPa6DL2Gws_nl1M zU36>AIU*eu*31dlfrtl)8yj7}fMG59fR^^{Q1Vym!)F<+D4NKac#p;&=pfg^;$G*! zly{d@qrpw3@(EK8-$)ZtF6sZDr`!+bET@ea#OhI6e7?WyPZNw6|2;_d?*A|T79*-K zvWQu;_x5EYKO`W`cQ;CB!5BwW^GmQ?!y}y}Ds3s1tS`X#ulT{1&;flmbYDZ#R|Y-? zUred{?QT#85&Wj@JEt}Cbtb^Ckmp{R1)w~nmHg=HbHk{V z2vm9;@?Plce=EeiZxMzs#*tz}Jv;(XbZt!Z(pM!&CBFD{0|==ljZnjGCAa6U-?Irs30iN`93zHM3_9D~x$A!9dUd4{)Y2OHi*# z5x)PjtHgU9mm@qw7n^u4bmUjNHQG?AUT;@$5keOVHy#uvH8FrlVSpnO!UU$R;P}WY zv@|ZT5)X@)Yu%90Dz6`CA`1E5ntq`TA%F2m=B$Q-^~>@BZC0e3SMtE}`n+!;YLVjQ+vRN3 zN3OC1K&%Mi|B(+sWV(v~Har^sFA-u?2=4{vc+PmEV`y86+DJnDA}A-*lJ{yg$aF?@Z$x=!cAQJLsE%Uwe# zMHPY>rtv2QB`gB486YmM0z`p5Gmx$DYl@Ln94pb7wjm3n_64J5YHK(%)RgH=o}Z!P z?nm@E01SWsCy)P88XW+0(@Xg^AQ|vS@#7~ZnH1!Az4kUD`p`lWN>2YLv`39GL_At} zE>P$!8b}Bb1y8dzmQFcX+e?3mO)Y8%W?av#2PdZ@o+JO_2@(-MT3%7;&H)A^wOw8x z--RH~%tV;(BeF}N%SKYsu*1beaa8iVd#++w!3#!`rP{Nb&*J*RS#)Ot9Dj2jm6En& zdzouW{{v{&4}Qc*(-2Zt9`1=2CgEh~PsIjJO-;g&Q|{_zh~OfIB2Y}VM8g)VJNE&6 z)L?nOgcYu-G?2s&%nwsuA{&w$sS^C;G9lV2@qBKq#vNdFjdX@|c5V2{N!@>2NvA)) ze#M!$zUQokXy2UK5}@e^rU0nq^8@eyaMtJKB(?-@Yu^MJRiWQ=eSB+5)^Pe8on z&=nsVs6)wX&S*f&mdQ$v+sT5iowtfEQ@9t%eP1I-h?B-%k(^wH2B`xQ13(P@S7(AP z{S3P`sdVvoV$hQGROfngD|}o1$i9}U5E+C>E$Kgl=Zj?}`kt+8V>q6CtJ@?GStDuF z+ScM?8Aq+*o%e!zAiK(qz`S&z%<`e-Umo@T&}}j8pQ7Ku{LQMK#2f=Nqn1!MoRA@K ziWnXUE^K0fAeA6_cac;f*lkc6@Wbg-{0n`7uk3{$ese&ce2sI^TI>sTvX$`ZA-Wzu z+mLcrR9YmWil#cHN*?3UG*xX-R$^tmZU46He6lsUrDY8$tNGPbM!-9P3TZr`TK4=Wcp=GS!uAT)OW3rY z4_56A&amI5af>eSnZh5|b)+tIk(CO6A9yAlZ|L~kj_`K5Mi=ZMxnNUfkUK=m9ZblB zky{yo3DHhXV*8Gr)SqQ%g-vOe#=($WL^G}GtQ_|=KAN7vzB_{#tyx5#lFKa@Om9)6@*)i$R8Ox0Xi;6rmrv$v+9MpaQOB#W?=Hd0?=TTgqfJ|6 z3Po36KUVH7#)Tqf(uzh@B2@oOe919Ds3tjmR$sKEgsOxzA`+m9_P1S`-NDoEMl++= z(Ww@-h5`VOT2ptiXM6w^0jf3uaz3ojjwC3=o(1V%*p5|pgCBTXs$DI0Q^udRoi4>I z(Pb-+VL8zN4s{!ry~apHy|JEtOC_?3zh6X^CSjI1+_}9uhF#7fZ|0_WvT4xi56Fd3 ztU>kEZ8X&>h0LG{>A*cf=aygALDml%8z21DI{$6#LyU^Tin34U8C>F}+8a(?;j_{j z!{$m9QGW*aq`rg2+0J(2ZJ$vG!IRt~8&4FKZU7}OlX=e-tRHL5d7< zQR%@~Vpmh|ABnrj2=!B z8_^NEhkl2cT;-mY^KEWvStKT#w%k)g1%^DoW7Gd?z)w{&$}fSnnWTR3Ff|@*tMz@` zR_f^<+UE-#GXJ&mzFZ4}w?XxdP2n^9#$j+S-jrpp+QOI^)pWE&QDB&%ro5X_QaAp? zYWMw@w<&197ch6ir%t(t*>R)-vlB=h5~?_HsOf3yFNmhr#rH<->=QouovO4fp}Cdk zXekTd#p~pldE{b~b59-5jG9B$d97wMr3)f5Wp#x+RDzXo&?6DcB6nSJMlQjh5`HM3 z+_rS~+4yN>nB8D*sRL{b-_^oW%MMBFfJRW|VMfVLS_6m9=^wPzlS}ti zF3N3PpQaN&MokBG_L=7_R@MUl)3x{B4P5vwaMin6!F%ZYkoZu`7`kegcRsR@v zQODfak-LDy>%e<{<0~e&YDmkP4Xjy53(aE23r{Y??YqRpJSS^Jqs6yT0jahesRvU1 z&^sRuDM}Z1^0yy6^^5e*;n?d%{2EU`k&&f;Ja@dRaRlPM=^o5e>oLwoAEPpF0pSQj zqDu?J_r>TEjOr(BfR6(%<2K1lY5O8ctgiQLKLUsFx!Yo|?=PFSBUL$tP{K{^+$-~I zm0?Md;ee!Jl?y4G*^(H%j4B&3)Ki8u3s)dspTT;DEKuu%D7R>L?LuX~ILf@a*e$r!Bc+UroZntW4wU?&wnx4i zcfM9x1&<3fj{gCkw$Z0yJxU=+?F2s*!}lh;dpn<_ujV)Fzt$AT*G_6_4%&j%&Z2SF!O|7y*!S30TOtJx zefV3Uehlb*E?`53ryWAfYS!W0FGO` zIs)m{Z`CYLAAFYNI-S_8b+IxiELlrRxIoYx``@SNADzZ>p+4<;TSDw8MbqaAh6#BY zg~;gCS$rfSRk8I3Welzbn^5a28GhJS(xI;fp)oq!wI_c3MIjqmi{vR2$vPQYNoPDY zXx=O}J~24WWTB00&j(1>J!PaxsuPobwGtlvbcy3Bky;&M&&smu4qeKNKwq1a?6)gM|^91l2r?))G~NKA+n<*?*0d` z$!Z=MelmKz)y>U;k41$HUgEmi{4?gpABaaD2|BmY^C`V9Bg|y=1 z2X^55Yd8hU`Pbp9qE7k(T?+eTQk?T z$rW!KlUT+Ue=xr!5Y)qdR6HsnmSIs^1PGQ$lKuz%c2ASIExaKRP-xA+cu-eKlL`jA zBD5ZPMRp*VyY+&gvWse{e=XRiO-}j^BD!se;~{v^5~CianBQGBmN_qK7k=ytjB<8W z5XA@@$411|zns#D8ZZP@wP{%~Ujpv{*7OH&yb2))9BF%f#&D8%daav-at#p;)!R$$+EEf|hrDuNM9dtp zg|Yvan<_MQwOe8K!K;!Q-N$>={hWKqykY0#qa(~c*mWz0t$#rr`sf$A_N8l+rCLj3 zG7^nhlgOLqkMeRMda`aB3h+ajBEHtAh2C0ct@PB&R8JVQB!2D(E?NA@YHqhHGlgr= z3#HC3x(Z`Ma~bOEkaL-6l!k8wt=wQH0~YRQ3p+JH;)9t>qp0AV!>BrB22J#;Do>*B zSVOGE!^W>da+g(k`>q_K&47neh>R2rpkDCT((YJ9!#pQf^epG99*N5KdZWkra@xT_ zr9~^$?cNj7FbMkQ*o^EnBbw~+(x3Vr%cVS}HwX2se(3i;acgh?ZEUyZ@oECDtGfd% z?&r2kvD{P)ja@)uU_W&986hJ}{~4AJj|OFxmR*H&Fj#}NFvo5;d#9$(3cmt_Ri_Dd zhtn9cpTFaNU~hD>iXfu99t!|HW1cIOv3FU;&LO*>?;^x-&GPQK-ZtFkvC3Wp4=Z`E zD`g-MEpc|?AUjcwg$yf7=+)fko`S1OI|>fX?^Y%nX>^rnc5%J2Wtv13-QWnnQjzVT7|*FwO&5m}-v0ZXf@w_x9UR zC5YMQJ_xl4`xyEv0!_}D=+V=d46?YVQVaB;@GdnVLpeqj-r=&~;mFl`@tyBX@+sUU z0kRCAg0zqkpD<0ph}p*K?=B091l8RZ&}kzB=&ZxPJw&a(J(}Cdf#VDpV0B8tB$ z6GZYp_29qf%KKJh1G3R=+sznR-H0R5COCjc#rzn7Rt1iKcoke;)PJXXp{{{7xNUod z*#ZC5*L2(@fA5$V;BYWXt6KWlq;4WsXUtu@ls3C1^!EVj@8f`?pKi%+n6f}c}n>w$?&B~fuX9x45(ARBPU=>GS66+!I3>Qr9J{JyReG)4TrSPuMDX-9FW&6^$i9$zY>iQJTxg>-ZFr z*`s9C`K-{%*W(y6AdMjB`W2ymT92fV-bIQ_2a!7LwSjoy{;#W2jQrSh28~za<5Fv3 z@XhVv0tUvyT`*|}AK!bBiwPc_Z^+;O0KZR{k>$s;_~>g$8yeN|b5xJ<>>YO~m#E*NwYl;AFOShb<&)v?TM_>X@irlJo=}>}7#@ zAr@nB_tXLaSpK-F`IsWK41c-{at0N*K z9gwoP_8yF5{!$>W@PR6b7Lg*Z`yM;!*@fHkcRD4XIxId7!W4NU!MhD5hAok`;Uzsuu^8^-&iZwgc7z#$At?P}qhD0Po1-^Bbi zwrE38dBmcUj>TSkxKC(;l@aPtH7UMzQvlex<@@1e5T(~dp6|B{G~^l&IfR&YOuK5` z7A2xqehJ)aoNeOJd*y_PtVvh}Q6FR9`0D?;k8DJFVB21Gc}{*0ME==9Pl>EP`0KLw zcY;&~$CV7SG;vZ;s?&WARmv2sgBv*Z|!#;m&&wBu-Lk5W) zZ?%z09t&qQApi)WgwtZOj-;)N-R*Ei%|MgP+5xqMyj&y~$o$OP?*3m24memcu){qj zKNU}nHo_>A4ycS$^q6Ceso$lsa>l=&3#t{RhJO?9reL}@r-rbdmj&=YztrB)ys;!d4gIRM6zVM%y)t zd4Uu$xS&8&gS+9z1h9uOj+A#Maz6+(^s0KR-M9e`Z=CBB0u^c&vqwS~!k?2kO? zG0Ds2eva2yA*etb->*rhQ~PrZ2w#O7f^c_2#6=@%wRK_ZhwI`qWXODnuAnySbsO=m zXa~c`>8s*)@A8wmg|NGPW_$NG485b~)T6Clx}s2QYsu^?io>=OQO5aq-!2*O4IteS z4>na}$Qn%C)3JF_#jm^5rqRK%y7V96tKyOK23qCA|9Tqv}0Gj9;LB`ZVC)e2Y z|6s>?uIDgPRC&cDybk6p^aFXy;5*?cs}yZh_jj=*X=i6Z^T3%^!H)7J^Pj0*DxY^g zDICLXD$iyZ&7nDr_k3MorbDh}cIxI6Tk#ZVvxXp-Rk)hvFW(5C`g%f9mFVpgWGq^D z-88?9Rl`$HN%CSk%UeX4{m?5<%(r79tM%NB@?GGpPeyS9n z8Mbg+{X|1hW1Y#bvA3Td#3#YoEgF(?t}fp#KA&1(M{QsnZWL>RPRVN2FRR_8o~WrT zYMC3!oXF=*MM2d_L8p^Jw)9=Ou$9AhK9##5c?4MITF9ud!fC)1OVxG5_p13-l>GSLO_(BiEag?hhI7y?B`)dzfL`+r@NNF-vNK5@+Vg?fSSc&Wis*r5o)Nv zZ^LHB(0B?DtGabI#KucGMT}55SWSD+q#YYeK?%R0P-^wpYU)v7I$!QKWr%L9hQ$kh z2!zRvIo!87wz=6yR0hPL${X~9xhW)CyHqoc!M86p*s64U5Ha5>$29*qqwVo#f?Xlt ziYm?BfjlMA=icr`&jGddu0E;p1-0uJp;shv6yiuMyu`$s$X68qkI4>e`dtZk3ppdo z1IY?qcjCQTy^q+fn4*)IwIj7n$UUq}_6+be7$!-`UNIXfL(>T{5_10%!Ys-==+?G2 zjH~6QisD|)f_uB$|&VVBawL^5Ne032nn&ms5XP6&l*znk&sXME`520HMB_>Vc|MnF}$yG>*uj| z+*@bq29r~+xdYidv^X&58a~Z6LoCFdN@6`GKcptSDbe|tMrNUDB8FJv1^KF?Qy3`Q z4CLyUVQg$5OVADau`wBnlf1f`{cAinGTM!!uv-oAf*IHoK5wWcix?|C`tKZcsEQ}N z2IV60c_tSZ{RasDwwT;nu2#R9Xf9*W_|#vDqtD_FA3q##yZhU_7WRHBlb`oQDZERB z91463Y!-(91DU|K;#%ig*y78i=p+p!**(Pi zo{7vkYhd&`&`PbTd5l?GDwtTy`W5LTJeggIt~<82Bl-!gP6a@hLoKKtlCK7wOMf3f zDDB-H2yE?XZSn1S&!mn*|2rqVP%g9eMRMOG+2S>T8~vFL-Zye=;);&uI?YngCI-4s9GXg_XallNw}|niCaY zfZS07fm@ibUeDx%5C0DUGC|G0*U@axNEu!kANY^a@2<=-!EeEw_DWtIfB1_BH7QEH zV8dWS^BEHOSmGp3CH-N4r^`QYhTWB>GjH7wa_HF{$W$N zFYym@I($KyjKIxE1#3Y-O1QmD&twC{LV*Bq(ZEwNt2iL$IR|MRZNd+JV}6hQ^02g_|Zqhy|Ia&mOQl zyM-EoT7V?W=jQo5<>RvK>pjV)o&vtoW%_iOGN0H}gQ zDJ3>01Wk!OoXN0-6Oo0U0ihEyxUvX$s7%?0%oqaV&!v1Iegw z!eQqSuWo8m5*aE;YBP%hF782g;u?s~c8e~|6aR5s#sCaAJ}!mVzG*vqKhdFK|QLv686rxsKcsQpcmV{x&NF*YV)l9_WR z2jeyX09{`5vj7r7;%aI)=J|3HjhTtUU}lv1nQBPlO4d?QD}*Q2xhh)cAxH^ImBw5k zl0*E9{xE+MIk#>dv~gDzyv)}BR8BTW}T-f_abAPICP54>h zP9W^ZwIRec*#!=NQ^Y(KS&~y4=V+6gJf}14iEL|0Vv$v?r#7H)YDeaC48<<38rTcK zVDl+*5@b$dKZS#e6Mj$c3R1m#ZhhHl*!lQ;UJQidQ|;rA^IDYgrh}1>PI#OD0Est& zJ2uE)Uj}wT$9zwot(Ak}%JrK(sk2c@@q!!RA3%uugXyjB@OiBh;|9-tl(tp%O7taE;E{{Y1K_@T_uD(OED zU7zI?&v=?@r;ana=X5BLl2!izhN)l?u`PgNb|VyqV>r}EPQULp(?Rw4G-nmrq=M5k zDa4mxFsX3dsySPBc+xyo{{X~)_?*sA<27B3a2sJkV^e%Vn>d0F`{hVcfyk&6;+^pE zoZ=;WxK!>B5pH^Neo-3xAthjunNBVk{{V(XxdH*9d%dHZc%%Jc`&#Xls@#V+;eHq3 z)n?4UMElhB8HQrrZ8cj(nu=!X++72ieN%~9OD`=iq(+n$-dYYNPIWeL^-7j7@FF8u`@AvlTuQpu_3gp^k!!OM>J&JtTRQ#RP|KuJxaYvMO6!~gHELQ zQlzw`1U3{PX{o10I@VB>DW?!hml9`=G70KQr$eATw0)y5F9@)ZUbO&ovD6wDb4K0c zadD3xWiMs)nS~RfL2R&!ie;<^*ic}}QArw-y(Fzo>02vO)U+rS6iB(;EOaNWIq25f z_lYDRsi@pWyMQ%3+vp>hdqm>;I<^~SOCUVTq>uR$rv{h~j!Du$O4>?LqLNe-2N?OY zipb!m#L~WR)vFmPQdtajCbX&Di`sw(+Ii+lX9f$ZUOMIBg05)91d3p z$*0}m0%p@Vf*&67{{W-E99HIeCu`lDR=nf+e+*P)ymyt;DO!U{&HMbKy$k|DZM0NT z(9}4TLRPwmR8pl9ap@oN!*2DVa$hCXb5J@R}$ zU5)g0+ccmFQdOKu%}@YJ=g+{+$)x0J;&iENl|MF`NOm$Muh~2yy^yFOwK{@>qc5;a6D2uM*{K!pMU5+}na%;Bp+Bnpz33JwVrvWAAH?am=yE>dKs zXd{7M?8LE2JV^tEVbW?3x=C=+vhv=+J3;(R{sZ{$fO|crL-kbOjNHpT%`2+E^0{_b z!+9TMy`EHO8KqW)Eq%=zRJO%CNpaVzG_uetQfHRu3Y9lLUJ+|Vvyc3q2L=@|D2b;Q zL-7)Ls>G|kiH~ghIF*@X{)}?G0F;l2n;Y z2~(sv_M^3XG~mzw057{a;GP6EXs&aGvRVw5qcCBmFwJB1`qcaEG~~F33e^pFp)MBE z6r>?aC{Inn$QVpKsX&z^AxZ&7%1Ls>>SzHLb4?@1nbvT7MVsR@6NgR?W-~NQxr>wv z0{8$bS^}M3uZpQsobZbKMNyczh$?{qK#4Fx1|XZ6gFOty^0OHA1}A45>h1EcH~~kQ zzWoXIkERH00YXFpKYxFA!2~7HBFCAu8&1Xy?|q;N!VnGOL`*1p0t-Hdj?7r^QTB$& zSg;Zxh_#RR<;q996dPI{>c{75Yij63^5zME(TD)@AnH6SG^J|;P>~bA%Krd<#|;jW z3m)*#=AH2@2&9QVe=b}4$F}&8C>9%!yZoSts!rhT_IK;H9RUwR5gksIO`%sFUwwJt zh*lW&(`aFs)O`)M(g3*hk5f2>L)xaKG3(^ULQ>vPC-?zL090@QEpm8xZE|;{41v>> zT)=_1TmI804T+#iNVxL9ua~Fna1TCBy6FM)*r3F z8p0$1w4X9R{Z(P^N(<=K?-%l2j4YrNIR?P-avtin{+AFnr=0CG>muG;%GKMNf(Jk60+=Wg{PwrUpY?=mbNicoA(L{I83xwme4-=vKKN-Aa2R!WuR+tv zi&%NT?$mGYa^?2$+7Wl3NIS`k!0J3xZgbNOi?I-hu^1pQ6$mHdFhUPE`roV#V3BTD z{{R>vs|4vm;{iJzz4rIP5kzt$d=N!QP5KcVY&j3@ymNtpil15+-oADwC6zi}91@_< z!jM4#03LuA1A1HzLMVfAvHD=sm(%STH~|}8eSo(kZoYTxgdP3M}CE8RWcF##fMaSTHYj~tG3=y@M`+uW_H*{+s9M!zW8-idHjdl&EY^M(I*o`@mA-zJP?F|atByyMwuH}5`6DJ00*>U zWP78|rZEsim^KsBmK_8Xa`fIeo_{NkJP<)p2SIBO@%@GfkpNV%rn>p=f?$V|2B39= zq(B^w=hp|t;2wOr_);Er1_Y9C&wJqjXWETZr_0O5JW&VLu?OG(0OJS*kUHUFq!>-) z^D*Q(3=l+t1Oh&L^uNOdN`e5=rM}PS;_%=k#t46(KV70iQb>UoOrLGzslf<4>$!ia z_(asC06Ex1+z8qyY!I;3f}oKb>JdJi{V?>90-WeJdcK;)JYW02^MCP$M8uycw~)WR zz8ID&GwDrQ`aSl4yWR|uCSoJkf4%*%iaaBoyZOVdQZ4L`X0Cg@n393H_WuAlSe6O| zkUaU=dQU0A1Qdu_{;1OidI3JW{@9ce!Q6hoXva>h$U#Jo zLJEYDb32bq3E$tA#7-8<)cZZo{J{rbp@Q}34)k-O1Uvo29V25KlRjIB>-KO$!@%kM z;DQ`fNCS|V8^rSTu*BgZF3(-#8#Zy9!r%axVSjm_utS&`}DyCO>HG9@mRz}&4DN5bIjT>M9BmS-{|E1 zPr@CgWkFH^e|e*i%QZwJ~nG!5THWH?!Zw{1J9p{T+KKyN6C|3ELL@e zHzc3Q;4yfFwe`Y{>P3kU?!WBi?Bu@C`hUhwTO1ApvslS!6l5I7HNZ$(m1LxpV=h{W zEc_}Xmwy&LgH`?=J7Ir@sT8NUJBupE74V&6lJDdn_`nbntV|%;7NEel^%Cb`ao-FgVuYbt@9NQl%_}u*Bh} zC=jW>*OxyW{{V+c{)as);rmtD%p!+i{V4XY720$l8HyQZc5@`p&9X!RD5V%&uMe4H z@tAapsN%rqQ-GGK7%nMX@Xcb08kY$hn_3EhxwtZ!Hn#CIj>e=7KnKF?Y6m0cvEo^C z)Ju{|SdyURP^eJ#6=6ZNco+V?)N5B~U4&J#38karjKYO#1nDk0lh?Z3gUzl{!BSE#DN@`Zvw=zo`ceIx^RV)JciW7*NdY*2N7)3l)~gb8 z%s^2}R7#W(W{|ZcsVGu{R-~w;ADHZUxPilJk`kcRsi9}sIqGX4U)Tb}Qb7jSr6^4+ zc)HL6R-MVU#mo&SU|`I0`eIo`umJ4l-P&%)aAKtT09<$SeS70=@Zwx-ca4jPDVe?X za_JiaX5gKpCnw$HN$W$qe@^olS^oI{0F->pZQQDv%*q1ZbGurR-rY!1REz2}sW1)X z6in*VBMUQ5{{W;hD=2;wc6S3ozbdxEESc6vol-8XiWgZ*zb^WiLRNs`(9u{yZ}pd4 z3GjjeN}x!vHKxg#?jVrY=I%{3sTI~TQfCFESA~ZQo>dpV&8C{ho;)i3H6CA{W@S&S zICb*UNh(lN6$wM#=^&LjJW`v0qS(GM-k_ldYRjGW8v+Ar{Q(@}>L&5b^?_4s}5IUEy~ejmFCtn7k| zhVu7W&-0wyiEA^g_lxS?W|^S&V@$f-!h)Evx^=#XsUK{*j3Y#3wa!toPg;NzNttP4r3AuK z=snV2w}dOa)P)!pQc2qoB4A3wf0$?kJ({FZ!{}Nvey@!r7NUr1*a|h6*O-iqrATvg zr0YoJ6ou$oNrN^6S38))hrdcx2d0&~)UgE8z<})V%C&)Vbt1KUidKT6y-ZxFGc_{m zmPu?9pi)vHNGkCp$p%&u830I90*Hk{4??(TZ1i%#RMS%53el_@qy@EYM-_WJnhQ19 z^`q@B4dchSKZv-0jdJ&kS(gg&9}aO2UbbA&;wy3-){``*Qd|d6P^hL^QQ2Y86(J;I z<`)>e1}hDXlHZuZV{ll|UWHgCae&6HO9wX>1~p#PWU{+M+5Gdf_`*&TJj-(|v0N+& zPGy%Iu#nHf&P&t=s33;>w#!{+qLkGV9Hz`@n5to7RFO{m3_1ysV0*CHpFrvV|H{q^JUxy?_PZxuf#agOe^)%;r`+h04arOxaW=D3qyk6tpYWBm}7w zCakRl6IBKZDo9$z>QdCTgcLw2Fcc1zgu-B44PGddw#1x3CY129ABdXxKXZ`7!)VE= z0($2OlLVtEBq<7_om_k*r9@B!l@$;jz}4c+?Q3i$dfbwbbwm)I{3+;W@L-)W5-_s? zT7v=wI9@EJxTRLRp8 zvY7!B3g&^sEODYzW`Q(1Stv)bsZnwT3FIy<%u!WT;ceYo-6?HBz~CuLi(n_nm1+qG z!hJ7K61AbYgD`b*B#YdaHTVMhgB^#m*aS{NS|$(LtRhKF&svg{sZz(Qb^#?z;$lfb zNHpOudFNJ$e50adbC13-Q&xVg>+{5ST+#FWmiomqEg++$IdttwJW zd8wk!8Oz*4zzai-JnL>Cm>?!(>C~KOYHo2(bzygtOLx zor`@2GHk=N*g2SaXNi-GPiZk@3{pZu(=uGOdeo8tGSU=yyGUxWZb1bG@7NYZ$KQr`h4%*&Ob(&Sz=tW`}YFX zcF~)6s{;+9S_6*M))Cw^ML{HKl%N8U0E?+nVPic^VkD=A?B?cxYI^VHu{_F#OVNW= z*N6|T+>Ii{{e4DnSy7!;q`bv0lv6OWN&*`oHX{BMh$@fNp$6EjrJ1Cuy^!qXt7q%0 z1FLnB2oll~JI_i{!J|=)>Uwl%Z(X}~HO8Hrc3av10AziPH^7;v3guLfarbDMzH0Vk z#e8$b>ZDUyn$pu13(Kc!o|*xPok0X)x@E!@equUnI|DhF_D z+$(P9Yk)3k7N!7#0W9=msi)KCY$6xr7W0(_lz?f`-Z&(Uol9!cIg(sS1EC;t#3c?( z3hFC&<=@`i;ZU&5O*AH*-*4#|_Cn@#`?O3{(^Wo&7ShYtO4fxEq=dMjK_rr-l9gPA z5vfXAq?9HRVER7VY}Y)h*Myj5hjU(C_iA0-UYv|>ZYy@bCCe#N`_8vi87h996)#H3 z(vn>8rWAm!J=h@%l#MAdi__B5TGa=Z)cH}qhMpE+GZ-Dg`8_H}wXp}e#-Dz@{J}|a z)fW>APK6MqCs8Dw_tYbEGHud>5((M4`C)ydCd7hv>*j(nOxOhK@acDRF7*v(O!Y)yGrfDF(F@*qTmdomY{%0{KjEjnmtS`2`cXL+O-H0KvZMBdJ#&$ zIod80?SFUW*I)rSnd}aNf{sTaM;|=Dv@UWb_E*t}StSbXZgy0JjwO_*vslk*C`l?% zLxJo9eV_vs0DZ!*aV;h72`UMB?*w?GzFI*5z@E0Uwh^~>n6hR9S-21#>F$Dga@FJK zMq4nWFIYkeBNd7~0Vj#Qa)X4BE2+`QYF0G&X`t(63>p`+n(={psk_U=|^i$9h`ylxL0li-o=O2S~VgVoC0^B91Tsfslv%Bo3^ z(QakJJ278>xC@PfYETpXpC{(p19w9T^KE|ZHC_n`Qg`a2h5+7a-bI#XNK8{KOjDD&ZOAU+5$w#KY!y4 zQ-E3Cl2Yisd`$o=X+&Hj}`m0)Ev6YDk+?}+0i5!_9JghT;icH3jH z5ItuE5fy3OUZ2PD@xckNpFT$P9tc}NRk^edUf3c6&RW`eF&$GT$9sAbPx!$J4Z#FG zV*6Wdy?#Gq4F;rY=A5ZsPw~F7n4#t%3I~{(u)mSE-A}v;!qjW$U2XE|NKU^G7N7^_ z;8&PbBzg{Z6aBdxdD{yJpyQ*qgTI*GgSrHLBX6b%i1cfGphr-FHlQ-FxX=M08#e( z3=8*6_v?ZPGJjC_`Wz5Jut4NkkrWbCsl1aXU0){F_4D(KVR*QGmUhjcD+BsE;FMUDN^NS{xf4tQzzli%`+_>w(Y4_&{^lV0lhLXDt-%trRO8~I|$NK4e(UDAPw!0AAj?jWv6Qo#m>+lomNWlvdKuG94 z{W_ma5JM!A3F>SvU_tWf&!pk$9?+rY;?zEQ?MTMWnr4ywDGt>->RXjOcY}TY$K{C< zI7d78@`%Nb;@$=NQoX}yG68@NJNrxan?XWEffLr;`3`>I?{7koYQ(#9<@kvHyAu2pC zAy{$>aS8|N@1?K8ypP2X;cL<_WwQJfiQT&F>QmdFM{}knDV}>z+3Y06qbpSPOPr}V ze$Zt~%yJBJJTq|K8pN%8uVHfVPY_|@JZ02GrX_bsR-6U+w*Qi$=N5&b}` z>w<^oC4OxSLeREYS`-wOAw^s&LXzQ0JRpK7LDi}G2cIgNhZqul3a zt0_{_tHM*>scIv_QlM;Xd1s~kALctDKM;hHaJh~WZlI!Et_e$a6mMLTY5*x9);+iQ z1nn31Z>H=fPsWLIvtGyJa+iN&5jPMqIn=ewC7;^JW)!yFfciZg+1Ueh%bS#WGM%+*fS@ z_}5P8-WfAVMKf?Tk^+JCFd)SEL`V{E(m~6lGC)250I$f#5^ZKFxqO6eE);#XD;jR4 zw7E>~rraAzPzgv9ZP`d;uq>?~_=wRn(<=;XND-Ap@Tu_mgW{uP|Q&8J)3pxl| z6C@VgDIqST00LY>ieN-aN<>qAl^6MSZx(pEg>h+DX57-;sv^O}`4f$0bk!7X2e@vx zF2GMEMq`n5F`XZTVi z!%X(vhG9V37IAB^A;8;15IC2!;m0`>6`VX0a~gKi9;P8(;$BE0O*oh;1f(_zi17u2 zK@)>q)XJOc)O28O9)#;+ZYg%@la!7y2_DgYB~(y7*{lbDIMdWUi*fx$V5g|bpQB|f zhpL{^8A3GxwIs>bK#~9g@C;00rx7J2uhtqG(2{%k`1nMgt!h^V1JU1406K@Wh`#Zb zRZYX0eHRa~;#W|DkfIb0m86BOY9@b5(pI7%0zn4YmP)-~<7UtUaBMZR0_VqgMf(o* zZL58}Vopm?a}2Skl#*O+1q+K66cUs;37e@Y1P-zW*tp}>ijd0hG$X;=mXEb>R;M%t z^dt&Cz$P5ENWhta@tbvZ&5;dJT&qQYCs*t zKf%Fk?gTzm#XIFkS9sgaApu&2^&o`Dgs6~GrzB_+PzX|2hMjD{7CKZC8%hha7OSyn zj&&uf?nBe44qb>FcZ~#ofIKqQMIQ=qgEbo0`8<;0upv9eJxphtZPKy;5lxMB1q9>naep z{{S-0Gj7v3@0`0;=|dfg#HS08%$qLh;t6GnPn|FD=Or3R0v`$0Y+1i(1NkV%ITcl%kO7V;+(5Utk}D zYVH}Oq0K9J%O|NuS(nwnnPps56Ps2Vy62lfck^^Ld2x1~tfMk+p@dMjTymnAwH_3! zL6_UE*5T*+>3Oy)a_Y({l=ZlB3*6LKNcEm0@a^bh6@gY?R^|uyc?I?D7IpJ1&^s_Fpw4_FQF&NFMr@RK5}dVs>`fbT;}D17@A0 z6K?6<5L8JZA{#I zhnlY5_H0Qm!Bb8f6DDesKxKfT!|eO2F76ooaxK->Jcirx0U8JwPyh>&bb~jO1l&P0 zj~{L4&Ylta4HT582i_N>RBu9CV0qFa=Z`70B9$bhE)?J%6Fb41z%dhH7>a3O!V48J z?kDSC2%cEf^KPw6R^V&hz4M0XkyP9HEBa8kN$%7~iAZ)7+SB10feO?Ji3BQbIDu(0 z!AQ6?1ZY7O=0GB+YeFVs2?V~VK`J{R<5kq9x*W9EFywbN46P+4B=}0f zI+JJ-1Ou__Sy-7N{i_QQLs!F{%d)8Si*mTg43pqV19Hz!tiVRn;qJyWjEgOHr#3-u zvGfQ>LPRCD$&Fe{jq>IdZoOT4(VN9Q^SqC#eQ>`q<%?< zS)t0;P>Sr;ma&pVgDuv!aC_^p6|1yR_34^DRG~mfccB1OyO2Wye&x(r^12~Wo3h;- zC=Amn46?4^IHuB~kQ7@$Fr|ye89M9vgTY1Z2^7zswrEKJf$U0r@TP!(%!QvOE7}HZY1hknw^R_0EeK` zp6-N?Iv8#dB9pu9Xw5xLv(>jK?_%jMRz49ZO17jo03Gy#yMBy&&s=U z{bMg6Dq|#_NPSpTzc+el?RSh*IcZ7OGy~w0Vm47G3BITxgS-N7BM6dNz&bzbHowBQ zhY&`D3SWOO(lz7%0IdGoM3Em~nA$KYxD~}xN$+diZ}HOb58`2d_G|bx$nOl$3|54X zp?~yL;brCBpMD_wL3OnutlvH0{tu+6Ab^*$3U{+=*21F6a-WJCE+m*zP*Qqh(Vm5H zyg5&3F~kA+yu%BG96)@mTBWG_LyOd`<1kAh_ms4Rg}RdkC&m<3lpT(; zjU=LV1zdpyxmnRcJU|0`eTEd6*rRs8|ln_C;fS8XNW3yv{ z&81ICt{C)sl3A`x4*hrgk&*Ug9ii-QGb^1bBw}!UAXt;NiG3^&jefg3MtdvI>heC( z_;9j#ceS1z$@upzaf`CuR%qi1&ncw?LRQXhGX<8w3T+Rv787iJKlC@+aoV24WwR7a z%tl?ufW?ABiI*Zw#b7m{P8lI7HUR_+7W|8TIK33j^hxRWwfGqrVS&MRn-?RrSVf@< zLQ^_^bA*$dd!Qt#c$v8A%T*nCkW@>8t#H~zs!T?m`HT7e&N`Cn5_kPP_eYa3j6GwJ zAA@21(B+^L2H+X;JtwI0>M+Q-KrH$fh)$p+#^BsYu(`G7U`71!L`n>KA1wh;=Y8*U z9;bd*n??w12#eo+zi9bjf*38~J93!6-zneAY%xjHTy3cP`PwW=nfwYr^1sRcUJqdn zP%|;g=al}}{!Pv!HLldr{2Y_ySVf2m(>Lfo{`U04&{~_Hr|3Ljh^tRNzc0V-i6DrM zwtrCQN{8+~bAlsUbsheGa2b@rJ6umeGCz-hEJn2I2VQYw4?F$Z&cOuO3c5+~;qMy= zX}2wpxa;rbh~ne`h?p{d;(u|HEeBdiQ-|ASg)&jZXT! z1wurEZ7~OYBK_gh>S3(kC{0?zx87~)NRNgZLpL{W=KID zvSe@8;@*Nf7&u6pAt(bsDH9*x1R?m2WO%pQ0fH6=y~p3*_Q4RWG33-60Rd37%tikI zkL~M%8v(%tIR1qCoBN;bKPiki_>VEU<)yqiTU?GG_bA+Pf2mCUU`yC~{{T32PVg)c zb%2mB{?AA_4_@8hm@{t=^C&BpzUHeR|*g+tUOjG=eQW?LOT7@5uiZW0 zF$4)o)IE$Ib8fc2*TKSudN%G`=L9NZEw%m=h+`#({n9_p{-r`GI=x03r15nTB9q0` zKnS0my4&DAWB0@{35P1Q1Je`}^|0Ob`fiKT!Acw*YXnNj7i-gW?Q?TL}@j{g9e zjCLdU%l`l<@IpjNZ3Q9;o0znG_vME|BsQndkI%W{LIJ$&W8dz6;{-Y-B{9fO{$t91 zxQR-8JA4i5-VFp1DJR~-)A!})(8d}YSMDHk{-bKtLV!(yI?z2ghbWT(?;43V9QmG? znY>3lNQAS}wjO!^07$KfUWNYvok~68?|YNoTf@>I0n^v_1AG41d{=`YbNe61?g;b2 z1WaiINF-iK6Bh(WNrNWb_uB+Yfkz`>5Y6ESNPq=^+mb}uZR%zRToCIGDcu84juc>s zfy_fer$0Yf#ja|mgs6~1QG*KA^}QgV5*fumf8XYOq0LF#jr%l zOLf=c&~zm4)BI1g{B40sAw4bhevjb@SQ4So_2pr9d+I3%LJ1}G|W0$OX`e5Q$ z3u@N-J^Zt@I9kKH-hA_qM85-6iBtSW;)ns>CG6<{kg`AhA6=SeYD!3vA+}sFuM9|3 z_;KU^0L3iOyjNlH+ha2ow=d3ibwIGukW>!ICcwwmFYM041ZBRN@!&aDOWB;g*sx9` zFYQ(lRkyM;5>C{3*B|OXT)NI9qRYaP5SD75xf0qG$lWFsok$873IN!^0b}Opr5s}a z0KYgb`FmEkrTG)_4WxdKKM46;rFLr8*T2_}CHB2g*wvbAer(9u7QlUh5PpX%>Xdn4O`os8F^pLLIdM(OiW3d8$ZQ1FE zAkJt zdT=LFHyCU#P;AOkp+2?0Qo z2(KYxIrFED9w8HrJ(i_v*1g8{a%kezjfN^IxRWPtmDYfHrFV&|AilQaSyB?+Iz!4T zkXE-l;{=dXY*(5INq^UCX?>#7X7H(tz1wkD8#uk5f#Vy=Gy2{ZU3O7fM9ycCW?Goc zvV6!BOIFcoZKY#f>FV1GQKqRtL##Md0*XmNfhyplzjPX!ySbnv`U2eTtM;Hxlt^dW z+N+lo(V5wcnrV+>;iVv5mYtq)L5 zNf&ySXT5}KbJ_lFnp7sMH_Tf{<`<%+#k!YjbPo!cXd_c?q$rfBTFR7&l#dpx0#vm7 z0muwqja!Yasqfq{PN4Q!)`XodE%F=Q(U#S4>ls2D6#ddElGvsxZDGwWHjfEW(hMnL zH4s9zE)-)92lidXgV?#FI3!uFM)rt^t+sO4sBS@ny@4Qfiwm^rQk2^K3b0C(r0MB4 zGC>3ren8lgBPtDg)X-45z1&>u^mqETgJ4~%NO0XRZCmRXKFxSGr;U3|?DK-D(&O)O zuO9Gs2~)O%4Y>aR5%CsZleUE>6SAuN!~X!H@|}(Bs|KugX_jCEl;d+OzXfM>zAH^Lo+?~|iQegqvC;+Ds?6J4VX~5kDCsI1a40@;!~HFGm-5(bE=z@3 z^_aAU%?=#k_(4L|R|=ehm96W zjFBm2DNM;ZCLt4~VU&Q$mX{^gqaI*(bNJ5V44X8hs%ECGw3+^zak`aY0_D@6fF1eHLaCZ@B&nvxA1NgVjT9m8 zr7KHT&A)s_iIF1{C)G{>hh*890LAKFmZQ^Wk4gUk2wZr+J?!Fos?eeo;~9ev(bXeT0vA?_#?X`bdB>}L9cRkVas-*AerqfV!Q&_jokDWN z@>$YXC`>|^uyKc#;IQ+uP7tDr@-qD8 zSWHZb4Zx)z-GeO(R&)5P;ni_h;or7?94#7$a}Le8){2ZH#!XX{S)!Rz6L|jspSw|# zNJ>%?qEte&-KxTjcV{zM5{I=nFC{^$TMH}%b0jzb0_r8j0UUSG#&Dj^dR_E=n&Xlt zNy__AgGL>4)D#k<+MH6SPDli_w397rTHU3~;zF54C*rb=q@IXe5X9H`E zqCN$sWr#bM7CmTRMsaQ9WN_UNYFQXdbXASE(GY`ELj79Mg`^azUBgd}S_FWTAOJy; zinB7@JmozsX>HlZ%$RYP zDR@bTPYiH%TS`J-SNrutk;K(q%Z!9n7I?8dcqS*~eandWshRrL}Dtw;8R6K5n7V(;G9 za&_eSQd21(vJFDiBE^}TTF{-1L%%t2`ZGF|)YayU)zM|8rG+$*8Dhyo(3jLw(RHms zDN2H#aV8WHvlC`Q6g}>`^KYLka;;2IDqpJr5l3ndxieKS;493XvoS1%MT1y_-VC%l)G4 zs#Ez^#OexdNkUZ;0V03pi6DXunIOyyz^6St1RB`Wqo0Vj>`Y~_47X<&8;{|8p0wq7 z*ZfOP5lZ8RQ=v)oI!aVE89cv85gmC=bg$YiO^&6xYt!Ktu_)ncyB6`kflqzuXw|*^bKKu*#AB>5iX{nyG10q)kJq8Qyiq0=-o19z?u8 zW0%Y+mXzb>w-m*Nr9L9%bv1lRn$%ZUDNohbRXtfy;Sg5y?zrQHBlM?ImQ(=nq#Yu_ zN6!rV8;Hfv#$)j_)VxM3iyFCw$x_th)RXNJNiSEs+CNPmkMc|gPlC$w9J32A1(szv z90mg|ktrc5lYqb_D3v7$O%M=}5|oAkm}uUF_C3Q3G&JjNFj9pA0(Au)Boh`;(hj5N z1_S|*IChg9*wdp^sM@48x3_eC7W8kIre<9!wY4Y<`2v*|ppT^g0K!iYyT?C+=CXR{ zL;0^_Tx#B05~P6gcw38-x^=Xrc!zSvaZQx1sUtzCaF7DQ$G86g2wkc(=HoIY&Ml0< zttcE+{PqL(p;fv3nlY%RlPRc=g1_5G=z>lPMnUNZ465 zFhIT+pPlx_u%gC?(V^*E$)X}s{^ z%HjfE&l4He^{U^n;?g~i>F?9jbeDlfM^9YR|KuC$9`FAdf^n7ijYPwZ$;F@X6ur=dSI^GYEF-DSIvf!7 z-tUC00{Lvr{si91CY~$0A+<}yZ7$r?#%G}b1Qym+X7cbkE2}cz^d;9a7c&~&0RCtt z$7iQXj!=8Ibyrfq8h&VDA2#!MsKypNAY(XRw1^;qm!$cOZ7Jalk+iJZqW9E=ROfPlw~Lpn^Fe$xPu&u| zXt^GEA*ux?xyGybbE|kserB7jBUU4z{*)T(joIb{oc@vlwT2D=h7_-t{i{Lio$T#+zTMc0UE>3N{goQD6@oT&p}>lGtkQ}zTd;NAIc ztr;7gitwlDIMJ^zdF6D_vVkt$-KIi-U%j(0=L&rovb1T;=cu<=xl}bEw_kKM@2)m6 zWU&S}I))9|#h!xi4V?bJu?v~xX3v+}BL+07!c46mG#N8IS?Ozu4jhc#oZC!4Sm0wv ztmUeck$ASjo$;ExVfpP7lRyQ%yx9$W;eBNFD~<|BV`qWln(g~P_~T@G;u(Z$%?9hb zo}DVAeUfFK3dYu6?1+`rzVH*3d^iRbii7|-P!YY4kPixL&LQMN=#D|~+6}kT@$QBC zvfqgGJd_71((dO1QLfxJWa17v3SaP2Rf&QXfHm<2Hvw;nON;DYPdS5IFz6vhdPGphyEbm*0+ zC!A&7V1R6jq;iSyVtEL_g$n`BgvRX%+cK4Vsyc{YIv7s>zPwWNG0_8F8O}ED2~N0f z=6=)b>Y<#=qC}~R>=8Gp5&Ttq*p@p3l3NsYOKZYoZU{H(B=3uAAT)azkyrHoNON_Y zXEBuuYdDko z=2x(!2aEg-t>~+T`0A;_3;2c4v+BRUCbl zLQZ^;H5`T|{yVNjUjN%qmh_^KtSc$r$aKqz5!-0AqvB!@5mbS7w|>T^pxd)g z@Hy^wHu6u2L#l6hn_6kpRTI^>Xj(E<$l2H>%jzb9vd{yKl7FfWnIkmCbzpeK%H78Q09ovNwv24Qlw;3pl_ z0i=&{ymvT$*T*M}i>vCGNC# z*$`1`7vWSby8-_o92e@Zd5>3uGk4%h#LCviJ_do%vS!Ng^c(HTD<)w@wZr?$%L5!| zp!b^>qCp>nmX;m-87TX`Q+RRVu~y(sQmw^uEwHxF-#j;DVuzm-flDVXvY)VXJw1sH zIWKvI^g!kY$>Yk@`$|rhNJkWLp=Qgb&BVV01Ko=}5ALLD9?@Yi)}EMFtJ*%n+F)|C$I~^e$Yvk+C&v-#wZzRg%GWH`8ki z5lhWjg1u4m`ue?Q5Nx9EnO^I|JuljrV~kN%1c$c=$OZ%@q(bH3Y=N44mxXV0={R7SCb zuDk?~95g(cZr2TI<{2H95wd5U>`}ufAgOPi5r%@1c?BzjC+dE`BWqX4vXJ*ug1oU8yhl-X@~TOroEIjl#9$^8P^g{p6L2t&9zP+sf!b z3kHU4?=4?JsO)<)4bNW)cF!|G4_>3LmY6g6GG?VN{wT_ftd0P4~YQaCOq{Vf$=OS~NH>@?e} z(7<^*^B*xt(DCT#_Yq664L`~4IU41pzxWUQc`2spBfNtZ+=wRz)KOqm{ovZJV)V)W zW~BZWqE>0fk@&sU$M`MiryMVXq0TH;s2Xh#G(f0&)AApnMtop4!!Ch56YCVaTAlZJ zq*G`;*z|-TUY$rL59p*n!Pue|E3Vv1QmpV}4yMSY27Ijm#xjrlfR-x*931Z~K6G>Y zV82KGr5R*4E_MpIB8jc2nJh4={Lpq}VvN_S&(s`RdCp~036w{a`)-*X7h0^TM6E&H z()DX`k3c;u+z}ZmADc_l+ogm4A0U0VebUKnr`piDLU8mK+(T^xHZ@{Ig7HVH0`8z{ zk(7KI-JU{YaYT;fQYY)p9K=5U0CuCNRyU_}ci`7iCpoae79*rwtHoER97)xPc|{{r zPQVTATXtl)AaTvgS&frcdds$h(|4`EG;J;7yu$CR2fM1-X1|5EFkP)LGs4ul`3;g0 zP)If!uh`;{o#ZWJNkQDI!q}U2T)l$#@){QB&_ubQL+!*skuUH~k*3x-dw|ljo42M0 z85}L`5|4neVUn zyQC9phwzb4&t+N!{8MiF7g_D2ss(lL=+xmr!Z}tVYZBhj2sSy+xAO9Fh|qA`bP9x=I#Pqg!P1&UdPnaD< zn#E2f1hdQy9;(doNIWgq&Wj%q}*ld8|p1!wgW*HrML;?LdVoQuhiYMlSil2TL&JWzm}9r zj^TaxVk^Pq7So87w7@8Q$gX_DYRg*eLIfe@NKP1J5)Tagb8X*3uViIVpRA2(cq#d< zX!$G9vlz~U|AJmxr-CB@`OeCYAwc*p-t*WoJ-TJzb*{319muH9Gyr9?dy{?=31vDr zacr=U-lQ=4`gFCDh`JjkwJp;6j@xCEay>tos0n_jd8qdDv@`g3)>sA~;@g5d2{pgp za4~7UwWt^u@Tw$V#ok*^YtN9S;W%BP+S6~+V9WD>`SY9*c9cp&_&AVIa|%zN-PjS+ zz#*)q{{R?odZUbh_#7m{MUuV;7qCCw1gjxRZ4+}${m|Ql2CjnN1|S32)_EoS@R+PB zlYS=S~ej5g3 z)N=D5G(Eo9ar<~fx-+}MM_9?wN^V9-vitVuE1tI|x%+MORECnlQnb+@L1v7u0~tD~ zKKPG{*X}EUGR{9Rmrm!c#@aldlUOFTi3qdfAz?xo)&pe@#;Q1kO6Wk)T5e1@+vcYV zA1$`;F?vTg$tB_0vgmEP+#)n{Fm72?)gi{xl~0CYeYaGO*YhqEJp<}Udf542CGUQg za*}a7z|Z@I4O@I_Izqqo#kxD_gWcd*1@b;KI$B5EZ$;MVJ#r$ooj5sfPmGNt`v>L` zPv`AFzARyQlr|vbD6E+cd9tZHa34Dv&s4Yx9y{8x*=@U)O#vNrF@TW?SjZ?4LH{P2 zP{fT=Cz#}lyALa4G-j=L+NJ%*-~m7$56$rrX^!Y>!i$K2xKC6JUYJu7e~ z;hJwM0|sJv!jCq`*@aIgKe8&JqL5-iL2nmLi*Cnoaa1?GT|(Mc-5e>y%QZU9R-N$- zu&OHZOgf`TwGMKPtVXizTjm>n`24>jy-uHVp^jmx=3@G5Y4#srfm70c=;NDc{_p-d zbA*-LwY%7%PvnGb1L;O! z+8SCXkGFM%Wj(B6K-Mt!SW<}Tde?_)Xbwt@ z8;79)>@a8)i$SFldpeOhZMfwM-y7rw%?d9kfp_XkYucv|LB6T~0NTF_1ji$f;oULTn!6fF z#dkofCaMwt)!v@->Q54aHl?<=Qp;t2t2B<1=*vDhf;-v6ll$S*aNe6hOZ^6A>4%3MY6(cY0NvYqUvzN!E1~hRFdF6+EjqSd^@@a z9HL~R2|1_1b!$b~{R$(~2Dshu;Lhp5tA_Z28HxC;G0~pk%+L^$oCCiuf#&Q#<{m4O z!vZHL?kj?mhUokrdCU|jCCogLtZJzcn6;6P#^!Rmn3}ZcD`$X;1+hEAmN9C7wwB_F z`V{f>^T)ScErcT02uc!+KT3y?lJ3E#YID0Z11!!zWoW|xmSH${zQt!9Jg_AJUj!nP zI+Mq8UcxFS^&1F?7-`mr|1dlWcOq2N{{0ei=p@}wt9^pPATTaLiq%DA^&}*K_kN$W z8JUTZL7*Tx4~q)d&zdaSdsF3`yfi@NXTgY7n@2FJpxaQo;K!pjBB_hxO{)%`O}oEL&V7A|;Ba|YbarHuu`mGp zsw}>41fu3r;prd5)=}f=8E?WW-u7-s_e?L&y`4=WWkVVM6j1%#(A_xWJ! zHY|NSRXh3+b-ZrqMU^xJwQ1@ffZdPee)9Mq;N#hoj7OU2ZSBU5Sz&a(DKrW(&3)wc z_sjKw`=@wn*O3=%#5o}J6gz3=cPyH<@(iK~%8YE{w(W{N^y#t}h{jBet0%kXrTP66 zsbfSqDo<9-*vnDC&?4%OlTE^gDIY%S%~ia)?z*2{4c>r;hC`{quCqA&I-do!FZG&_ zxZGW~D%@?UXwjEyQ8;fI?p~6Zrf)L=d{mwNjTW%i{rKYCFsuFKoP2y-XJ?Sb2`Z}5&Tk5mk-b*VuPXK1WekHTz1P#f30Y`I^qL$BG-43X>AKu^#*DY>ueu!@lb>Vxue%iz+bhxit zn0=x65SbeA-BHYwR|td^+m0umeQ)S;)em6&uiW4E=d4lxN-O2k-Oounx0KY_Q)qC- zUXUb2gweCDz#48+QB+FM*52TiHv!WWJU`!I0x=p60rcq9_FE%Qzhk)oIuU#0XNy35 zH8{!hk6ZNc)}m^T%9-w~>va{8H=onMO^TmzYilh&gS)Ovb&JzP&H}$KX*jduMDNN$ z;kZrIp}KyitiuB6soOV1g8VY9+G=0+Qd(sk6RjG@7Dpvz4}o^LPUdUyhA%zw3(mP% z0*5uuhV9K~JlP9MxTsPux^;LcmS?Z=J=&r2RQDueI^teW&8`TT%XHbC1$B& zl(kU0{;S$&Fwckkc)t7(Y>AdF@cPl;WVU}Z@INcqfc=44k<*LH>L1=;PjLJxZFCsY zw##g=h5LtrxI0BIhai(Wo`?s$Uo7t``ip5rX(JeYZE9ct0ak(dvc5`Rt(osw@&c_F zHhBAAk412(CUuz2 z$us;?@x$pN7-n6&HA|#L?k`2@^w=py1+TUxE!*gQlYz1{2S8eCX&;t8rebncHfA(& z=$VAcOm;6O4vPi?!Gd(~#lg2>45ZG07Rg9Cafb|m>L)qL(6zhg&-4tNxBc_!)FIf{ z8+?Gg`#Bt~(NaZJYqzU&jlh;wIoRoND zAu$__07fLfmfDam>ICX;Igmo!Yf9%qVZF4hN^3>`9wT2?nN#^$@NX>r0ZpExSxuVH zNokjhkG)DUo8#kUcoJ`Jx~5E~T4VLC4$6CTA!PKZHPdz4&Ya>{S#h0-uP6*V$MD64 zNLb*+U1DT4zZ{6O`IS979slDyMCLyDynn$T^mFqH&OHczaF)As55=@1}C4{P& zY~Vs)hNut~aDT?`UBsbLWZ%B7{R2#)d8*1DP`?CmUA#OfHv`(GvX1|Y+e(3J2((t{y!L60u zyH;tdzj&VE+w@OrFaBO(bjid z7f*{VQH-`!={Os~&B%Y5aYN_3BlXe?oL9(WHF|LNf9astcfb>A%{mwvzQ5_W>K|{N zO`$_iYbn{>fm(($F)?p&`y9@N)WL0&m2Aa3n(KK|Twncz?h(tm?Ig7tqXwBnjIW}A zk&y!ZaiywichTSEAttq5as1gGp24#cvwP-St4^zVu0SyhWrPI!a9Ur}@oNy9gY z_T&@Y%E4*x@iw|!+r3T6i%NO=>&d=^?(V-sy@#YaEu?S7lmbsHx~gg^f}goi@=D4#kQlJzjrDhX&|sI zJ`WM#v2o1$Qv46zxDWzTNSu){e#k?fO$PnMbZtr0wa^#HGubO0!O%06c3=CRBQDu_ zH9C}MQ5vmg%-F8kZuvV>9Q!Az4;pNOnuCdGLU&#Y*3t8=u0lFIh@PRvNbK*egOrNM z2;LXk41z0-3fA#{P2Ft_qu$~`QYe&&vwA@Wbf5M>_c|-nXFILX*=_;T8r++O-BQd7 z+huQmUfvDyxE1*(U|TLYur)^u$2~Dfp^~s&TV=xnTNJ_;c_RkmhF_8a;T$kLX_#Eu33h#&PMAo+mh{q(8gB+oj#)vN^%8Z zhXjq_6D@G}SQL-ISjG}t)O`a7R-QMums?ysf+1sQ5oR<|l0Y!1Q*;z=Y)8dT_Rb`T z9)DswB-emVrA7(Uunlo6@-R;tG?zv!(94?Bi06xakol1nS`-R-E~Ayux`1a&3+(#R z=>{NNt#~EG>2$Je>@Jot2OS`ue5iz<)Gsb2WF|N@myIqVe2~)-O^`OGUh;LxeLSCy zLq4_L30;pcl`x%2Ll0U(Y{X{g+%Q{zG|j?K^ddlxK!Z8g?J5%HA0126f_~80$|~VK(B`hY=ovO;Lyi zzdEYw5fdg`>w0|D2(BNfwW()Jz3%*IOrfH&iY$1DCe6flgZ~dep?ZM~05Y}sSkxMP z50w~X`=I6`FV5&!nZRmj$f%px)h&ZIqNG2F!vuxpyf@Z&;m}G4Mh^S0=z%2qvK8n> z^cE6%v%czAu#~n=4RosgI((aux{v=PP#Wr43J}*uMuIaeh)Ew$-l(@cC zkb@u}o5ROhG1=67outqdexCSe(BJqh8s9bZ)T+qfoP4{Voxx!%E*qP@2@0rC5OZR$ z@Y=mcHxqL#0%-P_$pz7}3?Ol6Ai9HQbTkcru#y`XkhtPkh;QIcC@^N)Z>d=iG~6oh zh0!I%$0SB-Hw?W?I$vvIN;csBovFO+$iS zUqrEE{}v=wHR-cO(xzIL*3zTG=o3=tFl0-Ly7nyxer(EA=dR!yz~JR2=Q} z5sePT7B|gw`uD!sacJ3C{{VVjYt_?OI=6H^>FNRT0a{gWjq>>9TSz{%8D|yEi(U|k)kMx3?qDW{uw{KMa{7^Si!+89 zI!2%^-&bK`cUO^-K_Q=u=@kzpAmt)#vKkdnUM>i`q ziFm7($PFi*1i))8kkD<%Ttg*GJwf4vz#LRcc_O`xTC4Zy`ce*nWG|8i82R1hQr;>u z15H?ASPn)?f{#@(3Z#w~dqom;g0+d7d|IFjf2v>GK9It6U%!A>d?ozo(*kM5^AQFm zm+1N%ismHUJDYZu{dFrKQ`F%ts*Qv;Pjfpr^`{FAEeX_q2J#zr(;-k|VRQ(7wvg75 z9%5&S!ZZH$*`Iiyl0r4341FHNxNK^A@RM3PW20R;M}LY41t(&2vHIu&WY5SM!pLk+ zfd|DQbI^;KZhic6s{lukNqia&cvA|qc%?Ye=Pn)dV29GtRhbKnuA5&r0jT`xu z$@l1=+1WA}8Hw(8r8f#$HTj7RG&g_#>DXfpLMHgq$EBXpLgVTGw^M@?p`S^RiC#J^ z81;o-ZSn~>{x2~PVP)2r3AGdnM&l8Tx7t~%=`3ViXrJo&oeu?I7p%xU%UKx7gl*fp z8I_r4WkE$hm^88F->&qnzO9yZap{WW$S9lLI&Z@f<4-FluRk6yJf4zTWOF{$EozwY z5=Sb1u^gS4(=b`s)>L;uS2JL(eQS`;rHHLX1ak$*A;aT`Zy}=coN+aMDvhVtKLlhH z;Qx{XoNVWlCUkYU-k1^UHhSUl@}WAiV6O~f#L(p3uS{x~eKlWRgBN38tp5SBo~3_c z3Qmjq=MPc(r|0-DON|+fzt1+t^;M{^{Rdi!=<9S7<3xj1Z7{Rh^OpH#(>Fsl*Z=kdU)|{pfO@|#~)R!Uq#K#0Q9F})#U`RSt zA_FSGkx2HQKjr;Vb(svgu2OZ9BJca+$a6iL1Xmy|!KRvtYP ztMSZ~_zg;+@);4fQ6B&x?f7lgurf4>B|bI;9yerRsVC36K$V0X@QdLrbV-bc&Z9ia zE9Z`+eQ$4~aCexK&x#SbB9m^UGI^vl_tk;w$|CR5D ze93d7mc5EmUuS~Od%^KV6Sa4>NpWor8SCgWnU$&@jHQ*%tmFiFshD1nBkAyKg=IS{ z-ur@u_n>#fu^AoQ8V0^Z3@FiWS*Q_kNW`u6qxyB3*yY?J30a$a^;^V8G4<57#0h`9z8yUz`L-lIuuHl& z>+|GIXJ=%Nhd1W|%(kCrB?5sUk&iuV3{jtEWpy1lBQ5%CJq`<7mr>vEj?0ZUI;f?r zV)a?SCkolR!IB-QB*4p7SlBL`$m55xsSQ}OX!Yzg@Go@mV?4uZ?T!wPs*W;RkJvZ7 z%V3LGm5cdqZ99>nDCK#^5Nm?%PzY4^p*w58?ej2I{IGR-o)!as!t*{K*ki!JG$$ub z@1rW;W7^-l_l>7hwqzfbAeKy)=VTvIQNwyHlara$Gs9HTQ~|u&Xy^a|oIeu*jH8fS z?^#5`dPi$|RUOk4ijl-^;9ou)fr1B*=sg`|in>JC>N)h~j?R$#q$nH;GPBlmA9s_> z54TVC8^m*&dZ2kxoe}u)qkK;~pap=&4p|~#TT|-DU3pd^=64YK6H;yJD^z-jYUwuG zZ^!?(Ud?JpFObx@x3I97Cc@Zr_O^FL2`hTeY^j^36%QH}-FQc37)BdeX)kAlWCR4q z>vWynzb))vU2BH#5q$0bF2bSVy};SmpSFqPF|(3KQHnqeFVgn1w^f1+N4PN-SZ}WfsYpdy)$aie<2y$xxaQ8F@_LF9^PxcxzoTU#3D~_6Air>* zgSj1Nyu=2XU}(5irU7)|OK{=lTu=ij6$O?wJ#ufM`^liMUJgsacz84{MOf5DZYeec zP~6rnHGJ1yH%@@~n`!_bz73{~Wmwv>?2t9KOxe468CoJt(Q(?3R0Q(T`dvNtRRB~e z<2FOIo!@=_5&qX=ss=%age9m5C$M$RHQ2_bRt`S{n09#9{G!qW5Be3;zGPp%irtXP zpXN|m^qa}R=n|^ofz_+}4nBJM*p2>`-G@sG4=MD4Cb+9i@m~9`aPJ>r4@rm7>{{Yj4{l}3i9sWTjAc|O*=`8DYNg61+Y%c|AQv1CX z(HT|6D_-<%6V5-t@eZzmIS*3$?%H#Zh!hVN^@sVxu@(KD%+Pk*WI&rpD;>*%wUo76 zy&^-yHJlK5?Ok_$u%8)f85PDkdbRmsGsN87v|bUTSjv(;ac6#*=Ku=FHCl!fW;4(D zT_=W7*Sm!ZA(t{R`_#B8uyM^K$guwjK4*%U4SZPf3wT*U`W2HO@Ve45SZGn$^R9+` zt;iOU_T}Mm%aZO-;1wQvvtQcv6Fl$PNs%qXBQ-M!qD|QFxs0Wr4XkyOmrZX!9EWd6 zC8p4515d_v8Adc7*FaM4Z-Ng`A7V4)UaHXQI)8)8uR3IKKIaLAK%at?$-6!80tbZt z_X+@>)?#P3`%~ii@_#lU3C6ki0VIoVeoM{hAr((SDCGb9#tD-PCkcpLEj_9XmoaL8 zmweb8BVPAQ%3R11zA*G;ROtn(_5$^Q*?`yLvDLE!DZk+idRNEv+r0V~uX65W#l%SL z7@o4H+-)$UGmy1mCCO#eTzeB&H40Z7ZgoUrfbVSm4nBAx-cnwd7eYxuiuLtq%!(&8 zxJ%dls*-AD@Bm;0o=TOW+v}vzT^jl~P9{ky4I6lnRa-_*4i|qUAz`8?mkORSbq%Bb zbM}yBNW1i@{uQo_dAa~80$kxV5&Uo_&DaVTqb}DMa;!AIUA1e}OsRvt$hty?!o=$s zztiJ)b=r}X!BOxxB^5e-D4fl|ApZYM!89y*U=u3^aq;Tdgy6C=$b{Nc9;Ir1UC6a1s-6v@7}isK*XaPf@OZ zLD>AjPVk~K6WIP-&+#CR_z%FKl@WtvIkAgmru1lPH~?`uTRb~>T2alwSwhkLirjw6 zEhQ89e|MSAWK6G+HB#`qT)P=S6QB-fvEhAU1QgSP?ms%z{I zagm`zi#(D5iQ{&M$hr&Kq*B6do+3CC7ozC(y#&3p1q#{s0arJNp&7svpc>7z*umY) z-2kvXSYgkQc9ghdWwpR(wdU23OFL<4;mZxl0@?=X9@nohV<=lms& z2*ecF?H#?hZ+pn>2=q!OlXG)8am2Uy0~dQEMh|_iZxEuVP0- zge#@Z>En9vZN_>|tx;Zsf_PIzv4dE}^+)5d=`Ek_p(kh856OO4cjv$MplXP;0PnDa zpw|~(`Q9_z@<%Tu>N1=0JA9J3Q9Z9>8P9Y% zU8G*zAygg`wrF^xe~!3N0NSyaWB4%vkEwc&O+Wcy!vH5xM-Flbyrj=h<9_FY)c5Hf z+*kc~l$tDB=J;~9;g;Y@0ttRB?+LAdPwU&%SYj>`e5ud+=FBbMALj0dYMzH3)=aD0 z9Qr_DmFGwpx-%O)f}F#GUL+_v-wBuoGkmaEg(2Uj39Loe{+?4`%5Zd5psCRAe-W?aajfD7sm0Js9?xP9OtI6>jV1Id^FUEaA^vC%UogjvDFPOCcXbuGur z!cxdL-at|Hc zadPDwsW;+tWo6L~l`)6xD@^4%x*bt4Rljd}D%xXL6Vr_kQxu|Od>rWsy&7;km?qX? zXrrCAA)teq;ePfhP5q(&+s2mwrbC~4zeUR*4(TJ40TKm!X}nn-guBeP17x3V_enp= zp%49r6tz3~DE1KW?S?XJxc1*|W5)bY7&*KWm(xObh`yTmQsYnoZ&M~5TlUUh`Gm1v zZrovTibqRD?LXgIH63T1XdJK!pvkHfOR{@XK)NYCps0aeq~SfkM)(9rt$H2z)3qgXV-02+ddwf10q`ytVCgtF^+9JO~z96ZSU9+ zj`B8R4rd~1t6;O+i;CILV`Ju)cjI29eg3l7AHJ@KY_8`(BC<3f?xfFMstBZ!?$yuc zc#Ly+gUsG!sfdLU%OyT_{ir*jcXl8tvPbIy?H)U__tsygZ+%*fpo5bGcH^XePRdA< zMzS*NRYqBPrGk3Q|v?k%UZG^IMaufD+nu{oFYg`>_dq<|TnbY8?IVUUXdkV$ z!5hS7X)DO)6+v#q{%N}&;xpeeO6BP-Wvx}v>7z6&RvzQm^N+UY^3I#4gwsp%Ip~-^ zm^jYwgv_I}$y9*k!a?Y&uhqb}p^M5QKRjF|1}Y4797UIY#4lXeraQjb;A0nme6-hO zQqskuO@@9iNx^TRpfdJi?crr&){o7|mz{5We{a$z7Ps;4vU*4u#Vkwj7pH#Byoi=T z^7i#1cwhv9w$k-Tgk2Xr7_Pynxbf2-fljS|FI!$UU)uH+xXny&4wBOy z_^L8<=JHZ;&SRa2xbOy2TtKt+5ov3pm8)d*Lk8RB7&}%uSNXWZrSbd0tsJnpF}ZHd zVxb;{QvVB;{d{|dUD(LwXxdv_EB?PfF!@N|s@}x*TF%3);J2Ddu@cORz{pQ3Fg*0y zbBokMo%&=G1<5wr@LGYlFh@-OkUMtA&_WfO501fHrS@q^SJj?{+Rp@KuG3%GrlFJ1o%{Jbxjl_5Gh$ z)Au=(j=nU_5btFEj!mAe4W{7iF8bO@bL+-j0nhKyz2((6%l?`m<56RWwQhc|MdG8M z=CJSO7S0S*4E5_Ja06a@E9D@~=nlw@EEE6&eYawU-K3TdsM3$)NatAl?Yl}1QZS2``*K};erX>andESC((RNo2gW1z)@rpSsfkkoSsD)1-|2DykEAWFd z4gxW==ILPjVgY_JxceH#m;r3t82nDGSCs8NLZ}~(R_*qmVB^ilm6k+Ib4d#JS(G#% zF0=2^*Vr=?^>plEaw_OjJ^lOWv<;kK120+$d>Zx_O7A+EYqiy{bCA88q zN<^v9ZYSsl*uu=@7{MVw_Kl4k{HBI?TUC{|4vZNqIz-4(CzJl5yf=Ub zZVSUGF}T(xN2>D}qAL_LQ(h8RDE~2Uustz`XQl%3%rX`WRMl_P*qE>8ZwaAkauN3VEBIHdW7n?!8w+(F&h;U zNhQ`g?lhPa!Q;W_|27j-A1@M;sH{q&i=v7M%%XiP${ajt%f0)Y9+=#dX+}0u&XkQ#9BheDHFT6+6AN%8{Si?lUm9ni+p1#|74B_ z=hP3~s#15bYxKQx4pjIF;x)~`C-u<{kJK)sD#B4eR;w#_XjEPsL!I8a zqjQX}{cHUsZ!|{#ZzKe9&nfh@UjF)>1y(L;_Ggt>w)3EbzU?RcKbu4h{3lbAc}@5Q zqw?MLihC!yPI={rwi7PDrL~q&z4RSs4#Dc4S#`CVDdupoJc3`<&1?Tn10&8FDk|WXma!kjVVY_(7n;nUyrOTU zY!SfF)pJM@y%E#1afne$o1wvFJ- z2M1wDDvYecA%V%VVwrOEOB@=DFMemc7X-LQ|`(7axb${osD@QpNa3297z{HS*<_}`)RpadcB{DEM z!OkxiIr(~abTYYR&*5t6Af3Zhl-*iI!iXfbU;;PCfk)ZV?9Hcspd6A0wE$`Y^2&5& zxi|rhluV0{R9f0Av4mrLvlau>B(LLU3`OS7Iqt;Uo(}`Z6z%eUaBAN3!{yDP-|#YA z7ujHZd0vqrgoT^>@U@?|pE7(0=%Ix(KZP)C9!EuSR#~JSn;h@9L1pzUll@bU!+)X| zGkhS2DEQM4Ty~&m-4q1zv0B%18#i@+6+Hi{eym-2qxt;qA0X;Jw~nNbGUc~+mcWH% zCj-{Ij=)0_-jkzLsy|iaaBzBILCMQniD)7Vgd$W-rLNhko{iD0y7d_UEt){vp7iZO zI0z+7ym`%-Hr#KsaO2*(mrJF4uDg$=|J`Sdr|;pak{Svm3mNXd z9)-Y;V*=?wyenjgA-g;&qO-yQ`8lkD09WuN*%$)-q_?|U|So>`u(t>xqxGAI0?oS}-lGS3!=?{a!3)M9T1 zt_Em&UoD`v*5Y$*#Hdc!5eKBM=Z}-Y6khck;4eiGP=6KGid>(%EyU%wF^Rd8fDs3OG4sZ7W3S|Wxl`2`c*Iz<+>1fSSu`<-w zzn^z72?m(Zbg9sLfe5nLNt^4Pa=mCY7SbV_lu|3@%T}kbd+3lKXIPkU3*$ zXowL;Wv!uh_-elar;QS#t>-o6cURwa)zdTEUaGZn9o)e|2s*1y0Nl)nJUj&;8L%fD z{SW;=0Oz}0bLPG4I9rh#^$e_X;a4Jg7AtyO8KdsuKq2Y>l(Z3O8){v`;t>A-ltbjF z@*n%O7T*K9tPv1Zx(AYe80K#JCH{|H87pK~b5N&#CkYo0UoJ^W&Zy6l;Rnne(3E|~ zZG$Dxh-n#IhCKL(s@*<*=ES4KQ~ZpluyXLH5cYtpuG-Q-&Ry0*u>&M#4lH^eqWqsk zDXK$l9b=4#=>s~q9Y8B|zq=IGk*f&4_oXw&_=IwX)7QTRm19XQQAwdv;q%)gRjjCF z=*<0UWB+;+0vCwER=*u-H9QK{pI@`mKL3@L@eLc=rdAxl=R>&->)Nmf_4N|?uDJZs z>@uWNdM5kp$OGb4JU2iiSUfxX3kUoOM-tUV1wM533{cUptH}hI97f{opD33n84qe0 zN)O5u!YLSna8Fd*t6PH>4$#mtoY6I$Su+o0;Q^H058U~qJq_e~bW(@Ql+2Ne->lW7 z^iZDGR`1|w0RUou=E$epT`68UESWlo6=t8XIVg>#+jV7hdeBDRO7q9d1^oX!X8*em z%akjTV|aI55O3pu{zZk(#HYF~UvXv-&xy+^caRSOT;{tteL(X;2b!+ROKRm$gODQ? zRk}C3`!Ny<3Uxufa_F%r+Cx0AVj7<7Tht9oYXCOo5UKfC7Ofj+0J73e{Y=4kPu}ZL zK(on)-?kE7t&J{NINwp*cZH1+ksYNp_jpb#nAZTueTrrEY&&R2`n)W<<-92{X{592 zWb`@k{w|+HBH3Bdi>CXouYSoF&5)ZIjUybi$p(B3&NaUsh=rpei1?gmv%-TK4Z}Bf z{MLVeg#sh|7z-oI@18!Z5zQbWRn|o$eN|OWySU85yH5>$A2Ry7*nI_5ZZccvy6C{O zM3)KK;O!9-zM7B`ZpM?h?o&ew+F4xDqJ(s6HdA}XzMuSU$*z9qAyYdxVriIcDsO+raBHb;m4_l+!_`CFC zbDtXlII2>UU8h^QHdwL;6d|sW!LNq?px2j>3bj4mEj{WSJY)G(U;zlmug+KGqDOSLRi956YWZJ$5!_pPFk4 zq-uu_jgN%}1a#7YFWtG;qJjRfdjxb9B=2YiH|PNoogR1b>t-4o8_6;g%CH?7++s_! z&BOqo3GC*uzsP2A!OuJHSDb%0FxX6Wo5?i6Lg0GU4ndz5B#}&chi{=cALik>mm{l zyB|s3yHaYf^ybYUc|GVYF0GBlXf#8%$F?R;jg|#%chVinKfr!Q6al)|U}yz0?NpRv zDuUX)93x^ye&{t%iM4)e1|V5(JxUhle7Zt{ZNFm?7e#cNzzGc59U-I@1ByJTn;cWx zq;=4-WzpWhU!d1oT}wIgX|@R-8bf{)pxbXg<##HhJ1QX)Psc>~@hkd5S^gcY)a9Iq z>VgJvplP2clJx`LmO}SEI_&>3^_Brm{g3EY#^@1Jf`D`*NOwykEzL$r zNh2lQAzhLJQqmotd*9#x$^CQ>ceZoh=Ze>*TVC^?hl|@#FsiiEK?j0qLg%2=HlEm) zQf$}w8$Q{=m9zcP`73cX8!zS=;tIbd|EnouwKfhxpp~NwZEzBKqF~?FgOgr__UNkn z!2QMXr`)HG3lT>C)e7y(Oj#B)z;(qRm4&);1E5G~Z*QwgIN7_jN@iUuL$yWO0 z@0S#eJ2vlO+EX7Y72ei;&3_M3N*Jb+CSEvZr?Qt*8Z5_cN7C}~agmLX?IqYPNi?1& zg^I3VIEl>9DXz*Ah-=C#=Dg*WYT2BmRKTio{&7@bT&;i|ZpwOHO!w<=&YfL=>}oLC zu6x=AWh5~^AFnpP6Nu?&g@aN3*Yht!;BEb%l*yF-lGl@m0W*+v!N6TI#1?6Jq~@2E z*A-2TsS#U}GQ##4Bmsx7c^_g3BT4h*u=>WZg>3c^l4I0^?waf)Q?U3GiEVkDB)xS^ z5AnNvINV^?(7nclVgOXiTPZw0I)(@^ODlhjdmDH^KKo=3iR@G`C{MSQNxYKcUKk%0 zGUI)ZoViTf+i)@-#WcI`AV^+9i8_*9q6zS1_{;FVhpH>~l#K*XyZ*^X1B^f+o38b)nC~#a);=3;pbxg-3eKhCj`C!Marx}0jqvLvV3-3UO`j+!C zA4u7T*QQ@Q6+WHjXxHXj$-C#4MAT)y)`8 zff||eyc1=sL(-s?*Ud-CQ|EN60D-ln3ENIR+@|eln{scL)4yty04}{T%@NEg@(?(F(dMc>|DTdhxV$`|91YAf%L<;HwgRR*chn+2#6@Y zkl}Fx#>(^<-9oM0BVTXy=VI|HgP#UQ`gYRPZVY~{L{;=&2lOn8biE~aU%N7q4Nx=8oUr#za&MU0q(z5vHLw5SP!#WC!nY5bX3e%nhL2#R3vJ@Cx zY3G%Md+OpyHf{8KTiWz-1|tG0TUzBj6X$@h4Cxmm(4{VP=m=QW8iVMbvO7OXXH2MZ zY*DUDAYO?wxWXv0P>2$fybrCXk*Ly9eSFcJ`)5`!(EC`mIs%~Bpcf9Mkzl5#T+ z+ODfiWc?z>CzTS_DTmgvO}l^J-JDrF-kV3_Nk(svZHV@ z>^tN`F|$~0xFA|;EAXk*?A>%K64r@{viI(n>#vryfb++9tL#o);jfed0@`+T$$$J9 z0H5sojX$2)_-Prznw&U0ihkSJn7=P4y|uQ;`E2_(M`E{52FL3}R7a;DOVwl0)=$)? zq22cb7e?kbKZKUJz7m2=bOu-JXDy%34KF<^*8)zUW_-?iI9+j5?M?}L^QlK*x-l-N z$Ei{47Z`dWRym$8bhW$ygM5uh&7HK^tsY&+AfCaGx^+mQNM*Niv>V zzi3LjjWuZ0#*2?Xk&lU`exBGBsIs{(6>}FOtr~+DOD}oRDEv~?@FwN4EuIn8&JQqN z(@=F_M`gM73npVCwpm~yM5)&?ti}5Hwk%wx#~fzN|z&F91(+xW# zCa+-+%mL8>)I$`p+bh}s9h|S+yKTGn`|`zzQAAz-2yBR99CygpbfE@O>eWWCxeQ9o zY5I(xN!GW5|2++Ir?pWY2pxbv&}|MS8R?b%Iy952#quJzTSwZ3PHh_ftzNkGT+OWb zI{v%DbQ@sE%0K6*=D>tD@E>3#l9&|EQy<6*cIU_$06_~wVul~{OWe~nJenPc(w&6p z1`zuD`r(|;(V$AMG7z*#KW`WX2SH%~Y|9MNqHu+jjiPk&Q8OGww1J^hb|GXSkk`S4 zX0DtwyWDWW{GBeC7H_<`#ZV4gfzCKQr<0Wlq8!Q8vS80RNU)`~NphF;VM=iSZ)17G z9S6#aDRNzVxlP(5!QS85)%e73##P=8C4D#zqwhpC<)B=SCLeM1Gu@x0hgVbf$|5=R z3VEz^J3K2iLkMg&XN8Q8>`~IMujBm6-_?}5bsK1Ex@cNLsP(FtsTFIQf7{{Pb9iWx zpIRH|XbdH4i7IL;%C_m*{Ys)6%EzV2gL(U=MKbe6eq(u?VV__d1s|Q^r%8f2Nik6!l^m&S*mQ?wbWJb1A?o&C+gG_!7wQLA#q+>y>IHaqgNiN>p_6sKr=PS_3m=n z+%j#V{)tC1!+_MuBQm-JR^P?An`kGKB&nJSvO|ya+aLpWv!EA0q|jm*l1HUc&C{W# z5nh~~scyy>DhJhOg@0Mquw}2On%_#{|6QEfPNlz>Vq#9fFDLc^q(o0E_@X61JS**k zRqake>_IYbxr)y+Cghw{T{%lV<+)njHD7`0fkdV$vH`5pi4v*^4b5i;UQH+nP_k#m zr^?8Gd|DB*Nt3g^oyl4}txY}3tE0#`z)X!F`{+#W6pb{}Z_R%X3x!30jQJ|Sm}2HB zVprQSPLjEApgdM}^kxm?_3JTp9yAV>i8>5~g_Qx4WAj0^Le>k`0hF@mK zebnPl!K@in>2NlFRGBMwV=9FhydI_r_Yo88Jsf%cP=vT=aDX@=kZ|%8NO)SwdLglg z2MO?e(ObSoZtdgU${|^o{%Qb@^+;03i`O9qXAE~XSORu?daoW8@D8YJ8ydA#ca-;p0kDl3fi zbmiC?i@|j8$}!ZSkbi9!#++RvYye%^ z!BYd0JoaT?x0NMS3HyzT-GVv&0;285z}l{PpzWk&Zm##f*Zg+haiR|EW5FKF4+kYy z>y{sywOYeaA0f#&h+_J-(n$IDqg~7W zb%^tK!P%TN+dqQodF9boZF-^R!gP2Fo!u9$f3rX!n&GrSlrjjhTH@K|B_d*7M+U+;Xx@3YeSryV}#v z$_F+SVJcS~YBE+M{4uMta%}9j@ISUM)h0GmLlcz>m(PjBr~_Xf!XpXws(UD8kC^gr z!;Qu`){C~M@W(hZ($A-n&%fdAJ-pFW1SgW>*@%1d+hndeG}Vt;=;$bNxWHvvh-dZe znl@4{@rNuu7;*=yf@K^Bdf8*Sga*mNhU}`z{M*T7>OrX><=IfH;Ua`|1|!g*8hRJs z``%_-5IAN|L6t{XsnwL%$aXflwitxoC`Me?1u z|9q}jcR9$OaA&?BNhyOnBV<0vWIp_II7)EM5=ouwV%oJMAn&$==pmucBP+T;lwVAX z&lb$kEAnQ*iWbA9C5YvWqDZ@c)2a)1uN-M*y+!|IjJ=Kv(LJddkL zj}+8rSVC6YS`k;#fipFsPpz$|)uSP>40HUowg-V8X0}`_yV=mk@0yeyhSQo{kW~7K z(S}W1Wh|E{oRfw7dY^=_^<;G?2}XxE%#5hnYehMGeqY831@b=nNQOV&9GIVTU$knB z-G2~6Yl3s>cA@t>Zb)RE5B73aO_1rA#VOCTD`tDZb>w zoxK{+Hx2JyF?wfhNYQb~2XopX;hkQZyL?Ss#iIsM0{`<)jt*NAt6@ zDV2$G_ZmlP%?L@Vddl)^eE#m1GRBDcQnK+jVN)puVukb3Q;%6qe+mjng2uq03A*C}!^-T{mhIy9^S?iB% z{fQ`I>tZ5V*7Gt2vw3fkeb){E|C<6NH? z<_+>Zo63{q8$U}^DD@HK3*)KpB}wb=3tAH+pv-DecgLNQAQI7?7&Aw*1S-tM9wn<~ zO`k(jp!Ea+g^Sob+Ob^!opm9IxZ!>n~byg ztMvu&Hvep(D}_t^2gnm(jX!+>O?~H9)k&qeeR)h~3e-kb@|Gb%D1!X&d+(oZ#S;h@ zFwm3#wC3IuE9u8E%CX@;SG%2dQ`h}3u1FcO$-Ft}+5g+HDVm7+)%rQG+bCL4hc$xj zX@a3UTZM)IlUX9PQ0CK9k5H)T$kl%MOvKqi?eli{6ytw@e?8w{Js4e)MciLmPVuwU zMp$#wH?ON+`6&Txcq4)mzOf99G9=Tu@nRvrS)MnK^sxfFq!2yWK-Hs%XIY~Z7%2ac zNT2b6uDg8Soqbi&Z)V^FQ2_D=*?qe`t?$40G=)Yw$ZOyN75+&a{gtn(WNk~6B^t74 zssC$7(?uqvD|z|CQq(ry(6Lv3FO(0*O!)r&wigqmg^*+JSSLS)R)7--#iwS}B-}mfg@{*pm!qH1nTu#ppPOk^l+YWE|6(b9D0}$H? z^x1KoJ4Yirf|La_DsUiP%^KjbwKkb?hmyMAo?$JS@&8Q~v` zA5r;9=gvgJbbu7M`DdwKp&8BdU)tnrTgGwA-pKV4kHPTq*AxRTujJqYIe!g?7~Lpx zf=`RDJ`Z09rIZd{a`7;_RXW4gLhvvnuoKl${Bq>rH@kk%%(`3@L>uKv1dI8B?NGER|w+>O?-f%pG*~cTz zuAhAjMDCyZ(S~{>A#Clz&~5P$GMlXf93K9Es~&`83!!x*chWcZXFpMF?Y6bH3!sWb5Ie_dUm)=p z)B!A1l8*M zCF7AC?YQ5uWwi)1m`HTl)x$~vdDq;~H#d|xpCy%L@jpQDRX7_`gv-HnUm}KF{MYF# zkF)=2w*Z__rH!W-#EMO52VT{NNH}Kd*lu1@q;}Jq->PWe>Xy&kiNiM$*Bwmb3l6+k ziajJ*dC421gE-<7`kNj-(y~Gq&9NT=CADuJq7P7!Z`)5}qyj|bp$hfF@0D&fL=na8& zHQ0qc1awPpc98P0iOO?aRuF6np!fH{k97M3j{425~b=e2U*3{z|))2#gPZt47{ z;e!X6+$xMHie<0w5cM^%c9Mk3YpOKQGy*OvVUoIDgmA>og1@g!ZuXy1%#J)Gamxv^QWTn+2Cg69 z1s&hQzPM7a7Iz)t)BySrpPrqa)Q{ms?UO$Pab4M`-Lwogbe(B91pDLs$DX%pTxdW- zOkNHpKZE-mpPo~k#3gJAr>bjWC>Dx&KABLtd7=FtafmHP#u`?AE1H@Th$TMs|wOF0KQUllb{!SF!QB`RW?HdhAXt zM(f$%Ur&>~v((e#dA8(&kWK&C!IS%2OAFQ}2KXpax9gYNfXQ0S7~~kJkQ!hK?TXy zHBAu4);GM;Dqb{V3*wM?=zgVzRB2fZ8Z(YPX>Qd{ zJgJWz875E6T)1ID4O$J~lio#P7Ta7R&nj$ME)lng1TRavFrbV25boKLbi1)6FK{{cWk9Oj%JxltEz@bLSd4z>#> zGd7bku@s#%8%i^^)w)NA+p9@deyb&F4JPqr)oZADYLT0F->*5v! zTl=6vX5~{n+t`e+;i2A+?IaXg{+5ErVV8qvV8FWaWa))8f6l<*o@P{6_nUci+orUl zqeK*J$%XBe#!Kk)=Mm!e&@pXW*ME|t;dts410rALxPn*^Nx}<@mNpFm$~;xryj~FEhfFV=*r+zj=%4V3=Oetb@+l6Y zRA%=|q_kK)+53rW4#UxIymv1@OS&*48O?EcPmGF*Sd9xvJ59yCqcFM!O!W?@OV`_* z-eGxB^eq(&;Gej91=dtfIHF9pe105U+6OaDoweI$K=i^Oa%Yv}VWf3HQji0hz_n4O`Nd~$5_no^$#}{pB?I9yiVphWLt6lVE(xZs<((mjOHdLG6 z1wWZpk)Y9Uy1&ntwCq$?G>TOqRbc$K*!ptrw=%pPALcnlx%FInS+keEC3C^yKqfNk z0Q<2slYSTP4K`N5nIhvr6@x-jitDs;eIZ&+HBwRz9e2Zu=}7?Nyw61Du*2ZYmejun zctQki3iyoJnGnJi?d~mM-;KP6A;HWk>;|yM<*C0k}Tge zg}t|PmFevTh!>j435uwKLL0Df(`mb0jzLHesIE~gRtijHGlEsnTEDkc^o`_I-9Yz+ zR3&QV{OL?`wsi%g$wl#1!Y)62qRNT$tAUnT@`PZY$h#viDH!OEw1kx6ovzL!d8Rk8 zeou_B%7)bN=G%^sTgKEPoO2C3{5C5}wbJ&zcEIf2`rE8$k-2sKQ{I3t2QJZ|jR^*2 zW#}QD33Sc>Gmf%XHENVprm?Djf7AzNQ`6jx->oVAWE|&`YbJrGZ;28Lk^u6!I2IMG zKX7P{>E9lZEf^C}?1LwSaGOuM_S*VD3z`o~k~PC?sva8uiV#fCURMa|ZDggILWcN! zrQ#;AZN_oeZYB^zY;Xb#=#Dbc&|RwOAA2Qv;jz+6;ITkw@Ww5bwd{!aXrs2oA*xkp zZID;uiK!@3kj5k;Q8?Np$l@WY?H7*1;Db7=FwgAMyqUpMDTRdTJc3v4aRS zClegvgF;#w>zLzL~D}Od=H7D~6l4%+lcUs_$4qYVdhda09MONba=(T)!sh+r0 z&|?kB>1XhVj2C|hxBm)E#E+b5uVc85_-ooA-IEr#oIvcEH>Q^(zDRp+RbMN6MM82q zsa9H)!D>EY0*3jVSu-5b^?N6NSGCAx;qh36GQg*D)X}uc!B7(~_z)+}FaBeHl~goE zNZgU11x%`uIGxS;VOax;fU;(&BiDRX*js)#noRI%_x#JM8S07WxmX~ZTjq3Tst{dd zX2KhBbxj;pxV*y_7T_E%&j=(}FZe0l^h*Dv;qcjYMpF1>H!)F9_R_#$FXv-38fpbs zd!>It&Fxr6KZ3-%cp|q9cys8tXplY5<~x1`2qh+oS4yzTiDlX+tpLnh%KdSo_q8ZV z2Gb=CBW_dT2){`*E;mHr%_oESsfGl>%$%eQB=*HIlk}1!Y$hOTv0@}p1sHp$R&-5TqeYNb~yeZdl$5P}ivTb5d}SVn(iIr#*v)a&3&our?OVs^AJ zwJZy#`-?^w)@ifnul5YKNn_or5B~BLz-q>^ZO*1bZm(bw)pkYOB_-#c1}Eh>uSm}# z;>^tQh+QU-6D>rmAfKVX1NYwkzOu6B^t)y@1dBt6Bja0aRYL!pd$ z`qH>;8i&tg9n_y zTDfARn@%TX2khU5(~BhA`)?&7`=&t^Sgb`~;rWT5q-ft-_QcTt?BJ^PNY#FLj)-Wm zHj!+KWOe)&RF-S{`8%fpHC)BYxjL9LSUVbZ4 zHhzWnE2R%=iu2z_8isqAGMk8-s^189<_o9uvME4myv7R^?Ja` z0U)gW>4$L+f69$mWkHKq$?Z;V{mHyFefEjl*469*OeiL+P;G+M#y1i~3A-&Mt3bpMd#U&{Qo@=Q@P2taDg@0Wf0;g|sM6ym=AN7YGG_GNPqD{DWnXYNKPgSIF_Vo`A)ZH^#5bt~@{N z|C!y))K`A;y{3_n8)<{q7Vn3nkU~7v(D>EV*sUwjfX9pkq(&o@+h%&~R;>XdlfqJY z4wcq){oKsKg4%#>B);(##_rx`)>EU7V9HAR@h95dSN6af%-tk@+spzrXd!G==SH!% z6>A=9R_c(?82>|wSU0(Jmr^eq8i0kX_b6X zUT~uMA|&l{)s5+dW|8fv)3#>Osb~Ujr&*q$(h1it&LIhBW<0*wv?#m@Mb@?E*T|M; zag)q{{$T}0KH^wQ&UrHJ24)D4dqC5skgVq&tS@UAKO^tR6J1~sselR1#CR0epKfEd z*cDQ{WNf{p)+6F3l#gq9^RdpgP-8QGo zvpCl9!+DO>)WTWHWsRiH!Wo&8)5IS{sQZ*qAHwTAOJtoD{$ID>{Xamva!AfYfMfYm zH+sE>7X{1QK|sPAw%935nf)MVnO7r5gj)C}?M+|tz?3r2=;i@YwK!Xo_R=X*;&eGr z>%o}a<{I+L>+XMmA@2Xlh5t*?ikLEJXUV#j7O==X`Uma&Q8Djd0^Ch&Hz|FqiqR&& z|JoXeE~O?laKiUJIlt?IW+w6HLvn-#OqSs34}w3*8AHSn;zN60C^o938uNKpZgV8( z;DG2f95bnyW#)2o?%!{B1!_O3miA?{zwgBuMNnOF(bN#U$h%&`l);uzzPct=wVr8I zAVTZGm|k)<4{3UQ9|sG5U_dM}LwaD)R+fp)!txeEnGR7-M~NW_OZHc_o4%yPgQ*uw zjJF3bj81QodhBj{@r+B|{=6_^GvaLcdG``2LOy&Qe00@j4yJJ?Fz$c*uiz{@oTd8k z3o#}#Z=j-+SKOCB{2(b!Lwl)*tKaxqbA%d`52JhI54VFJXIve7RJ20=u+;wmhO^44jmFO<}qIa_i?dALK#HAJp+J*qoVyoQYXDPB8ccaGC0{C-n!N z@vyma$)X2rju3K}vRi6q))ltTQ^**I;*wCGX9N~_S=fnnLuz)d4peHDGoy)++-!(@MQuw`#O|DfyqK4JsOC4m^~bz*Dyz5+~U1@wcpv%PTI_`}U}S2r|dFh!3B5$Fn$(M%)xD}zAz1`>YfVT|W>{bL4?ZGpGoNy>8!(J#PiBP>67U!N^?#^P^ zHxPkR4~8EI(R7|M5txDO0^rIxvpfsbBqijUk;c4QWZjChf{?h29=X+LAGo~U_HooB zK$vt-Ahn6J0GN}8l?@P%#K{9p-J4W3=4XA@`$E860vDybFbiV;*tC%asnYC2-}{_Lul}D_ zeyxcXhudU%p|o#vl{!tin{3 zE%66jVtF(z>fAXaSDUwV@x%Z+F)|)+uI`(qu0=S}TZ}y?>Kfrxe|L-aru6fuIK!Xro@%N2Z95$2h{oU%~ z>&sAe;q$KJpQVns4fSHpOT(M z3XM&L4|0q!4|%g`j=ovvqC1$`XX5-hrFeABl3LWtxs?8~aG;y$Lfn<9pmt|XKn5tC z*#0oA&_zb=DlWH5ASj*+8}K?znu`VNH}DRQ9*OI$dG1|T`1~4K2JdTpywR()+bG z|M>(?s=D{^WXg_c zZ4^zJHZC$(%WYPbll~>Rf zl=$mKO+xdPX_7%WQSW~-NN}P?wOH@)x!=Ptw8{4${82LebPz;_bSB|5afl(jvkjyv8D|1j z7K{{2zt<<`hH?C+9KEc_Ij&hwTRO8_+@$Q+9=WX6KJ#3sj$htBU#jQny{Vy2r2e%b zvx5pRTXF4bs?q8!r>(ALZDM6ncPZ^jO9!QcwA+a@+6w-P7m?#mQ_5b66&&7&A$YL@ z2Mz7mi09%%DO<67X;@#|M~^3ob^vuEef6Bf{Z(8(VizGHS-`s`JgslBYB=wk_c)L> zN&*(0Sq$JoO$n2bvL@1;u$w^D$Pv4G9(%P)wS8;nSuHqRIqt4zCo{!>%i!oPj$VA^ zy&<%Le^PqEzS`rh+V?D!W20{z|4nuO3;2q<8I3gTHxmcrn>4*DeF!*>NhS9HWWbj0 z;U})^wg=iDyBb9uSOV_UO51t zs{~nuVS6-jz>cxiz?^u{d((H1e!+?Mt!&ygtfP1{k}Czm1oW~6dk?by3qfQ;AG(`a zht`$*nZTvS*qk#AmMNquB-?tE={yj*a%qSTQh)Az%UOj7yqWDHy-zLOK+ZUi6TWft zXL`FW>BC!lUfP>nl_PVHBV~iT?be0jG5;9JKa6ZRA9#?0#s}a)3>?I7drI`mf%V9( zJQq*Z#Gq6#kC52ZJVE{VqO>bwVb{rhiEwtm2Alho4?AUbo^TYR3u!JUXz7oYz1G)- z_FY%_IvmsJeTFQi(8mq@8j&_FXDcQ4p;7j0pqu!CJ%4n|bZElA5oyR@AiJH{!$Fnl zty&Qz*>(kP-G#ku_AZvXZpvP~f)rB@py+FtB$IPrunww&26 zR;?eYxshITTG#JFYbaHBYE_n;z-j*cYLu@n>||QakZxPEKmT24#}SkR@{pKK{}I`_ zN{sk^q<7?3SUFd}yZm*yYh_EI9iaMMIOn`?@*f*X!KzzWb5)%`n3O8apn>1Fq~N^-Z1tg@D65<^ePu*nNAXT z=JQa2JZ_9R&;`eRsDZzk9Z&C7H!LS5W_2!Bwi%kiRE*Vg1!Ah%gLZJ&O_OQXv_)dG zIK&}&9W(qMZ6XJiZK&%*1ruwhlA&^zC$;d<=**U?>~|0v-#3IM3OP`0fL0Wq(vdT( zf_@2J;e>YDYb8Vz)Te9``c`wby0;{W1=MQ=xS6fjyM4ZHd^2sx=NOL@|AlwfPFD2A+l(_3tkdN`wW_#JxE z4SGL4TF)A4_j8mRrJmG7!TXyF(ZbrWy$A%P;*wA55CvSURMh_mKw@Qjx0MgYEkk|B z{;5<1Bt_Zj+H;0^|M^Av6dX+PE5@4m=ZBp4sB4#ht&Atd935Rbu~>|H_N0qE3~gr% z>Z&H@SLaGir{_UG;T)0Sik09nP!xb1oK17|fY{pE21=tppS)qaSdauQ z(~ocK9mm*SKGd1Bz6Zuw$@BGlDvH<(!xRl{EprW&P2)klWTCBwJ1HwOa`PpnASf%8 zF(o5@S8K>>f0Dkz`S1#{#W}$J>~*Bx13sqGQjh);cVpy7=_kVYYN$o8M?vk5(H%bJ z75_rVsZ_rOg_bMy0%4Jk2EC|*&20FaFc-1;?^j=b8U{Q9|M{C2b@9+OHmrQ8uOud? z6Ol~p*F8bP^Qj=9iWtL^*HwVl0DXy}S}Bn*fLwi8Kt#o8sFsM_6{+#~@5u<7X6p69 zC?~pQQuUOPTXk&G#sasx$r$f2EG|xiINkdUPU$}d`C9C{XkA4jXLH$0waIYdqRLz9 z#$l~nr!VwJdFW?w8aZ@g+%&Z7uX1f`zG8EY&kAXd$|C`nr8BdGx%t+mw?e!mWmtKl z6+JTs|CH@-@#w76J}7p$LXn{Wmh4#I;{?(le=TuySxU&@yTs~EClZ$mb7_W*Jw~;( zDU(x`*_z+YK6q3r2K`); z7R5e1PAW7lpc^q=UQ_YyGMHiSnUm<|lFZP(LP7ZFM4q|BeI+gPX;56on@vb~s~TAb^@$Tv(v2jbQWs3M@f_xP;^N!qbte0O300>$Xn~ zjUqsXSKP{quMJC-0vQa%cAe(SNUSCkTn( z797NpH9L3!kqrlhywp_?JfNxX^91b$VI7x&dg4YD9v6tV~n?2YUcJ8mY0;{MCKIK z2#02s7o#S5Ryx?|#A64Hi}^`pF8)wv6Oz!Ytl=zQ3&Vv3m9s^^L;2nZ zzP@IA@~ZO5iZOGZezcr_xR&N=)jNoLADfIHY?r!Eaw75YIE9&LZL;Z_CG#oN6nxCS zEcf`j9OpwPQ^vC$@~Q4&`cd!T2en$n$R*-WD@ysnICh*^G=4Q0Iao}U_NkY)nJ}`o z5NY$}y_k^^sj>B3R-pzC-%--mbEJBd%pHGW?w0LVQqlIe|IJD$H|nTdAl7&4(FFd$( zmdEnIkLJaHN1*QA=A&2Bhaz4Sbna|cC7drK3<*yR+}uRC>kMnCOh@alufhDdg!Qbi zb-FJ-&P3S#^v$7sSv{L`+w*yZ%T_-0rN`k6e0Q;kgP-gXVO5+=kDRPuA3@JRq3A!N zjMy>4(u0tkgVWQfNc_F0%84t!(mOX4qZGd9fEV`ZX;cx*pLJ_Jf1V$HYV@*%bZ0Yj zI93zn5Z$n>%J?!OK!qGgFhaaPo0FSldo_^gGcR6DG~ES z#B2Zmm2xqH%|Fh_zbc|r5g11b0D zP1lxo_edxSIsBhHvzgrQnFRpxFE3Y@Km8uouET*ZFV7yd|C3sj!mJ21)BxP_%blb+ z5&r>-u##mvEPDoDEvg|ju_A4tU4s7N^MyDy5)ztgRE-Dk%lHM|P5OS$5kDS=Ra29+-HE;*)07If81AcxjV<9&5tFNk(fhM96& zc?>_j^dKe#hJ!=hmPUp!jk0+$Hv7lRGclR;SX~bt%ZDqJF;Fq zm;V7O`9tMU;5`Y_41>RK65MH|saAJkR5XEIep`1BqCwAZ;sDsV_G^R-!5x7bYFcY@ zdqKU&2hT&;JZ7O64D**T8K5w$g327m?sNg^)8XM=IH{76k;|DM14Au+`L5JIFUR`T zNDLSG1InhqKW7hy!gqBYH!u+cxJ_5cn^Q!F(zt2|)4B3L3-lJxrRpyFdeH$VNCM%Q zOst?;Bw~#MAFT6;eWmPmxg!B)K)c%bK+6!{-65}-dmfDvg+!3NG z^q*RDAK%~yI^+k&vtmtxIHNgcGAMgb z^yNEHpF)-$c-hq{IYCF`PmEC<5l+s=pPga;X>IDYgFu6b8Ex99FjC#wH7N8x#F_B1 z$GiRbIuxe7gR9zK&>yl(U|c`@`o>;ljNGI=Q5D7X01QSuG+&*XgU}%pbGNQqdsU zYBN(Xa!|~_;OBFt+?EcfZG3qyELH(OWLI19;gsYABTG$MlqL0`&Q*acwkkJlN;>ysGsTSv$dC#6$q%6tsgH;tNbs zMxMpd8P4m8tMV3IwIi8J*$FtxY9b`BN6u4nB&vi!$gy23)6kmp0-a7HGgxE>FJc0X zt(=GIYWUu9Z9YC!!s(-c)isf zb=`3kJPL+C`^8*n?lPm4?e>wS?*tZT1Umi&DDFl(gAg3LpF_{2bv(A-@V;DRNmBo z81rk;ZGm7ClBQ%=Q(D^jbM9!QIlYBVldnXJ9f@mzZjkGX{k0XSf~=VPx#|IjYt_~& zFqTPHV8k?gKrY42F{q~OYx3$^9&y($#m{-@c>=kO#r9s(o53pEa6L<;R^yfeqrw;LB zd$SlZ#S@tY$bW!8x@CBRkq*ZH!o#WLu_07;&^&0LylE`zF>jj3ubFNpglfny|Bu`B za8JWI>SAfdN)1=L%M_`+ch~XObg1snv!-^|zHx4agJYGMT9r4N1CIJ<#kA$w)wDiy zuop*qMx9Q^n-n&U1y-dPTlTJ%j&F{a6qKDR6iG(e2eC`}M!C8Vm!yl*M%yyIzN*c( zAV>RDP3Ywji1@`u$3Ug{Eu=~Cm=y1g3D~64F=bLVV1rw5OCLGW7)XxhO4EjSBNEJY z)#>z>hu{tgW7YffLjy3sF`0ARCICLJiSX>d#5~P;e1#3X{in`4L&XKn!cMH*)L3 z``ilYN9=F9R1Q<{nDoh6iKn?t+by)9rxct22E6*T1@qx7N=7uCU0<9GMJ+lw-haw? zbZ&R;u+W2s?it#eSni2MA8v&~cDI{Fp_kd}Hd+KE4*_8#Y#I>hH4Hki$Gkx{(mRou z@5Z&QJSx<|g^SYCCujApINOip=&HRd)}*B7=mzTGS!2sqbx5^Uj-$Gx_aXkP-_RUo_}#6%$*`yg;wAQlq0M-pdMFn6k*cy{1B;5!`mml8?6PPn2> zzee5?4XpNBBUcK^g3&1v<7cyhYa1eQ-d4g@1$uI&M+M(X%jdPXwX-$Nejm^C8TO)b z5o*TKQ~zdDPJFqHski;d#~Vvd`%uq-BXidK?|eWB$>Ut&pfze!DDAusNoBC6oa^*9 z4L%095QUzJS#$SLSiuIp$)~-wMHgKiSkn|LPL&PO1d3c?wG@LFe=lKg-cV)s+>x1 z#DGTtK6OUDkH}eVlapcY1|PxCR!4nD&gTdI%ZolU?LJxqty`;v)jXS%ySuk&ai@53EAA9`J#WtWKkNN+&wR?tS~HV7zdd_j zo0vx?-Y;}V$@Seq&OjkUYkq#FrOGz=B>B%=x=j^sO509CDnNCA)6CHZGl4r5qSk^> zWO?$(gY@N{>~aczIi%L3+*^$5oY7hktzza6dh93Ie<~|OG$sa|v<7|M64=<> zsw=pzSljv}Sdu_sAU{qZq1+vJXAMb;(aa8AocDTM$O=!E|9mtJeE@8#ABSO9`L?Sg z_~YU#1p*C6Et{5jBTn@3?~W_RHheP|kWjf#==9nw4isX{R;IbLFdDRi-An3n%~%BSHf)(S zG?Z%(V#}scGfO42jQRpdz7W;!X7ZEw|f+WE7Jarw2^&E|L6j?Ve0r5P80TE zX1iE~P5+@rYj8~U159SDjRrFYl88I{@20&BnO*wrQf!oE!*gQi4B)!C`}=I14_*ReTer@&Rqx-d$kb+jV6O zucVnGVH=K33vSlhP37#eje!iSD&1LF(z3q)0;6iXDxGj`!&j-B6VPOaR9kVCj-_Ga z75KHJGH+MRvq`y-U3=;@Hu@xJ5t^5X)N}NYM>0+z1vSJX?KMt4cPzlBVP)Y(-MoTB9ZH=}h z;8v3UqS8dFZO61(6*enwm{|$;xLPtQZuf*^V!WgKt2nz;!V8(45OO`qsb<2MrS%oo zrzu-%vZ0BK&Nk*djZ=;tOcqSF`ms3ORSjMgAxhuU$RsF@;_Xg#w7Tl?L)vo7z9MO_ zu7QkWT06K@c-vQO&=DY-|Dro^Y1^KDaFnHIjTtm?erb2McBkOq|Ma?ffjLVmT+Z*$ zMF`V-iC-$T=6yTPR9ZI1xjE5}Y=dsMSjY&eyM*9j1H$e^fGZHaZCS&}WbzrTB!Z7Oc>fqcw&N$as`S%#uQRm<6|%r=m?$B+e=7+44t$?= z8bmZYlA2-lijWn$Zck%l$*9+YU~+&ZDI)ZcqBYUV!$J4aYTK9=-`#9izb&!X)mP7L zE-Vty7t&z|)6h1dd?yh{b|6HNR7(ITS^RY*!JvnUfFhg_2e$Wpi$1i|m3ACE>|TMg ztup?;@K64*U%s;7$o$EbDH%@j0|7;uaf>F+*}=jOB)Hd$R7tv*(RtRmNr<0TajaJ# zIry;-zduL36@vj8cxsY^g#K0iV;`qawD9W_dCy20vQ%65+VxvtN-#EZsI#+#lw#YX z4;8SAhl~?`bw)0|1;(XD{L{FBJf1lj+|oaQeismHhqM_RES5Xp^91^=bId@g+!d%0y~A5N^s=j$sb!LI}p zNhB|9h}?zct!^J91rYdIg1U_4g8s|PuqUIYwWu&|FZ)p?{3{WlS zITcPQ8zUs6p*_hdh}G|oAFZQ@(jmO!)8w!HFo+})Qh6k58>9D~jNsi=YVD6B zR25**iUK&D9SWJ?{R2RH3QW{Mr-K1t7V(iA9suIn+_Crm3j7V2oOh5@sRbC|KW(Kcs`5q3B1KTVY#tqipo5RP6)dvNU5#kHth2ndzB5egs z`Zp;s8xeu=S`FK8S#}o~lOUi)#&-4+ z+x^PLFv%UqvKF_V^+q4!N8PCIbeLTaz z3OfG>FbVxXN_hVS{X~DI|JqHp<=JX`S*FWU_K}>-%%=TR{s9Cr^AJA-m>t6lMYN_b z+wZqV^JC|#dTpM)QC?AZ&ZY#zJ8+o-Xfopp@)h`9wFhslBkVVI`+ISX_k&^g@_E^# zu>oR1{;!vZfrk`$civ?Q1BdgP$HvZDu!t4M( zj)XQx`Xq)$M-@<%+B0yw3QC7z&5_9Wu=e1MnuenTXy)7$PmLy&U;#}VGb#rbsqI*r z{Qf_4E1X7$>=h98#esPownr5ii&vd!I8)RUXZ#6x6&9CEXIC+nS!up_x5W?bt%B_8jdJK&A)J+qyo~uE$ievj6NIlO=@`hUUlZ{TKIv4 z+tiM%N|Fwa2J(l~d)0uyBNw zCrG0K=z-&6n9GB^LKtaZ6SgI)(tboRA6(QHANsvX^K&mbw@+k&wbF8!v?%o>3Yt2@ z2?&%^Hj0v;qY@A^+K&b}_zmwkP2LK_t(WFBRArky^1&-#9!AVL7=iiYNapEg?XBN$ zk>vKtVRblL`eVXaP-o?iX<901qWG|-eNF6r$FVq=+!0xMsu%ju-h*0r%Njij-88TjbbdvVIPtWa)O<6V*aCUCgfTal;0FMA0Xu^YSkHMCtaA%W0!(t`{; zR0-Zl*&FHoQ#;BbfcYhM{7OS2o0c^d3cf;gEl`Br*%K*x`D9EXZ+1^?dH+!ov~DKP zN<*dUQ3|YNZQJ=8UnjX)t*;|xt$bb7QZGMCz~It=^+U@m*ISz!!i=Pp_zA;k(9NPk>!{bI#PL$ZXaCHj2rpJIhq`{ zvkuC3WNyDHUFb6h)>kxp#+$!TpOF{y1W^}_=M;^vloXdh8>uGk{!~=CCU*rCtk?~b zO)ySY5t|ZybeJ!FxaB^Lw*6E}e{XNg%6HQ1H(*5JOigSFQ;T$Tf9krl5&83lV+UOI zZLDj(J8|`yF|~5++c>*D`=r!43)CT8MOEo;A68n%uqv4{DX6jgtgExLpkb+{h`~$@ z1%2v9mlg%nVZTVrIW^42XgrNJG~h!1Fivm>Jq8lf%26|= zW?iKiQ5Vx_OzOO19RN0{T04HeaJH5d6ty&&sk2zxox|y|X z28Oy4Tj#g*DOnIS*uN41kz(!E`I(-P?@Pu*x-UNs#jBj&xx3xFQw(duGSygVn7{?r zHpsw|I^#?x?Ifua30HwfJHK>SD$rLs!f2N-^&0c_j9wj{^`C>H|5mmA-uKAtlKfNc zfX=59OPkiw5aT?yL5WsoPuWGBJrf*8YYj3SZ%bu}6MTI55!yO@_sx+SHF{%fPL56E z;rz6%#Fu`>QMqqjZNTy1o8}>RaN#P}#CHp+aHjLD8LdD1gpvm7zm$XZwRFC)r0-yt zvRlFHZ5qkHhY!1w_BMeu)OmzL3Nfj$@$VWA4~VM zEH2iUVRpu)NpsPvHkZ4~6r@7KAW*7YQ3yDq|I*KDo+6mW1J^&?^mILU5N*?j7$=3~ zs^{P(NH!IJ)4CP}KYJfYKCUr-E7^F6-&9mf@OPiBEizecx!yt(fwhMF1eX9K9eXta zQi}csw+?7M;jy~iZE&G4gKcakMEmS#7_RICYwHnb^VcAnD~@!xSScoA)!>`p+D)l^ zr4;kemi5mJh{1|j9i`={W`lJ(%q^_yfLX}Ixk~*MfsJGWj2G*6g(uH>V3Vz!WgM++ zK8(u6qz%ZpIu2Nu!#loc;%Vk-?jZijsvCL_{P}r-Vm$~pDjAncsiHW10$?6FdbpZ5 z%3-0x0YgetVpm-!E|$vt>cjLWc3k8RL)bkbKI-RABfND4RA1Mo+JyoyOH?Y-E7;T$ zyowWbje=-r463>CY_2eu3AL7*!{>`m$43DvoT!0q=U?VU9{}K#Jv}=IZkLMKXvOqz zE#V*y11tvEBAO(yCZJMEolrft+|GcQj-HT1ubWN3nA#MiiXo@sM4mJ1`sowzCrZ>5 z5kI^}k$GxK(6-*79lnus$Qu|GCujb}FBQ|PRY?U2^#f4@U9AoeIh^Og4&#Kd+q&al zK%47uR+jdK+ebF^nRg4GLOX?o2BpxZun?r01o&0V-dze&8_6$T>5sXsZx*JBKoBQw z@U#Jm|*3LNK z{7koMucvF9yaVrHKS#s_8lyM2V&_DLIXa;m$e>!FS{*6Ojl9(7R4R)u<;A|4%!(u` zrPM^dtkMu?oP*u#qviTA?m5DvR7sJTpr<>$>1=Zsr?B$Mn9gZs!H?^Lt{7QBb*Nq} z8o_zK%HQ7oVVii52i4;sRVC%25Q?Q?PdU<%2MP8W(hR~qN9>aR1EA_Du6J^E6Duog<3ABf*W}tpieAiV%`dGm)#NFx{8ccnm?p62l@9KKn;(#+iO@v9NMG*1Uw?|U^3?O0{n%SK`8 zz*S4KI2cSa1?b)hjy|?H9);3RMfvZ*vv)*GIsR@t_faAn1QZ{FPaFe<&V*va+uw!> zXnQYT#PVE-L-BSWoP-NwAnw<2M|4;zs_ldRqSHiT;hn~8&lhXHKb8sazMxe1w{TT)8lFiz&5rTLQKnQEqMpgD^BZe;TT zqCf9J=0FfRcb??umk-a|TQBf#^a8yGk^KH4C;G`8nXxbJsvg>@ohVF>_Mck8xfAEQ<9 zeVTihoe1|5EX#Z9JKaMP;p6gxg8U@*DSW2YcHPq2V4NgS1LT zS5)$JYWD}}|EMq+v9+}wJLnRi)qVQqRsRx{-h$)Dc%goHW<{2F$=3h$e-PIKUU!|19Ue|^{%_{zWG52vdwad?J7=n2g>sIb7O8L$ zMc<#E%}y}0Jji{erMcYdJ@e@=B4Q`wLBi3U{$e>&n}3=N`hzxwfn10v9Qp7xg-jX! zGfI7RYpC87tAcBvFq4`8UF)l`49g8-}0e$ zdU`2;8dpk(@nzxR_;Bkt3Bh(laJ-bC8mC=YQ z)IuaXNFL$B%+SVsN8GGjFv<&8e(gN}?hHO8C?}`&w0g~Q-tUM-s^mJ2Wn_v^~GVZP(EAZ$-KxU z_Ebt{umFmhCwD#dueDMeJu54G#_;NURTtI=2?26FGZ$n1%;c zmScQKpE-8SK2x3wE^dHV!XxToQ}k0oc2@)dSd|=n_I#lE6#{9n^si8{XM5^B`5TVV zXP?U$@16&bcgG7)d`IA5f+&CTzl5Kdq7Z|*44&0u#d9azL?WEF&9a;0(61|pPKy@~ zUeUOU+8YZudws!4h_mATMI-eyQU#^dU~ti85`BSim(Tm)_}{}M>p|2rTkK|8I6H=JR_82H@Bf^q#>P((lTMLj{$rG8iu+hO1*k8xSH6a zZ-MuRP%S0pFYj8qv7f0v-j|kO8%~A>3*Ov5eSe+92ix2G=lU)1oIN~OW1v%z9l7*xuF_ArPyt(DPFXZs$Kk7J+Tifal(GTic<`SV0npMS6KwG|gMFQVYj>%0Y!~ z%!L=R>Cmn}EPU8%`wORxvnl&4S-NRUwbCQ64l@U=3=_fXh^F6c-+qo`x~&MmuyH17 z$MjlQQD!7_Val>0127NcJ^?Vac&G%+3o+;E8;#STWl}LZAccO*x1#ayof9kx$!WW4 z&fY^Tmcf$FAB!Z%TmR0ZxF>K@j7(z?i?WwZk|;1#T7*F1fvH zOd$x1ylPz9dY1=twfH4%zjjR1gyGkrA#wx9G$d#7FLC99S7ilcrQ&r=vbQ-1zqnbI z3kj{S8H}{jZ{uZsUWqW`<{#Ytl&yh4e(q7c9@FICchw?$pqYD+f>}_gJl|msME0<8 z#th?6g=v!C`$9f5=3+CrO=FHD>7cIO!@hs)sD9Jh>?1LLBRr1UAbzk@@I(L1*%m?m zn-nXfJOka4PI6FwQ-&C(cULSvlgL@evd=2dw?fM?48|Y65sbEO4Ue+AO4ZVVm72Lb@g74vuVL3{ZX5!Zo&^orR7^3oE zjq0!(`s7NAfmeiI1QYCg8-0|?3_^#og?wo6L|6@*@6jX&=7(lO23eJQB_y`=^?qt? z&z9q>0g{m6KUI@qJ$jc;cYw~^r~>xWzN0@4fA5?}h^VA+Y*e&q(&0mJ)TI5byeSUO zDV~Q--oRxZGeI3mTEIWKJ#54YpZM(igcOllyE3B7x`|ZWC?;W)c5kOPmNUWhgD22} zWAfyu2KxKY|8jpxjWtW!tV==!R4RZiluL++iU^) ziJNp<#k$eZvYG4IEdTX&u7sw77;9)L00fd6TX%ZoMp-3WsyRo=Fs}YY>!qbcM+{yW z$Hz}6wMwhkVhfT-AN1O2*%KTpQber&J)HB>avnRdvPz;*j3Qrb z^a%_K{__5r_qT?|s#dOYDH9`s$O?9^LkrYSP|9Bq3(&~BI3woqxiGHvOBXrlCrI1V zs%bErI{psX#s!1L6Ds5t6lg+V#l#_qRO+rCt^*a%xT?l#7rW&*CbEoyH>zK+V|i9b zYz#$K;t>u>RR;xPx!eUTN$MAbPm&@A1k-CUY<0euw56n$_Lo$OzZW95uzDvl+Kwj~ zg-?=pc=IMEjI+COELU<`;)az~@d8%MiUVwZZHStR;hDtwcF(qb<#vlmyVvaBwzXqH z?^L^Uvkxy+X_-Z@?gV-%Vk`RM7nKx{+UEaBc0<|B7^2VwRUizvv{%+MdYrcT2NKcs zzZ;;6aR`BOEI$Sxg?hs-`3G zW_jI}DyUpCLlFeq*L>nv9W_u zH!_S6US0b%^r?&Eg7KIOX;Uc~I-?{K_cbaUXhnetGjW0DpXK}ODzo6u^`{Rf8C3J* zB!pd21j8V{gs@?6)gOlBc9=xAy=+cgCt5kms-T)-0=z(>3QM^*%5Q{JU^D`q@gM#P_h zEbvbvV*|=RmSnr9HlNG1x+ooiyurCLErS4_S6S8AaVy`wq)H(qp{|@X$Fz%I^qjqi z8bOSoxF%85;dBZGgQtvB&z}|l7B`2*}d;B}gWPv{o zMmI;G0k!?iJjY9$-tY4Q=}p z+tE~>JJJsm+yxha3}C<)7}V2E&32`Z>chK(QtAwNr9)bus(8&GP8g_(r#h^5&_kJu zYLwkFW2raK`eskNk&F-v(QT{1FJsA?^4m7FZ;CP&z>i?k82-!{$#(VZM8RVxyuzAK z{o6C&4{G#?NQ8PM-zG^P)`G0vGpDo+ij`&9Uhd4QB=L0;K5AMMQ1^3}mI|(^5VA(( zUeM6LEo;T;vWzlK0MVqa^i<4KZrJaS$7Pe9n z#iJ+zE8E&ZGx)O45*rc&HIKV=y)GPjK0zFQ3sZEL)8R1`KH&goX38qC{M1Wg*Cx#+CV3m+Pf8^A z;nXLlA9D{8&T6TRSa9N$&dure2&_UFX}WgryE7-fm$6(mY?+dGd>e* zf*h_qq^bF%*=ED0S@w5}5Z^e0j_2%ruDp|Y^R`C#hiou-E6>0F8z!fDa(GUc3TyG(4KtSgu2`T_PT;_TewX~LL@I$@IKioZo{I4JB# z98gALnGnlF;X7E?H4FY{N{O|7KXS^kgWgTcY~Nid8B-WPGy^-i14#+X>+YIQYzS#Adp$gDN3V@on9vYOLD?KE~judT&meD?KfURebh>AT!rRk>=(Pv zKrFwYHQ^wq4u@6NMT;x&h`AfHu}#J=a0rSdEsz#uslF%~1rI5*-b(y%GE_djZT%L( z*;vjxQW2ECEYoFfx*^Uri*(Ude?4f!B8luKz8QeP7h>c47cDQ}1Uafeg7-;$Z4*~o zjVH7G zAHl5uPDbS~!oNO$+3H1<+ym@KEMRMkB0B8seROc5oZ_17$llexs$97JfwMHedAPI< zH|k?9ZJ=8qvSzoGgN+0gEQSdPp@Rk&yd+Dh3bPUn86TBu?-T+2! zF(Fhn-d)n~n=Qou*$2ek4@64>fU>Ai0Cvf=e5d>2i#ZER?dzM*gJR^{H8j_fDw#k- z0?8Dci1WnR|Ci6~IZj=E4(_g^UW3OWaHDtZX@1>8$DVJ*+MCaEbu}>Bj={wm^xX=P z=-W3zj;oCqxqtL~%fUg1gU7xkz+(p_E;%Kuf&1D&6cs#!DD zAef7XOrJVseuHm*x>%YX&})a$JqBhFKHv5aXiXTl3Y`LYH%nejqvV|c{$?eP7J=Dk z6eRif*y5=ceh!z*Ci|xlWb0@VQ@EZ?3eBh`XgT8lCqc7Pe@);M@Y@mim+cK*dM_&) zPj7KZpt5(TE(%WwS5L7o@y#Wh8zw~+GN z_#Trytt$vdte%i2f!=r1sRR&&_@L*M9(B??LYz zP$;2kjI%$s#Bjf;?$cIuf}#HW%iGIypE}=>==*MEo*t#gd3}G+lC2%yhp!iV0wzIz zP=p20Ud*AZK>{P6aP20avMoOBZYX5oX$W1>_io|j>kq%^g|iMs2%Xi22@2w7ka`5` zK>f^+XYtVo%o3phtVCS(?h&W;prDX`C8*H~&u%?1-OKj?LX28@2x>K{)~ z+shgzY8tN4KY*+Yxf+Y|_a|Sh&tgw-Ux}Oe9qAptRiQUOOGifm`H@ShY)ccVlF7xz z(;d(gE5H=9@1kB$P$s+;ZohFqD7B%X;Zh!zGz#ypME=?)rd+4cyX?R86#vJd#pK&a zpdp3HH;2n^8l!lx30V=!sTxBe_2u6Wwu9?}bkWG~nb-Z$+Oa1IBv=8t^!3+96jsMG zxhCw-*^bPD(@5L6Pc1naYgLK+#saEhXvkA_5aeocO$9)(-}~r-ey~!ouKf*tM2T6nq@S5g~zp1 ze*v?cbMZ%_^VSjh5$R#cq8AuYLwD`;_+iwJG@RQX}mg2 zSE$##4+s9ru%>e)?TI}c*Ig}QE#5~3phNRWVHD!a_E#$_S9@VQ{8Aki#$_juUx%=iApVW_X5#u&_T4R^)1g1^`P?gWZX4j*AOhr`B1HM`5; zM6pF@ysY&4qKHS)0HvMWHpH#m(>MB59fq@Uk8&RSKr{-9xP+TXd-x=?I^8&CU4|&z zqFiBMg{^m)YXw@AjpPrKc9xy_R~Q&67uPXiQP%6h<>tU%ICA>D)NyqUkRl;IGn1!7 zTckO7|xq)9ojlrL@Ew^~aXKcm6EHPsPWdTgDe0`GskKg-e8~3F5QjdL(*y zc+=#Sw_Z7<1B|H*`D>%!b9C|tbbt=kJ*psI*T27e54kvIOX28o-Wei%(=jA84UA4e z^1FoDEZWQ@;bGMq@^p)PnWur?$3Nr^iv)04Dll$>P)D*Gj_&w3b-&=3ZZrj&VmizZ zD}pm6(fi;-8jhEIV6hUBK1BcIsp|Vvfi>~E>(T#WU40^97b>Cp%`gc#5o+hFG~v6$ z(1n-jqkyU$@|DqHQlo%X!{?ZA1Iju#V#M63@i*9#eU|3ZD1rpPv>{1Zkwykk(~fpX zO?i_Iiq55+jFao}Fx!A+xT-Wd#>&WMNx><72I3zjc`Ex+J{;#JPfqMXf3PP00XT2d ztT!v#neT-t7mpcdbzrff7p~e17c#Sm){}(~0Kz_wsoCpCnGGMrO>Nl%jyJG&V@PK- z4|!BTCyR^>S_A!s>0|~LCf-jzu?!BxaFn#153dR~K<-FVT?xtA`c+;iBE9ws=Rnvn zI=;_6Nw&u5*PNGwwVfLEas3Z)7W!HvEBsDQJ#VoEqL%y)!;sX zyU_i8R8cGyXT!o>E``D%snemTG0)FE|2yYM`RlShlTIxAce*uQwgT2Yjs_$L>JmSM zO5enX?|J3_(F@v zj|2W4+qB1|N^aIFTn$?}u&DeHmj-C`JO+mby7X|W5PYnz)!hGTNi+Yz$}g#NLkk<1 zQ>!2kQbet-E_Y^lDuE?Tu!fXYhPY1uZES3%v{=$x(PcJExFN}f4%UkNwY=V&Pej{7 z-}pnayi5v>h>f%{4WG$AgPsLX4u0ql&dzg}BH^C8bF;LS%^*k`dZe;2I`}q1g~2%( z^?PfE0?De2=heG!pVvp=Co*Hye4Q6pld>SaUC43zJLKqvqiwmbS*6dP%vQl^i^h`r z%G5rkRtBGR^{`8;3NaeSn3}keg3=F|y{s>?*!xon1fzr+A2srD80p7N&Tx^bf{EZY zEt*3$?iShgy9u12WssU&N{kijot}dasSrA~Yf4Ym`&LfQ^tPo6(5yNWR&(+00&XzG*EA5#!oj zAyjSNU|2{Djb^8-+F%Y)j?2a@8 zK?Orts!2yG7EuJg6(V=$#%wB=rpAlSx@g(&b=*1}I!V26Je6MNeRULDY}T#Li=Q-e z4z%1^8p*DkK>f5ZEHw;m1kW=`*pjWlsIRade>aKPnEO=n>L1WGFrfjl@QVt9+A}h| zh80Y)ruGdq=CrWrj4+q0b?42fZ09Z*&ko$*b39u1`&taera;Ov2mmappp${h4qvIJ zaCk~jYQt5wQ?8O9Wp2yRo5Z?7U7RcJ*K65kpUs*Y+OGiqEKhy}v_(K{cGWY{lwS}S z53%JVYK%)f>|7_;-6TOIpbpFJ{`nxVS-LAVaqyy5{Mf_7=&Sa-s84pitu%q!tQ{9| z&~RvQx%cTq^Y6TQz((Txzp(MEY+U^3II_Hw{m(^F=$pE1FKP*E40mR#kbS zBzCUdYzienLS4UKUg;yP+yG4!tjnO)sE%buG;+||OMeXpNbbrh%r_e=DmmGb>k6+e zK8!gx6$%&q$SZ@H0q;BakgHqnD%omC=MhH5q31bMg&)aapZC{)q3 zA?f?N5O;)=wVJNDBjy?o=>QX6<717n%U2IlS=Qk_emAFL99=tyz~48kfh5~MM|4iK z$|_HA@sk0S3gC!@1Qb>fM~S>nCVWVoS@=LTQA{#(OfHfnM!R=A>m(UHa3DHa1r*da z5C>EiNjNv2ppXSbY@j`=Y!iJK1?+4r-Z)rK`KeBjj&aI}W(IT<75dLx)$=0HMY{)=!ln>WZDwnf{mI6RMUJ1mxLk$w_ zIQmLIA!3;6eHAmMmV}ZSnNp%MR6@zfsQ!MAk|YjavWs%stuS8>69NMWdLDZlZDsbm ze5t&68ad=bFK42S>~Nb5qK_Jf^$iW`afulENp24O$&0uadObjROD#9M7(v6K##`=P z_Zy)Pm)_!3%=`-`La%TU=ZC~8YFw3y6Bd$C;d-vlo7hO+eU8ls`Z){9jSt9L0tnDUPl6gAyM-YlT*r8Ul z@bZ1aMjVdE+Zq9K~zU{&OF;u@VCg zz1CWZ+_m#jtWCuinl3hnZg%R14W&1qApie>twr|~5Ay#7RM52~k{rD@SVfHuwi~%+ z^$rh^mQKUoSsa?#x?c;4>qoKj^Nvt5IC|I&Q7D=eI{>d04_A)^L_yaNM%{9|O{l`a zXaBLDIOErb&<)+EKy^Q?mlUX71{}iX6m$rM#P}ROnQN9)-%_9A=(gRyO%V`^1VSib zHVSy3Z$pCg*7k+Z+SnbABZ82gy7Q6KF*lmYw72{HSIMR?Hn*MPNIo*eoK5rrZiGG% zhK#YDLeK_fE>7>D&EQvk$PYJ%CTS5LALif@Mp-Oey4OD=?l@%9cug3Q<#V;eWY`rQ>fM`+L>0l6V5EZP8b_;l{*QQ<^Ow?8kh5!5O6=%TjHy+4`f;`XLcd-_-S>!-U z-A=FaQ3+fLDpT2|&D=6iu{D1Y87I|nfB(n+_TM9N)h2bX`BsqP+xVc${^I|94Y>b1 zU=9?{47Q{gI(~7Z38FAmN)_sQ&kIb9&1F!e{0fmjNOs0jcUT@6-hF_M@KL67t^G*U zuV~Y)%-B*xqvRIb)9?(Ha2c12=m?PlU7Zta@JcK@Hnz}=H3#9^;l;!2cG3RIyAa;& zLOpMT9wp%^Jd|2%5l$om=Ty6s$M&}Q)blt*)3@e_Hxn=fZC7K~427n0l7ZwsMh(&x zRskO8Z@|&0hi+sKH%d!C_v$rnA3n4?Qi&di+$`GuGkSrQ7Abv%` zPIV8A4=+CD515Dg;;!lI8si6be3v^CMyjL&4YeK2+(^B-UNTqj>9g zdf+>Tst)At9QgNlT;zY#{sGjo{@Fx=v|8OM4@4kUQ3ZOtMx&NhK0~3jdNmQR9sDjL zYbJkwY^>_j#Ew)9CM+i<2Aacb;G<>$AzC)4m~ z`yk!ci?>VZq`ULyK0=xLMgNw;f6s^P)!_@HSO(EJsQ(n!va}vu3PiP$AirexGk191 z7CMy=Gs^VAO3H*sW;C>pphpLVQ%wy_>o~qS;3g!S3Au%|rq4V%b(%(^`d9;FKg;&Y zjbHqo82#77GR*6PXgdmww<;?BgX3n3`Ikhzb0n+m(3(Rur{O5BD7E(E>4xab&}1Wd zzsS=YR4Y5f4Niby1m}N>PwXS$;9&qjALX`XMe0F2)IFjCON)-hN{$I@;;Rv z+V_p+QuUfRkpzx!ex zT_tgGsZO-~(ByA%F2-SFy!E_XPoFoa%=(G${jfCF-xav;-a{y!I`y>K_~^;7ewR5r^IwG+&ce-f9Z`HSl}$Y*o-X+tqiQY0UHJ$J)pA6W!i*M#{yXzY`O{Q~0 zG>D$Cw}Rf%wgB-V>lU)4N%e!Zw%k|P{R9Pm`I95?Em@tU+ zdWsp83gk0D{26Q;LfSgVoKoE&P%}VF(}LXmu}1(>NdP2{3DqGU##uSUw-Oe}owb}g zkj=`}W=)DoFtX;KJCaSCl|P59p7$%zau(nwRnRw-%CK6?u{fB&v{_&Jrlp23H>2r< zRJ|<}3k72+1^5M(A(3m}Cok7I40xDP{fMA&NTWb+*~zk}EYFaNO3_+AoB={XOVi^V4^8uh%3>W zLF+hwTeMz(%e{ndzNaT7W)Gm*Q|?pG68>g*^)Lbnb=Fu%kV--+e!04oDG*4EE|+o9($l`2HJsq-Wr2pcowsRu{ymp`t;1l{f>NGzMDZs7A5mu+)Km9*%xXR^xfDE0j4HV z^4Yy*cNtlC@_(3eMF0#2=T)(_obbh|Md~8r)P0u0q`I4vgF;kJH^@1xJ zm-mkxj;2jE{LjuXcN+@24*&BbK^ODbjHuL_D7d=RyAl_*YPi1UX@!?y&TnnncH5JS zd6+sex$gL^O(og*xz>VfidLm;OgVXzSJ@QyH(6a9^$`dI$KmtOprn&i?+bZ3@91@pM;IN?P1tTkC?B3|f?S8#S zz+V7vk51twuh+`N%@5pJrUW#0QPoOu{zv+pVry4{TCe$DxovvjS?zpuG=2NGZ~<*< zd_ERF9fXiwK9?%u@N|I=kbt($n;dbugUXZo{oB-L3uJ>;y43#c(=8m{b(s{yg=y zv$iMfHbOp|=4m;E^aAVbhr5+mw^YBHoM!)2MJ1(2V5()TcMijwh-c&ZB&ju-^wqad zO`TooL1QY32?zogj_V_9pewRzk;J+gEVKa*N&W(&9`31jHu<*uNE?G`S$9;a#<@tU zQa@iRluorEdK?-{DGDVEk>lIb)7{tDOWL2T@$zUcQCE5!uvXkKkJUKB1+ny ztD4%8J24X*At4`{Etc$Ts`^tLh4n+JZMAE(No&1*+$jFoI|cX%lIV#U?oOelocAeq z;<(0Ldq&QKDXks{zhj@JGJpVctqmhH6B7Wz3Og7o)Zas&HQ!4d z#Jrqs%@T1kK)cQL$m3?{k3x*r%KcW1hYot{7wedv)n;rcbIn1*2T;)m*L4eh4w?f! zM5cmC!vG{D$Dp^X#z$~4`X{trk9w@7b4!<}eYNkr-a+-p@BFgQ&}F5t)=8`~IE+E$@9XdjJthWV-dfsm(JxL+X4v^AILXJoaZw&{6w zfShUZ#iD-ZPu;^ajrNH;%hzGNKde}wdFrW0$66-=3G>uP!xT+A=GyFP-!OVi>;poM zf+4zYBo>HwBh{Qe%&;; z(-IF#0yBfAk^*aa&ES^~JUm`}7KV4&EO=D4oYSu~ALh-Q1UTeB7YoE!_^{no*Kx0= z9Z?P$WgQ5%2Nn=Z&zrgJ!bM`pgi&e&b#}5yM~-)xbJ^_qFClucZ15-?Z0~n$IH#Kw z_)C0GfTAjl3N|F2em8fXA{WhF_r|qo1`VJs7~5p=JQ0Grt8|&~C`Df*EP^4PMIS+a zAR?{h*kFGLlN8%Md9IPk-Ml7;2v>u-{ZNOIGY0|`Cm4%QxX>EH8X)1JWi_R`8jCul zF~L5trD)=+v%Q!0L-x2=hWGb?3pGI;4H12E+P|vmqwDpDwIYW7y+}Ee<-l9>+Wfz z)ho%`=|Vn{W-Ox?F`$&erizcyR;H;MN`71K>1wyjb1L4-M)0SRhD}%-(pAJuu3&BU zrjwQfa%3ixw!n@0Qo)jCER?Dk<~}2ef2R{_A1E6!^hvY-3!~D)=Sd^o3Ay)kgsxw< zWI_^Jrsizk#gJb4vDojp<KB}4{bQHEu8V>Om&;3DXwQVvhv&(}0rK&4ec}Kp5A0cjF9J2-u z=c7=BxemP!r}=kS%vZ4y{>*|hVAz(bLEQCGH&O+Az;e@L0Wv`<4&W`ll=Q6ZUED79 ze|s@yZ2tn36W3g&1fRF@Ej=AgF0Mq&(|q{bkWR1(=u-Vt)=s98bIi6&6YrCxX(7nh z;OMHC|LBM5pg9=d7mxvB>8S+^s!Oo{4TLj|wju8F=@g(&H%%)JGx_Kx$|1okL>l~? zcjE`iPn_-M*2I7aWyu34D~|!3dhO>PMb|7d)^DIu^@7lQwfmuXw_;T>4O`@`R?>^P z&;XKB3oPiWq2n}WxI1AniePj2(`GY#RVj^13E{uv~WP`_1GFT1U@7VVq=Ly&{kC;{*OD7kn;b1!2^z=jEM=Je1>l(>dV+1NfI2l!lk0$LG^XomI| zAO;qpX$1&_dHJezo8d)_!ZO4PqbAKIT)t-8LQj&Rhgq?#^&HKp=d*=j&SD%im?QKW zbl$0A033=}H)-<(BrxY~x=g?n?Z3+*vv8;K6#HXf1WCniPa;ts>m+kY%O zr4LYDIusOnTmEQM1ogrv2SC4hd9U;7IhRnzA&qk^A)=2kF$4JV8UMl_5r~IS84hLy zs<;*=koMOnPMrJa(OKm5RK=s|M&r@ZKRbF^)J0SCYX*IvPAh)Jz0z(S|0iN&{y$no zvg`IW`mJnj0!tB-S#;wsVCSEh&Fw!P7@)a3i!_#U3R5e#i!0*&mLHUG;pl>^A)=O5 zAWg$X_`a1XgHiSyX%yLx$DzBg@m35+1c$O5Q)=zi4Mv-MZNHo#5{)WjJ}{mV(~tMT z@MKR$AZ3I7rTUL^wei{9k`xJt#%BPY5?&P!z!&j}M&|M9SD0|l6-M`w$-%;Y z3yHQcEc#XWi|mEeK!VgKL4&2c!3gQq4!;_qAgCz+YsUH`gxb6A#wvGMPO)MdmdVUeVYaJxh$@%>LK8W&Ml3lU&|zB@XimKRzL71To&%*pva3Ikc3!|^78TjF*?C<1%~qgKxx`eya^ zV9~HCX4ySbFof@N@k%B_gbeV)3FiSg-Jh5y0XX2yX58Qn!HM^j$ZKe5n0rEsw{Lmi z%)m#ej^sYr;@=g~6V#a(n!XCjrjWE_R|`wh`}x(QGF9?PHLrmBpxzCv51sm;}q z^PW#Ap?O$6cvAmYeyiwz!pG2G zSXG7FG5vaOei848GElV}px0%2Fzhf^1jFeDro*;Dt~=Jr6k9jl8RQfuA?7hiuJtDLK( zQ~0o~eZfRkZY^Cu$}GdotoS_mpXC5;fL%y>AlJvGJ7M85<B^;p}(Ofq5UzLZ0z6(VY-jViq{r%jL$C3b=9s8d|P>w$DamZ*REC&@cRe9bTG zMqg@K_Cp$|s$=#VAt~^U;ei^{dNbuewY6(MC)5tGQA3(QJ6>U7*s-8xml+_)tab~f zJ2;J-(+wOM$y-PpI@es9R`vEo`k02|8 zfDC6y3WX|S4@aZbJ;iCARB?EnT3>#>vZOX`C%O~Y9B92y2P;!Pl1a}c6$k;(Ym>;C zCgz2Yju*!&3Ex+5j=4?k znK8$v-VmZ$xTLPqZ$kJ~e0#3U>OEuu_btTfp6GqRF|4uVnoA2WUI34u;VhptKjSo~ zqE_h}oxOw;oBqj!JIQMFm&%#aW|guj6u6pn3mMV~&Jc%Aoi)t@tS(QV)wHCEiW-l? z;-Fdx=C)%H#2C(MIgbB*YER-?MRdv#exVC3=c601O|T08EaRtF#qfDdjXfv-9jgzC zdHcH8_1bxKh;`j9GxwBy4WF)aE)HDtD;ypL^;HtIr32eX-eN~&HfFKEY_8ru7R zwXTjG77yAr=XV<6?jo^Kn!M0V_tSu)(M&m8^erLRMY`xD|Dmaog{SBSb0 zYXH)sN5@s=q_={_Ms14QW~SbtIVXyJ=My~GaKQVX=M;{yWXgVmcg0yuTGALo9OBWc zIoGgi;Ixy50aDk#-~@*Y>d>v`ENjQI3d&EDS?!G#TLb<~8*1D< zI&V`#k3xQ%kC44HEV>IH3}O}lb54?N-+egNhA@q<*5G^n=u*DoT2N=blV)oGI#+0w z3q+M7e6xbHHfOrSCL-l-Gl<;3=rQA*(~i^QMvDdDl5wi|)U};%sa3m>l^Gf_rxV%4 z&s6C?`UYlid1HjN9lLMHvepbrXc~Yj9ttF6;E+iCAN)_|3bnWIwv_Jw`~?t+!!Q0+ z4{=VYTB*{sJ~WD^?K99)3n-_NG?=JWGE2ofaYU5@7|79jU9Kb%Qe1bfti#G$p~hU%K~N-ykG*CGG6of9A@D4-!Ka5Qk+W}H1g}la zJcN(=wSSStY2U!{xbzmmE4<58E_s8rXmW_>D|4rhYA!!4%Zb9sW= zca248?9FG_s&7lkpHDQW#N5Oe-d*C3ZZBQ3rrv!zFiH5TJlBjqn?rA`z--CB>y+uP z&0q-nV+Ybu(tu!aNfCN$^XY`{0W*A{)xb7#^S#mWVj&tLt(^*ZaE`aUw%uoFh<=Yk zrv+_J{fWmdA#}u9pbsqw+5*qD)gKJj{awULj-%)UHZ(wr3a0V)pm@;!RRx#++PCJ4 zmFh|18XJChfiJDt2?QlPA>Q2>$oS3_BKfC0Q6<7R`=(Y`*P%nQgOnxRi!)23S82O4Z5N~63BxJA zN-`-ZdSaEa?Z=v4bCyY_&SA!-u_FltH&igw-7T z1wlkm+pJLkkFQ_W5Z-sA`45(O!#^aD0=@BJ)8*>YOXFV+tjlYFm5U6e(e=iUunIm2 zeuZ8OjS>1*$FdTKCbANUeI+`AxUor{VD2fR1-*`qYJaA)1uID?H+K`3S!`?9FQCC0 zWFBXMwpA0NERvnz2%6{BAfYId@#<5`Jy=oF@~>-SE}vvNW?BNCK$AFm{+Q)WPh`%L zc82U>>a9%7ic(tX%Cd{7=*;)<+`MEQ1pB@SpO_qSKl$Y*@bd@1@lmacRf!XAH0%YJ z-toTW-?kD*Q<#rHYp@fe0}W3_!@e6kI{Y!E-6_VZ{=t1nf$>!XvaUF+Kl{6gTDePe zs4KmDJi4q`n5D*{u*>Ba{Z<;os4HgRNEvTA@$3N7)$R(9HB2?F994LO3bQ=xC8gtw zo~AWc9R#{X4p9Vb~#-bJm@cu>fY|KU}LK-nR$>B;b8CQ%T;4XJp#;Zoa7T{ zYxn?Ms2_*al_cIzuvH;JZd zem~qte|c#;W^mMR*y$Mv63dKYa;a{eI~Cj6^2uZ2nZ?GI!qYeT1S*BzcC;Q%8d#Y1 zK4cfuoz%PZ7DJ%+Fj1?$OIWm2fkw&kk>SLL(Y)7+y+XHIqq5F1=GrfXE49a8m8Y5K zj5f{XbJ)Ax>tbu11-8$&@g#i7MvrgzUh-U+X7=dqF(S(nMr26Fntu(&LMnM#ye z)(;3WxAO?M6?|nkC|L<+KKVH@;)zHf9Cz!am(W5AYd@aXTO0X4q_I^J4ky%?;d_o@VP>n>^bd zy|eViJkXdy`aC_AB%*fZR$cJr#IrP)4LdHKk}q#rm&ZI@*HjkSbikjeEQ4(Fe zpbvSj01E`EJJso_0LL$fO>5n0-Naq2Za8O1zy+P_FJPr2r&*R<>`LV}cy1L5XXauW zLu)<)m&6%TBGg@uxS{^D5*CGBV8hIwdZmDryHsvCAU_nVlf9=j&wN=|zu0JMF{AI$n*SSNd{E&Epdxyz9 ztk%Xv1=>yZK_e-c>xCwOMe1PultLJ5ICh}9R)OMzn$2ZD7jub+ z6H6Tn!TsSWT^Al=Fndaw0|L-_*Y*1vqxE|k8q0iV;1@*_fDX}f^Qwd!!W|IGS!#96 zADRn8j=yd!)tRsh;6MX!)#BuuFWVCUZUDQ>@0Mtc5e$^=vSfj-T$15ltj8gm@u81d ziJpT5!7$IytBBZ+OCu39d}?ZG;`6t0N`%Lf`=UY9kj)L13sMTODD^D#s6XX#?yVG7 z4-mO%i!G-E=##Gc?j+jZk&SF`tKb;KLpkSDT{h~&L(JK>MJ3ljF^5(e{ti<$ zlaOLR>dMxwD&qjpsCWgqlI!Ob#~aA|O;Ocz4>^SQHmOkCi5LdH9GF4Z=M7 zSa?o$>dJHtaiq!GIwIf*3+Eib`nC$ia={w5hZtf(jTaDi7g}tZd)No6umE#3UgH{< zJ`)@rG4FkyjoQT+Y_G>6kc^R3%D2CO#BKcNIiRN6L#RRrbV23tcGvhK2iuTR`efe1 zbN(a)-GGE>5W5o|i@m*zlATtoj!S1Y60HtM{>6Q1tGw)w7O<5&054~HBp!4j?Tx~ukf{TsHV zAn+<<-ej4k>v})ZEABv&yAPQvEG%(awATkLUcOyBqp_Lt-H!i6Ey}71Dmj*a?4Pg3 z{Uiu633XjV_4er{-EQ-p`^pf81OMpt4Q&8M``0t2#^guC;U_Nvy=ZOA^?+oBnT76l z4tkGkD$I!M{-TEc2vRtWf}6wrPKq)dDBN@X(j*O^OLzJdc0#I_P15N1IQ!53L!=X& z^QF4 z@3Ke&#b>q>;6gjMKQVI+u_z9@my%WKoxXfFx`vG1%V7VzQt`sJDjC`J_PiF;ODZX< z=bwRT#g<98ksA?t#i}w*5s`;ikUygM3($;D`**wY-wINX5RM2jDMeWB%^;HJBo=u{u?aHuE0i8ReoFZW<2fG` zd!(vXuL-3`y8i;iq0hM%A3`3@V%Mp9?ISOll^EO zIwv)-=+AhBo@)w+9pw1*kbBD3(P=F8ZJV;qDzy_#_%Jpya2E0JDM#G7rswNN=hMj7 z!=g`mT&&gP5D$iX99Pq@c8XF6$&94-Q!Pa*K59m>E&ihfBv z_s|{{(q+nH)0?w+bk&SG4Y*%e=dlol4ryFmv66tv*C$V?nFbloMM^wO^l<;eA)a74 zrS6`86cUe&bR@2KANRDw=^@(|Ts+#)zUAlPJDc+Jbjx8bc>BQ1WRbERpQdd4muc}< z!w%CCjlP@oMavcS*~s0~CKKXjf&Y$PNxGw4&0*k@4*M)!`S)Vm(5e}XCh1R(G&sk$ zzr~$!m|5rFzzPgq7m-#Od|Tr|DUc`gUmQEMHkkjca%$r86}e{ZP*4w~HNTdhy_6&? zM*uWnXDv&>g)fih_~#VOdcrZ7K~OEv0Q}U*T}MOwRZ7$rM!eCb-b#(Ji(`wu{O2^L z-bz++%FvW)c!5+l!+Jq(Sa*R3`@_21BoX603cM5vn>85M&`pjVql_*;59zd>0O26# zC)sZ)@Bu+ql-c-V7w3;e`hNjmjVZQ{DL+6Khi8S*p5r()vkmV`w2FFOX?{h8mHifsU^Txz>B9;6ebT_gsf4}R zCgZRQt>Z?TD>PbD2=9ofg&n-GY?OnLRODlVjwp`%7gasestO8ABD z+ac#PwpP(tf&{b2v4e|Z38e%NPT-5=p2I2J7cq+61Rq&B-@)>|t_woLq&Zg9$~w0c z!Z(X#o<>&po!}}!MinKCIAO>~Rqa)#l}%I$qM+{~UH}vrwSlB;G$`J)`z}{TGn25|P>OU{y%4{XRxYKcdSe7k?VxFO^(%Sln2zVl#xu1?`OU z(ZM+Yo;cV8mh^V}a)o&c*VuGiqcc%2M#k_lk!|o{>M?36UW&&HK3Fr-Dyl^rONA|8 z?KyuRBj@Br&dC<}Y|WYAV=@ zuU@4%mdt!&ywoHzH9$8H_=N8;T1^B8Fc_CxyTtZMnN67fNTR1Ep$?MIPe7S!O-+7X z_4dVZZJiZHTMchUkNjc@B`B&nvB33Qp~`dCKEImEY0V8nqlqnEED6QxZbRv0%BYLW zw^nHUlCMOQW9a@l^UhDj@f{=Sv>#Zgq>M6u^0<~8vAfpiFW`vh>v1-T_eeeolU;$~ zN2bH{>nb z6u-gQtsjaDlsf&uJnk1t{r&36DLZ^v!|LXTwTrgUn#W$7e_a*=wLh+I3z{{dc#q zGvB-$r?k>f9gi2=Q=l0@La`e{v(>5&(7E6_T+O?ekvYY)XVwjxbA6F* zuUSi%rDENTq-!LzkKG+vvfcHWzBp!-M}79XWVkr`_OUC_k0*)OZ0c zSS#L$B|DAQ)U_1_FU~XNf8}FVp|>Q?FD{*`RzEp9FY;!Z(7KiMhTiI?W~k?*N=j-v z36YCNywG`@e5&1Eo*`Dq5~hHAY7nhJ5lBQZr^ZBVI zM1~UzKyrh#H6~P@HfUfc^G1h78G@on2fI24M6Aqua>V^UJ{Pa&Ur|$+F+0g3D6f4N z{+8u$YGp*MHX^u0;^s@Bp{+IBH1Q$`!F&8($D71ZEv!lmBG`3mdPO}>$WwFkMAdmw zW%{=EE>3!eE~hTM+F7yT{&Py;ESLX>94$L$JO5OJh##+(VD7ge0kxK-F2MsiP(mBM z!cqwW;)WD(ri4u{;*-q4)w&XchfW_N>ui45IS|{qd6Q<+W5rJ&(drYKri7!dg98gp zfW3*oi67FtFUuM;Q46Uug7nnp%-m)rRdhHcS`02UTP-JWuu3MoardaaWICUjX>|>j z@jpx~8$@KD07sfm$-fp;hZIGn!(rmrr~`qS7#Xeh3-B1@ibFQlOd@n7+RL2}p1iOX z;5>pUyg!lgp0T0)8TdIz1Z(knlGJRlK*|G3clzA|2pE27FVEG=~bUUEW@ID1Yww82>Yd=V)+*(9JPVY|3-jkWcrT}Wc;Tg~W;E-Qb%PVLoTj9dc zFf5e%gjZe?mSkBKR*)LN*-b93S=mS$qV_%>vrNJuCL7CmXmw7rNttKz+*J^J?b4Ja;*(-n9N?6}om}Pg3 zTH$c~z02Q}Hbyf0rihxwT$OT`hDM*B^A_Onxs8TsqvT(E-xMBKuD$J25th!auFAyFs7di`Qi3SoH6UjCn2^s~YtoUUzNA87Vw*xbDNlW7~ThVX20UVs&zcNlPXX)pnkED&i z1?FE1A>STeG_RC()lJ%FIZ=*jZpN`9y&1&FFb&e2b|9c6;@4OiqX-=4B|Yi~*xI~X zZtxm{b;uWOl)R@yKAF|2|2{tEHe4~zo~T|6EO;%fEm8997QUfqv1;p%gqJ z6Ep9HMI45_r6BvuPyx=ueWV55;!-cK1)h{|D1I(-iq1Y#6edpiEQ8NLkdBb9%OMm=i5z~+FxxNIJ;dB`Wj#shxTXEFr$f_ zB5RYH8L}#b)z>NKCU3lP#ungclA6ph1N!i<=KmM4UV8HY;a31<*}+P--Ss?~H!*hcxC%jJ zWtjOQDRQYXQWiHZ=10=Vwf|uOF0q3zNx*U389u8EYfoCIKGJ!17K#X3j)B%G@6nYp z42xbbhCHicXVa*==7x?G5=uL7iSmAJXFcglTp07W zCkvE+DmV6IO@@}vkDwDPct0-7gP(T)>6_~4`P!BfUyaV`U5sI-KpdSbk&_J#_sFV$ zGJjmr^w6AT1ad-e@9@`0f3ujM%jzA>px8W8Cm%mdWG@6WIM~HE*J*eO{Uf`ih98I1 z@N{D4a8vS=FJHGYhP$6qk_)~Ik_l4OP!GkrS9>#o?Qq*^SLGY$?gnug4LUIIy3SAP z)d2hV6sCRTjX8%m5%9fG!*F1r9CEzx_2r? zaTjtDa)hV(3sAg!N@BtOC33*)xu3wY8xK8$XI2N})DH zp~3oek_LwaUJq9F_wYqdf3W|bu7-*9w707A@Q_Ha3Q=`;-YlAsL`&CB3>jQcOj+)) zAWnu2W)*zPet;~NNu;BzbqIjjbD+BtcZKAg>-SUQNsO!|qCq{-S!1g_X zp+VTqqAiSyt3NNA9w5JvPQN;|n`D%e|1V$_tB=S*aq|PiZm?Ll-CbgY;A6=1D7%gFBvynpJWUtLM2*viGZW5>Z(4e-_^? z{n&AYrP&blAM4Li+r`L(Rt0gYs0(S_&t(On8_m!mlbs$brtIQi@i!p3{T91e*FBr% z-KC@thN#%lVQ26|J3z>He_1YDuzMSJtTM{2{aJW28u)V(!OdH;-(#7*0Bx#-B(omy zI31U%YtSLEDpcDy$yMpXKq^5UvMxvoeMCZL zG)*X~aix+vuX<=n=*o|y!O%}w5H{_s9nW_$(4+cteB}Y)P(M=L=r5T(eFtDdgao(M zZqrpn7gE=FUfOXoTyd4>n?l%$h{C)hShD!a_E$CM zW*Y^9SuU^@c&XqdO=M3l%U$0!eQEX!GJ(St7@=B5a&{VStCDjVog%dFfz$XV;#*+N zY^|a(uf9U3T+$h^Q8u=+ zsmM<~?d~V5M440YHo}VzK;L!nv?Qus1%;(ib4Y0lY4+6e&39_hjd4?!!zA_q`85d4 zmGZ-FS)WAFb>#?(n;!IzO1nYdg1;s8^Z?fl^eXM3(k-nWMJYg*y%a8Nk++WMghz!k zP!frdj((JEkf~8q@|KcXVaP@xBlgZ3P8@G9AX3YkQO{IE#um%{S`t-eKcVi)(@)7R3Sd~@Ow!-Bp zlXLE9^T)uF^tCSyh!QYt)OMJ{h*IHH;VIJ8xkyh1qd$5)@G81*KX)&usu)b`&WdLK z1xUIGm2azVb-|JO5V`(fPZaJGf-zNsoVN^tvcWer@F-bs+(oMdb zDJ`6s7yw3%?TMIv-z8|);TS~tpZf$ug4KPZA?Dhu8X!a_yjvx;Ze6YR6|1JJ?KFi< zYGp1%CW&f)k8J~ECTcldMHyJ5HkBicr2ZsE?LxO=4akbDCKA_}0Tx%~tSKA8Y!;#p zIkNQpiQT=+CxR@DC^B9JP;Wo~ocfl+f7-}XFI?^Y`GQr`@*8;YRW)l5DO%gujw4j4N z`?=zH;lSFvuDV7kqARl@p(;@|+sGlWl9_37HA77%Wl z2|l+SBFJI{;+l9>Ua7&0nDuiara{)EZGD!5Xno3tl|eI=){iLh7eIUSMk@ z=f8j$0iX6VQ!j2L>Ex33+00$z*#fm>vxYW$`u$eTDz(6Rzsz*<<8T(VHI4Q8I@H<4 zM6PBs-p9$+ilLh9JmAo{RGam*k^KFRm1U=*&Gz8ZGOU^)0(Row*JV~py-T`cM!b~ z+wQXwNcfbw%F#TTXN9OB{VL$KtS;WB!rdq==;iS|jmX zX)RZaDb(qsHt7E%i3+p1+7jvG-s3^J8yVS28YdtL#UJx34Tm<$78+P*+;jpB77&dg z{sr4AbF)@miMj&W+v3i zS8pkK<27=$5;&0hBl~wqtrm_-8WHl+wCAh5cZ!-}^ z9Iw^Px!BZI&Z^E-pUWbzaF8$A-L7kEFop=b8W#BhI>@-6KRr zAsy3|!ZOUf^2}dNRxO={{F_M{8u|0r`c&;Fl01ZADi47r_wnpJPhF01YL*un0HdUbvt~_) z!UmTo&xaS0)+q@4Yud4<2LPY7|2=qw>V) z@%_fQyzz0Zw#cyB1P4Yes+1I8qY5xEPy569JpIo72~<6LaNlvSjzEhj9JC*D^}l`I z&voHJI8AIt4Z;8Ju;#q#c1&Ubqg2i=9S;JfQx(Z8z^PxPYZaA|OzB4z6UtA3c zgl$d~Crj%dN&**!C^RB7!iOxl3;h++sD%9D6sU~8<`USlc-2EAD^V^#js;93Jxvw` zV$w!BN81p!7F8)Vv|Y3Z>@~kSq-_E!ji;G$V^Wb(wP`4GVOmlewxUyH(ml}I)$F;^ zg+v40%@XEbOfbh3>4zz!MC~8F-d5B`l;0L@9d4Lb&}ZGhO(2>CKcbYPDrXhr43+@l z$2z0R5V7@5Du^m76X0*IlDfXbnA5>@6veUxYMUHg*N24A;hdy9iGDt}*1&jR*78D> z7XbVPI7t)YNwN_o5BOfY{B9FqQ>pj4qPu#K@6TohHnB@gy&^vjS&)n=Xrz|GKPUG- z27+m0YFPXeedBO=#&%%X@V-T;X2Daa`r`(cd{_@KONuhwREtrG6aUc^mHML-sWi@T z6G`GsxaZ{)Y;2qthnTJF*3vX`9nN>KXvj0Pb#zDkQ0*y-^PAOG=UPRdd%tF!l8$iraJs%%0uj&FFjEKi6|6HHv>57 z-@sx)ojshCxTuYXrd3FGQjnTNP-8I)uWH+mq*D1j26qoEK4+JkL-+Vr zAmi;4thR(p>DeW&pb&Ez^?ADKCzAFjnZ@t+|XFQ5AN6ZKr(E&l>K8`I60m!Bvr5uU52KE3A?MVB>Y?RvUe zJHH%+M^gMuWwU|DV1g@)6nj3kVigV@*2=&(s}G)ZJy zjY_{@tp>luAVZo;*C{01(6f%b_usZ(w4Y)XTx zMqJ~x&nbmumP0qoUbcQ~3FM}`jnjGHq`i+4Y@Gj=skkR`eHb^dI1@gt7Y$^7lE?kQ(}qs*GbHGVl#eONM)ZD;wbX zXC`zq7bK~%kK^JF`~JXf4+_}?8}YV<7)hZLotI@&L#MA-cMeQfA=h+pIzxkdPct2ftB`>AU936_<kbujeZz~m!` zn}5$hDym#Rx4atq)QK0YcDxGCD<_AAzvh1gh)KhEM0{STv(leU~itSlULzga^d?r;fp_^(X^_?oJ{GkN!Qp|c~tU!G%-}MjgpIL zqRp1UlWJRv1Gr(U8BhM759r2zE~&qR3$9wn>TD+~`1lu)MZ@$fPYHk>D^3m`w;O;LVh*#p0gc4Ij-AJ@8 zvEAgkHH$?D)Tv)xSzS?WQz@c#%QJ@0lMgbhp{i4i>|$cjw&TR5-=Tx{zhve;1OXZ7 zJyM>>Jn@eYE`--i32uWK1_NdqaP+cCVjBD;1X=~^Y&*k?K3CS zvr)CmiDQ7{3jdF&bBc~^4cl$lQOCAzr(>gH+fF*RZQJTp#dbP&$LXMBt79Ac)ZYJp z&bg_px~Vl*jkUh_eV#cXDrKoKOoV!_6L1MzGhDcl6xElkb$v5`q=RIgIAu+h7b>OI z-~-B&LMOwM;ptUh*iY7fH;#8H=32s1$?Gj~vKk{G44YVCILE3-Qbtfenz6tqtwuct zeNtZ+rhcG5>bI&LgLHCUJYk7T`hVD2#go{FX6DnFh6*yMP@+Z~*JhY<08+S7nQc{f z;phv^ezem}>Ct;2>B-yK@~5g~l$erR0c5<>i{N1q1%L5fKU#(jAFhmd_}rc%v48bW z%-Ch|XrB;n-)-zLSb(ME>{q0{ql%K-g8?q`8<$q z(<8^w@&7r+E;nf@*HhzBrQm_C@D;g$%xpPahy}Qgs$G4_iVEvQZa zqMc`9*EScvM)xn61Z5PWsZF3z>$wc%g8OxS?#O)tWG|;SA2?N7w_Oe5_i@dU9i6>L zY{lqg>32%CgGLg_UkP3uStAkohehy9C5W;Wrx2b!Vq6JLL!^b{Zv39f?p+2|;ncRt z>jyl+$*{Xv71E`R%N0nNNeZ%H((guN?N6UpFWSRDCKxfG74$0)Q?1|K`rcD-1Q(3l z@4TK-S)R#*`IZxi!=G9a3|#d$GH6y2xoLny@xG63rrn>sHu+PVblNVuDu)6#9;fX{ zyrJ{xr;B49^XcX{Jockq0*`S}IuvDj4InNPh|f^rUIEsNYSxRfCL@Ro$wjr=?n$N} z6*Z-HAX9{CSd1mT5j=-6Q${_i9^=i0$_V~_W{|CuB0`fmmUpm#z2b!{-L~GalJ{zy z&Mj}{-@)a?a^SvyRehN2X`B5P4G6K)V;Hv*BNxo73sH&r{yeIvswA>(7gsry$iQ}J+eNCb z2&^ISATXJuN2J%47$*aG`)130lolPkfe|VDCj^sm0uElm&m?JNQ<+hm+68b_fcU=Pl+uI404uS=AT}~`ig^qJos{M;bkZybaPcThJ%H2cMIoeBx5vYX|m1J zgOxeWmt;!tGi5{s=ei#Kz-K;aGY@+=%jicLfWGqdjDBm1tDX&H$!2@Icbg`krKZ8! zTWgDrJ)4|f#*kB3tHe)yCXeW#ecWwri6StEhU~uAoYpEgr!GeNZL{oH$!gXGM=qy= zU2wgCNLVNMj(qT6WQrD}`hSmcm!D z0K1v>Jh!@KphiXRVfLvf0uHNnpAf@!jcQ8vA_s_yI#$zG?-#tPdbH@gg3Jl;idY}p zBAlie2#Fob-Yd${wMHOpspx2&S96Gi8j7XS{fOi;V zK1}p`xcq2#XtxC3!I_Iu__~O8*vaW(p3W&jLN`igA3BOeg}c)0HNFw6q}H~T)3zrZ zWIyaTY~_8ZfvIPVraw2j*vqLF+6@)JxX&*@>PIyDig5=q40m-bQ*Kkaxs6DZDthCElNh%r28zTSSy|Lde86%>&rUjz z;&J)tDsFBTzEeORbw0gut(&;sefub-tVYAvRE&YRye~Z`V&wj_nGH7ETr2-r>(4zN+7RiifVt*8|75Tocpc|5nZ`sccFJ%9VUm&$yaD z!_Qrn)@b4bE+PuYpC7jMtm~}b&b3Uw4-W1Al|pcpn9p$Defo2ANDD{jxi2r3enS|t zwR(Gsv5k0xc?w@~K@pjo4yBL@$qz7215e46<_%h6<#cMa#xUj5My(V0ZG0=UwUF_!}!d?xTjB2jHzC$2fvA0&6%~45_l1a*KIU<00|m?{Tvo$BbKiIv)+>i$w*R3LG{2 zmvx9eA^fR3-$1-^!+=eM%acEuX;pqrQhxD}Y0acB zAK}zUlGa?`p&GMel;0W-DwfF9PCXr81u$!D0tW?{_C&zJ_B~RjT}jwN=v}+ayGxEK z+%HMDTp5s~9Am_2SdSK`n9|{!q8*{4N*(vJB`8AxBN`>A9N8}lp~;Xik*zZ|N3*v{ z9D&RqoG7E70)(rH)3dwNjU8Xj8vGGq<&dSELoZB_xNRpF-Y+xDnWyYIPJc>d5rECZ zCE{EqHPyG@DN?}38Z`(hZcIh$ugS@21i(tkqUM}0ge*e`FLoS-sK0`|D6Ej_NZ#ux z0O?W)`oU#Q-o9hIob#SeYg60vxZgRI)=P<5g{I!*bAC;0CKjA!WF9dN0Vot=tk;y_ zzJ$kg*lv(x(lFIhrAnGRg4MI&{cd&wA|;)-ayFI!*doz?jM8v}jJKA%pa;C*6NEJT zm2GqMHy^X%v~$h8=-7L>=^wJ;JFv7fQtcfa{xjPw9k}4bEI273!Z{yyym%j6Z+aFv zs>Qs71LH(v?HJ`0lnEar`ZK}(TBPFE4C;}jFUWwVzClE}h2zUku=yn_tC*9$8T^!_ zB|gjr=SlmNQrRxKM=1IL)up) zSIO~-M1kXQpH2^qvy(cp&*YLbd!i>$!llR&#@*n2i84@JHCgWXwrB-46W+A^J}8_F zCoJF-An;C$Ug0O*#b)hQ(gS&h3;dFq)nl{lfnFP@Rqu__r`o-J?&lE6aD2KAkc{no zZJ$rUyR@zxj(z&Z_9`yP+b4eP^D@h-_|b5%;!a9u2i}BF8j(M2U)9edgGrX%B1mr| z#C)jJBIfYYJFi4ixI0MXWGilWFnG}0uFCnleIuW$Mt66wV(1NV4QKakBP$-Sc<4#k zW&0^a37?o(-b3Y^{Btj_!_Uf3MTh@D96}3_N<{w0&mA{W8rWlaeEM__`nSI__zr+p zb!>(@kcve{mK&PRF*GmM%VLPEm*4DG|5#~GUnkfw}pAd3PUF)E@jhhLpAzsQ7R z{;^05TZUnh(hVUF{0CwbUYxiSw=c^N4+S2<Gkp`}tETuk5(#|JC)t;*qWek_52P33Ur{cR5*a z&2qs+3nrBjAzkus!9ykPZ;3qUIo;e2pjB*mwZCilKHr8w!E1Oc9bEWbUB48%fonQi zNAI)GL;r&Y356q}nv$h;07e+RWIs|)nUPz%zeeYw{*N9y!uUjZTQ%}OrYJsmf&`v5 z0Z(BD2>f5KCe!_P>AE@`{hnMSMTmbPw@zLdso=7(Sp0e=3DhF-#s&Wex+yxdxJY^L zX6A^bki(S*?|NQ0_WE1%GUp_S4zJ!!LJ?EB<)YIjHbysTzf0<{w4F@akR3bx4EVg8 zV0-vi>HK4uy~DA!+%KEr+gsqX9rxwy&~F)E0nVoZb}BhU4~QHn)-O&l&<{n?f1N3x zqPh78H6th4r?+nA4R%axFYNt=GI{J;NlTZSos3@u1w)(@{7v$ZN} z!+75RZqpFo{Qr51Y^soC66<~)*d0txHX*qM9vVV)EEIw#DrR4k=7l~x$hK!fX<8~} zG1$8Fwaxo@diVNMT5Y?now(|ph@G~FE0{vJ~EZB z;Csj#l}kStdAjziEE(u!p`(wotbRO~n(Wu1eNes(s(gdrU1-u=gEAR}FaY!8)4L&8 z0cOrQZ&)X3qyAqE3p$M`nG9>xb{%%$bNzJzFU7GQubXQu9rq>WhtSf^)*gJq?DFH& zN5csCqBm0E;gP)uu>GYFWfS{rT`!z}Ee7j6q2l1$Y@aave*E7c725?gTnfm^JWZ+| zz(-397-tuZXnXi*q|%c5S+sD1kunwV0X}?#sV2d^$M^ToA!mY(1mkT*!o4uh_AvNV zQ_z@^D3@>Q9fOJgeRlqfQKpt?^?GKrsTRa&6c0$5RWk|ol~E4^2m%tFdjs8yxh3RUu`n?Yp8gsP+7u=osB!Anj{Mh?;<6m@wrDjN3>a9i# z)ce*?aw~ilBLhx<>_CHu?ZQEq*uO2{<34_v`&*5_6cKlf%|$8`#sPsex7`6 z&aO3Bw_=JO&w?V8n!HV|C4yOfmvvagu1R?5$h86A4)bLrxf+3!`>SdU6-Yig95qlf zf)>jXT=S_b*(H8=v55%{s zQ-)z$`2>CtvSJ2WmY#~|FE#Ui;Z#>lVS}1CWZFk{5+l<=qF=*lS6|9%JD+9#^3ILf zVTX@c5l2mxWAf`^CQew>f8>Wb0)Qj2WZD?Z0Shh|`i3)Til)>_ofx94)>~7{PpqF^(AA6R8i=cWZ2rfD!Amk{aR~vwH zEv2BLqC?*ME)|^j_4~6=hXA@@Sq% zFU^gR5MMBunhN790vhYLsaBFM;?T?)rEjTU%8qj98gIO;k9$D5`6%jnW>zYu&Uojh zH2tM8r{E0)ylZLcH#15%PoJ#(`3iF6r;BB)dj&eCbPxqd9ME=q7`2+pBQu-> z(D|_18v6VYP#nl?nyj;Sq3%>K`$~MioUoL4*piVmqDgFNs3+xG`8RCoB~Wv_1~knK znugR3Gzjn0J6&uWy5bojX(x2?4qHk>Dg_&gj;d(@$vV zN-_RK)LIbwEn4O-Zd0k%GS7DCs=i50ZULJM*EP(s%IZKeG)dru;$mI*+jKvXd#874 zbtZxuR^~|O+i6qPPZ}bhpITTe4mDTohNkKfPlU=xC#TWkF}?{^?pXp$yOFZ$T~IdX z_cx?Kg`8?JSGl>wT#+OOwgOd*;el-% zO6H}-z>J5X7TCTb{Ya%+!ExGwsv1f22oVk#cd+WERDk@@sK|Sn%KVh z4&^yC9?3WpgHho>Gg3p?w8CrYGoZd`>G7dLMXP`DbTN55cfh74V?K(kXyk_A9_Wd4 zJ0nT<6dDkeT+mQhMvPp@T2v(P{^x}7N=$0 zUl)&9C&c^dI<87jA01vC??-2|gZV@~wJt@uvArrlo2A>VL&jzg{(`?|2wXf~tZ^(+ z)zzD{rnT(Tmwt2VN@yD`SOUqv)fuW)P1O7>*i`z{Hm%7YBW(|eJfw$ztX4!ORqT5E zk&|tuulCwWKD1mfd-44rh~`V8cZNs8kQ~Btb4z5E zi1sJvs#EV%LZjCOs7+|;>uIZUqA4S&W%~_>LI|XlrPFpdOXc9nr5(Kj3DQJ~CvUF< zMbBiPEiQ^~2#b#UV!&9sb|suTpKqajIcw*!d0dWFbum8jrtZHcI0*p9IJp~HyptK4 zS=RWBhZ>r?h5?Rhe_d02mC~}!V#bsTW8t>a*Cn(zo@Dg;EpHDFo7fnEb9faU5O+d> zW{U~>YlXUU1Dn9uPh@T%>n|>rC=zSR)*x)hx^m{-8OBm;zLIhD9FUxEDcjEXRox_* zBagpoTHS9EcW`MXauDW?^M}9)-llRh!)W=?(cqA3KC2V_nk;N?n-mID@H6Zm2!1>7 z1jzEh+DY%}H|eQL@60@!{i9g=D!#^_na`=Y1qhZ^j^8FmXg?_`Y}@8D*UR>3&Rj2< zL;?Ecj$I;CO%q&SI4bZCCq+NHzJ1++P^Nt@xEyU+1MVHkVO~eNK7;FevGYgg@~+rh zOx-MdT|n$=!|{R#2oK_?R7!{b#B%i+wN%5e2*s|OjMc^mlAd+_(>f^(7Dwuc8*MWZ zjS$FQEWb~bcgh=S-z4@WT>_l|pzCGiMRJWw7npX#d|L4}aSy1Bg}Jn?E{F)BO~_X* z_Q^Nh0{QW#)nCeQo-`e(JsYgDfZsH8`J23#eq5%kCoDp+`Ta0Tbtbs9+T8U`PuacrSjpV^WMa z)AbrJ;MTHr&LfBG3KSBE7?L-Byev)AI#{NoSWb@KdL0(>_lKLKp+#mp7^dE?@F?#tSAW}=` z_^|1s6ckm-3YS0WyO*z!jHM90+=Uh2nKL!T#H)wp0MhvGx@Y+i^t!b?u$^H&~h zughq8f&NEd=8+c$Mu=uQjJ(*q zFTR>XVRro*l7&SY$+dykT?j3`pb{qxxT{ECO1g1`Wz7?c0Y; z-MZ=AQZlp?x)U5Q=4I^adf4q>>d6Xt*aDU@9wXm!`anCH6h(AyqmTTG?;6NMg0xQ* zeP;#hfzb|FHf@x{5C^%%{6zfvotI9w?;}~2z+rTFOr(Mx*HhV#q#+E}r?9>QiW%^0 znvMK&WVx}oSO3h|+#Wah%W;wov?e#$1MMl4SG)glE4ckeyVQMo6Z}d3C9r@SQp9jm zL$Va{P7+;$;JSw(&>vw)Q%&jIUn-D2B^f?FSc23p>eEuxC_1E*tS}D^Ou}v3ibBAY za$Pe}yL{>Ag`%*n|Bk=ul*If>BfgbQYUxu|A$u!rjeMn?5#oteUY z`}QP^_z#1{jRhqY4%uvlu-gq%drZ!m0NL;GcVPnC^@Z@{Fn}Jhhm;`2o+B5d|0f^* z-~JBwf4QKRKKCd8VK}s5wk0{acA>u4-We*e3iH^T=lF^81d1#SE??4(x1G6t(3yB_ z>;i^{I`m7^S24Jtb=cxirB7*L(a+}?^R-jbllxGrq+Vmg9v?FuU)C)`yPnsR7Pstdj))}ezPK~^%M!5TNnm#J(qtu$ERK_Dv#kXKk!#>rXGUlq`kRL7~Hv%^6hvXlCB&z5V3Ei?Ef#cf zDA{;o@Q3uXG{nP)!_D(uL>O|$#lzbh9yjqn5a3&b{65-+wA1({Xfazdn{vn6{^53T zlm~^mwG;-;r6Gkxhr*%@?(*b|^ym{!(CiQl-MSN3O#A?hTTgeq0!if7J`58n3|i=f z>WY34xX#l{f>hzPHketT2Pua4F80y%3xwg}v>B}Iq3@Wi;r92N?>`XnO#kYQz9h`P zehB~M0uC#4_O2b3LtI+xOX39p{RpGJi~nfVxZH-Hjzl0Y99jTX=#xi?fCN%suYqu^gJ8hy^8Ty?nGewF62 z8_(4sVmIyBU3AcbO0RXEw%wEs`Q;ugAVOy; zpBAYGkJV!!60b(sZ4$Mix6GUmx+1NQ=PAHO=6^ZHY9lI{J4`b=UJjK|diDmRa!ULYy-M1ATN9#Gn8h?OmV6ftoQDFd-O${Vc^R0j2P+=ySyUhJZbo4zi9x) ztK<2MeD1Dl;`l8QfuBKTnyB_uc=u1U25ZyxECED+=?k0Jd8!?hejTJ5@)V<^nP8`;-t*ouNC?pjH4+Gze+q!|pi@L(N8;&(%DkKl#+9$;W zFtFhR{_t6IXOzSu{WV^UPYoVRu_#q$sggu{e#xQ}*G4x}ITwa=)yXN&i)t#5{ly67 zf|=f>CXM3;EmqHerczObul{RUGfTh{s7*T_E2l%@7>-ABM}#^LSc;MK4}{bR;S`R^ zYcm8-lGvW}Y8`;e3Qwxycr$v`)FsDm{!q6}G!(0we*_TAR!nXpfbS+Abs>YxV*O3i z3+{%;X)vb?(VlJmw2E#E@X&@efu%N7tB?WPuZP7XrlC(_I@>h;`T4$!qP;f7DKh5{;nThT%vt-h++CUNW;~r8zJ5K)N}&*z-_U(fOOr^e1lXx94`~3=sV@I_v=HUgekW9 zUwH8oX0DN*3|Y}0`|yilVTf`}a7_}e%_P6>Mw|SPdLtW0sM+Nt$CE2G0~i&$kL4Vo zewHAHfW$3`yiO3E%mEkopm!r-?<0|Rvn%^#HfNTqS1W$2?bY{vkJbz^jRz!Ec}fZ+16e7v zbgEPck(Sz)vvL_yyePEv41BXnD!U6YZKBzD-zCMhBm~c~)d&$R=UI%pB);jt)jBg+ z7l!v1KmyMEy zW-7ByMJ+ovN?<-LGi&i;>Os(OOprEPt6Baf({;?XEJss({w{qrJ!2ujJwXCLcM8*&h<*K#W>5r|FQ zwa~+>rDxfm8W(?$0K(x_)L8@#Ljd9_03h~t{Z`))nI7(qHY>gksJ)$KnQfb+Uiv+w zS&=n2f~>pD)EUEcuk(g=-^}#mdcF^abc}k;``wzJfL&FgUv%N&gVyjvB_kK6qR^Mq z0FQu}SjPNTLOQ)-dzqesy0u&xK9_CV%_4Buvy^Od!-Xs1H_Q_A&_58=>WmejDhn#q zxyUKIinsp@<*$AE^Jg=Y|E*E zsw&T|l7Q3W#eO!l^~>5n#c_6FuqXh@h0j6-C6trU3i zElG$*79-L>d}5c#zgxDw4soaz<%Oa!k)ZGV*_PzGrs*45*5< z^ZV(ABmW|hp25xS5FDyuXjR&0MrBL99S#uxRL3Y%Yh)>ACzivuGC3-u^E$GqP-e+} z0}ah_3nz3Yf_5&aPQmYC->v#seG5zW&MbPAfHXy%6AVZ*EoF;j?V5l?W>=O}JePYO zfzjXJ*dyuXTvTB3dOT}KjE?o2&9P%Y)sXX~`QtjT>s^vuMNmYdP7AqME;@xzKZ?T=JbwtCqfzG{Z4L@@~;5_YDS*=MUl_ogr=DS>{b9Q?Pc3%zJg;}jy6_<{% z;ui`@f6?iwVU(~Ot=0x=r|}f0w|tsvi)p>ZFKP*%y5QyOs&%NJC&>K}{W|qyu>Hgr z(Rgs&U=ZRlt5j3^5emq8qE$UMsr|YVR0*acwQ#$2)2_f_)^on|9OkPQJO+M9AwGcd zrPv2tQ{8qgXev(#H^=Zt-vV8d1=J!>NvE3(K5`weK5`CO;S#1ohH@~V0WxpW6q7nk zI;ykI)maW+U(*=)!1R1ivKw?QF6!3J#Gd7(oT{e{ZpSlKT|hi6CW4q&<(A^2YzUNp zFdjn8L&!|GlD+DbYNzJDKVy9rOsv33m*eJBLe~iX-ks5}E-G$^wx7L3vu|QM`E8}v zI=fR%)#UrY`3v_&>yOIg49T=z9Ec(PU$~u9Xc)za~FUcyC;T$wt7ml_n z>%O6gzvUFLtEy_}i{_^p`SVq&urbo9GMLFu42uhBp&0@he7-28sx$wB1i++1Hf2;o zPUGha!T!Bs9kG({KkSeTY8|GkxsEEPj84ufu1>RNaZjpr8d|n^5wR8|PCeja^5W|B zAQT6v;cBa(5xat(Od$3zE0up%-q72TmXnBIB-vP5d7m`Le545f<)&Iod|TIM;(8TZ zF`AP5#bH2$BTHdtr+Z@EQ9h9?b=TUvNdN4{P+3;)PHi}!O2~o?Kmg2vsL-T;m;h0V z@hJ%$H8c0zzFWk|X3i=;g18o2JWOg>lp}`CZ(!-(e;UwB(HC2*QZ)cPmiJ!9c1Wpy z(TSiJ#0rfuHD-I3uFd#!qM+Cf^yRW3bO+^BBIbdC%yR; zOR5;Mc=wtSOT^6SS7g|(UdKVffE7XB`XOa!Q7+h?Q$7ctdj04>#MQrfM@gebIotH* zGrFk@55JA8V6<4ShNh(-AtWnsm7mfXxn3g?#DxDk`wVwe54%tuP!MUDZoqUN*1d7X zeCY&Wn`yKt18mhAARfcLo!-Q{u)2?byK;`}n79nw=^gF*l_3^abjh>3TRmo)RS?+E0+8<2IaewyZb`U7uty zIP@kjX6DB(B_ z?Lv*r=-JzJ*jZ$6nv$~kf7X_`DOXVUQi&+=+QO zHM0dq%PJ-n3G41gue{zTN_^Gs7N`x{dv?NQ8(KLWYw-I%)xVW1LBt=fM-jlgT~1G; zW2@>s_1!kWz%@WhFC%8ru-5wJ*$sQX5nT&0tbxsIJ>P9;>9X+DEBd=cpeUu^@aS*e zLN)VUFpc-}=?^99ytRYe!>Z2zcySuTTV^W2Ln9~+BmT56H$G6`6fb?8CecSa5_>xD zKd!C^VBpUK*rHB_%?PR;obujbO|0B$PC$an0 zz*M$EK*TRDuvc=9i3tr>Pi{+LZ2M^djWDY#%SmMZOR1k?dlFxg{!VxZKIZb%01P_^ zAW0T(+59F^e=L*Hj-NC;F8hD;O^EgLfiJt*mqq*|pnoGgWHkMSS~72nMg=^}i8t|yN64bF^4W$Abcyj*6yAuwok!;)Vh@+=5}I z+X~t-mw%arl6(~t7hCk;NF460WkctWi&I>V&ucy`bN^Y`M= z1$%h!*}wB7a+MW}h#GFY7+q+gI;T>hPoRtj`MtA2kql#YbOm(PqwR^f_T7TTa%^+7 zLB==7QPMmptP{eY;Hyc55Anau5`3t!KZ3QN&f=k;5YK&UG#rqaQw66B`yx;c;`Au`!D$1io(*g+_j|_S?a&L?nolUInbqLt}&Asrwa^Vdx9qOTV_iLlY(zD53ccJOPcXB>hEMsh$6TO8Q! z4#U7hsApX}!&LKEM*PjF;C+i-7Tej&d3~*CBNs8@BdIUz>LU7w4!;*}-1K#jrZG#xksCl;~f9dH4rr*gb?s8279LuaJH1p_bss zz@)dfUkHZuwwdoQ!Q-Grc%A(1zY6f}FTzKk&lDDKrOMn#Ol#?(jW)G_gzz6hz-$c#NnSp?2Rd?abkMYH~>$-ZfF8<4OT#Q2St{^JDF#ZMO_~ zQYV)G#zlBTGpBJ$``q^uuLP}r55~o;k)1@d!3lLLw~ziAz#!8qm4Tbt85tJ=USL2L zn)HiM^0@}=Zr`#geXR9?YXuLG#|0E_H2v;WNye<^x5IX%EpiLd;Czqza@HC>+|gyA zyHlyuc;gL_=0~GNu3Ed@X2`K|yl%|29-Qh!FU9}Nh~!|<0q6ltH=cGUswuc+ zZpj57?yOmENCx7k_yTuYe1|!R3BFn)h81I|7QWY22u z<*d^4S>J$%y8B1np9MZKEz<$~5l*rkL=R03?rPEQGkGosRayQP@q{`b)Us6?FLS8i zGD5CF-FzxatSEkVjZxTss_?FcB)d<@M%U}W{G;8YhW9H|_IxIr&dI5U9N?<#mK}Pg zd~T<$2Zdg86BHIU;Z=Jgp{8?yRPmk$?k$_P6hV zxW95Z5qo4iB ze-f>g(4pZL9$IB_s&Zs5YvE(4m_wi7((s*=z_nmdqp+2ROlOObP=4L2w21mKqEbyv z>2puM2ASen+7g)5XngLSUfm*;_a%gn2oUSSNAEZwJFq_p;_wc&gepCcN(?Hk1I^ND z15U(SdD=2+GSFl>N-J!|gXEGCAX3ca#!ywX2yD$APBw4eaLq~b3n+tgqRF_YnPre0 zRv!*YbOY%H@Vp}K0Ex7dw(t|xiB7g0G8p05UFOZ2tEAh|fP_FNi z3un}R;koox-bim!A+3`b%tXfRtEh)dyxfDbyF;uWO*(QOe6@Isx%xv&6yNlfR5S(E zuo53wAiiXf&36x`7b_%QSlU_K>k%Dbxtwjwtf~Pm)7k7ki-q?vD%8i-lxIvofy0V49r>AKWNzUNXQ*dJ= zJvdr_b8Gpl;tTrIb&v$_^6IY9qL0`}yk{50S|>;i5QujNmWvL}jtaI57D65CbE2K_ zkbZH~IRj*9YjUSE&mU@1tgAXFgHElAfz&MAAMoOD#l(`#NVbDHSeLL0N$y_n`_Oo# zHl5z;h`=g%j|`qx&|q?GdNCiA#b8*bp8PQM*0+6xC_=vE(7aIMe1E;t61#IB@l?JD z|HcI4u^a)%>mUf#!x3gq35j7ui+Xu%*opYTjMMbqaBn2D1}k{PMG&9>U;_2EnJ+(! zh~?tendP2nYZMzgD(3G2!Gh97DbBQad4k`!ojQc_@w@!#$U?uE=Y+>Jqc;@pJWWR0 zl1a255urd8OIhGdg}#ou!=e@(lu`*hsXiT`wtg}w!9OZhPo*^hI*VsJF#T+IPAKtH zKtTLQ)dWZ3Z%$YjTx9yW(eJr`R#^iL&xaT7EyalUmAM#B0qHn?3(B%Akq{%}_~DCC z*0@&fV#PfMNzmSL=vk(VMXL}SjhDD-!$$@oK z_O}z6o4f8EYgH>g=fBs^M?pZ2;ysl~hbOEMOGGWb289(7Sa_WOt#>k*Lna&y62`FHTEP)x113Stm zzl9LY*z939Lfy?$qz~vf7iIAj;&2*PnhBF%9ne>I1XnuR*Gx?_+U_m}5s(}Z zsO2DHVBz)h75j?O{pkm)6DFH`@`X-Cp`{l|o{gu=Lvr#~zcRp9Pv?Tpw9N4fl~+9j zjP+$?6K~23JtGs^-Kz*(7BzX+h~{&U4B70!nYzqlAKrZV7itFJr3B)$8`bi@PW?-c zSvWDQZo$~QKAQu5j~`Ow=P$nXXg{%W*YjWvX5}NhA8c8flgdFIe-^ZwcIQN~=mS>qt>l%6*MSYLU7Ec+>Y2D$l8Nzb)=f; z@R))2j1A4r;&kYq<(oi@@AMFh_-(1#%DUKu7Rm4&nkL{|@||8xu{D!^xGH0GqQ1^Q zM{U)Os5fHz^uZJN;AtBN$YA~R1m=q>a`jn{^zq?^6PhReGu}J7Gv?BEOzaj^*|got zMD;W?z-DvCwh~=iCPjNuQ?*3u!>O^z@<*+WxJp$lH%&qRYB?Nw%oL5LEEQ)1b;Y_$ zgC?x7q|`Z%rU*7`3=ZuUdC1Wtm`g*V6-XWQ$DZ;X!!u>A+|(cL*`}m)ZwqJY)aF@U zV9H*nVs~SR6$>B5qrclt^2`Q|t3JrN^LTjaa=%E|Ajgj}J!sc{iOd@_b5o0{%A_v+ zBL^L!G=uO{&dU$#+%K2onPu{;K@R8C9u{5k$WWNG^h3RymY_6Po&bi(VFbmaFi+8S znd}{~C%ce-@f;WodX}NrPtOW?O=^%Kcx2PFDAAdkeG$&M;_h z5gmu>U*JaJ8${;qvy5z2?D@tIj9jKKc#7$-!(x7q#>UQxfG5k;gC+>p$Y}y!Y2AZsKhoelL?f@W`ed>rRg(kJ%rdx6Z5e8G*8cX)Jg{$5h}?dsZa zA-0|gi5#A{EPM|VbrGQVk_8gmcXdi;w8+rvYyWcTpQE7??~qJQK_YQAz~X88Rx$z= zQ+l@Yx70j0gkZC!qe;lJ%3ugIISB52+}5}=FZP@Pv!q(lnjyACh?HTi?Fo#l4aV*O z8H-R|o^G37BqRY}4ImaK(ifSU$11sR+2oX03;$~XHejVtn2TegcQWAc-K`xe zjwxG&{x|?TCRtbp9-fV59~0r=Xpw3EOkpW^m!1dHSDG3o>8X$i1;uh;tGlo|d{lJY z31vK5sA^p5MU$6O&zV@dLYm)6-*u zJ|2jy(J=9TOX5N-Mp7h@su?G-ka6L^8gWa(Hl<6r!kaQMdga?u6+^S71*BS=%wSDs zfJTRndKKFja(vl_@SFIMf8w~xVgIgq75rW_q-jWNOPqUbKx%M`vyFbBTd2^E3#2YT z)APG^f(-fx!lV9jTP_-)wiyy{DZrRFJ<+Vt%#o^r8$P;oM4K#LAoHp9Rc`)GI_%`G zQAg>|qk3yV_DIQrwU48DdVLJj*LTTMS|m2L%B)!Jn!O6>wc;9QVQV?<3@hwT2M<2G zK9+!28GRjkeGjZjbw>_hHvgI)k@!Lfor-|@UmgPg8Y>bl8u6s6{N!Q@I;;vPEG#0d z3b0%bHXRF|J87!WA#{pEaDiij{aH6?3*T&+!Wa+ZkF~tod?ghfEJ71lGMbvo_xp4a z3(}cc$qrW)W+ZWckGGG-=vlKl61j%}9m*V|753YsEbE_A#mIPDowkbSjMjwr9v8vn zaiQUN)8DXG`wLH>D_Apni|u0Ql~kz%lW}ld&{x}==t(EwkkveuAL>=O=B)Z%eEd&& zQQjTpUy``-*+&+b^Ds(7*|+iR-*;Le1COzuqlqUXBj))V@9gH8FNxb*rvkV9cBkuV zeq$kfo;bJzDZ1I;e2;DU5TpbevDebC%&@=={#oVqm9_T z=M-jD9=ZR%l8r96Y6#I6>R(2|$`?#ln_pBesvPvaxC-1owh6t41;jy7 zZjr4&-h543h{yRmKCb|yw++qP|66RU9=+i2XdapR;hd#2C# zyyx@&2lK~&x3TEk9B_aEfE8~iE6OY>W%eE*EY{!|qX7w^1n*NyH-Q|57VAP>t0 zzbS7Q`dWw3<;RT}xYjNzv#Hb6<{swTNFEH=;#iSRg5!0W_v)_UZ>;2$9}<;PN=X7q zHg^M!JG8%tzU%UjUWi<&fm(O;FlAsbo7d;j>9E*vjUxH-4yz$g!gf>Sbz3M!K_L4^ zR-@M{c@?bJ`j0l)jCdF-a@5?I1QxIT;!c>f<_;B6v^$ z!dX06DMScdgcd+b<6B~28SgI&`Ue2Q!$qUCk1_KB3ktNUTLrp$qN zeS>^64iBNVJ(lTTv-zI^IuxCp`j4#K`C8G^mC1jJ76>9cLe}d-4LZ1f8L`s}>%Tm? z#;jKn1HKe3!ba~UV}F+2z1_$2{fA_brTMw;PIMLN0+JR;r%+^#fbB8|XXL zJh0hp0hKoQ`O5$1j<89qDGS02KKMwC15T#qLy7r}oC-?+g7p!+2OYf*C2_wL{q_i> zF*$kqvpGe95~FT1KR>jUQyfESPwnv0mo2f9H~miQbq)Q4jMUHx-O+sFFBt3^c<1%0 zQ{nb4(v2c87+!c5TBV1qQ)g^&Y~YgO`u)o1S13x+qVUVF7*(M{AZX2q!~dh-AMH}c z?`3_ZyqCrAHBG2HW)$KhaRI#I{1>#BH9^0cffXra$3 zlT+3b!WlnPseh$0Y+L3zt|k#gxb#rZqy8ln=efO(Me4BPL7ouYyUva=p>mV|ma@dK z;_$+PvOq#E9IR26wmd&+I%ceyaa9A*A0=9L(A3p=w5d)qRCf01`4X~HRM5;IjG|~_ zkPZeK7!B*hvLVtAvw3>E;JuS9 zfguaI!;?lF{3_`>a+^q*MENc%jJ#VeC-+^jhHVC`*Bgt7Suz_`Aw#bY%oOqV3}gIfeSh z!8$f}?u*&0)Ydicm!evYY5(Wjk3d(0yRbCI7q@(GC4q4F$!7v%+sr zqd&sTqqpc$0pdu0Q1g9cYn=i9FZK3DZ~b?mGnKeF1hiByy0G8yVDEi_I#F>y_x`NC zjJh+ZpmxmbXa7zby7c|+=kiOlV^2xb&`Pv*%T~$?^_a7%q(4Heg)lMiK>WTeLYkt< z=L!9>X{5_B zZgDhdxyODOAq3EhB3?9F3|wq>(TGA&#llKXg`-TXM~IXDHF!vxL6H5jU1ZX0TFIR$ zxE;~Bs{;2y=E=aO(Fiz-Abi4sI`!NwPU7o)i`bCC6j96+<>K^-ESF01lDtY298`18 z!_y{5N+e6jtmC#_{5hfKFl+$RRJ`7MW%e^5e`e%r?n?_bodjz9Aev~<`$3akaH`VB z!aexGT;SLpV@tx_&UQaaSLvpb48FPMmc)>ahbZ`{KfaO6YL%*F?)UfV`@Yna9r`Rf z(j3efc0q&0g*^_(Ib|o^gu$G3)yJ_tyyA+J=-6PPQb;a}Z=P?=V~<=y!S_X>2zwKE z@H&KBKuVwwv$G_yJQ=}TnpMG7LGUFvlp=)AO|Oq=p5c%E2%nO+n9c{4x&l7J*GlK6 z!}8P;crkb=IS5C#K{tbT`<$gEcAb4OGEjF{dBgyx%x|{GkAr^PyN!aOKaw^Z{DwH9AW^%fXYCnDY>*gSqOV|ghOdI zpFj%C%mXTw)CNpaJDE1<8p2IwQ@Vqub)P>Ka`|E0l5Y_Z_!(5c@q}QxlL+4|FE%+Y z)?#*zTSaaz#W9scCCsThwi_IjAt=ObWcAPF)H1MX39`rgd$Bz4;5XuPwIFO1^zGQ~yVB>?eb~(Ywi4k9|=FK3c%a%$m zGAI39LP{0OVtEuR%ka)kxhQRAPN_$*Y-?*Sp%5=>RLcngmq)P?!dPD)6F>mpS7N@L z%XQi&=Fjy`2S?f?MHE;O6f*j0cehsYtj@`ofu9Z4miq&$Zwd<{*CT67uG(ZIJ2M54 zg@sVuQP(g$8iTHp_!lfEp7Zbq0|QZA!SVj+Hm2*1F9`i8u_N2UMO$8e+FlO9X$Z)_ z7*AV`5L}!}Xq^FF4E)9eM}DQS5<*>??o|PNz>f9g!Rp?rTeWNgMkkPs=9n_@2F5W} zS9z8eF;syD%xOz(u~r!LTkVB6ODL&w!}L`TV)ZhCn(V1K1*EjHn!73O1x&TwoiGt4 z)Jh^F@XUK*;BzP1t7GY0t6M=Sm2Ab;73E3_7ZzLx z`W7f5NZh^TQzLenr3TfbqHb1AuZ8*}&6bqGD};J^8xiM*Z)3r(@fNR}Om&@M2x?HB zDb?KVKTJ~vg+pC1`K^&fkyvc|Iu=9M;sAq)Pzl%a-%^(#n-LP5A%XBL@#Rj#2nIQ_!#5E7JO0MgDWUxJ}Kl6xk zoZAFA(*uAv>@B|E@tq!z=iIU)^J}bPY;*%=#mqjjj#e%+^K}q{?&_)%=r!oo!{{F~ zVH)=Lhx;CZVUnp?@l_TpN#DyX>!pU1cd*&&y{$eN?m!_N`SfWOZ&@gciEh3~<_}>0 zF0+r5`iUIs5xjUm8%jo7>hT%52ZBX47Ym73;>phhx+spGh>Z2j4l!H)$HhjIH5Pbz)<+F>4c3R#yi@XSAX z1)bejzyPjnHb~GcdQ3L@53}d+_pV6Pzf@S{X=*XWFGj1jPU^!2*bB&Lbr=8$0#h4` zST$M?=v>p^efBUdQ>gWM;7Yg|$AB-wbM76X)88tB67Zy%KRR zK^$r{V(eYwG~s^#uot9-fl#V*7W&~&kuOUt?NfJkX_5JJ)&;);$trC~d9+hpAiKCN z2K}|NI0^9!EDAdMh45?R&4h!!4zed|D30Yxckr#|3kLDZnstW^9;PnVRIbbmFy~ z2b>zoEl(Mdux*7BEe94Dvz@IYG;F|cW(vC@a)Wr@3}$)gSy{W3dZAT__J)j&f__hs zL1rJeWhRbmzD84jtT*}5<|p2COHCHU9nwKdhqqG{FJ^r6#aJm~n@0|f-M3L)UR_t9 z#&+qg-r3jpc754M%ls5JjMW;FsXZ~j&c|pLQ+iqQ6K_R$3~J?Mp+zpvbPh zMDl67{!MyxtsfapS#jo~i)ZUD?P~%BC|K2!*UpjzlIgMF%8t7$kIw)mW>a|y*b*>J>e5QH^L>@wmE1*?#k%ug1 z=55PU23F0?QHz6n@l?b(!}6<5RLZzuRVZsW)C5BbbRyYq{7mJODcKD{9nSlxbSdB= zyv0U$3-5HTr!O{noOI%zU3Wo{TPy|~5$geAwMl^+>8)GIo0EdIi&1tKhq^w~O-tS` zEi2>En&87G{r8IoXi7HX1Zm^7{YZt;Cy(>OX&e3;ml*KBR7zn!l|*q#Zyxak_9Eb!3n&h@5W=lJy)+^$l#(0 zYMCAO^eD$A#*xQ&42)X6~IGJ*6ZMRx2VsOjLz>fFkNa# zyEe@FarqqkVPEua)Cdza=Nf&vB^Xt*+Acnx)n(SSY$Vh8$xcY}nKY?y0k?3esbf_7 ztqeITg9bK228IM_Caqp!E$_1L+ud->`Iu!_B1q-ADPZOw$0+ondx-l*aOiZLMLQVZU=>OBA{+c?h`FlISWZ< z#7n~ok3^xKrCDboS!C1M9vT^L|3tcBv$roUXgM!F$1KkJ_o%o&z>w_pgC0(8c^NT7 zm{7v3u%w3bEPgDABf`~3xWiRKl{+-5kE>rYn@e%9!VE*_=T4O6vFCI2`fl(Xjn9;? zdDhTRL=#zhN&_Z#)g1oRiqx;kP!Qm_%|{%(sSLI4%VtJVOpkGkhF`!q2S-8B5JMWk zi@Z?{vmd*Ma}CvVWrV>5{JSHq6Z8HQuXJ)22fukGJ1HDw&n(`|io*l#BuA2)FXK#B z?jsEtknY47X{OJ#@N~FHpX!X@opXj9jOP~^GR0lQbNfYl;sGD+*wp=S}x zBP>!(%hHSDNaz5~9bP`SnP8AeT{b|Do^2L|UX>pl)JofuOX^ck3`HL)RR~dj)$Y_O zm`LKCB59T*joe4FG2{|r`4e!8hNCnS=Rn}6Vjy3%9cBt3yhmEQio#UdoFzvLj|iql znjeVsF?gF-OA~5UCuZe2%I^4gy(#XM zyRa>|_I50OF|OtF?E;Q%5Or^)z|UTK6L+g9@{@z*!Jb0srW$XW`i#@2LXEvY{>!gC z=%GuMmpZ05p~$!`-5lyZ1J#gMYzTH1`2K>a78cD&!6c*|LHm58o3pm?xm*MM=XhRTtjIpjxaGK<%7Q7eh+jl9QA1i(7O{4gdP5 zr*=x>DUt#GtRMUQE9;o*z3vrHj)}8y@L+#nDCT`Wz7gsTK}<91+h_o;rbLQke>;mo z&`i5cXFZPCBJ808KMMkH3WdF1e7wcxi=%&g6>T<9#I38!)H%z1x5Y19v4b!=6W$ez z{!gJ5^Up>tjwj$S=XFf_j-watL@cam~5anNeQ7~`xi{;dP$Y#fyAkJ826ub zD{|QPW#sbddfL(36{I^F#Ev12en!m#ZKcB{dvV9*`F-%4_MlTb5jiYf&sU|)ht#`6e=Rak&iE;G0ZQr)%-Ec9|-o*o`J~zJh{B{&d zC6#2Up%<@*P6&(D1` zf|^^qmzOuAgHRkw2s!Q!iZwGQMl>mOM7D2nV>ihvGfBN$M`4&^U0dt!S5AonXrFD8 zq2mz+>&*Z}|7uhi%DKlgN2|g&BZr&w(u~h<1YnE?3vmXaoa^>Z7%8W#Acd@n&J73d zrvC8Ke;HmFX-rsiF~NvYkW-Wa?PJ>4z?US&m=%K`6mmbIuuaX%g;6$J^=MnCQ=*Qj6|9OqLy{Yxq z1#TXm(u|pHIs!vfs}`lo-U$(q%2wX(@P6r)(opGz#~FmIjvt*yjttyF=(q-IwMo4u zkek`&w3THQ-<`=mJPm7tpu_=NL@yf<;m=CvJ8%(#JJi*c4E4GYwB`YN@5w%CwLe#2kxIK3=4lq3*v zf`hR!os4@M8Lof~@MWO2x@`vKPq7b(cgjcH5x9NE+O|X*AfJ{}CY#|C%8zB}ibmds zdJ7HO#;Yu#;!*4U71L^4a7WF$g*9}=P{FUl%(sB7O2#oyGp2%<>bo_@W6ad@99QQF zlH3QtlmSyRO9BZnC_mBys*=M=k*LOKXdznU@G2`jfkYOsSGOsdpQ z%s*2>Ci#tEar;2I>`*bF?BYF*ifONd)CX;Ql~M-PGcM)#foF>(FEqdF)HqItr~}Z8 z!5+iu^?^=uY*TVG-m24ldM=U?{Y)BNGy&euwVqf=W&L6^sRh%SjB;L!0|rg67XB%l zBNL(+aP{M^XrlxDmd2u>Wx=y-*j(MppJ_CD{x?QyiHg=HHZwvlYoyn(_xR~-(`Nqb1j|*RpoBm1M@pR(2SoMok++r4jQEaPbiB9eR z@IiF?Y0C5_@6qkuzrh1}BFLkjZPC?jz6`LW+R$#L4YbT~;-(8n|HXt87ak(uY_T5Lo)8o1_SSt3bUGG>ro zT!eM8YppKi|=vfGizfg*Z8l`3Y6D#TPi8w{FgUcKWhikXyxLHt>@t{=Tt za;fWm*GuH2cjG6wLNM(RvHYG1s+@~&tg4F-odpU4eu}(ndt@6wqEl4v9?YvZpbCnv9_gDnw?Ap$IDL#r=GGMYsZZF`S0d z&C4B;IgclNss z1$@;F)l#iC8_`8{+`tkOUR$imlMAnIC$2`W;eZ_HRC!xlJKH}>X!#Bkx4a**) zp)LuO&eleduG{w9dZ&H?r9#I=0G&lXlQ0xfV|DcG5W%MG&Ki(h8*FoPl_+#jxX>+C z(+H@2E)v#sK&+~yt)8Lpo^qP34(I&|J-nSRkkyQnO39QSs+#9^3N&ZENAg!(eLXi4#!zgCrqf-Q|}4=)=__h z21otY(rd`463V%YZ`&_!%Jq|Un(o<2a9(Swh^cj-SGlk($~8cu*Wv6SXX(j`snTT? zyCLpjghNvXEhesn}PzqfGNdo4TAnyt!FQk{B20DmUQ+rQ%VvL>logyM1KT)}>w z_hGZSq%W=CS$vc5VnycDR~skPRZ~6U({??Qk`6F@YgEse%2GUA+flEU%xEsBP1TI5 zCarOutiELeZ3P;B*ZV?dTmekL!)LYq+UBho6?0a7cGf)viE`tFWsYE7^qg|HlVrm7u*bUE&*}rDgOA>}zJ#lVpBoje0Thsw z)$H}$hnlbLq@s6V2~Ptf>w=4Gl})T7~1q&V|=7^O6SEV$hG<6>IjOqr6r(@Mivl9H@uB$M{=zRH{!qY$2R?k$?KJYa+Joir&AEk>Gu zf&Bg?NP6L!Fbzs)IE>_os)r7FMMIT{9P;5#b!2gwav+B#Vox zYEw=>|H2|G)II%1mo{I(@9KPj8V*LwL@J>BoO5>R1w2R*!QO7B`8J1I^Dg9UX&lA- z7Dt!o*WO1}?#^^ZuRT#!dGQ)sQdZqmk-pjW88MYCg)yMMLaS2$XQl%&#iE(j+r4!; zsI{i371`KkWV~S${ZCPQK+}vsz=rNiJG=dGNR|N(wCD|FHQ4qQT$$Fs;O*W;uM>9$ zEgp|r7^_nk%s|?*(FdjL;|ZZNH~F69w`)77B1W7SlJbJu-4gxDvGK z?ZC^c?m4-^L(sE;Y+=Bst-LhlWRW>xn8l4_B9T8b6Y5fU@k4R)*ain%h~MrJSeE^2 z4**)|*yG!}=&X_AClCI{KWWhYRH(Wp1{LVYmR;X^Rd0)^yBwNUF)3?Z zqgW%=^#1BqS3s8=vJNf|Zf66$US$7_{qwNL-dcAH%XEp-SbPZ0FjXPIaf7y5Wtl7! zCP7*aM@l7Cj1Gj#h6Ywk2VK$?h_C)pyMPUlPCQEd^f;+|Rr$B`^R0w%-J=+&$Io-y zd5&_s3*%)0QRNvR3d{+O?BDUuV8R8*fSy<`C9US^g#9TT7C!iM#xLw8=(_BpZJE^@ zLUb!c6CoL#I_n+ePZd5kSvi#;OJRuMJ@n*2=$Ex{KU~Lm6DUV90mNjamn*rAJL1!;M2QDs-x2ff z-jPr2E4*;&~bj6raNa5m%b8&0tJ^~KG*DW zS*xJ=IDQPGn`a~#5pV{4qu2$?3LK65Ky!};Wj)h`d z<-P2X6mfI;FBq>Y%YsBHN0iB`%8*FD$aDYnLnzbS7rpN@{*1z+etSO6Em4Ck&u^wXj!QgyCG=Ir!K045`#~tyz}CF9{re9} zKSP!d12}*;oGF}b8z;}LUS2 zL}I`Z_0yC2(e%>tby$ngo#kYVylRpL#ppXqQsDKn_lT{j+%Z8OLdX2DcU4Fi^~rkL z{pOY_?Zp4nZZC>rc;fcK<&rUvHsR+RE8rlLi6L63`HiD9zsd}O z^$xtfwZqv1_+HEX6-|(nRG8+i|6e{ej1l0l$nw#I0N&#Ddsn9m++bb@(#b6-=Y9a? ztvx@2sc|e>R6{c4Brv6mMB?M7X%y|3Tf?lYu*^enK~l;8m3BwZzWN;fK^v7sabv?F zSA3b7AaK1K_)qyn2{%thb9FL}LW!8nKR_>vO+8@Hwk}>n8q$OivJ}u;h8-04-Y+|y zM;vv8ev2fAMgABja_p-(@qEXdYS_E0qbdqpMe4ZuC;B-N1I?}H@=P#Z|6*UmPWpvl zS;#+{tVTn}>3;ZEh-*=Qw+VdTg-Vp3_aFwdT-)Jhpy$&FrDhZhkxV=~DM7;1F_PaV zChy6wz=di?!}?mE!;krLA+QNn)2Bz?1kzgsBKJcR^=p*BU@1{$_enw3!$1DZV*Wp@ zN*;jS7EO__RO;U`$R-a|{_gmLq&odz9Y^5z*4~DDh)-UTVURwd9OLV_7bOQQ^|h0L z6kT(Gno8@t==K)cyqj&buO`?;Ylx#IU(BDUk?1FxpqUZXTRZnxE$pXON2uMd!3`?+qeBC?pH}*VUX?z=mnX2A8t{h{)ZzXF`BuM z^nsjHi`lM7cN-2jNxMr>5*TV7DLu0%MR&iY;ZmS}RyVRRLCc&G{{@o>rHo<9Rway0 z6ZU#Fls&;CjR_@{UZ7aYgd3o^C#t|=YiL~a!<9P+#As&HaopMy?T<rc(4m4huCY*o~+=weV> zQe<^Pu?dCdF17FEBE)phEfg4z<~fPq2q{;@8;2@#0!6!8th#&1mmOHLD@zV9SM(XZ zyL*87Z`z%IOxOn&t=$w}DQE&RFmI8Ha*(8t2p)D2iilcf`ua_oOx5i6We^4mD(=Pb z=_s(sE!>F{rv$lwWV;rMOF}MDbmO)^U$nE}8@)nQtQ+tZSj}lN(-JE7*j^TSUMDO4 ze)#IUyElyzy)TZ$L@9k6(CD-aMawAFeYuMnf`Q6(8A{FNpcD2%FU%Q9y6&TL_qKPA zP2>66zcj{%&IQE4UQ$c!==}>;2+H8)L$BBRy02HMxD};;EQ}_?c~*AM56?A2lqU>E zD8&aU8ohknl7phxQAHi$L0c3W0l#m~HZroL1%1Xj4|j!SRRcgYo4z(2oMoeO(d1tc zV<8>b{7*K!xUm-Avb&W74o4h8Y%yem6OFmB&pnFbCFY96QN>^z{<9xV;k=wdeHpQC zMXNOtNys_|q#_<*f%~Sve>Nqm5vMp+Hh#TnCfV#O+ z)Ji_WVYVvpHc^%|*#Iy)w&L(II+=tK+%pr(0)u>FM)(?RhMig(DAQH5eRR4U0NwQRQExF?rH0&H3nIZST71QDT-{f20xO=#B4W#n0X4U}6DI-!q|1aK6xsKAiB+ zs7X#bw3cLU+OtnHQ86`7+la{%NZ8Um-4zWymm^77r&S%Vqz9nGa3a0D%6fxnL4)qxXwlPiR2%@@|W&;8ar&da1$o z$<`6P(*#)7S#J;MbvBCL-0m>z3teT!fV8tTyIluH6$aNt&r2Cji=IYM6;dIxh}xgO zn6f;bww%Ex%^180zrq*^#479`#c-Td)Q6j{kR~8vhy}1L-md#0hUC@->V{JWyOc%z zhF=c0R*}cq5cU_aoWv|(VFgU+0?rUHN~#hZJ1dqmhz2!~<-<&iNhj~1w#4v4d*!S^ zcPavRYf{Zeww}2OrTn@HYB2MvBN{_7cyzgK;3)69hVwk$CqZXBP$YS7#c1Sf?(%Zu zaFz>!J(EwS=;8_$(h0wm1*Y60it*B^sQSu)S@RO6cj!A@b~i6F5X3q z!}$RvKnOO&u+oFa11)6Sd1rw?(ddH#Pku#B602oJH7Qja+5xiHJa2hA^c0>=X-yV3 zyf{W%wVW(SpiC4u$9Ct!>yKl^gVDo|G_OsrI63`b^L@$!XbUHYq$LAqDhK3n{h;%d zpqswoEA-hwswE#XrLk!4=gK#uGGQIZPNIMCCKI-wbTzaQA8Cblp)C)mY|g8ECP>80 zWnB?yC>a|N6VJdGH-KiRaiNyo$|f8TTq6j{iOipI1Vakg*MYHe{mP`x2yIsJI;nFC zpBDwtN*$%oEN-oh#)bhH;3`zH8E0~vdPF&WB&*%_$$y$;Hm}cQ{{@?&=vLa+gTs&Q zuV#gabwt86gZ)PjJGt~G&8l-TJgn1&MSTfZGiS76N&tbtsYg~KoQPI)j%}uJwQpQGB!xY~g9%tPoI`Q&L zXk;L2{&v4BH-F%(ovQR0RU`9>1O4b3I>;{hZDf9&O-TqNnHV?*af}G48mU3eU-%65 ztF}a&uQn~*S8EB7V<-ZJ)eM~J^WQgP9i+B7ANrtSfRU1Q+(00#dND*1@vCVQ0Q@P- z1nB$J%g42Wmwpqb?iCowS-i+4<*BIx$f{n;I(FzA;-pfk{vK;woFJU~DRJ z*R+fO8bH*;08$%}3yeaf7L-1!TBj;#MUK19>vK7dF*4ce$s(XEyYdt-Y^EXm?EUG4HiK4PY_FD}FeiJCz>wr&?l+FRWT_+i6qZD}-ol~POCegQZJr=I9^Hp6w z&~KbDvo=Dyy@W+o$>3=x;!Fa=bO=ZF5KPQzv5&39Zk=lM8gz5XEmQO7=&rO6@$e?t z1X+Vltx;9XkHnW-t%RwD&q z>wziYq;NVH{-{y@GDFq2E2Cmy$x?wZ1s!| zRhAU;cA#D-2ewYov{v=^MEDx&V*H;Cb9T#rihcFi@tgs{;8oOR1s5pgA+qB&i;YX9 z7+B!zbktC2Ds11r$q}TXBLL*<&2BeqhnL%NyjF{mtMKQXS`Z}b?zp_TXWbZQ#|hQj z+LHL&ux+X6)#RDYbz4i={(?CiD`r8xsQ@1vc8&5mlGYuhAiXr6^a~ustWfFb5yeP@ z;V3V0aeng?;Nc8+syE$4-ZI2&Pp{>`%+Lij++e(#L)1sJd7lgGC;K`;utJ4*f_H=; zQ0Hr&e|^2X-5NwMmXGOR?qz}5rg6cgU$Mb;H#8((qnV*4092B!j~XPm&El zZ=HdyytLTF(!~C}kZ2|ZA>kAw_3B+CIVna6j{@l{K;}TZXb=#l$@k=ZI7=-i+hU*% zjiLdArwe_Xr0U5*mX>bP&)P+5>dr`Q2AUb_o>r^pj3!#E-~|W5`0(G1AE+Tp#TA|M5>&TOgB{MW9{3 z3+Wljes~UXbcq>_m!>UFRu{+Xt5@2UkG}6y&!B+Ka<1q03tjEL6jhOZZ3CGr+fU!9 z-i19w#<>`zgtWZr2~1XsBk204J4-3C>1>!YDxe^6N`1^q&EX$-C0H}5&K;0rL{QA~ zeXEEuUrb!bAL-Vi8%H764U9zBt(D-&(xVE$5K>Pq8(vI%O3%B&9%EiYCrhu(3C2goS&UuF-4pI} zvS?`?!h=zLhml_HLMN{$DlfaguS*FP>A`1(WJ>|@F6L58=(iCWkY{n_^Rp zH>r>lsJdXo%&8KgxVlO>)JaDT=hD@K0*kL zI$HoY#v$9}(ll0Cs9QRgG9y(8@R)U?&;6h|9;#%3n5ies19W?>Rv{`ONI3Kv$mZyr zj8coD9cJ^A^J2!A$4{ zO7g(Pp(&~inVkpOa})N`xeC__Y<0+X^wj#~d_;ljZh2#yoh!^vnm&{B5 zswu_sWZ2C7aFI}jU?p;dVaVkx#|YJA7?f*|(}lAHIe!eGXYS6}*WSn9DvLB7GC9i6 zP3!PYa_i@J$l@k0H>DCBu~T8mJl8)DC+%cpk-hil3YU6Pzai#^*B;-V_kQCFY#u;K zJ8v18+!fJ-ibR8vO=k zLC)X7Dwk0H`PnHC4h8Kdyn6qZj|G_=P89|!@}ZSA3`C=;s9*$zLcibC%4N}oxGH&m zL+$2(#P}5ym{x2haw1o+qe>DFu0`npyGO8Ffvg&@VBBiNUNcK8Hgc9Wt_RE*~@&znL778t(2A{-xSVNSUAw%XX65yMS@N^fqdxbh}HV;J!kcz(}7rZ9kmn}L$R5Ubx5}UYe+bULhMCS_>UfPqB&1iHrRBMsZ zdXPPun5tLW((@dQDL8SIJ(`5})OJAo{mUk*)M#Hj_uWHmcH@&} z9dG{S)w9Qv{BWwdz34}m@8(H_bXn!v7(L~;mjnXC)}U2N(is(sKOlEnX|0N#VUu<= z(aMsR$RZ^qDPGJ-G+D6kYQZfnGl@6zf$EmxL%)jLg9s918&qw0uC;RR`{f6DJGq+h z25c)yG-9Dc=dvxo7%SL-*8?`E;9syhZKd2+>QDC$HE1z3ST^BOl_kec9d~RPxVw%@ zK{T;NOTM^diX0-Ls9`8H3n)B2(eP|l?(9T^gM6o`5o_y zCZ5$qf4q1ouLE<}NynGA?Ji5=Hr{u;wk0^h(4}PF8oy$>j4qjAESXE;>%@Sq0!7s1qWI};0x@riPu8ia;)*iNGIcp8 zO}LM^!)gEJkkPj!-~YMf$z4&>Q#y$|Qn|OJu1}6yl3G`+I=eYKnA36Dq|GJ}-7{`l zSWPSGJj*hDmp7O-rbyEq4tC5@l?}81na^U#kH`UHZ8HPpd~yOt`wlTt-y5#;wMM$B z8Ufp-RtRQKq%TY9k`el<_QFHEjs75$#0BYl`-}fA$Uvw4 znWBRJwvx$ovC?=^vAM(Cx{u%$3J?Pu#B@8RsCbMsr7W_H_cn^QpBuRN+K3Qd_{AJz zi9U3+0@h0A4iVMj04M_bMCJNf^u;nG11Wc~z+A=R{!~77BdkFiU}+6>kSzB~pKfU})l zgK)8O!mK94?87|)b&)W}%S3uJW$mkt80ER2ffrgN0$Ds?U-UP#o|UntyX8WF**Hska5 ze{7R;Mn!>gu1z&qf5^kxtlET0N)588bI9FWnc(v1-MieV6eZ{leyRs~u>%%Q^{5dP zva`e|9_6YN;77OQZMM||4hxJ+Y;JM@vT_hov3YdFWbog5kX+-}a7uVqZXu3pj-Pgq zB&^W4{rJ+z?HhHz1tFfepX(uB1B6rCss?QszchVgh?7yti%+FhNv2mksp9a#fC(TyPzIw31`{_*UMPpxW>8#< z5=Jp!lz~uCHYgy-eEeXZ>v9!ed+&O2)jD6sj{V}BPm)PXy!}Isqf3TGc`er@jVTSS zDa}568=1});m=b{ON&^H=MyCQJR9e6EPxeOjD31%0);BShI>7$PbfhQXz&MGeC-D7 zuo0E3WXFRqaARkgldawhCL zABX+cknDn9&js%yCcV^$EiL)60CUf7mLb|-I0FjfQ)+Hm%Bp3vnilm^Kt(}jG=_N} zFGT)T2L)}-Odie5o<)6SbA5%j)i7h*%9# z7d1ZS&w`I#Uob@zhS}<^i&f~>7G7ip>CQ{YFdO=OF842<6$TD#TUF&A`>$kD{|72T z)xJv&wv_jT1?osxZNz}s378m!g4|`Q_IKD`)E4Oxm4Q>m-YeIq+OebB;p)0)%Gy;* z8IW9Nx~&p-07LAyqCqpHfTa^KNt_%3K-?QOoKS*1Zr8CFiIBb*AcE{K)W3I4V%eH- z{RKr+bqJ?JnX?Xswv8Yj`2@NYRuUReT2cv26#;bIj66zv5=E@^?qR0ewIQtbuI*4X z)4tK5^WG1xq|9n6D-w?r%4Uw0fJ&)Cl$EJK5=Oc2RJxVILyAyBBG_uBIA(C#^V^k} zTe>Fo#5b@xRQx%78kUTAGE%5ZN}@grT?t%5(&~_ckIE<$rERIQp-`KLd)1X25+IrG zs)UvfMztGTLfW-MF^To1=;r;}0B!XajWfiQXbE}dBq3y}Ev-dMWlAe>qfxqnPK70Z zsX#*SbyX=%C@=D0)!UgH(t<1_-5vh`5_XL}m}Rb%sbmRCd}1J%6s<%O2|MkyfFN?i z5A3a1HLj`YU3quIkg`z1fmI{U?O4JpSyM$2NKz6((hR{!wT#Y_ZO9@;?}YWHw7vbK z=C(h0lZtNYh-?+I;B7`m;Q2`fF*guGqm-N53^j7v@4wE)%hlp(Q@9)vtsA_HYVKYo zR2c;Rpg@9^6CE{7kYd6w0yn?`@9FY4*Gr4IiPWd5pdNnFZC;e#ObHuo z()ZEaZ`5){k%g9(@<5FZF$(&19u`>}vgDT_i6DhSE&5uuv&$ zQJ%uF=$pJ)&Q+F%8dz*42VKHc92$~z!Tm`|=P?(@2A6=Fj>MUSDcPzG7!F#ycMqmfOc3GC}D&wF$28lQ!-s-xV{y46i%(Eto4LSkUMMXxSZbmlZ z5hg%1VmUJ!R+_N-SVttBwc~Nqid|e|hJ%nLnuL_r_AV$RQTB1dDpz+<>q;d>Ga^jd z{KQJXREgR#&wE3f6r{@sTKhq|8ohb~{NufPKJ2piB+p!@AUXDJb_BByrLHx1-p^2K zsH4l9NeX$!nMedqrL7}Ln+=FFU{xj~8n#yWsr3f7U>vxp=U~P57ZKI7!kh*RgNDuz zEdVChuTa_u`VH*wix=59;gf`TQjNt^RA*Fem#Nu3+SH1Kf~QoZ&XSa@I`R?;Ctj5a z(t5Wa<**?k!Lw@A?|Kt%MzQ88cpvXR(Bg5EK|hwFO-Nwu0W9Zo9-QrP{6BEgIH!$T zt_bg3O;c&6TZ}1EQl(TnrqrTCM7R^)mn`a1lq3bX1}I^X>!(y200qenTU65TqgJLi zoZ}h!46T71F}nvW&IkpD^>JYy?7y35d?(qZHc7%1PB=3v6)~LV`BITkGTL)3y+pvL zD|IfrBvjPtN>b96mfY0@`d`roN7?Dv{{Zg{<1oRZej2ze9N?L9<+=X=Yn-K&rC|+8 zPE@5cf*w9S{yDpQ=}!xh&g}1FdoQ1>E6-KMo%WwC9=>K53B|;mzX>KtUM&+brCzfa zH7E&27)(U9%b7)U8>&{KB+M8}gc1ykCw_1M>l@?NYGq6ISdo4gI*+51r8(E)>d>?+ z)u{?h(`jl+3pNFiKvB397bAGJx^AgA8xQ+MjO{*_f;~!{E>dNJ#Fu9$lef&py=(DJgjfN$n2x7wUl6HBi>Jl@ z^xt@MI3$L9zKzYfJVaR4B%crT?|6Z|?YSm5oE&9_`jR~P7Kh4|CX%H%NMJwWYmR^l zw)8ZJpuiJiWc=@MIpybutKL3zi!wiSkMn=2PzA&tWYmhl}#Bo!Tp z*dF^}g!e%wnLN*y(YZyu?0pUuHyuPn4obh{Pf}Gs{luTB_QJ2;J>MZ286UbwRX;Cj zMz!;8VjH9to6G^XrR4KJYW>sa zQ3f6Ta!WtF<)|Gh72A|F-hAhKO!fZVaSUW)I*#O9=nwhAD#op9^Ll(?4FS`iJt966 z_%DjUB(KEvpEQvSNJqH;+5>!vF-- zu!!nu=g*xX$|gkLZ#V=KKKpyd5;G7NkARQ;Kp%#ye7<-fh-)ASoBqeA&VRlLLsH!9 z-Cvtho#Egnz@DFfx6=d!u=BY1?Gec)Pc!x~L=XaG$-x9ZKtP^nq>bkP0LMHb<+0Vi zK1Qa&@>qY|qx|3MR0&t#a(yqn`)}nr{{S!2G9zkEmhf9Cqc zf5rZ9tOZ*i{ol3^;r{?P(h=y~-_O%(?*qBp@9*x|LDBs3^4>6zf@0zi+;M1GO)^*E6p=0`~0vk-96ti3Tfq{ zm1RJZG~1u)1D`Ty<%uNwzIQQ>#D4XE<^KRmVaeQEpYA@EpYMsH7$BLs-bm+bN8g?Z zAfg3`jlna$!TCfO!43>4LtGN~s6Pf`&|$wj^q*sG;NJ*sr8jQzTR5am1``+F{{UNj zu!&}%ZG88FJGN*c#z`g&Tj)9a;YGYfIT*)c0rjgaTiL4?_fjuw(iI=5d;Ja;Ac+Yu ze2)JBJP_oTJhg*hiJ<8+CL;tOr7jPFpM+ybVnJ}%d-!vvyz2!)x%0on z`J6gto&NwxRTluWRQy=KeOsZ4V1u_Wxc>ka!Z=EtR2#RR{{Vzd25j%tK^-~IzTX3d zP@q+fJlf(2a!i5q^~8iFC4m9?in%P^EnRI$#w3ba&vj|@CY-6H5gYq|d?JH_Jp5ts zVYrUCSRmrj6VP?R1UW>WrV5FH;Sxr~b+-Odi4m`N)+%D$gcd9~o|F&6+#V(c0#qbg zIrRX~P0V4*_eX!s!cHK~TQLBon7z^tnrYn!K@l9s^M^!nC6jPHPTpO7zl-7-!P4Q$ zp!mO7mO%n)2_O$@ri0xu@JRRH&+c%5x0?5D=SK!8a?gYwQa^b<+_sZFMjnfvpHbxB z=MZ8GAQpB|@MGuUtT_blefs$jMgIV9qV~j*RhZcKlh1ym$cYF$j+)5{Hsox_7rEqyiD!(RxLf@asW$5B7# z4hVEy8&v7PBg@n0Hutt11BG&9^NQGmS!4cj)7?K9n9k>|qtr#Q4g(>RE2C6#I;_kxdN(yu!)Vy>@ zW&Z$&j?w!_BC>}ocO~K1;Y;ES{JBXUN z#{#jR_KeP#Eec>$`5d)qm5G@#-2PTb!r@4$T8U7O)8YRB!{=nZn9~fyK3U@|{iKz= z<#M~6rkT>%D@{IHg#;9-dKAi^Ej5{l`RN1JV>1F#mtjc-&JQk2VZ)Jmh*Z)G z8HI!SsrZHM!-&5O{itTVYm`#eW<8Q|W@5KEqbQ+5mYyEWR-~7+yuFlahAN76*V8o# zQ^-|IV_0x6%4+R8aaoQj6Q>#I#1dU<4a835v$N}?X&+~MA?Wu7o_0qsz)Qo7K53L; zaJXqpN?A(6VOXkKXy?=q-2fcu4Pz#u{7G$z|Nl-~jz`^F2e_F3uEW4I1U7E$H`tQ~@ET^)#tO|^kO9Tp*Nu#j# zhMoF6YTVbf_FH(4kWjS{6iD2_2{Iss zD1MT0%yZnOQYX$J5)CX!I{-_*qV=iJ$E5oqm`lvE*;3Y95h2JWfhF_?rGW8cXvK1l z4?U)FHw$ouBH?^h#CbMAhx|lSHJWBr^p4gDCR0k&>!@-BB`l~+6N9oDE+e!ZrpWRv zJBgXDKLdipP@W&n;=L7t5 zvTTnTn@p64BZ_3{xLIhHfa;i~%K#NAG3oCSy_Y#ySjt#UHcLRo9$%2r_iR(r(B+vn zUgb7bmnpX)3^kU~Q)Q~AFP6I_0^W@kW88%UuWw==>k%PeDaF}d7v{@Kw z7;nnqk^r$n2?`&=UD}cO0_|sL^FK&^yzMt`Gt5e{nTKdIZ0|K*VKMmexXIX*;xQ)> z^-5QmOGNoaY$hNjQr(_B#QcA!FQe>(i1>k8o36_coX0D9z}jmnRB0GqhZ}ocTP&B9 zmy2O-iEujOcX4XM`bWUvb1uYomjjQKlZKy>=1CyG3MM5py-;vit!;*wgHWl`N=PhQ zul^bH%=@#Sh)+ZL-(_)lnOUTtNWF%gETJII$7R^O*CfK`l4l8njTjMu$K!vzB)U|r z7B+HNl$Y|;vd-0m?Nd0nRENr}d^|I0`IDa#6Y9~^WLhFvE0*LS>g0e{pgtnzxWpnC(Ltd1|7|QJCb3cm) zvQ8TnF+eI*#R|z$Q4K0%dk(^0;y!f#E&l)wNbL-aN2DIc;DFC?(uC~SEyE{ zp%c&$AN#^eeUz*MxzdS3{l984b~tjfwVA4qT6988Wa%)z@>ml zQd|To{IK*?@v}QHI$Hh$(A1t1O~Gn(@@vmlj$_WaXFbU?3W`+GAopole>kWsQkzPa z%WAWl^#G-nG*YA$HkfQl5Fn`5t)_*?$Kl@-AP|sEFLGV#-1lO8^%1K$qbg@Kc;%E1 zC3Q$pIh}r`01N`8Yg8#IAnFbn4j>5e1!<80tsF+@O84Hi(vJ@eLyK6EYm>cnA2J2> zcM(u9%g$0|?FUm%g`qkIq@*OvjyWJhXiA2s4c<}G*a%9rz@+dUxCd*3R^^tAt4~lUbvDk#A+)`_1CYF77CR{ zRSj-kg=ptq=J{aN)JjW>Ng9a?TF+QaBzZ=VL5qdRk$f_8Q$PjWovC)_cNkgaxVi|)~CT9TqXYB$?S8l`T#}(4A2M0p?c>H5__D8_#&GQ<3=QC)omYu+=Hk?d{UO^}G3kUxI zl#}{5mjE}6X}zk?6NQp+xK#l${oq~0a-|P8CiG_Ro{4)4J2=PT^PI{^I9VRf9$7_k zc(<-CV2|9F_ZR`TgVAmsuWKixMv@SuAtAK@0EF$i0Fq`TosxIQ1I#6>8#8` zqYAlgd;R^9u2xfplfXhiVqAbLu7nnMpxnj7DW&3$)p)Nd;2sZYbve#-`dcp`6{d!i zt+dovzo00TG?fd9J>rod$s?zFG4y(SMV;W}s4Mc)YmF*NBZRpHMVJy2ctLI=7&{pA z7vit+8HMyi*bLh=l{xb{#O2vsps<;u=pj5sd0;a7?n!g%cTmkvH&b=abr?efsPi=9Syu@Nt zo~~-KYsL_@DH*ZMKfHvVA;O>&!7R53lCl(C#C+oR$Jk??mi{EWU+lpua;Gccp3u09 zD#)S#nQ4Bk(OJ7G8blN*oAmJj%Bcjb6l zj{^k)K?HAa>^LZs%|ZUY_&s#L5_45Dud0EHQANkYJ? z6q1z=AY*W38ABOW0u~aK>2C8Hia-Xy?q{L4ICeJ~GaEe1Iw{-j^t;95dO(Q`0sTO_}jQ{{V^Vl2TH(GT@j~ zZK**1Vxl#^lz_-WRAm>4MiCq#y>KLw<8~tK!j(4C@knL!^ZD+#GAdG7kX%zh4Zo|$ zr5}gS#>Qd8Ow&@-=Bs@OLRPk(QFXVLq-k1TLdix9Codb`VXij01)bRHdJ9HT24CWWqmPc*+G*oPC0Y%sUZGa7DUtn z_v2Hj1OEUI4%CkcdT0C(dOhuk#IF_EX;>V5)Rdg#CjS5fjUmN2v=-$kFWXLn3 zB<5*3Mma^6zuw@RlH92cu;M(#e6IAb_#E_&*nx<~c4A@ewo8Fa;j*wAHudSe! zn34bqSeb@Ea150Z#T>C$OI4Ruy+L13QAtxywKnZ7Q!Z1~GPJ0rNO_l1ml{wYjV>iB z8xd@Exl?4~gn_#hcTvi^F zaJbAo>^3G>5|YMbnW#j`VALnpB|h)~P?&`3#1ajFf@aCyGkzft)9#ir2_M73P2YyU+ap%pMj#vk^cle)??j^u6RH75- za6Kgc$LENZ?;kowLO>!?0R$1nLq_(LVj)zJNh7R!m>jts@Yt3DvtfTx{Qm%a5RXy% z)2`j)rm+##q@aL7xSwtR0Bc}|DL{q?UU3OVY$Tbuk9gb4-uRg`0G0uReNBbw5%dJ~ z4r$0jp!h*HiTM-$aT>kj=Saxm#|)&nZ+M!2#IL8{Scaksi53R&(+z|-Iz(li;^HG? zwTH^&+VFP42!0y*{R`TV6p;$CHW!VgAMbvhCk$SdW_|fuEaCqEjQ;@3UH$ETJfEC2 zn7$(blG>QD5st_2{CS>36TuMOFq40V{$A6+KA0h^EIhw=-s5ee$$}!r6c1R`Er{2r zf6^zyI1?#Dz{hsu6!EMzUa__PE&bi{FPBP^KpFn#9QzqLa{RUP08 zFb~7hd7Lqi8~490zMJb3#6_FOPwjyK-Pmn+`+VXWu;L}Q*3@UyzMDYI$mx0b8~H{V zze;$BmktlDNoo!!Hv`#LwuL55_dgRDNXW!@ln4TMAjd*A11H>If+4S!`rD@1AvHbE zPP*8@m9g|cOb{x^p@3kBASp;x&nyu%scBhP61YHSs05Px?A7Wa(+L9edqu8!&sZ^r zkmL$e#B}LLeo=BjB2obaA8Lk&-Y^wj(kFQ)I{bOg141r948S{mwD8kepeiOT1&61( z>*pA74EMX&&7lb(Q~GoL1OZkiPTa}mz6ga*2?V=uv@*oWOXw2K@2elc|rIaZ*Ev=rJA4tea_YZYe*tN zAP!@D^5q_fc$;;cHF|gbAoJ(X)*g@4J^qIcN4z`!MhihfF?rf{Hy%@eE8m=92qNH) zu)Q6$ZT21g`m`NCqyBGlSI~?X{n5C@;z#*06ZEONR>V_@LyeGl2e1RP9ULFe)Nj1e+D(cj%zG7Y2Q@}Ivv5W4AS z<~h6SIca#9%7keq@<0*qznLZnk--ZGy`W5xJ-Ye&V1g56E6+?Frrv;C)2;zw@qOQ^ z(X~Ga#^M3>sw`Nu9SbEc+r!de^`2b)zWMwn1SovRpPVr)XlnEz@^T2}@;%Qf2XXKI zI(^4HLk>ZA>Q9gA{{RRwNfRA5>+RQ=#P<~)LG$s4!BQYw*Wyk64=#4T5h@A}%vyw4 z+g}rVn4>aSmPZK;r=e#TH@O{lHMBSwJpH%%`QgymJ_sP`B!S5Cj+~E|_Q3=hFi@n4 zD!GW>RE@m=`r*mkcx#WVI4fz>jB$1r}o9axK=aV+d8^o>PBe9)5nlSdl*H z@A;UcI3HM|)b?uJha=1mq(P9Y&Ewwux^y$oZdR8>PW$)~^YhY=B?{88dT;1{BAz}Cu0V^Wsh2v=UW!XO+cB`5rDd7x~qOB~u-PUGLmA94UNtA(CS(zy?+QCzQ z`BsCkX>(SMC{Y^UX**w$c3ThY^K4>$VGl10ET`AvL2ucw4jhEk9yNga0~_a~FGHQX z^r@N4$~!HQ;!K<@<>B!;wlPyxm*px*_H!xOsR@`dM1lVR)9|8l)5ZHv2Ct}Hm;M(! zHteyAJmZVpr{g{AN+} z1o;?g!zIep#WHZq;Kbqs&|?;o2>|GyVb8c21NFbqSKy1W-HujSnP+%0+WynY6wNak zMs^PZwE`5Zs|_$MGL)e2Hg6o_6v|q)%{argsya4}Ju|FRw!#vY7-LIKA+(rE8*Kq2 zK>;BF2}%G-2T3M;quMUrlBvt}r-+c_hy^h3bG--D$J0K@dM))*u%q&YD3f$ZLM7p3 z7NC_V1_U`Hg~dbd9Cyb3sZ>{&wNwN)sS8PR@UIYs0SPH2#7PQ*Nr@aYIPFyF^CpE) zBo}JCm(r${YJ+ngp6s__u=rF-Y&}y(C@eWz!S|+>tZ~n2kJ53bQCXGdETX!OskfU` zZX;L(prO*Bum+`rsFgyy#VeS;hP?iIC-I_U!@u(fI(6pScSm^a?pZ4>oF)d6`l8O@ z5aFw{^)A)$16n=PaIQOQf{3q%{YB z#4xQjG0=a4{Lp7TwB_C5arQWChd9TG6toWTwWA@f%5uttg0v8}bBy0DaUd-xT2N3@ zxH#TF4;_zIE7K2UW=@oq$;)wA{J^E0Bw?_4v|&oN3B}F9kWCyUJ!8uM0NQEkGFN4P z1%8+D7+g~D*!=eik@jB+GE%*yjNdxSCF8h2VpMWz82Se#30j+lDkHqJDyEx5)vX~Q z(zjVurK!i%+S#LKu+cWQ&~>JjG{LE5!BCXG=YII7K!f$!QkeR-p&(lCu9qPEUE}e6 zmdz6IuftFYAd^6W?1BYCcvgTz7E;nn0w2eJ^cSz4+o=^P3)=s;zbSZRJD@?7M8nzL{2&c{z2WGfwrMhK5&7gjb*)P~{r z44GJ^0!aiFC=vt5y#D~R)3Or#U-*v6R;eQqgzUaSndhi85)~{+!uERymSN{9UgB|5 zbKG7EbijC3Eu{qn@rUtGvwk?s>&($SwqHpF+GiCmGbMHqYounFX)>nGdHRB( z#X>YC>OY{ob!8@LL&SMv>1xYwQ`NHLYH49E2vSOud)+#OsQ9HIks#xq%2p!*h~q&i z3~opzxoAVyji`=?io?mqNtrHmv?@_FNFck`g0B3Zn+U`5rqhsS$>HqjZ7(jWDn>(? zysZsF`=HiSRPvDGgWc$Dsuh4r(6YkRd?h~4^fl>5US%RrXEU5Y{KTMBfX7T&3YZ0@ zO)4YXc4?-7pL*o{nfzS-8YX2kv3`*|8-NoqSgb;EU7*VFQk0V-b}Qb@aMI<1rTXvu z+_ZwEF_@6!f+S|yW3-jE3ZQaN;Rlio?N`r8h9BEIX2mwnIBS8T3GdqED(G*}@McI`|W%u*tIYihsNL4dxHB~Rs znMEp;6r{b~mrk^p(p!~6bjcyyYTw{I{{T*}#+-jl)<(Lz9EU4db<#>H+k)bQ+7JWP z3k0^(N`lZ-H6#V159tV%I}!2a^B*{w1KAhZnAW7Xdku|aop_#tt02iAcQXicGpjv?LZBj>F(1SLOpysC6VMU@QO#DNsQr zI(h<>DII)1Q;25&01fqG1J}E+P2y5b4J=$ZW#%d-Q4jqsr8);a;-aq>N|Xf@R#BUS5g!y{TBet!j7{6V?)Yoe?9qpwx z>$pAr8K5g@X_DG(0a4Y`#DXBCTuhnv!nT#=Kp(!mUst3nF;J_fss+8BxUca6^o*Z{ zIBCr4wXXRKO%qBqD(FcdN@C6^Y$YimoMCf}(KrbvS_m`E$RY1tPL+Kr-Mw}v zvQv9Wm|^D4;u(Gkh9N2iD4s|F@VI5rKeGP-0R5rDbZq6UGK3{9Hq&W9NDvZ039#fe z>^VorjpkU1xS5<&4-@TBXdA;_ZK02D_ICv%DZ|Q;nJJPm;nEN;TaXm4nscelDs#8r z0+yRnl%fQ9w4pFSAc94d4%>);ZZT5_mjbU@Fl|yf3w%BsMoTj783;1v8@M1`v)cT; zdPk&x>2XDrRrayRRW)j6t2WB1xGA>GwrVNb&Fc7!EHmYQkc3Y~>R|N{9+q z_X^VF7C|Y)qo}5DXuV z*y5RPSgF%JVF#HmDmOZ6+ZvOvCgZaFtv1 zNMs>I6o>-$Ccu~@a8^y{Xt0cE*xb!)Q9MA6sdGo8S@gZ%NbMOPXZ=jMNJM~**W;52+fpDte)4BfAUT2wBsuHVax!#EfAkW+~!-SgX?D*dMWdsc7D% zWFC=&d+cSm#fOWTfbkX2JDd4t831UR&%%Y|t|72Vtbl&Gw>Rtn{1WEckxm7N(1 z4K+kP$AgBW4o~)GJF(bobgmhgR2j}lQ2_J3t~IIX(d=j8+wq;4WiC@wW?6&lXap2G zhsuIfg@mtKl1`-s>TzfS6cXLSO0l_RU9B?Mu41oCv1?p`yYsYgPTG1LlZ-^w;qc)k z9Z_cDU`$cM?fndK0P)56uTeW zJU%W?f7$Fhu?az!D>RoU^7)Px3TI5bY?O(lSeZ)WmUR}6Qtf~6m&ber-A;YN#Bt9C zB$b-{rI#GeC>uT@{^da0R1QMD17WiTe{^Of)7B^) zBE{PdwGbr^QxpgMU`NQGOuvLK-}*h7ip;w^EB^o!c4VX#nG&WeKbEAD3b9fpB}v32 zq?7Ap%ON74l~cIn2bN@o0wCNG1QI}k2(bzT5fd;4@zlXd4ipF_fdmj*>_zB74WsQI zL0q|rQA*IXDI_GNNm588ummVLDp6oANnjoT>F}tL09cqYeizLr0v z5glgH&d1o!-w&UjAN*hC`oRz#I+vv&m=SJdT>L+rIwDh*Fms~{yH>+*ybSW5K#zGR z_Y>uaRqr1 z`}D&RM|xWH-Y-OF3D_QAb8&`oD|6!vQivdToqM$T#AQsMU+w%)pSB)-l(whO>G6gd z0-czjzr+r3&_&J202=|$V|(5@eQ^=e+WBW2>$L%DTEc(4{{Scd04S_1T!?OvR$}lX z-ETg=xFCqDGJP$A2olYsukH3QL~Awx?)}XX*SEO+{{A>3I2H&Y>)+P?y${TfZ%iOA z{vLi5YHs9#n272(^pUXG+o`;7u^0tv&28QJ&^u{(OTBD#Zv#``K>|@DZim~?&~w7V zD3NWCcRqty0*Qibd70)p;D~PnBt^C$OkDnw06PQxgMtVewy^?E=kM!=DNSvi_8aX9 zdwBh{`L@-HOu!vvpZ(H7pEv{3aR9oGg_!S1$>H2fNFkZuLi|d?s|X55T#*VEQ7hAzUD({O3uFM(F7$Au$*m_`zl4v~LZ3SM_(Zm#tPZkA zKtPau;_-qOCQs}^-|ze2f)Q#S5Fi_qGZ%r>k?U#a(*OpyJ|93dzTY}{zxRLU{{Z6- zosUl;_wR`%>)rK=fkaunAn0Gx{Kv{K8`#9{geW5ABdPr&L;@?J9sIP3Ai)Qr2lvC! zt|43ed^_475&)2$#iUz#8y_x^H#Xl0f>KHil;X60gxTxk5loaOwuCz#x*ks2{NXUE zuFW*qwK?R!;$nWjvi2J+i_iuy{^ZIN6u%4%AKT(IK;`+AM zEq5>sY4~T3(?$5PI_H2Ji%2%#v`}l_kL*75QuhH+^2QbZtYz zL9_w4meIZld`v+|CP@i0qN5un0PnB~0F%%S;}cOp0t12)MUN3A0BY`CfVrR%j+~~L zwE~ioo*=3e@S<2!)y4d%NF`<2AQD&tW4nJ2pM)IWv_|Q;!?Z+I@ZSV37f(VI;~BpZ zww*;kmP*1LprKhoK&((p>PDkZhR=55xW8}xE@1KmoVy@eF}Z?EuO6c%gQCpoi8!XlFo^V1SWo8Eh4h!fh!c{ z(=I`D$E1$td>z@B4>QBuDVF8=9#KZpny00sLK^`|w|P>y3w`$`OIwdAN-A`aZU#I@ z+b-WvoruI^a}09vIHiJSE_Bamnm-$;Frod}w(<6Z(Ql!wrzOj9c`ieeNXl}2G_fZM zfWk=NmYE3*3c?e`HxfVfCakMhEgY%GeXyE^J`nf)H8@fb>;QzV7YhUs0hJ~I)O6kq zk2!Xew#%HUuQ=7%J@(d($aQ+hv%4ec+EyJBT0&W?Mh&e9=hpO&RN^i?da2i3Tk2YX zR)7wb>r$lvsE{^^P$U_V;u3itWuHx*>sK;riEm^%axdelVL(qP?^#D>Wk{DMT4f{< z&IoQ0)~FhgE2!r2s&g#e>Vl%Rtw~8%mlU7^m7yvK0Nwy4Fg7=m4mB)JE@I782rp*V zqT1J^2PQX+9tu>NB|J29#Z77$hdKdrz+T4}s&Q*sx|XG+q~69-sgOJ+OVco9Y9cq0 zMR=3M&hJVGZ5v0X*MA)D9$dZ_A*rprP3}31Zu;MfI9h{3m&{Wg5{FqPG+|2<4&>Um*Pk8aK%+i z+)2W0yOrk@?k=H7e{!d&}$jv>LTlH?8E^?q@nW*J^*kgOva zEmQDGwB)3%a+&Ecfb(=TdbxU&9=(63m19|-7xtmVluKhWqHD5M(^Aw^mZ}?DnXer1 zO4%q#aVUFJG!GIjkblEuxY8UHhgL5r;32{HnM`b2 zmM!v^`(Sg$;eQI5R$GU}keu_v*)WC6tBDM?FEl%MX9po4N<3sEuzrr}U8z+=KM^;L0vRm6M`!>DY@MHMJZq2-rcZM9S=Q@o~FOMtqxrERGJ9!>mEX9F(D@(d7Kn-`9qG=RYH z;4p?-OC>2_6_^l6u_-R?A8fy|@37Ld&foe{?Og0S2lq!}@VPet55tS?CU2Bx^UCah zbJ=;)l1NVx6A}R<+3_a3gmdl~u0~~-cg#4#^`&j<2_Sx?ts=^8aAgV*WRW|MBH(lB zIDx>T0T%>;eZf6nGhIj1JD&Y4SE{Bfb}gM582;nZD%36T7@cyIj2HH z^vW=o(vT9Runm@;Y8P4v79k+v*z7!+^4&lxuR{78Gt}GZcwCDcipytk6@FfK^E)vD ztZ8doPH!EPz#X8foo%xE^_1wA%7`FyLIS;E>9Iz8w(V2yvt6OS& zi~2*e9jHz=OsNyd^}^~Un^)W&-tBnlFz#E#`A}k{rQY$fLXrV>rKS|xkm6-XS_~AZ zLX)5n0hrVpR5G0Qfvn#!qir9Q&MK-K0=F+=8dw66)Pz)4vfP@PTV=$AJS9qU zebeZp(qU7W8&Z9i}2lz^lh8sds*4M zGN*P=7m70`;ZZ3|Mp21bCysbRNEli6mo-f;Rca$)WPPP(nUyk_q00N9ba&3TNGV(H zWTDWekc5uxBrQo_e&7lUNsANQCAyN!E=d4c+}Wyb;kVmU;?;zp3j0KWleuM<#*+KU*8XRhIHKsKMx_^9)}njrs z!MCV9>M6*4FC4kWnG)%WL%^DfhqdWRvXrMBb*A7>p6RdoR-gNpr2z=U69A#!gm33w z=w3Dd05QMI`uU`39w5t2D@`G!D5VuGB&;SiH>d|Sfnas6rcX5kXd@6b%@j4$!%rBU z3*&HT%sDaL>7#eh9W}GG>LEyVHc@a1xYQtkExw?2Yzjhv-xXBjdbzDA?_EtFSFVo6 z)V1-_*0*`TibcL3( ztBnP04Fk7Ic-rw`s-#E(BrOC0LAH|$f$kZNv1{i0VLfG&=4DNVoxr>cF$w!lYF|M69zXdopuQjmZGKmNIbR&NF~-frOyf~1E_t0B;C9x+ zJ$cz|gA0^lWl57E2?6LDftt)2&Zf+FjgL&?)&otbB>?Gha}Xr~K$Eco@JwonorW@A zAJ$Posk8t)>1WVwZfs*{#O6j)9V!X7pfO=`TiNkw#xwpiMp)aF88d-ZMq7Ta{{Zoh zX>9XYw~}PuU}Isy;1ZMTX8!Kn2Jdfn^toBh9nDtS4-0K1tGu!j`=e-H5{MNvj&@KRLDYw8@a3sON#YD$PW?f!+a zAw8qau&FF&WwUWiDqjVQ!0l2;=2y{)CW>#L{{W9HkpBSqOOt1_P^ErLwwO#JRm1H6 z0QDRuV1zn|AaN@TEF?JMBm}uqW9gHKYQOSdnNplbaZ!}IA?DDgg-ti*&|D?g6wOK# zUre{ZHB^r6z~jkJcCn90sPIZ)Dql)dN{WJZ&;hi3jKR(I%XTA#%Ox&2fe7|pi@5cH z_=sW(xkrcp0O~VP#Xfu4K5Ej|^WW^;al$?7u9Xajisltw7hQpGeTsTs&L^vdIC20ikl04iw#STeM=4+b10Y;@Zm3WGmDpFDe%86Bi zQzV-mryQpPgHRz7<)jCF%^cK?#r52n_YQHLrb;JH#m*`wz?S;es7EF}b$xDko%V&8 zBrCnA${SejSPD?ql#``Sa+tURQjlOxjxt!hT%5K^aP!HbN}TVlPlnpYuYthk9j3*F zP2!V*udsloH()5m?mjWN_+y3>NZU=Jpfu?c0D^oZ8%5%6&JO!yo#uJU<}K7f2BxCG z+U-Msq<5~)c3Bund;&@s*_gR{TbuQ#NZlDONVNB3Nr9w9c%&y_M1eYrjE#lyr($y# zcOX{O9nEug)N*=`wUuP50>H&bpZ-UPzRArOcdI704_n%zJm%8 zB#6enKQk#ihdS2XC_Yqp-f%e*O*nR`b{jt;2ghjKoI0oJv;`BTFe1j3*ShVxZVKL@Sj6^-n&W+j&(E1)hwKPp^V+}4NC&c?qGd6r1J zWTv^+P?a`Tg}E)1p@M`2q#J+AQjnPtiJ37whEv*wNL?nYV&{HO@GX7CIQMJ48e#Fs zoVDR4tdT&#n=xi`Z}9N;SFm5jLMvLOrmL-K#khneO+MzJ>PQJ%N)$?5BwIibuwWAg zdmcmDv`u8HVuaG*5@~%4+rN`AXz_ofKSQEQ=Pek1DH?>Js9CH9h5ma+&+S|A$=TO# zlr=n8zz$IGcOKQSr_|de4dhwl=2oWbX_`7HYaVl4RaslJE>5bJ zAJl2;86>3@RPO%((x|3%gQ4cinD*Or(hfTpxY5>CodM#o3GO)$om9jUrf?#0<3uOu#`w-Iwl>P|eSfh|6?}e-6Nj+SWl_G8l^WfZ4*nC#7R?uN-+=0rj)x2{2HSsZSD8%h>qV-d+AS~Sho?@ zK|6>ad-MDCGN3X?I>8Z?sgoNHzrNA;F%ak?5*7adCZ&JsC)^*ghO6B^b%P8*o`Pxb z_>?JycQhQ}WXU})Z|Cy$#4(V7sLl4d5%Br{00X87B09fO!Uq67>+|znZv+p=sai<@ z0e(<_ztay2Ppbrrzl*Uxd3PkSHA+#IT7lftrnFFF^*sIW%66EJacoGk3o!=&07y!bSQ1HkcfA;&HHwWgHEIO? zKhFCcM*Nw+JpTK9PD)&)$_0r$Oyu&g_?T%i@F(FP&fA_?B3Ofh5Ziw~ZC$GXJD*d6 z76Q1FdtZMdMEPxkAT=cK5!9)X9R8g?e%K))5}iQJxqJQp0GuH#9ww71GYR5T0Kf+T zsGSDB0^~F`Q+tAXh}`c z>}cKp09X=~4%^;62!rYMw?l=7DL~rwZiF%E^QOiEslDuZm=XNX%LGRZP>V=6_vg9i z&*j$y5G4=PyM0f?a8`^5-=0&EEalmyqiyfGILshAUQExF<(`=h_+ zV=R*3A2)$7^&fAb#B2ADcgRF*_m6kTLjjUUNd5gkeTSU7UcKK~N26CmUPJZpXaU&! z?f3g)I7mPue&TL^f459d03QeLgX8Dv^V~xm+rq~@ypA>1DOi{ z0O{oseZ8=z=kD*%yulL)2b9{^fjp69< zex7xL9gX8_^uOJ??dOGGx_s&(z=k5luU9qrbk}hYN9rGZjw46BJN`xl`@_HFVAz@_ zgj7YrBYR0F%HOxs1za(0^wO2n>kwiqDM9N>Z|w~)6aksoKmjqg_vId#l@>jI(T*U3 z#4*^=_(2dzF=#W+`wjkp4||M02rdX9iHV-G1IyxegZCIBM6mFl6H0e^6>Zx{SXCJ$jf@fQ1{6Iv`u9dN5J#+9 zoN&^*Q7L-bn(Yrp$JhA#_rVcC(5^?@;esB5#g4kYuLQvo#DGi#v5r_F2a9>}^1V^)IJ{V$xU>@_(fRy*uFOi`wGz_qXrugx8?%*Ofbne&~L-(uc+ReS4sw!I+<1 zEG$S0G1H%((H~!~2q4Xkc2R{OjVGs+a|1{{e8wD=6#Zd~k|zM=l%I6n+*s}$14~t} z4?^KEs0!(_kC`05j7s1N2idFVtWX80rUzFnPcjX>wHhR&bLsnxEIJAa-(!7`oM3_q zgFQTlKk?J1*ok3_*0=J|K~i9VcK-l7_1~V@YrQ+S?$x#o+~9>NDN;p@&g15Mqo1ZF zN`kHPc&9p1K>?lJ_>)wc8rgv#7J?**+HD(yezrIFiNm5GU%Qw;SMg|&l>$$ZjrSXG z=X?;cJtn|SzyOJo2m|hLM8UZ@01>Eb{9B>4R*-@q$X70k-|25#VZ`wcW8yXY#R<}c zvN%hE^eo{0Mvm7667W{el>q+J?VpVi__$Ocb@N~ZH{jSS(VMsbl~Nk59h| z--Onx>8Lwc?Q1Uyp#iF{65%Swl+dVdJI1rD;g=E>`E_Y+RCF$+6@lNrneN(e&mNS0 zDJKacenXJr3Pna^nIT}Pl0CT?d5I+|E=Zd>1dSyg5=r_c_*wib&&=^zXQV$!d1_}Y zBp%9kT6BfWmxq(Z#*EuGku?cfnxGJ=I0@)V<&c@v&?kDicz)L7{?e%OYIRf7y3;+| zWi25P(h%E2Pj=c9833U|;(Vn#5V*LyKob-R_ z&|*;~Q!wMytNooAbSht}6=whjda*h-tsD`@{ixJcE~C7t*LMnAZA1W6H2?~Zkd&!0 z028H2GBsn&+5Z4)*E`oKQ;g$K0+azgD`3~>-yY8Fk6`6tkt9iw5X?^!Tj&TUy2Ys+ z)tm7%`GVJ}YD!e4AxH$10u-dEkJJ+Z1$`jgW0>RfIoOpDRy;#9nm+uAvF2m4U~**O z(IgPDQIcZdKobTI*u|Nle*Dr-?p&Q6$))Ye zftoW(!}_yg8HiwXJpTYlhPM?7A_`NhNGX!2Xlgsi5&;04$;2v{o)S&N4VtgaJv0=v zyi4kVSYcDE^4gZMdcO6BC?!Y%0HMeW5~86f0YD~VKd7n%01_ujj7GJk+yXVA)`jUx zQ~+D6xrrquNq!{-3LDn@2RgF`p=h>MG@}5w8bW|lOzP4IOprlH(xNwjHE*^dm`N-g z>R3_yG}oU;jD{G2bqc5(wJB%PFzoLqX1Rs5`v8X>hIV`)&e ze*&56Y#E4m=}69CF_W|=i)#B`ID59w!+#EOr7c%`moy(*9FiJJyTD{g zG5UZ2Ccx={Z!w(8^ZZs{G_>V6BHCzZJUNO+@q@?V@zQ-pDoSw!X^hS^bD*z>CWwFO z!Pwq+*vAomA*r0qvk}8NreE3Z233_U_5M?oLI~tPb#tD2K*(gW8rW1;^g>F&iWVhT8t(pVr1vO zlILRNVGu}`h2V02HkfD-kepJuiDqzGpdU!HS5)H)O5Azn7gX!1NJB&=!t$G2-Qv=a zv@`;k{L;vD6zNl?OKANET8Z`bcl^Ut)Dv!eRc1A6YTuenEV4!vBp+o<cyp0%56pRe=b3% zWN1vQ!~Xz_X~t$Nvbc1X;YM(vk$-4prP{&53lVlTpab+V{foFc61zX?hc;4KSp3(q zSR$rL;zYG};b4of%G7? z$1ZEyZonCT;$%xmvydDYr!ivx0HkBQ568~9<9SYSugPZ&l7u*+p%q0Sh)u*ePwQ;+ zB&TqZv}16xu}Sr~cAG5KDN%CZJp)*{=}KuDMh_K>%P|I2@?_1R3IMi-?Ry^P8iCk6 z0on?geGIT6TAuLxz0$G`$zyC2DN9P)%-@$+dDhjG-A{F zdDLe1R6sXtP=WNj#{U4UW`~Qi#+1w5&S6NDRMOP8q2Dt4bge5wh=B`26<|S)kBDOh zlV=#-&*l=bIBC-`^Q0F_kl<**5kfS50DZ(|J5||?r?r_rJ1EWYQgN6JZWQ40Ql@aL z)+K)R1BD5c^=((AP*?YahXE*@Z;F5lQm6J;G@1F1&t@t@o=G7*SQ*n9k(gl+l&J|>;h!W*nvkicVh%R0JFCy# zadhUNhj?|R0Hq|Mmn#yslBUCTO4CqdSfg1Kgwfo_SN#YI&y<3rvZsKgFz5K!UKn>+|Y_})`B}-p+uI&a1 zC`W-ZR&^va5aN`N9u-wh3)$OW4t0sh+S)t1yPHQ~SYGj=^Zp)bZYwaUQdX80w4owE zQ_8Z4z4WLduppEqtVm2uBm;dR8L#I3bq)GPg5qi#eKqOs!W(3;+^7jJA!>Ey#`jWC znVYFf5O*;Ykj}%sytjvZC|N_0TYl=>&at4gI(C%(Lr6}QCBo_e*!WJMkUylYN<<4; zVr`1^wBEGjSaLl8A?;YTB}FMsHx+lOBoNlIy{{NOT@)oDM3LfJN_LA$lej1#C`^TO z#6^K}N#$a6y(|p|nh{vDEV=>f^D%3yp+pqKg~$Y(&4ur}k4Paxz}j2GF0@a^pCVje zXnV0C*o%!vQKVa%%FRQGX$S!GFiwyO)C}4La)Mw1&lEAZxp{sa6K63}&#)ibQ@6Mr zm1+n`J!3nORDf2*4M5NpqiWEXm71mN8SB;C2ba#@2p z<+HqEx%U+*VOf4!y$yg+>*+A36$B81FQ9>S0t|ZNV8P)LEETPZXRYeYz_ESo*VZ&l z(>R-mUj(0I8UsKJ4K$%018qo)!yGSHp4Bu87x{9jQq?4=Y9G|0VRL=VgNoUvYbOaS zi6n7s;Fr4ug8nb9?-?G=_GcBFPje)v4$j3qA?fbXyM=yn>o4FFhiG_fw+OY&ktj`>>M@xnmE_?hW1MXf&3?D_@g3R$=N??yBC@x-C(&{_6rJ?5>xr;z$4>e321yu{Ec{N?qP?$8%pyS+YD<~s?0u5UC zBtH(jX#Bu$&lVJfiDdv0>QG%(03;zNh|Mz*!^8)@FO9ho9wYw%sZMo4!`wgFc5RsV z%BXveyJBXGEm!4(u2d_9dj@)-p%mJ?mW$4*YFo$tr+lXM!N-|>H(-@NH_9d;w+xI~ zQp<+`;K3)0Mk(aPZ(pw;+4G)^m$Dv_C1Pc*Q)lNnO0XD2q#06>q-O=CnPHTYr%^uA zxYV%ZGkFo0QRkdBX(u^K{T=?Rk| zM0*3bAp@z$FwJv=F-By`iX@jvyI<|UQpPZv8#gN9 z;1Zt!Fhqhaw3yz&n33hz8tnYStP}uqQfx+qaR)1HnXA-_LDrfDV@0#ac>YC+VMNaTQ^4dzbsjqfJy0;SE8(@Oy5-L2dk znPwE#qmXv9u^4P_7H~|I0$i~Hz^%hs#)C~G*T2Kh;yGHWX_r{FK8*!Ut$LE;bxV#m zQb_P5gai;3U^g&3XJGI@cu z!M&gUCAclTn~9LqIlBy(IDd(0l7(hjHBkhk#*?b5fn+39(L7Yzp(F&U0a)}OV*7KL zADE?Hp6xFLkXFzo#?OU9aqN!FcB3opRsk5y1j3bqM9euL&5<~tWR$sCmO%b6 z8AZ9r#D7ZOihEh;o3s-$&e37etQ?S~;PZJ%TCf@BMu}uj9;qnxIDE!il2U~dq$Cgn z6sJ<9I=}Xbo`+48pJIGNan3QZ7fZNfL014ccRfY4FL>uNC5ffaxB|+HQ}$#Di6=sb z_Q9%a#KsXpxF9C|NBO|#bK+w0l`OdRX|Ajr=I!(85#C&QNA(GwP-JcN!a}Uj0kgj< zi(8+{A}I2-n|gZSh~Ns2bMxNY^8Wztx6cG2rQ9>`L(fn8t>Y9(9AQcW7=1^4Zzfqp(5IV`@l>^ z%ttd zgb!t$5dQPu&7daJ<$+csUA4Z~i<>Gw(d$d)#eeH|grsr`n-#EL>~wrnb<;ZS5Np zU7hPr@W6M_SQr?&6JdGU_CCP$j-0o{S3+3o4?^|SYtQWukmhnr8WlMYLQ<>ku!=jI z$diVT)5GbsWMB>u2CeRrHyTCAKIC)fI3S3M*!}H)hgrc0bl2z9R5NmO0Kp3YTuQl% zn|yFY2Is6tHzwDex4%3PfD%{&PQs=f29QbKE!2M>So6Xf(YAMXZo2g6)-Pe?@cU{N zKIk_Cey79^ur&|`rbvO-Pp79J6Jyi&{^Q|-65rJQZ>&mWgDJQ% zG7X3Z!adFiSdt8Ff(Ut6ghH<|z3nr$2&;)u-~cE%Sa1z&tF>XE+#R+P($P4Z_O>zLwr47c|trJf?h5Z!3t_yYCrc%e35s$!aj62Ry|1P#b)4Ml3?L%|~8;DiP0 zAmHq7Haw^J!2}ru*zW}Idx8k|!30Qa^;pD+GXT!UK0iC#1S)U0!^586XNp-VcT^qH zLFxbifynedF)}^T=TjK`ON>Ix7Id-Djs$|+{USnC6agIp^qyX#Fz9G62TG>Lb1~De z-=+vyAS7*ohoOQB6b~`!=g*(qVihH~*Q06Q`auTOter=GduTrP29&tyEnO>1 z7&;ZtV0!$z{hTZ^CA8rLO~QM%o+raf-_^>~#DUxr~oz#{h|VEhIA;|OLH zP&1wU`Pvd}w9TWkgYcSMixIPd@*6`EuNVjVr%x#a%z$^Dt>iH&ZW2Lo7?R?E26Ebb zH3g#@NChc0Qp2Dc6^+sjR5^zc=x5-M@ZQ&HS-Sg7<4PdD1K}*uXv*l{Z1;Ry#1v#H zX?QFS_bkg+0_#u$n|`S*Q-!Glp|Qn%EOzYVSXJb?a{mDOrY=xcGGa)44h3U~B?Tmu z&zzE#A<9)LDk&fqBidhtZ$Sgn4sS2F9i_p8*AVq>K0MY<} z0r~#`+kVxYB)RW2s;@%kx{wLC>sEOCx9HcfpPN<%GNc3`3X+llW+b~dlzz33G&@P{ zASu20`N?Ou|%|hzgaablzr6#Npgdc!O8H^lu7AUj;g3m6DW~l2TWt4msg)EG&S^P?wxa z)AS$k6ty1E_S~quWO+${{{YR^j^ViiT+qkGukFXh{%g?>p(#^+q^r02`bbN={{V4i zj?LUQE&!>Ej&9p*wKU=A=fOX(lybt2~|U7Mby#_We=&BfRCpD`zFrM zazzy~?VzC@#hO1hxond%Q7lW60ALTaGLlGlNCSmo!QlxibL^fUpZ@^Iv&GKeYn0h4 z{I?5aOouzx97>YiZf^WTDGF#MZ!MnXaQ2I{vIA3y;4;j~VITlmKx3Ba_kaph zGnldV5ybS$ewvln6W;1VNKpVFz_BLMr0sL#07RI_gS%Ok%}?zmB=rEDsY+}?Hne^9 z>_&Ab8-kT6%q|6uSi6F1^$txQ{bSHNt^~;V;~`x=LrlCLWTYsd5|UM=l?4D`D^8Lo zNrb`2HpAm*;UaFNG<|fdu^(GUl|P7&!q#KMwUrsR zS4!$z4tJ<=q?G9(&ZKAp6bx=`M9fAy&t`jGiJq!PB4GeK@mMoEyH}a<5zYHa*y$L_ zM-_w>WmO}I?)6PY-<7B%n7Dh4vuuK?yr+o6s8dTyDYF#?YLu%=Q&F)p6&mJM604IN z_99LjA1jVe2r70C9a>R|Hs4Wqj>hcgw1cjl znwpVt18yXZ3vE}_l9L8VjU@T=$2-olP!ANe@T@%;gUAtMQ(p_;uVfkMn=jN)Bi1}lF{*|SZlKK!&%KUW}=zZi9@OdD=j(_M7CChm5GH5b0-I5 zds~@yH#o!Nu=D01$Ud@W7&=jk!?*6HIb%=ZEJa)fj4!1hMqQuurP)k>XLGD9xze#x z%*+~;0mNXZDDh!2D4K0LNglde9w9BI%wifj2RF>~L{$DmGIjA>)mdRwEuB^MPgAzj zK-02G?r96HlG~2z3VBYc1)5T{Bj|5sJ42WDqbR}V_z9l13sR9WB|wxdo@}{*RVt8{ zC<^xxGi*oO9|i zs|IrR)N&TC<$(aSVO(*Wy-h;nG>Z;6JX&Pa*X+ye4b_0zz*-v)k zSQqTGdpCBam_nThA+=-NEcOIg+RRIx6~%d#H8xv98BtqpQ@sBGGF%}+rk*lJrPY-= z-AC&i!jm|MjXV4g+8BGauxs}}k92dN5%=0^5cS-sc&R`gIzs`ptQ77NY7U{igM2k) z7Ge$lkddVG8YQZwPN&4Aszs+#RHd#2Z>TBMZEZnTi&9?~5v>i4!w|#0=|gc*smLWE zMbtnRt2tuhTX%JM#48Y_g|(!G1+Ee-q)Le?iSY#sM@UcwCk-KiwVPcucL437ZOCIR zzi>V3{wL>ul_Jn`_o|eJ7;jRXR-l`sDK54l6Ro;P5yZKbXYhT~1O|rMgL?Lz7}?53&icF_zHeZPBGE04+cg zrIg8WT1eEX{*VkHaso~4IYwul&CKx%SW`)q=L(Zf0Q*si4O%-stsc1Shhnh3nah%d zr`O{qC@&W=U=LX^mh~=bWdZ{;rJtN$+e>XGSK&%Vq6&ZsNcRHV`P-GTjR8w6Do>v+ z8${`-!yuOl4RS(LZFFI^DP3AI9OF8AtzVU+Dhz&=1kR^Us5VT(L9k4T0#4>LSZOH3 zvgQX+(drK8V)wl$G<_={j-QX4>ZHG^YNR}#$u-9XI0ei?1NhJu8l!Q4IS?$%kWubi7b#Ai~L7B zpP@0=zk@9F)P5&BF`{mJr|P(or$SVu>q+4p{{UE6M~g^Gbd-y%pVF`j71sR-VJ2by zB4jhF;|rc8h}2}5;sJVtZ)6vG29FqjXl`dG9r#!DiG)f)La|+uatov*{{R_wbu$!7 zY?P%+Dpf-lBmo~l6%^=kjJ2m$v?aQ6sG&XEYh67fNO5f|xl&qFr3Ee2B-GW{w$fwU5o_6)!ScOP^sPfPi>V0?0Fj?h;FIsBtP_ zPXPdjj}m{>YloQP8a~Pk$)_LfoMqZZrkkW@nQ-R`_CUhB^t7(>Nz2L*iiVkefZuSw z^~5RF`D5_|jb0BKl1xg7{Kg9!6!ocAPUEfKv?b^ZN7)bT=gFmKSs!bqPvBqWGaQU0 z0GHw5&Vp6KDgx`mNDn~8obgW!@Z&5YmY8S(qyqo};Q+#;`nI?tz!Q%j=6L1ISFBvs zJ#}(Rq;lqzBSdd^O0UCjTr9m zGZMC_4)uES4qSuqxUC(EmW~l)P~)vd>*L9}98-5{{WAd|C!o5CBteoiiSjqJ;@r8# zCg7e%y7>GzH;OZGrAa%{#;!Z4bk(Bemsh();#3?~kf5Zjsx*MGkf21*hEs91;xCN8 z2QgAqT}Gtq!*D52jYr#G%rf6wE^qA84vsF)Dai7)bZ2H=q}Jrs^vOkMc9zx_mr_(n zg4=|x0D@6+2#`q=AWk|)QJZkD*_u*0mS!IpsOsHgiT1OyQZcxB!_)%q-p@8>tDAgT z%?GoehYsEaTBy*vwH1`54j`zIv1bdZ0Hr#00NufzNdOY5JB5;D!V1j zNCm3eS5ykLB}66WN$%fhDGCyC-uoZg+{Yrv;qkZ`r3#k*vPx7vRxK<+M8Fsjp?9;m zHjkeEnY|JAd$)NGH?!TKz)G1So}yqtRwDT*&vu6c+AOL@ zVS*)UTFNIOMzTu3+>|*{;ZL-M5D0c4kR##Oq|Zh@q4Zzd40mSpqEm!alupa132F$> z5h3_@_~W_Pd-K*C0KOc6 z<=WpkA|liY-ox+GXP6`#Vpt+NmHz-L0k8nvh!dpD9iwUU!&PE6r8~qvN#UO?tJwpJ zs-X83sbRN#3DS~eYMx`3!(rDD#q|(NhvxAeX25yR!~Ecez!q$I6Uch~Fq+#-_uJ3b z2oh&d!$)s98m)BCSFXP8Dz6sDZd^X1marf`#gyW|cwrbk$*LU*piY=p(TKmLh5KiI;S&u!wy#^e8l+ye8@QBD@ z)?)8)NgXUIK4WA_5U_W!`4T>#97NXh=jPVDS;Bw3{{Scd04Q(Y-^&DCh_0wBVh@;| zgup#_lhOxV5JW{PZMW9r=z3s;XQ}YG1buuuJz!+OJ8jPdBFh76@3$z9qCg~l-vmPf zN9rE`07HT%!~Xdn=Klanq0VkD4Y_jXf0x4uDoJgtxcc=P5g90szvK7*{{VZgz21E3 z&FGr9E^YSuz}A0Ma|GXiKZXcJA7s+3X-@c#sJvWqweRK5Cj>-U=~1LIU>SmT@{3#A zed7cq1eoc#7y0|dk%9=M?PDOBn2#tuO}>3+6-T_T@04V4{{XsAJ|q7CxR0mC5r3rn zeMSZQr@Q7NRr{yA=K<-zkp0idLACD~jUMpt`525)*Gl+Td)QZf;e^U3?|eJ`-=F4& zz(F_ZBb@C&1AF^mM!nnf?(l=po{+eb2nxv5ZZ8q=!2|_Fj$-2R*Prp$EO}wlz;gGY z0-C;?@1utoFhKCwBHuXol6eC`3{ zWLunA%pRUxq+DQHv>pEdZ$Ef>Iz1e1%HitGTLMWsRIT&&{qRfIyXy*#d%mztNfJXv zb*Tm+;ak&p=bHB@R993Wn{)apFWcZZTpi!KY))h@Hgrx0ydY z5JSO<=s_PG5TLW{K^`!0d5jl{fpJ6nYs^6qZ@Dwd_KOd_hd(5?zTNc;?sX^_7}gN@Ht{OFTBoIN^+t;7>!3ChJI>I!9U=a}{L~cg%1mZ+%-Svvt zw@{VB31F~51h=}*K-InwjVL8oHX=RbMEvn2Q$b_b=cm>sgRvg6FSqf6C4;1h-ed3Y ze_Rkl!MM5dkM3Y$4Mm@<5NRKOe;$~E2y}t~Ga^SL^&&o>v(j3-?)M!5d)HU^v2w!M z%Qv5(hYdg*Ox|Q|hw&$Wf%`>oQn6r`+^g!v{9x!<7CvNH`A!`Xzm_p6wUIDLk$e0{ zukdja@7?tdXlI$O8zIWG4A&NaWre{drMMh(ihy7>AkcsuR*gv7dlA_T%eURC?Du5ySxK9T z&a+%b4n`?yQokpFlOkp!WYrN=ph_HRT8KzMmmEZp zHe`kapbe~kgFO-YI?H=Klw^IG%5bve;Bsj)@c2n+013;GgvyzWm<6WeF(cN^C6uQs zCXweCZC#@rWf{_3u2p%9xq#GX@cGt8D#NuZWHA$6UW!dYo z0N9HLHWBo@(U)T-;c&P>sVRDKQJ6iBzt)9?c!xF3T&k&cZmg*4U6T-HM4%E?F?9th z#OefsH{rP4V&_aNU$$y4`q8;=;fp&`zM8@1^6=7R`n0nKZWBr|C!=^%NXLu%)TI>} z0l=hnKq&+&2?7DRyd5ft#x>WgbE)(5jF+n6OATp4=FTm*m#au&ZIq_kl!>q+#1cg6 zo5>~=Kq)3X&~OTp6}h3IIts)~LV~MH-j@3@8yGuFDo~I3#-#%)ogkDzwDVr50FWdA zMevAU7506)zn+>xFW5^8)Sjk*3-jJ5Af(<^1Pg*nRCJk8AJRcc0FmL@LT3yrC5ibp z`FtDU6-oecMm49yr%QK3%83CfLWnm682uqhP%2O9Bn1GHVszftUN2{dpF3ZByXmQf z2~t)`Q*8DE_pdRx#s>P=r@}!{kVJvvxgIV-K!608Aw*gZ1=U2{Q{+3?S|yUlLTqiO z&7*0{QqLIeNl29t0#XPdLWu-wDk9+^8)^nTLfr8ZUanrdznA((N?0UNYJNk{d|DlF zf=W~rl>(qANg*H-1VNbq3Ed@Ig<<01B7vT^{;_^aN}R5@HV>w))KW735Pb}A*BUqg z*Z%LquAzl@t2G4WJZ{r0X=J5tEl<=o?{>XyIo;?`LbSf$(2u}U!0jh)r2rh{Sy2_Y z`1A!UV3EW@IubcY#Lw-%mo$1S^c#qO*h}<>pYfoBU;h9nJz@}B_?B>T%0Z%?AbZYj zd80uEbp@trl-p|#ooR5TBqg9rzB+oA+X`E1c9n>Ry4IQa?G_e9u%k*d!$hB!Wo+Km%9dPz=tHK??Smu^%@->PK0{`Dfw=pFYSR zezKjO4#|<4<<)G2(ma=|vxi%44K_705i2dMxZ{X@RS6DhI#c4W#Khbbj@4wCY*Nw_ zJ1;+#0i}d+%v;nI2S_P>SxGvG{RV$zhFh3peGGQ*6P2ob^IW3py@4gEwwoCH4cYF}&&%;46Q2o+!nmC&PKk4w=z^{u zJ-w1zuJ0bMb{*S|CSuDp6*Xkg?!Mf)=7dAW7l9r0l*Y3cp_^ zBh)aWQ^W}+?YO4DLmrXIv)m4QhLjA(o(*`^0UJ1Q)h>0dS);Nu46BFu_a||O+GXzf zl8q_>Wm?s-NsT2^pcHn418iuhpNPXvDwcRj(6}P59A1Xvn=y+-Pr|HAAejZloC=q1 z=t(!*oOz?$$Kg$eUa7CksL$oxD{!RNOi%y<;YtFG z^?Ht=KR1fF?29hU(Z?w%3Kd|)n!hupdRouL%iwM-;D?AqRTVjNODaK0UbY~_pv;fc zbQ2yW2(ZTRjPC`QBh}2{Oyj8_5GCe8wC0U+NgO;Axc4H0s{HUQXKQY6Al;U!UlTw}{yi~MSW5qh^YQUzcj3w7k5=p$Q8VNhR8x#590}w%Xf5kwnnameJifVI%U#-*RTA zbg46?kWxTUAc6{19SQ^%DNs-#yAVNkB0e0?l)Nvr&(%L{5(6j^;a|wL5hp!iRQQsBRt`?y$ zPt?QZ%i}e68(f`-i0Jo?Z0%A?8+|1@7Nj9clOQ;qO`!BCD??=32~p*0{BE8Tskv?4 zh<)lGN>^Kf3L=eq`(8E#3%O=HX%xrZZWM%)ORxZvSQyGOt{znl7v^eP0;Z{?))Ot`Aje(e*tvP#~RaB@7ZXJ|8t0OlEkD+r0HDVdhwiQ+*CPzyxtVuEKW1tC%;i84qB)(DSQ zlr)ko2nJ6(s&?FXMUK;B7}qfA3R&k}4z+%lxzt)ZZaAh;qY5?mYw2UFR=dU>n7dJ0 zu}e+}M8eiA6*s@Q1P!Eg#&SZXNcM<&l)W|%YWv=sb3kDA@9j&Pze#nu19N z;k2VW5veUsY$I$@C5i??001Cz)j!fEBwFC}wBH)y{ca%f5?kSBXSU~F=-=diUjr$V z!tCq~dDBx(K14)p&xjhPWlz*TntE1KbtzhPT0#&23R1`gNl4U8*jSNo1(vR1WJ(E{ zqS)}THLp5~Qkq&YndfPw;U&$PK6j}^vHK}$P)epygTR}w{Gi^tzW4kDaV}?_eldF- zTOm}L16hX{CGPku{{S)_?DLBnL8wv`g@pHxHd#P%I#g5QQh>*4^j#wfm-f#kM-rEb zQOyNM8{MOTI|GaSnzk_S2E4>{{X|kq+Dr1!(})*eVsDU5}=9bJ+kcJ9=MPK zLdghq5D8N~Sp7j}&}jyw7ZU9gbmma<%b{$c2`O?r*ASg<8YUa5WeITs4Awl5;K#DT zAu`v*0dqp3LLJ;_X;XR%R7b)rDB_rs2M|&c2?0g@qytdofM0|K3WLJ4Sn+T1-ug>i z2Hu?y4kO|jaGDB`OR8R0!ZB+9QM#S~+`yGP6(+ zr6x%!nDb9AkrF)tJu%`u=N3>}U$R_Z#F7gWV)xy{$K0;TWuCPXS24*gMsq;+W2@@V zO=GAyYatq1@PHs`Iz*Y%Zn|Jb;wIO)=Q)Nr?n%*z=Fh1v5A+e=GJLLFsZ0UXmUr9X zaBN7^UEh-As7OniK__q^-%&`D;Q#_HVo2#2)NwenZ&&%$(@uK>O=Ee%;P`)q?^Znz zJ${yG*%TRaGz&x$NH*}FOTwlgkVhemT-mQm7CNy!i)~Iov9`IXa`?` z1zH48g(pmJ9Jzu4xF;1BNZ1jgWav`Qos@b6@F#uxS@$shc$8V=6SPzk#TOGQhX#O8|-{UrbGiW zbwLwjy@io8OSu5*7zWY4PVm00bNGFs&QgsqL>PidowY#(Zy_YXl4MTykvM}#aj76x z!?WAvtVFqBEW5YOoBA7fMS`ZBp;9A38o;=an2;tTV=yCDk_E}d>@@P_jmvD*sUgV* z?YD$d#K{YBwQr|i>qsRHbCmCeCE}9MM>qv33M;tNB=aW1*N_f39IqCqxF)oF?_JLS z06RvHoM4l{J)JAFHF;j!_(x~<5!zz%yuytMsdYWuh>$@~fVj%kkP3nj3QQAyNL< zUE|UJ0Bl#;tj0&Jn6+j2@xzOj326wg4D@X=B=I(((rzQ?=i(>O<`>d$WwDZ^NJPHT zWeWA#CMc)dS^x;f;ASbDO2G^gsDwB%B%o4F!X^B!N;pTBFT9GZNmbL$^j!2|(B$sIQNeg6P?!2~l-vQsgpO&#!e$FgXO%< z$1}FTPuMraL=kXDScy#T$6JlhJ)h(`#B~q}jkomlkH4-YR*A51A5NU~)&&qKIUCrA zpxe&S(Yf`A^q4!H!5E4yVL=|T!5-^4A^qq@D@ZUWZ-=kNHz5iHaVzQ57{vLYjS_VfGYBLp%Sjltw5PfH!}MA&eBYD*Rl z=RoOGKG91)BWR0#aEY_hwWX{Bd?O_h)Im1h^ZR*xEO+t%^XC0uSV3C1e_tM4tgQ^Yn}n90&_5oDHqlaBs_hf!hQS!U2io zWP0I?ASAg0*F4}AEGkWax$eADOr(ADKYn}SD9K2qls!G`)A)=DN+!_}$o}|+N+yiW zq58rpMPEn|GNJsAK=k$A-6Gf#?+*U}k%Hbkf+^DXy)_L2!(kz*-BaZN_dp20(s}gx zblVdkQTO^klg`jZFZ7>psKcN>eEq2b!2|_c{7;q7F1P3HxWb_ORbhTjLw`K=f*ZCH z2$+t3-@XVT=4~E_()aT>AbQ@wVX8}ub32+ELh17#Uq2llf;rw|E&E>#O9EVUarCWW zAd+O*$Duri&^qrvQGr158Xpc&r0C_e1_+WQ8Mj|-B7NcC@-V6QhkwYy5Jr)GrhKk< z#EADtolHoNba~Xp5pxC&rqk&>hW=CMhZR^CZ7FBpsfyT%PRmI4tHcG>YV{r`x)g0- zK=a>o_wR|3?vDQenTkXZSb`45h6tE8CvVg5x%KHdbPz#4#!dGV5IwJg2s%vifH`@e zq<;QbB4n4*fYX`zH`kOreEAuVZkO}65cmB2b*C!ApC@mdb0&>K3&fVlvAK{3FM0dx zfapjRsj0ZzeQwaB&yh%_CN`MgYY#8KEGVtDv|}gB^-3f;LvcG2!nC8gr8O=!hajIa zI`R+)-_xE5iVldI^tY}EavO+0cUUe6nj{0U_rVA)=g-TaiH*oWnUD#S(2{rbkCq5n zDBqR3!20v)=Y&3dk0#o|4Nk(`qW=I7?lB@&!nYQ4(Y*XE7@DP33`U2gJnIEP8*S6f z^ZlTGaH|HC-{}gX>;ZPEAd{f0^?s2d3gmshk@x&zNGbP0FL$lFHt`^%4qqrTR7Hn4 zk=8c%{%?g_H6mYJk_Kgh7IYNH#p?#_=4mL!iN6grEDi9QT9p-`md&;~O7`-UujE zPv2iL*Ut_uM=iBSD%Z_U@Iex(HV4d*KgIt1qbX|8hoEwPG_(+8#*@C02^*6TZ_s>z z!aJ9x3%vmQpNTNOtO-{P{;!8t;lE8f!leq3BnT$vdSBc7;!Y&*^NM2*y0-voy;yEd z29&F72TGtGQE+*B7~2g=B~`f;9pWM&fJp=d`^4CF+rB+%{{T)t%iBvFOWPi8OqCz` z_)~=xv|Y3nrR?`GW=HBtLm6saPw7IcTPg=qapylyy`>V)2eQ~%ixU!*W;v5Hlr1eL zQh=x>Dk}+41d<3E$I=hL$D?708P}xD!w!;_DsN{yG3?e>6?m|$>o7qfmZ?*v%F89K zE&>eMM86tGSoY(^iKEMEm_w^=L=-K=ET%~WgvxwrB!DF#f-Q+V4a?ob}j=GbTz(i|Jjg?8A5H8RN*#(72QHmGvzblqGH=#YzfEbxKf4 z)hR9{jY%U?0Mn?X2=WhTvFo2cceD+_U^&+O>imOTN6|jWWaABmlA|mJZ7S?VMxoq~ zkuj&!Id6KfD&Y=l4GVL;v%nY&)y+Y30NrhZqIvjXVu{B zuSwDgGbU6-s2xB_)u^X(1&N7J5HQk0N5Fldt3g(p7Yx+;HkuAma-~{UqDTOd60$-` z3ko6xNRpJu8lpsXRQpY%@4xkm)6k(}>;;XrdKR_ov`-Fd2?0eAK?NX`l!5?A17W!` zCrE=Ek%_P-E0h6pM^+z5ad_Aeuyp>VK?lN0NRkQF1cb)+kf}IUR>#1=q@`CQZ5#Uc zz~6FHVqlp`8n#PeDLPDSH-JgBlNMt=lzeIMXndq_D&)T6)IWacvecy@Sa*ZOV&Ian zHZTMkfMAON0kor~HuzV@FUSgh7jCrBIc|Q>dqyi#>Oz(Jr65hhhv?G;+UhnZp#V%- z48-$&Kcm;;(VCKkgbi<32c7I$&3oe{;Y!quw~J}e*?n~Mv>D%ahRVtgTlvhbFBZvc z>0oE>IAPQUDFLDjYF<6t_!P{BAJQ*ra5V!P*x8ele{Oacv^g}1a+1T^TIJ$Okbvq^ zNhu5gkB>jwSJ~;f{{X{pVKamUDOoS2e40Ykk;ElR&AT_+2~(FbL4pv2mZU98Pynz| z{usCBYg4Y&0JOByi;UEll(dmf^d-LW+C;YfGM&x`na+Zh7@Nk z$`z$Us@?YvDp>wv@W+Z;h*z`1fK=s}f*h$N6qaWJln4Pqy{Pd+@l}&jXMMTx8&8zg zF!v5{$O?6BI!X;+5v3#qIQkoJx=IZ?66(_1WhexYvyYvhjH#2by`$`&B44w`fW{C* zfLT~qt5u$^8nt-+Gyec$8u0mtqF+jw8I}C6Yr8zOxdf>=+y-ntU`v)q77!)DAuPnZ z2;zP<{4ggqNBgEih0<5H3yi%^R)x0Wbtwu307A76TZxcCIPz{&+qt+Edb!8f$OO`1 zZHlYFxa|&(J2fb{?kxcZ`~Sthl;rOvYgSnPA#UYLo7Iu z#(K&ep9+FU=UGqc00fmhktcPel1SeXHzn;1jwwqBpRON?JQM-L63z#fG0Ns(`%T!) zRFgJb+42&@^TxJimlE0iLY;0vSzsEvni|y7gGUjX!(yb(M+C<8%9yk0o zC24m#;bGlWH4iC-f;Bh<*&@WGkYISDa5lz!1=?BCnocaKV{l0umg`QswzYA&b_Wo! zsbMlE79mNcOvnkrcY4wY@g+$GK!6Hl*ullw z(`Bk;tY#R2CCj8;fpI`@bI)CjQsgYln<8-HT7z+s?9|=u&iC<-dE)LBtfJzF_<7~L z#z;!DB-Hn?{Y40Q2oy+DvKwlNge)O5rN^CNzW`IN4|IAA?a%%u?C{Dl)3~!diRHS8 z;>G2Y6h=6+su=^iHl)gTgWK~O<{x>eiZ8Yi?)8mg;B0%#gssfOz5(uBw? zp|_C22mt<<0DU4q`O+}O4`haB6y)EfAsVbc74Y)?@y@-V@XM66bac(NDMd{i5d?*` zsfLjx&x^e%Y=Uhd9flwzH`118<}G2bM9YKQ8kW@0s=VM%xrHK7YoBfLpgi`#C|jDspniEByBamhGr zGbucB@aD6zOclfxhd;YjUBeR8?jD2v9Zc=VX{FROS##u@AQ|2TYlJelD z;6A&6Vkb(f(<(&CYESK$<(cy#1zQzLk`f>o@e2u_M%;kf*x~pYhGFAdlJ(Sr{l)d( zvF@zeOuRgwS1wXRi)&6=l118#{dIkPGmj{sf(nV%0KgMI6TXl{UI_wH1e|QLCn8Z{ zR=F*y6fVy7_{TWKV2&Y0-=GGsYBP%ITZ%*S{GlzGS}z32xv-ep;1RgI@8rq&wX8Zw zOByj?{{TBwQo-#K;PP}!9 znjA~Hs@bolB;>uF=bTSa-ebheH1xpgr=SFe+DQQE(-%<&R<#08Gh#7c8QKgc4L_I5 zk|xY*%qQ6e+>R?Y)v!8;Hu8T*xsG!_US--@%(VhA_*+Yq-^~8$BE*4KVAk{x@J*9t z+&}n}?4;5mwu=?qLN!7)DJ$UqA!SL0Eh$oz+loO+T6JsEIV9{}h_SQrev*4OX3bFn zdC;TZ`;#15&4{Da(t}sye~zq|1(yC3y(~+?NS2Nqzp|l^2xI>Mj=Lv93ss(k>>8#$ z`OoW-Pexo)R8{#TmFf*Nr737>3kXD0sr1yP6$lAo_cGd=l0p*0OGBgR7-wLx9GjIE z4sZKX!q*&3My@0CSCueQe`2TMaqLS4$haP$aRSTt7GT z4BLqmrqY6xg4S2Tz^7N{Y7*~oTnx)jCAQYlr$|y53s;hV5z|U$_I_9-w7l5UM6Fp# zmU#u&gpkT0@Y$p7m-cDl5i>3A7g|D&U$LUX>qK zuC|TAl+px&C;)z;BEYAa*p1I1B+pE1IE6a_$Y|z(=z3{!LmO0-gr<$nzp3fn8J#l# z1i&&$kM_AjP z?vm4pB`EF}y@29+N5OrTsi%oz4jqGQs80@$? zoM{Zi3b?K4H~3IjM~@HZ_#lPO7Nv{X=ts@oyEu;{e;r+$vBDjzqlz-xbkb#fa@z9; zLnzSM&S}X;a1*3LO5v*Dk!@AWAVRV0Z$zD-R%wyW%ds;Qyhd63IZj6r!c9;ZR3sAQ zC29dpU$jJ~4@(V3pwwI%5y_y4>o<>2-{HTOB!VN6sm`!MkFd1)lLQWMaT2IcFaY>J!X=WQ zTR1&JNm||NML;xS>L8J*1_bo#-p~YeJO2QG@rFg!``z9iZ{HD=Oh`7{%+IgfUT=aM z0VXd8+X$;`-PCg(^@Jt4@)v*;xjtw7P68Y=JCRQD2(NVaeBoFB02m}fisThq1PHfZ z-Yj{^U^NLl$k)Wa#_z!0MK7jcO{}Kj1E$InEl7g5U!=gLtjgsVnX@{QA+!&z1G)KwwX8SVS*u1a$9EWSNlMfxYYz+IY76^)0_@x1U3dv(K|;;xjlUFK?n>e zT1Di0Tj+k{5Ubj9)aS^Lsd$?X{xh`IB@3r8PqzBJI=O+-mT^qYdI+T{GcGlCW+^9KYFgpFhXE@1VtM0?DG(AeJO z0-qfX9iIIdY;=rnBloNB_#`|yZ_^T_jvA3lRlB|)BM~H4wiPe(@B74nZVxF32!#_v z@eF!2NR<~Lw&3}_D+UHYKTJ6^NP1rVOgO zKypV5u^jF`0x1(=eM0*lw#2I*-w>!Ny$En^;L$-Kg2WN*>If|V0M-nXAOUC>{zb3% zz6yD*dK-h>+P<%^np3E;9q*%PWU4I(kgyjw-2FOj)Rv_20ZQBYLcsx$56kb<3j`gv z0#l@t44CyX9)Lyp8Nm(;U`2Ngy&L7;@QzP-RFF^UAjFM<7?mV&ij8&VV;_i0%=M`# zQ69krB#?cC>>0y%TP*9xknwR%@ri7L1>ZcXs}2#^2-T4G29nB_eEFuWsS z;SOQXd!N6a_;GRoxcx$7awbXjv>vnNhAe{2LBG-#3Wp?uH_{3H$E4fX;XrhNe^FCd zAmL)yi|%I9ZLm8GCk@CTP;@j8$%)^-2sXG$1DMmy*n>R2Q{*uj4ncml6{e;3E@*Y$ z8ngR3fK-$yYy7Gjb+uw{Q$tmp<{4I5M1u8JXHdl>6z!RD+K1gmJ541!l`ZEG){+W> z)DseLv#}Dea%WGQL`h1PRH`H;Ad;e@z$GahMMv2H65vB}%(D@k;<9`W8DP!LmL&Rw zdsEESO_zsEl*vU+fjEib@w$9B|1Si10O#7XP-^Zb8O=~oLyMus%9PPW#QC+!%Hl0DKz0B z(l!?T1bQ3nB#y=QOS3(Z!O519mF2Rq^VEI?yxl9X0atfL?|d+ogq7*>r0I=6xE4S&ejFizqb8$^gE|y3Y5Z(le`H8M4TLB zWAOXKRWFNC$-b>ktUl&#JIPQ=h$>14i&4@PB`$Bu29O|gkS~YDY0`m6;XE&G4eD9m zy3>~CF0E965)zOB1}14Hw4;ZE})c`8#Gj0kLNGbetpjd4iuDvq@X75 z6fEgjfI&^XAWGl~RIf-T1`*4<*LSa$i1o@*kbD*WslAV1YQF^@m1awnm127(JF;D^mO?nl8E0rRygq11 zPwtIJY1OH^mf56Y zTdP@jTWkAJXBDIpl=EV0)vpTB9m86;79@T?u*fM$l`JTdLH;6WctHtDf|A2!aG(+u zvGT9+FqAXfpi&g_nXBQNBn2(jou^o%%oL#G3Pn2)1fR~WYE?=DkLgp1@au8U0e&{{ zVG{cnk4Xv&R&j(SH32+WAdVwY%BT!j@Q?|tevH4cuQ^b_`e^L@+VFqijK)|bJi{>^ zr!y5TG*Xg<6orxr3R+64p~aK%USCa6(?Li|(&9h>jY)+zkehCQQ6Rv$>TmON7c_;a zJxMx-U^IKPQ&!Z{KSgqn5?I=l=VE)Mf}^3Q9`Mby9$iK)|;>PGFdl!6Xvk zg(QPg)CAI|v=z9eGHFE82`VcDk936#7u*BOx{f1QanA_kxweejg|=ADsS=l4S^z?w zY><_(=9mf#M5!f5^?;xw7qD2Vs2mF?)VcP0>DIKfSZfyKDrJ^9nd;omtlERn8dMnM zZrgZVRAea6>6=okvTYSjrWTbt%0VkZQbCzmN|FqMu#*H^9e>eZYSr0p)?hJaa)u8Q zp(7fCSR5WOaL++<02q0Hwm?7coC=uW--z!m0USnL`Er+qm{Q}_;^qq&<*eep?7-3Gs9S1KZ49Z@k~9TMs;bfbEvO;dhLE{B zOO)-W!jO!8B$bI1C50q{2m~sGaK8@_Q1(dTbiGFvI(~lVVP?vlv?6^Beu>OKoz-3B8n zs|p){!tPUTHL2N}ETJJ?0bm1Evo`+#BKovx>h2n0%GW6aMQaV08$wfMA#NxJtunWo zZm^Oj18s-}Lltq8<;a%2M8zc%(uFQ$#gLRsmXPXNlt>9eLR3nW!8|0p0%IwcU}t7{ z+$IkTE@avBCQFklOr$Do<<6Y1*vXeI2~yWKZx9mDl$9)y5)?}t^SnCO4`of!4JFo^ z3#fj;IP#lGQC)tbA|M8*i64BOu1(?nvd@+-w+w6W_ZPV%tXE?VvIbXoE)bmu*sAM?LA

g%&=;nt}<#j z1^f0UgzarBi#VBRh5Pnt3Yny{ zX3i`n0GUWw(nY!Xjsq@F6u~ZbE&l*^f+k8}H%TdQ;jCC*FjBIN^Rz2$B2|{)34iYl zN>%IT5JQ|ig@Cz6pS_xps{5@QPlk8{B;nbqaX*~QSEWzaRQ~|T^NZ2|P$yQ9N=X0# z0?`CxS;yyCynPCcxd-@V)PYI>$75&GvA<+Jj><9+nUau+lhOXsio~?ZlRS|3~ zv%Z8zV(oKj1qcZcEMNm*M!;AqY+!+55sX?;LVyK}v0~)bf~+WZzLBwhWdU3u25tPf zJM*oKbZ_9nE>wOdyESp{{S)L$#81yAETDkc)Ric0E<+g!k48&1N!uKO(Y$E{JiX&jaY<&cOfM6k8^YC zim&$cA%f6lEL69pC=D{QrkaGgrIwjnYiY+4DXS^i%gaTihf-4`=ovsy8o}ZujZ{zZ z0?jaxT-BWG%0D*wWcBf2m=uI0(D4MnvY;32qyXH0%F>hYM?o7e$|?0;!7C`ys}&eCDYbQF>K46_c8X~*!YeAJMA(JZFP zF&DFw{{WX{NJOx{2sLFN->ZPT7SJD+~|w;}NlquW~UG>^GGjs47^PfClk{k~Dv zTs)~mKnU98T!g_kHzq7O5&*|J#;Aq_RQ+B2K5ZSZE&d?-AFXt+O~h_&tstQ@xd{N7 z1QU25@*0lc<6E8zEJKh*7+8`^0$dO&Mt0{LEU*GYYi^!j#xoXRB_tE9BnSddsa50w zX^646G6pf~DJ~h!gKy#DIg5~D;sUpGeVkI1^H_K~XT%KoTl9hlQ+Yd$?_r3dWY~U# z<>$UGRG=)~L;8dhaFmH8kN^OHMZx3+@d~};=SY{0Y#5Mq=lax9g)3JPl2SGxDkK$1 zx%CD}gQSSJLL_sGjA{t#KUj%u#6E$!)DqmTL{3`(3-h@#1V}ne+}uGUkh$dzLYjxW zltv~OusXNjlw1b4h)ameC9K{On-cE3JG&lXI+ zv20aJRNAyq&=D6M<#`f3WMW&i1MeU zESWfrB<1kAFJNA^BEvCDriK5z+Ex>kS@xfKm+(cUqy#_tOwiu*T3OKk;ac{^_L22`~m ztzY>zyR6-WX&R->t<_YNfKIw35D5bvhtaNja&FaSs9N}Nxbko_(xq^KwWu6CTSda>Lk_n3wypaMq+UFj-B}YqCiW(3|EmFi)>IkThjVYE8!6@M> z0H%f16utmM)P+oB2BBZFJ*q&7-cO&`koWoFwKfz!8gJb7+mt#6vtTXz`5*J`hoDrR zKzP*nK>=E$XrCyxx_mExKR1Mdpzjbwbr?xK2gC1p=Y@nc1jKYI0`-y#fD;$kiT1b4 z5v+pDUjG0{vxMSLt>X#>K@905`NP$%NC_m3%$>RNH}=GFVggK(0OxOE2HwNcFhKxO z>CV@-+L{I+^*X)bFR^P?vDQenT(uAS)GaWjgO&= z`JK7j4Kza<9gXh|Rfym3<<6Ig$ElNPj(%SdI&}2{*(n20rT%cjEIp!4cy4}2Zhg?u zL6SKb`|{p4`rwhyCh# zaX&s==PnGH4 zT{(_Un(OdyzlVJq1g)c~9CwI)MKO@bemgkr8-Z&-H-`fk7@5My zmj}h)cI|6qQfw;haiH) z5Oy>$aAbb{f4^Rb1Q2waZzSLQdwuVMBtgGd=cB*F;{-{AyaT@2A?Lrik@CD49*5zA z7tNX{r(t;_#PuV55JD(Nh;A<)WPv{7(aibA0jN6Lz|);WntY5@#L98&62OmpM;aZR zc%JA`WJuWINFw4XK~Q;oKK?&r4_l>Awz@q>${c81Xj;Ah07_}fBp~_y@{U4e+jD_r z+O6~co^*+P{{T*0;z9|S`E&OEKfVQmO#w99nT3f21;GXg27X)ldGh$-;Xy!#1?~;s zajvv66v;wsIUtKRub~=;1L+I~M2Q`IzkTh_Eps>$qU2bOjr{$hkReneqeBNx#iaaB z*1iOWNO#ua^)KJ!*L4N|09%f}d|~NO5D1Sy?dNIu;bCC7Ac7@O2Z)2H%zxVgJ{OOC z5TLN3BBk4EH`WnBBlN0fLFM)j>~SQrqMv9n9EjBK7|F8JFNcAATtI>xQJZZDHryC^ zbd52|FhSVH$E@|lkj^q1^bA4Ycf36-K>||;P)WJxFn98U(+yR?8=Npe0I$lYDWLo6;N7nxUXAR;!JUspzc-9sr zC3A9YK!f+!2+mk*=U5}I{{RGB297)J{{V&)S4@|3&J$f%7uKBtO)WWNJ?#rhu}BKd z-=SBFbu?l~IOjf<^OfY?jK``}z??>6Q4PFhBMOCJyEBTJ+MGhtfT>fk@};VBFy(1UNI%|~yuNTrA?D#| zT}=Q!Gz=Qe%UJZhnJJZrEJmh<#XuLPr{H%Gbg3uxcoss5Aj)K&yhg{wm@~;%Gm1)5 zN{f3uxqgqK=e$Bf$MGaJ3y+4I{9%i6Nm7tjAw@wX8Bs`76S73i5P4}cY(oD4w0bi$ zzain{*GP~f*aEHYwaYig7^jo00^vykN8q z_&)F;5P(Fq+_1UTp8K;KJHF?qeI;cnR-rS8p&DpyB z3mA@O7!ykL9yW45F4w$W*s4?+RFs`XJA#m6WL^j{b04a}sA1eI-1x;sOO|dpU%;?? z9eO>ww6v@$6cki+!UfPb(|8dBQ8IVaGHx)x?>y}lmW~jn0mCsDskmp+!LxOWE(Ii# z3ScWxN!rOt7S#X|I*gJ{x4@Dp8IR82*I~RpNGTc-)Dk|erqmXohONS&06;QAldQ$n zAO%PQN~Cm_*D793+ANzh!~{794Bq3ry(an<^t0&v$xP$hKS%ww$YceMB^<{y$g$WgvS3Y0jJ(En9&%aB zAgnwm@=)V(3E%qHutxY(^I#A2iNuh1fTmJxV(tt>LE;Wz8l3dCU zM;u9K0YDIw!rVIRsp|3gKEkC^r>!e=u3D0riCRdTC=z&#_MArm zelDn;9Bsha`a9(@QB9gZj7F*Po*hxu$$zP0Y zomk&TxmIN^QUS`d+zD9&{{SzJ!eb>AD1bs+8oViDTowh%5%$~r6FXjvqtnN2J0~)F zmY&plILt9vB=I^^n09+O$mgLF(od{Z)#cdibkKyS+LbP3z?7j2CSPakT3Dx1GvP!` zlc)rMK_id`By{JGlepA-WELDe_Sf&T+CNjW2;ePRj+83+{rX2pIm7#duzMFK2TMwyHt+rJ;xwkjAB-GlI5A*GMTETY4k0% zBo(%jQ>(=UpAkueGbHLdh>N;Z#mZ4sveXs1ZI4HzY~3g))*$dP(*sE@Pb93 zPK@=jj6Q3SWcl>JQ;Ne&RM+>&4RJ?MZiMXCk2m;_vYyyG6UNvuZGkW+9iJ}kZbzTV5;G}sqb|fy!9ngG`8(FWDPs$f|QVw_S)l?^O6>F;1d~z*{{T3%lloG{VHNov(BYScIvi3= zsjpK0ya6Dc8;yrZ?0z2nK;-%OX-26Dgp{oz(R1NiK~DPv5>2%e69b!OeUHZA2q6Ge z3meq!Q&7f&I<{TfyuUC(RJn-?+lek%=pMT1-50FeKg)B>s=c*TvZmC{Luwu8i$m#B z+ZqHE=?n~+RtgCL}1=#Qf=(H!@(Sxo7m zs}qOAt1re9UFTqj43&{~oh*s731B8TENLGG{{Rx5g->CVUOOxQby)>SS(v)h+jg zkWYU$B#d-mt^`yYmU{|w=~5b|)Dh>w?UDl;`&$4@5?C|Qn6tA+9hhac4%Fq@%$2dN z`X|T&0#IBlXwQT%LSHF~C%79y!$DDbjt>n91QMRLBo1b}1_mcc;Q&?sB#X7J%FvXY zJeG50=I?-p^7=Pvr<1h`QX5LNkaZ%RY`nBZ$x)xywH`I;hqP+T@>y9P5@OSg&H_^~ zhjDSo=y{{iCjT43o=wdH4j}ubKTO1Oy^mR3mSr)ZpzXG ziB^)JkdU3pBxxXCK))@s2Zxkj3$(3)Ft$y=H2Aky7H4;D)i*pX|@ z3leq~wkAxz8l;MVdnaR@?nDR4|3q{57uHc%10 zjfNXCVp2@ccd6FYs9-?^a}Ng(kb@I>%{NeL{#Q`L?o zt?shgg0{!z8l_`EBU;k7{{Sf3$`bRm@4A;8Xt>*Gex*-SsD|37l9YV|O3w?shaufU zjBo9bFIoT#U8D1XmLRN>QbLGN3-JyHFZoOeV3G?3 z0JIX$DikZPk~NDfZnsmMpzUdT-SoRZ>=vw#Vv zmcKejR&dj#rKFjGpr+(W5J@D19uXJj41iB z^&*D$eQz7XCnTjJBwKI+hy=+ZZ>ZkWZh&J~#Dan^KrHX8U#RFPYgpSQdxOq`Gd6%g z8c!tNCJ6mku#g~_z2Y&Cvi|_gsHdWr7Kc&lB-@ns2#Np zh#@8{B4?Hv5_kOm+W!DpyHdumqA#$4CeRJ0^E{901m9s^9j}XQ+EGuXgX$^^n9OLB z0z^pEVhy$yGOLLeA_$&)VikMG&XE+#kau(9(Q{i~1E5G3kTuw(?qKb3k;o2Q+ZFi3 z*zEQ+eJ_*8lZbU!^CmFm9-WPpExsB{aDNxoiMshfV7Y|K{{;zGLT^fgxP!y6A-KZ%^1Nw9l zNh*seK#N{vY%#s%b4hR*kB+-;{oV1;v)NBj9nP-rT{b4IT3*q=yCKib7I8-AvYjp? zVtgu;#koXS-Yk0Ks&;b_WlaUf!(u?cb$XQ3c;OwkzkLdt6@Mhi8=;xN2U2MQ*q5P>aJ z3k4+)#awYppH5iGq%a1rU!L93s=(B^F}!*0<%hulCL(#5@*AIrED(9~^6L>?b5(_f ziS>y151GCQiAs74T>ZbR*N_meE>4llov-M6UlrpnqUVt5Uj3pvj9LjNZg(F$V1~dH zVrGANKEE5`6dWGyA6>rs0|-lR-F~nPD6i`QEF9?yxIi#LQGs!=zTWfpFc3eK4)43g z*>aOD*pgJvOAA;qVX26UBnx$qt{URs!q?Mgf2;DmNbPuvwN1V!uIBxB7;0+oU7owV zLhdW$Pxy~^me6#D57Yz#4)SBu;&8vxF zb;4f1G33ScOH8*34Cb|abJ$nL1htfdFhA|{vE}4r+D-_3L0PenOa1Tua6)PW{5-dP z^nlm`dVMcAASHq134kccpNFOhgqJVmKo|PH?f1XF2uLJp1}^8uN?ToNU3oxBB+t@dSljF6zn%|=gol<;QT%>0a4{`q{)kUeDBMi2#yA`5@hWrHZVC4 zP|_CYP^w)-do;e@{6CK|H~kzpf;L2sgOBfsLZ)p2ztZAc7%Wxkthz?azK#A?Q|` zH-kC0x6%v@N0FQnEFB7vIrZj5z@H;)df^D>#*NLMhm~sxHrt!@cv57aC==G(`W{B} zg%-1NjCLdU%l`l?bM{4TX9PHUgQzg%8}q*1JpO|OLkCHMJ_pkTDoYT2dO^Pa;{;5a zfzwzrOo7(q`=7s;To7{e3h$(_>Q)bV39!e`^G&%L7gN$oDlZ2)VEW16uAM9h(8JbaG-?` znDg_2!vIAc&2LYv9V*!M7xd@n%MC}d@9N#{dqb1#yV33Dt#Npeg%NSr$mYdu@Xp1sA;?R{*YQIfR$;J5hI_dl0ZJh+tU&fkT{sT)|-lS zvo%BWiW86|%qePUMQ!Ym=HbKg1Vhq5vDnE6>m#kbdPq1>LW5|0*QLW(Yl0Z>QTqAa z>0@D1C07J}`5u^3Jh*=cEeifvw~GgZ1x>9nK33o7C(`&xSXl>{``_v?K@Jc(4=ZhX zx3509cM`YxM1HDY{+fIQ5TOEmMs$J&u1xrL-oxXz1T9PhS8DGOj3lP>JmoM0GW=U8{w{KDn^tgiE4NKAjV5c37D1yf>a3rs0LyK)Q787j*&qw0&ba9 z4aVSt52*d|C#^>R01@|yh8AsosdfJVi%@8N-#WumshQ<}Y)23!cXP~J!|f6qaDk>x z$A`!XC(mz&9FKT+{GskC&fhro_x&_DNi?}XA935QkgUR@gNA4wRD;}5w5-E0(nz>2 z)lng2Zl;1v6r6dF@dKNxW-}qk&{7kJm5avWfFSWJ5TS>{yHpg*Pa16LQ;HB{>9^o7 zvPhAWcC)rK60oep6@bX_3sO}{SSf17<3bc@O6O0Ig|RNIIZ5`Vmh4X0v!O|sRHsq> z3%D_OB#v<+KsSpR$Bg@9is6$vkHl5WH9l4NnELbR&oA})gn&V)Va?bQH?gMYNb&o% zTJ+u_<$~6o0s}I*i5uY z!%1Y9qXyQ5_ga+J(aoLqrPm5)0(>sy2taAK7QgvpjZMjnRcjs8_ z-mD2+3_uiiHn+UY?_HiRNLgA)P}E0=f4=I3=Bw`|ixa0`{1VQ`06x?4{8-Oe# zX+(Q9-{S4#7Ue${N`` z6U-1p1Koe?MvLV=VLjPW%&1C2017a)l}S28K)hTLu^3cL;=KrK)AZByj9mEujSJ8k z+e(d(lS3CoKl)Ro6de*S2!yDDLD+=AD3F;^oq)uVM4=!8QUf1owHf)=)p7kF-=qYU z0;me{1r-IT_iITb`eI-JNel3(k}yMkP!uUY5z5!+y9-u`T@EJ7JEc-)1o&d`NKiMM z128TQBK4x}@p_(uCzmNG1gP;3uEn2VMQK-yRZ{f@5|pb_RFFy2B+5dgPS+r6A}n_z z1SzO>ww;!-iv?y8T?iHZR`}RFXeK1UAaA>$aoMD!`Y75$^QVv zIc6SAu%BvPSd47EB*lc3NGU{@3kA8W@l!|nt_sTY##*Oz<@xSWnJMJ5+E+@&Pji-3 zIJdjSytJx}=v!eZNl|ZBrKM_BUr%`#{vq^F+UybmGq8DV%+&>nDxH)qSi!==q<&Rp zfhUx7Z|vLc@_inDFMSZ>@i?h+aJ{DOE+exu#I+zMM2^;E7>tVonWv1Dk&VY?fs_+a zh7~c6_HLf?ikR*H02)k1?-znZ5+oC7F}^+|W>WYu6Ow2OfK4t*9?ERxi-0aGX#KHa z&+G!&)COv}C4seFzyxkC;2w#jVSU;(g(L({p$EhvBc8&*9e^Gw6O1~Q#LmHk0ieG8 zm~E?q@d~Dpwjb1!p%$fV)njNnsus=2DJdf4*bo!}7B?hJ3wjBbzxDXs9rYFG3BV>g ziDGUUtXNj{;eFWjf+I6!#FRV)_hB-jdyr&E2Ed;P9wR0q!ow{801zdt0khiRYi%2p zRbCW@k5|`V!*gr3KS;y)i-q!R;kSd-j68=BLbV~Yr%>2|GduI%4fnAoWXUIqGgOjv zsIdfWZrWT{m4yxm1m_{&R=r)`fw~amNaOy}e+v8q#f9b7Wc27+QHR=WA;6_wJ|se< z5=6~_u@)HJGOpF;xF2FpElf65KFvd=_PN)ye#&IIx<}^n=?f?R6p|U~%aH=k`iSSz z_%%6W$~-m3`R+-jN>dDL{N*YYFiL6aHMRmtFf{Uk9Jcv>k#VYXiNj%Mo!2N3)U89l zm+_4%XQHg4N`cGqS%y*_%*-w+Uag=10MVUtwI9IE;Tc9>*bizs8j7iVNy@8ILuhq5 zwo>|5ttm>Ig;P~9&{2>QLw*?|Q?;)U7USmm@ zsSec97HKN#*hY%1*_wI>AMelPQ(5`q&01+qJDsP{hiotdJ(tJf)bBLI)nVUiD=n9YT2Y8nms+rX zq%bTk%7<9^#Vb6Yh+ZSqDM=hmNdyicTZEDr7jnYABbYln%fADirG64P(}^p1ql~*O z?RS9}+7?}y)ge_pMU!T{S;Fq;G#PeeidyQI6PF`8rcD-d{##0ldRiy*HyNZ$EHc_f ziFrC_WoH*HNkEgu`JzZjP)ij#CqP1r*bqyD4gUbdc$^kMv8%PBML$17_v%o{mc*)4u$3x1KcGSkhW zx5W}^%~AqD{L!c|B;&ySwe4Jt@3mO@>X5v48w|6SU`U!Jr_^AF1jVIJ-l~2h_NiDr zDjw2Gh5qdN4fI`)><^;MqXjl>oECAK%KrV$O7O{4(Tm4>QsqubJV~7+3amC2Xiy7G z%p}FA0=UH7?5xzXh38yDnN_y08rY-msVUUri3(C3Xc~Y>1V?;ykVfK89Ov@_+YN)k z;m@_~%qy8nvG)CLo&*a#ZtLunxst)eW;n_6$t~)NF!D%u4eGdN7HW0xh{@po707{A zCS1y%(6oxGqPnWGH6&g4wXE-}N{Is26Ne`qkH?TF&syY`JOM^g(yc9xfj%~B(Oy}W zWq|%$2O%V|{K4UyHAMda`^wC-`&fTt{C9mDoWObm1<9iL?>WjCh_98wlz%0I#Q1U zOLw?D>!@(zmO8X_uFv*uKn2W}3l`=9r4OG7^bhc#!zPo9^Q_eYC=?tvnl!JwO2cpF z8Mam3EJz`>B{Z(9Qj=t%P^2g&So9y^H44V|_c+DQN{VoOmchFvo2G7uG zs!XXRbOGJnuH%T3TmnF1PMkG0kIw%9XQWKU<7DB}G%U+ks^JAH0%%ActwA89DK+9v z6~s#DDINiKubrbg?SnL1No_RQa~X9VJ52uo(`Y|PjNYQ1A`%C4nP!#j!4s(pnIcbt z{VL-QGqkzlfp6PPz?DC;_$($8SgY!!C;_-Sq_^oC@RN~@lhH?`=Pek#%E z-v@Sm=+X3}vI==pIynPU;(&J+LZ{sM9Xd6L;{cjxu4XnM`7VIVnHHjuAzuqs-@^j0j|eq zsrGyM0UNxKcr0#o+|n~7j{&}rWVjGiOb{SNgm2DFvL#Qt3plMwxh`}*bvs_N^-6}~2?P)c z5eK*}(M2*hlACY57?X;Q z4Tv!Ui8rHpF?I=1Za?jSM1WurU=U!zHWMUH+ZLsQK%ls8`!qtfIw+B-2A2+Z4^gJJ z1}wRino`hH;5Q(NQgo72a3n|rp@VEqE~=*AXmUN#=UCL)@JtyA)v5tJ!gX3nBXUTz zNSFW&W$Py($G_3`?Z*BTh$v8-G`gJD9#u=~9UbP?!K&D3hca<_yJw zoAM)d%T^G#1PTF4nzn+VldV|SrkR_{<24UQuyIM-_M?}wd3rwMD_>Uz7j{NvqE!Z#Bs@Al@4 zwn0c9BXn&9d059Q^#>k??40jrPqZii(v;Jl;hv3GkDor4ut6B9>)}@uGc|dT8Os-` zZDZ2@+3KgW?h$+&H^j6={{Zz<)?D-^2a%W;jBn_gQHYXX?9cTS+`h5qJWPQ#Q{|T+ z5zO3gu9W#!Jg)pu<*CX$Va{D?vJu1-84h_M5Q1|&s)4$P1Pvez!49crU?~n0vW`8M z=)V@McVln?L=VW$oq_@C4iK(y3D}gC00n9TR0#N=__)dukos`XJw&O4`|}Hx<`{+W zo)jY=gUhnhFqRAvjzVXss)$&c&>GKf6DL5F9l= zKRA53UbK=)2n@~qMe4_3DEpclz}BLrD)J`XAWVHv#|YqDf!`ORGLX{^nx%>Y7=hqHCz->i=znZ)>r(j*kW~l&NH!Jfd8kvxyIS2sH zFlN)zJ-Y20$a7I~Ug1wW>qxhTll)`_6Yc?46?08bh^w)15SW!*+hOHD&pw!JT8NG- zQ8T>W42^&h^PCVwRT4?{`e20T@$2MxK-5GV`F;FwLl#AWFHVhLy?dfMj021Cdmw+hRCx4hc=cgmreXWJ=w|_>bn?hdp zzal!@sCwSifRn%6PhT(C94rQ#I3GQE4f=9`e7Sx7Fhmf=5o5{^t>8%XJMD%kL|gi8 z=adj4eCLq<(}Z!r&L3@W1W-{kas+83Ze-2w3>f9~#33O{FSr#fEGkT7WCQAxaYY8z z_=$f5<%A8+UjG2L8#7M-0Hi)*@V*r+;orM> zB7h=qGZ&7N@66(07%+kwrA3DS08A_p6KLr-JwE z9M2+o^7Qs2n8Zr|0G5{4zy>GBUSjovB}h9EaVvvj3@9vL&!3zTAa(0EwWImpq>K?N zX+u+2JrC;L&7@O&{l9!iFJAAgHeS8oSSBQiA(3+=5dlzo&i;Y~3*uBTO?0`nclky# zbiY)scv;4);#Y|9Ay;}tq;ZPWD- zM3_ED@2#*OZrX3p`a=8K^d{xa0r>LnAjE_iKEL1hk7>X?q_H*A&q`~&NRr~W9z9^_ zCO{m>9+GeIwj06-AkeuCX{DO*=#oNQ2~_EcgX%6X?TLbe7XgjJw-x0POq7|RIU|MG zj&~mc3?5_;JI9dz{QRU0G$(=FhNX*|UV-}z7uNRi5cHTZZ9RSe0JoO- zcxxSe-|1ti7l)U))6KrO8Utu3GuPkm`}G(YXnteor!q$|1D!|6__n-&Jd8ami*oDE z{Q6-bVZ}gB-2Kc5{ynfk1qw=ukrB*qGrWHHHa=KuDK<7XJh}DgKBPFHD@h7AxaVW# z(Eiv&%~D=~zg;=MIF4kb%?WZt8WGO#PC)#%gaqmYkS{x=*uXsRVTU;9;C|63o#4PX z8v}TrqIZu^+Yq|9J|-jeQooTt5GLVEN))0Gh5+1!p9$E-~8*y6}n4Vt0D>@G(j z;m(m#M4woqg_WT}GSrlS$|;;rt562KD`Ub~h5rBvrGAC~00;auI76^r31v+dYI8ig ztB9)-Q>8`h?xo7cih%(@ia*PtGY3cj*hV}j>0>(n^XTqYhk&skzJwZ3U;COp+q)aT2hfD0;DQHHe7;*Em}!J3dZ{nG3T?2)O*9y zfK-}nSa{jmK8JQg0uv%tDk*MEM*jBo&<(WZHDXs#vZDn{l`2X7Wg^1xonVBW#7vvx zM8tq<;>~9jv$MTYHor&clcj#h5(`q7=14lywJqY&ls8bA(n%zOon)Ors4HZHBlId{ zs3gsZAe?HzAXGRvCd0n1KwjeDmnJc5mXfbxgFtEUEdAXej#ULtq2?e3exj&Z+(}WA z1?48*AutkL153L2j@8;b+J?LzUA!-WTXHoa-qj$PWVyg(I7N{|61I;|vRhGG>#P+7Wk zBt0!tXmFGQ-Af7#TA;T+M6f)Wny^OdluVG4Ab=#M#1aT35+>e)OkWIr+M@2)dS9zC zBv*HZ^=y{EEl52)v;p^ymFN=nSQaD`T9R}+X!I>p z63dj{_$7wK{W{n6d=H2cqRN%5!jS|RAO`}aNt=s+qa=aQ37MpoC1Kba3xWfX0MV#V z&{5+(fMoj%AcqCr+kcTgP3auzn$|9ZGI<}FYLT*~CC3XYan-gT54OW(lPG96ShXYt z6}s~)Qky~)r2R_#4ED40-Iwh*V(}A<$@xZUCpOCw#3_Z$)A?LL2`r#6_{=66Y{iC2 z{NkO=)W7E&`*6JujIYu!q90BeIRgiU?I&mRKE`&d7baBIgk~L+%`iFMLpoy7`#IBb z*;ZxR3=x#1s%|1ySs`e7cv*y6yGY6h6LxvTc`#q)*P3P3^ld3rjRcieH~E@w_#^&6 zu%_Hed0I*ogekDn5(9qc$6%!9IaXzt&q-b%AB4u?7L&!r>hU-=%3A`uy0MF4sY;#5 zQICrqhwTP8vc0+Nr)n@ygv|1M<08$lN|7(vMq?`*mFAc^lgfCEv4{htuIWALFQWER*)XqHEP;*0%GKH-LNjg=b#VINwL|RCUdF&72+FnL5UDZ*n_c)Nye10 zqAmy;P&G~Wwft)tdGl0>jvat+Xd1JJ9e#$CfaVkhAx=mEU@vP#l)>AX5<##MKo`T3 z4+?Ls8qQC|uGHgK{ma`0DS@@rTsWf6S$X!4gCWIzgSRF9ZwCyzzMxc+yj9jWq@NwSGpoh>W1k zYSdf`=;KYBlw;M65*^5f?bP|)k=$&PsK zl9Bf~PdzQ9hnb+NsHad0)ZI%6Tdb`s1tls&fhq>jM2LyT=h*DZW?P=+c=<%6;juVu zU{Rss!Q%Lm#?Dhv4Oo>&KlF!`$+FYV^q)x-;U7pQwMH2vZ3~m<}gel=J3Io%=Q8Ms4Z(b8v3q@#i>?Oi2Wek&f?balev-C8OeBo z!%7wxxNCeLO*y=3^YGIZV_%8CR=*<_#AL3zbuF?*if;lz+V&t3VSUcgdH(=BC8De^ zR{()WV$WTeUXK;VSusj1M{v}>pg)5EDPUtH%Q#8ug%9|@s@@Z{#0dZxweJvT80M}StO3+9Kx8oOKv5z-7QJ*0Da~7Hp-R#)!RJ&Vio=`9hBu5S*l4&P>j0) zjK*RTl@Y_oIa!7kC`hWx3P2@`IJwA4JUC1guFmI*pE+qVM4#CvehKO@6=nhT+l<9Z)+ye&A2n&J?` zm);#~gqGG(-7?r|UG9Y-f_-(IV-teGVzD!o{vK3<93&Dr zNsp4gjq+K!)_;`ccsVkEH<)LLmpM|D3eF~cxogA9;Ylh~>9XcYT&W-hEfXcF0ZA!; z25ECMpYeWgooG=10C>)8^Cr?v=tX5)uc4#JC8RhTN0AVuBqX&RO6Ho>V34_7Q4M627AZ`*_ zgmyOpB&7`i2@sHUji-^g5f}0XNI2pd+KFAXt>_E6@gEad_77ze5h_bp=6Uzi!<2QW z2qZ7KuvMg!8-)mvAl}AE1bN>a-!UOHp(96sYW3Q**1V&-WI%%_k*nBU>H~a#PK(JA zr70SKAd-3#0SA8}2c^N-<4v7K7ulG***vsseB*nPM-9i1GZ{v`DLY6_)3{OSOvmXW zBbX%a4k+P)D1HvBZ)Y!6#0SwXa6i-sT~#&;$isgxx~KLA(*yNg!@U!*VSl zHnH>PZA=L&YLGW!;7*S8CBU}Pe#*&pN!URXGdBPm^8!MEJmz}hti^$$r+4%p9#@M6 zAXI=~Rt;x&>$s?miq%08-YzHt1^^kdCmsKn6ePO=W^N;$!Pn=`flEOl@0GYf0F=qPNRuEXBH&CL zSYH`zEz{J0@!#tZ&oyfn4mTHmS074HY0Th|;Q*LWB_xyKFcM@9f`E;TM0w+E!7JJ| zabP;s*1NrW3Pz=wrA1?<4=VL>Ms04G4hn=|Xp?Ne&UusyI;cioln)5wws5oP4Y-YR;S z(t6%(2giPxed$L=6{9~}KYV#D5|XTJ=_NH)9gSUqYHC=xChbNs#(#{wg|~9P$GGu? zK+JMn--Yuvgvnh_Oz-=p2C^&(qRpxj-1v#7N=NBVJvZn>G*a+c&Q`Wjm5Gy%mL9+2 zrUXn1tCq?68>3 zz#%97?jH`23W{TdRzqOHy0(!!8~lztw52~2u(vJuG4sZ91h(NNTu#m`#8RA&ur(oY zbtxwE5=of!w2wiyAL~=Hzv(cC4K&?IJ?H+=1NcQ1Xhwval^H(5e{XL*FqbbEC8SD6 z0!g_gR&u~UD_cM+yaW7K>B*LStZe&O8<*=L(Kvb#K2J?T9 zFSCWV&+#3{UXIn|$$i`4d&Zqv{9Zba;7q~TA8uUlf?u+bNeQV0D7O3cx9JRGQU?hn z17J%zr$=M-)*CR849=4RI+8tYB>ITM740mj`>Yx5VAZwD{37&#M5F=;KGh8mykIII zdi0(D0JrWj5iVXOC4-;q=J6G3uxzfx>#6a~K zlhlp;iT1xQ*x?qgGESd5=sJ7_;zzcU{AAc`AI$t2n$2H#@wu{fD20=l-kdH8N%@;_fa)umz3 zA1wa>TEJ99lQ!$^f(RPPiw@ zKS=xLa6t|`A9?c@v~%+x=Lo8U-SdT|ejD`L<=04iW+q@8e*0mFVD%a|JNe}a1_t1Z z`E}(V&y|7f3)4!ZKAEbTr zI8=M1&ZZoXba~XmsKE6xJvtw(j&{SvmDB1X97rZu01|k`gXpcIfd~2YkLK8ujBI`% zc*n!%Pyh>ya`pYLZS%w6f{7cMKM!mWK~k&VpdUK{%o?WgJeIc>ZdCgaK^?kAqzE$ho{Bkgu^r_adN0DJlG@NXw*DH?{M@aO)2X!O7y z$436ItMnCkt%#Gw?W9zDz+8>`j}kgVQmcp}Jk7th6^OWKpcIt{NeUKlK@YVeLI~<_ zv}}E61UMET?FI&JFhsz(1|F3}&e3mQED&=7{XRWA`b4&rWTrrnVtU#KruM-L60kuO zlx_lu-1$KDjpv+gHo}!E_3=06Dm({q7OVMG{{T9vyC0-b*iTG7L{ot7{5{)6B9#Vo zkoIsxwb|~_WSIl5+uZ%%4L-`LrJv7q6-fklbI|u*41;nf%ztm~=Z7d$j>d%fXC@~W z8bpMu00h956K^30)@|f44{0r^^ZI;Xhb)q!KqlfOOhQ1nEy?!-6Qrd{0MhhTE`0eh zkDiqot7NEy!(d3FMy=K$j0BE;A8((@;C=7s&nniB3OT`mdV(jPPmj3kgmKUQlk^`AARlYr&w17qElJq@ zb{xGhtPX(pZT5wQ)JHis^9OVB79Ux{w};oQf21S$^V_9AtUNrV+eo)r^FO{JQ*jy{ zHRaIxe&vgXQB17MILfm02zIkG%kuQAuXwP8j)W;n(4G85$2fxHdn*;}-tYq>`zx&@i$g+gEpN`M6Oqo*vB`cPMsH7x^1uIZow1DbT&j_m# z^v}bz?BpCDmSyzm38u^Pni?8#kpMSEM2ci2>_(pM2;`{|*XG7~jhT#{j?4-6vG|-` zEoV+8sF7Q75ba8{i%00^v6(cCuLqQ5@@4wGK3Rjo;E^iCOTplR6G8z1l$4DN0YVu0 z_xOq8FBMy8aDWQR$vmUQ%n|{D4$w&Fk39Cj5}=vIzy{>d+y@5Naz1hHUqv}uIHW8f zsI!_9XaTkAe&M6bt~Sh*-ndkvqNS)33I3v01_y;;090=wF?5V@EZS;7b%zf^3Fuz$ z^xi$Y*_=APSFuGF4X%c-MiuJ%)Qw9)oFx^iZDvy`(o9N*q5%>BJ8Ccl6DBvtt1?(p zXdcFua?Mx039E){J3je7OUphj(|U8|++$_4QWs7Zg#r*YD5$4akhK`uL@G#x#qJ64 zoMF_L0^xz>THI?(a@YeH70YF?6$G&y>Jr?Yymm0dTx967Ld7X_dheRADy&?5PT}rtoY` zoIQ%_D^t~p?{QGY^*0bD23q(l!dQaV)1x(>zyRkhri+)5L zAuQA*e&+&#jTyt2=NHg~l<`xGQF2?CA}By0R)IDgZ(B(_E1O; z-Dtt9V(oJF)|3>)&%=&73-UH4uhbj03%%lZ=Ad{;Fm5!0Py@1~JC6#0PLfQ>F&J0a z@+{PHYUSRh-AfA!o~#eyZxQEMavl9CC00$~DqvWj4>J*YnY@8|;5^1G&w5Z^n;vGa z&ea8dSpHM-g>lq{NitLvnOK5JK}kx8u>ctWiGT>aVea!?oLF6`ni}fmtN`&a^h;5{ zi5lnyzCY6z&K0SrQV>8;SRjHzB_1S%s#RqqX^AAML;x`&N46+v;J-oz`7=?;c4|fG z1tCO>06`mqLo;qF+LLCEWaCAe8naa@N|W9ctt(q<))Wd_NlFx?spk@!e*grfY9`vX zdZ+NE+VRFjm{;hfhbZ#ji+Tryv2u7 zl_VOByF%TgQK(y=mSw~(Cav((NLW*4BqRi>l`;uWfhX+Zcv6ID%9a+4kBRd}8zo`G`AWx;oMquf3Roqo3icbAPNg? z6JVuuoOyYCE&l)(^`WbMN$zfb;>E38ZdePwE_89G<9@MusHRJ(T2z34qAw&AgC+q6 z_aJnVZ5U^*P+1`)@?u$w4ulJyLhlr1&RP^!_R-k)R2p-)jA2=ZQe>D=L74<7kfY&| z7690Veyvm27E?3WVAar@JCA`BtpEksm&(eWcOW;wuzC&dEMC`W!)o%?B}xcEnSxX| z5>JIHShap#O-hPMyb^X2 zNH7SHX2u1NGqxp`qNlUzOMnf4C770``?xO);-O#*x#?TE8+@+dg z0Du4h-6ws#jOkH9HDIC{yTS3HH5)j*H(wfg+s=`V)KLV!Mm2sp5n@SuzEumJ2_{aU~<2VoV~{6>Ph9#uBEAoSeEknJ-rOW781h?N=|zEGI% zoJmrI?5U7>$W4vT%aYCYc$p<+7I&yTG^^+;PQ445z-O|G@H3dZ;R;v)SxsQtfO%6& z#*X}Rej7g<`QCll-x&TGdpN1>2Z6Y!@cG%6SH$h*b^JX^m}Q((kkxS=a)o1g_GOhg zDJpW_BFoiE^&^!jb;MOA!_>isIs(nb(^Z&FmdcTM2UPU_2LEmA4FZQmxf2R1_3H;xGio&gYz%ji)Um5Ys@b7T7b~ zYT%aC653id{7wz9@~B)~HAivK5??_^)W)IAczH`&rM=7~1i>arCN~mdc7qZIHc2?n z1TZ4|nvJM*{)9PwD;boDh1rGb4()bhUh7|5A>m~>1T`uoizQG=APKbifY=ze@wDeM zQeLBTTKBtr2S#|u42^499;_4q2UWPK*KFyV0P|*9V?`r7JT6{$|E>pqiY^EC0M$(WcVoc46By-gu88*fX z6Pbk*`$ZB?#hf(@KFaZM%^Q9LA`&evA?siZXxxQ0`bK?0kfE>$K(L4sVD2CQ2mq!n zJCoF$M9f)J`#{z17;WLTTSG|BOTi*oqE9*+z0HFbrLA|1`&n>kO2kFS^rVO-MogO$ z0fEYPnT-<)*QF~$>>i`yZD^Rw;7Px;cNVU@cbMxg0Lu_C-g;q;h#U^0~NwR%;})0XyAFbOQIY}r@@0$ELtID9J56LM9CpF0vjbL$Z_-3x7J@LSli z+FwgOnVpD}EmATwitQFKASGy@HzS!XQm63;3rwuT3nwp^F~Uj=q$Z1&ljTPP(w8cb zxCTy;Zhxo~u#qBSI&&EC?r%GR4B~;sc^VOI-&TF8`UmWW6_@K8yRinfr#(&E;T^5P z4J;**v;>INazFqN4#ba-JaP=)Vd8h7p(d=rmn}+~mu5D531P3lcUeL2T;$&vz2;>VqA4((@V z_;g-S&`J_jxon8qZYBtg^%B6p`twZcUwIXDD!@rh?c(IpCoKQ(WsN2lJ({p$RY&ma+ z@cRu-^u4d}yK2!WU45P{oDZ{_(lbn%vUJSGk~}g909+_V_5dsNro=$nl;7g+`b31N zdj!$WY>d*c&yswQF-Sx3KsOqJ9x3#rh7)As95RLJZEJlh3sab+dx6GGxtsP8Lr{QRRSt2LR4jl=2o1P} zE3JaAc^^}bwp`(5O!>I8d}`fe#fjlYek8pLiZu3?lkFsKLsScDBb>hww8K06GtgHN zsoopoY@zh4a?3fcU0;;BGx}1e*G3g2CPG5G4%z5e8S48;WVmc*4r)jxioYgPF(3|M z2HwzO){hZ>I`ULv{TENga1)5h^Q^LBCIpkfG4e6FI;IlLlp%QR8Wf_*3QAIH;yluE z!mTnk)v)UkG4R_Ss6rE3oi!Z!N5{NSCxL-J^?^Ujv?|t3wRJo4fj+x)+YC>-JN{-Z z^7H19H4-Dt-0#274uS?&Z?%s7Bd5+!FnVBu1hc1Y&fk4UPpG~C#C6b$_x}J|>_jG` z=jUGQ62<~xbDuz?`+Z{oAoCAf{*XI^!snt%uY@XqZ6--2TONdM<;>fj1^Z!+mTltX zxk;DA#FC;Mo=)F0;}Mjt$-mqHFvUBBfLej?@v#CWH2FkzstGf>G4QwT_k1^m6)>oK z=+|$gL|Ce(l(k1o>sxE2M<7haJ z{(QV-rAiWkV&t=)nhs9>L?JpsCu?nUKA(K>M6n5Kl5ZeL5H|yy4X2Y;W;fg){{Wi) z_T_>I5#D+O>whnA3XF|L|L%qS`2;sjAQ1W<83 z_HhhZ1|WmK?S?FZ%t61>Cs_rUgMXv}Mq~+*6BpV@_#Mw98{#%5!Qqg_8Bsq^OgUch z^P~o(o#BQB$ezBO{kq`;>HU8FHQETInE?5d^Tfa~17QZ=ubnj78l^>s{{T!Z5G)Wt zq!ZKKU2V5cf-iAw5JNCQ4N-t%50|aB!3|MhXby*=_}9&$CM517)DvO6-=Q%!j1bpi zrxf81FIqG6<{HphAf1l$%pddf!4g41kv~trmn;xLQP11=!2}RrnErh*KsD*GKRb22 zB2#iaAPtGMi{8V2clI0NGKyS$0MlSJf6@p#2!JF_?ayKs_QYxSdB5`3(8Qh>eRd%IV4jipz9fln zXl;|W$Ca(Vo4oxQp3&PWmO zgct7S-){nW+6S)8D{qUN4qzRwL>($4miv#sm^qIqC~`5!@lmrM{vEqfag1|n~9*QM~N96{3Q z9b3ACTlDH8o-LgX{{R;|@V1(9wV=NeBjA4FUy}?H2W5--T8EGaOjYSLPf_x zZyA3Ba(zMI5u1?g5G+#NyK zjb7TtOb;U)`OFdPc*k{{XA`_-yb_^ULS`u%wN?UqCD8SVlYi{-o2+ zkby+n@i*L%51*GWJR_gfl1SUa^bg!x6^G~m0Po{!mhTS^1kT59pAbDvZ|8+~N^6s3d0fx=upeuMMu z_MeBcr$u6lviF)>#I|MXWjLKh)Fwbu2lb$ls@h0ZkIihyF&HT))-t65?IasngM2x@ z^pDY}W-@7+z9l9>2%RRlWrqI%5F9A+mZ-UB-%1}U{{RqNOGj5)p|lxZ8_V8mYiDNGI2Vc^@0a1yd*ZZ`4~N2n5}=Z1Ksu1H0n$iN zIN}+(_QDh>O7=t)v0o7>UM90ZW9S!ABZtLR08xLQ>Nh5 znBANupD~+dH1#d53rozQV&ML3nJs`Nz^(N0 z)9re|D_&7bNGD9LZbisQ1nbyb6%abk#1#Ojh6J7Tt#%C0Ul=5TxO2{xHSnN&MY@`? zX;MYOBu|9J#F09`Nl@4c-BA|9<+yrXw7uvxt(+Ze5=@djZBG8@rH! z7QtH6rZs>HWTb!E2$e*Vl0b;O;w}K;z!ZOU_1f*-4-!^w!b7z#+uLuuJz(jnN|aIr z>K2sT2%ew~F%vQXNg#`15Iv9>yU;Oqa%}pzvERH@r6HcA5Zocl)YNTnFBZ(fVaApr zTFOqSQ5!8pTHYh9l$C8NfJP@lPrc;6K2WRt4k2Axg7nkQ*Ze^BYuu~aQaX38F3^4fFGlBL%nS{kI zQyLPIkO2lF@I%SW9iu@>SsI?BLrpgOp$J_IAxFR|m@8>4D3y20C=RT&O29|x6SKL* zy!*4+Mtz0i^?AlymgX|<6E0R~m&nFSmg89}#LiKm;R#VA*(81g{U~-n9q8-QKcpVO zX0j6^e`EVo+P=@Ba*_(>%x5 zeC(W*klgBexl0<*5Ok@z5%)V7`~j}w_rA8T(V6KPl2-{y79~a?DgY`msXKsBpb0y2 z1HRX8nz(!D+C-A3>J13F8d9IG;_-}T6loqJ1+^F=bqgecA_++|qzH{tK?f6| zU&NYd%6F)s*n$N+#AKyR;{*^zXvM70(u5Ls@r+)klr}YKI*zCsk^-0|v`)89IfqzPpzAn7sz zbHX&JohlX}B$9c=Vs0ZbTrR~@kqCjS(01-j% zzR)oG8+8aud*mr$ZJz?rl_Y`oBMX?|F!*dNug!o_hQiBO65OR|04*v@dovJy^pV7! zKxq7<=UGN&nP&OS`4}%+oL(;&t|_xwB`eGs1StDFD{o+e+5%&nKNUZU?#cfE6CIuQ zXr9YB--kHLpCSA~a2xrb`zmXhs>_CB!M&rgNNVZp`?rEBJGzvzq_d$;FyeLUPLs`j zHR7aVJsI|s)Tped@_a#LIQGhF>_Ot71Yeyazy1vJyk1xMlJtJ8WVA~Q#N)~e62f3P zdP}B&FWDhfe`N7mHO^s{2dI*;fTaX#n}8)DM3X1Pk^mhd+~?*>7E>~Y=X(wO2A4P1 zKSp!W$A=Uy1-EkB(!uT4@r~x0C|xru0I5Z!00>Yi0kB4$QlbeUDJEh>i$*QU#1zbX z^{YRfJPjikI}OJvnDEN82%JD5?@M# zz3T7HsaK3|_hJ@((#EtMY~Dx8T{jA(kfPbS5DHAdTpO6^0TD2pT;4gEnwo?;C8{me z>qfX1tHjcBiD6#qo1xgYk&{ty(oahd?$O1OBO|6Qidem+$yO4IHQ@-(e zIB94e?ylyIW7C~n51Ia<#H&{Opgd9AlL{YDi)E*5GLUA#&0ddrex;39vf;bnrb#8h0AFe zEYBJ0F!Sa!aw5czAO_B#XmY4=SnVv%mCU<^xPF7XFDos{^15|XxU>0Y>Qj@@(gLPw zLPFNJ-0z+5Tv19;^ZnkiW9@fgvXo?fi|nRTDITb0xh7MDP$8AYhQVP`iwXn=;>9Ln z>MmpO9TOSWpy$0q-StEvevFdjYtUDQNI@=R9DREILZ2qjb(@*T|?0^3NPwj_nXG@pI!IyS_E&(YY zmq5g;6`0C&ihAYc8GQD7esy)>Qc32X0?Wu)T2LloJmkjwqTtP@3F(h0XIO*7MtbVT z&RMq_+v6W$`ze4331L%F<6&NP-GLR3#NfI@6v2T2fFKl^8bVCX=51pJc?Bv^Gz3327+CkDzkO9WD{{0I7NaIUY5U66U#3_T}NNNqi5tTp6xc+ zexN~6fMi_wLC_2dx2PD$PE+)&Z%@0#MiXY(SX;2$3MDb&_Nth*5B!ChGixUHUU6R6JB(?R_gI@|UW@;Vk&q7K@gTqFy zwo&J3jl)EY*$Dzpv8HCnP$Eg>2+~Lw=X>LF$mWqcw0k?&hsQ>0ni|JF?JSh54*Vpu z0(2WaC@R;idS%&ue?^wi2ra!zg4#$^j=te$m6~?bAgOOuK9nIT7n$omgY1z> z5>zTpeVUU(OS8XqYIz9pAEen6Wqx@JN|_u+V~R@^s>^#6b|tIGN4nX+8|Aze*iQ~4 zh!oRr@osgMWVDp7D1|vtff&12y0c(_>cGw}Za6K>aTc7zKsXU_dMK7ZQ_1T-l6 z#Bb8^_tXxURy%cVI{E7e<9?5?pXmcp{*iKea{mB04L5%Za}~XH?%WmGylcqP)YHFV z5=ZJM-+S~Paqk#V=a?Itb>;J`LQqg#vj8ey%r!a=a7ju5Gd_{+&*z3X0)Q6KSH}K% z#p!a>ED)D2*CR^`05vsdFH&gH+;FIy_(8Dro1Z)>AY6ggB#5hBk3qa%dVh=#9h^Se z!V(3@i0UN%!=@uq0k}vPcJ8}GW~6X{=D{Mhu%*u-ZgAKlN$0hq#%JMgkESE@=i?US zNh$W$2NCx{6-cYnz?yRclyV}k&wg>V^T8IPE3RrV4Ti=DAoGilc^;S`h>BIp2~#CiSuVPeVK$ERN}p$!9{@6*b_ z3?;ss^Qkq1>1yAo01uo6OWbm>2k&{?`NG6%Rlv6Y06wr48%*>m-{0ao^}!5*9lZWO zcf51w?SZc^D#2ft~KqRmPo3IXBdP19|pCigSbmi%Ye~0|P zNXq{J5BYwP9GJD_%s`#Lf0tZC5Wv+S&c;MVf!-PFxN&EizbO-gj-_2kuB1xNd z_Q44ZkPIG^?f3W8f&yY~x1I%@MFDy=A|^g@LV<}f(1YuleBy}UQNyIJntA)T(!AZ&(hFAllq6h z(BUA0BlM4dslfzNqEl(y?-8~AJ#dP#>Ev~Ql5gkqK^|9Ohk`(cl@A<3EW%{ zKf7<&q)sM>B!X%{Yj-`g=3U+?Hc*>-?YAS-&k&}H4@=&fK%aPb{EQJFq1g)b+XN6)pBC~+fPDJjewXyaQSOfa0GWp* z*SLUoJM-EKJn#4W;n6L3(`|SCAgDjnZlm}HO#S-<%h32RfOj^G*VVp!x#)1AWROnc zB5psl&&v%C?({Vs4R+t)?(~LCC;)M!s zTo6G}PhW0hr``DvLL@;pdVEJuABRXNBwS#LB0^OVM^vlaOi#zJ_8^H(uULcnuQss) zFSK*G_lUs+76Gw~0tCsAVdv8K!i5ytuFgE`@QCJ16995S4#ab~BT+!&ko5quxE!FK zSC2#G>3{_WARmvPyhk=lCxCd;3o)ulDXj=98oM61qbe21`+E=IVL&QBGZX+SK798N zloS#+!Z9YLw|aT$4oM)F*6v%s^!q`92V;ffR{sD2_K9Koj6t{?3ER;7?dj7K1#Cya z!&Zc>GJ}9!oQl+FM&=#=04M$DatDuilh5{-Ma&K|ID4Dry^P%k=Zk9f+00IMN3Y9@Q$KEJrZgITU&TLnhbSBinDT z9_bgOzJ|N$;}jh3f2F)2qgOSfd-#*K8^n+0_z%2T^(+4X)tCPOS^ogGDLRiMW4B&$ z8^9mt*otXMZWd7>*t-DC&8T@7t)<{nH4s7p1evd(PmRCA9`-Ji@-m8%1j;Imw0q=cRX zjK43!;DVfk?99ihSeZ(yCMIGQl@O#D^po)e#!ltAHe$*hgc^5S@Tl1!C=i%3W=6yj zAng(I)3&*~u<4$Jt?O^3Oz zVU0LTvCtIsL0ohV+U;*NKPONA%fN`Yl00+oNtMF=3Xic_p+ z@+FdwQ7{zZgO*q+RY1`|C{@9ySk;_km!~qzo@rpHyoW%8`a-}=q!0}1HUJrsd}`Qf zhD7qMj-U&>)Ey0lY#F0-!>#`SW<@JU-n9pw&8Iily@T;%%v9$bBbY5osL87wWg5!0 z8aMU2!$f2A zsI$c&mMqLp?*9OIu{~6zkV2(G^LV&qQ(iBR9ib)V=F9uS9CL|5+Fy*6!&=NMSDoZ;Df4-y`r#?_x zTv6L=@2ghgtwMH;y^5mpd`kqRB&9(_LP-YpxC2e6a;q3I5|S1L7`+%#jV)IW-0H&7 zg_wgaxKnb(UX?FS@k#(I%0Z}&QgsokK}_k?Krzf4m?8;)43cI(6%{!Krlf)|*IKZ> zM$sY?l+Y1o0Bz$BOUZ!(btoi3BT18Wa~h4Lk@}%R&;$lhCe@)lIvq<5wbTt^46;oX zt=_lrzOLKt41MG&DoI+V9fKg5(nwNNq`@}?>ITFVfK1|KD5wEfs9*-p2xe|AtZTGF zQlu6LL2Gdou?CyzUx!%Kd~r?Qu*y7}+6wEas3j;u2 zv>J*zdlR&0FiUx1?&E4WG_6LQ{<{AFfJ=b5rxLdqrBZ2HXeA2?ZQ6=zbV6J$rNBPUx>gdPE=5GT-P+q5|sd5Bw#SOTz@W9Nyb@N3tEJo|GkAKDJ=bvD2|7DODDk zjvOeXQFho-m3GO&xte>HlbPk2wGaq*ym5WYnQO$Wx`q&hJKmHw+ExObTsRAgN#lQw z?4=lv!*+KZQhjz4J5~^{gz$-Y{0=!_@n}6(1tgL~0RHHB_fPf}_K{d$OP-XwHWw{8 zZ0E3q?eETSrZBU08aOok=pN5=1Bv zM|fdYlNUN>4&>wHwE1N!W>7mZ-)-vSyhrKZ7J?7~xN!TsG3FWib&ZgPYXiby#HezE zCsZoLX}F8na-3K3uRlI|Mk0;_jze8VJns~xXt6q?WLjbmAtVk^V9m6M#EBl*e`Hmj z*5+E8{GvoB)_b9blr+=fufi<#Ls)~Ii?QfIigevib{ZEN`VP&TJ&E|qLj$7x}*ckJC&`c zOF?E1PJnAd<-rx@6>~4zU}o^rptk+gAZc#m@#QN#C-@fa7m2$_9oQ`+}x z93RG+o^M&hN#Q;Z;cgV*ZW-+zXv?w+2Wl&8vkI)&G~)8&s!Pn}>}C0F8&phyR^qwe zrkqw^BhjyDrC^e^uMluxtxiEF9DqqAF!q)QRRAh$-v0mx_$==m_<{6qH!}4_SX@Rp z{>+2Ik&$L3r~p)1BL^&~*a8(tuceElACg_Qf5#vjKnXBZsS0f+K-`g}39zyGteKy* zZW6>3Z94w|n${c8$LLFeqEDB|Xgk4P}Oac#tTt_nk6bL5d zgCh+y6op)FMJwJ&_}#3~^h!$8PTB<(VAnrK0l!K~e_tMvPNYqE_0!j&!wAl16&gCLzI|D4FkIZAZNNRDY zgaNyfppYm@AO|HN@vlf7qd9&cw`y}=T{t6!GbZI&Y?mRMF$`8c zT+DV~jhvDcphV6-A`cO?@NQlZhLfanKc~_>G9q@o2|h&Q&AGlHanPjg&ssNdx%x-c zF3sTr6DOGsL#0CwK=srxzgX}^0atV)1SSX~VMQcBlhlX?M_3>pX_`$byaSfomI!k_F@eCN0kjM;T*RVd2we zjIg-sK5<~pDwG^jMZ#i1w8FJQqQ>$j(Ii1Li4uU1(xSD#yZdDA0&D1J4J~FiSO~JnpaU9pCmvCMwuR9g97I2kp)M4ev{-z zF%u}yOZ_41V#0>uxzdf}89#Xl1du>M)pWpu6R2NxB#0z~V4IzzhaeE{2(|m-&6w38 zwpOA-Nl8kI;OWv$z&8=30VK`h+rC;!RR{GntU{$4lIQPHru@SWw2ofw7mHh?cvGd& zg*ak00Jw_;rg?%vn@5$gx@7rMrpq%3RRho(S6!*QRMOPfGhE#`RRjeGKIjECqqwbG zQa?z>_;jkV0rbkXM~Iz7c!?_{Tm+;l2oZ4`Mkd51GNuHBLicBTxovPn(Uo^ul>d?>VkP?66O;n`#}RMD(#?RvA@?qb}~ zcobN$cQpV|ik&<4?u=I|%-PFbcD|w1GTSLGy6cK4ZH7}STGq8fQKd4aDqum5nA>u+ z%oC*)IRH5T6JyH8`siau&($p9aSE7IEg*3zOCbOuizt;405x)0UF{ux_%F{WJ74X0 zAk8zJ)|)5n?}c+V>as?&2Nue>yNH@4iZ^N7BT38YYM0+Mw@Mmu_E4=#1nKk_VCLbD zGEAII)?icu6bq68YB{4`jYeZ0CG_DZ880DR{J#Meq@j3S)fU{Ng;>?R+Q2__vpVMm;KqE=CdTFSd3m^a6xS3 z@Py)2IE4vJ#RRAaWe^JzBhc=^bBaDN$SQI=hY;;Pb&=L(PNqa?uAp|6X(mtq0MrCX zfB+zpMC_fO#wt?qv*oA-VX-(&SY!N1R}r-w8Yuq&3)QO}d(xIcFBCHjCMsE06EebL z@KX{}TZ0>f{F=&v6afzI`U-jvkkdBsLp`Mfl$XBN}pL~KyBEGl3P zCTT*qH*4mNI&@pt+`V}BNY=5G7IRI&X(JNt;YSo&efKvI36 zLxDv&xrY*u^24kpkA~ZnsOd=i+FiOBaJ{g9?RSc zx$0l29ElLcuTj_lCI_r<@WMD25B~rte-?{T9ZH$Kzm9!=pIjpm?*gs0ZyQ5NYl1=8 zKUed-M_HJFU`_3N`E9pT*9d4h%@wh>*1fsI6+#ZKfA*FhYYZFiJU8EX5j7c?DuYtj;W?feo1%zA)Dr_U9^HF^ymoBolJlPxmAtGhWo zi5Czul5*X)btkDoH=TG3v12>M~GO+o=2Ga{q(?)c!dtSaxlnZ?-70f z0BkDi&0GB-fow-9pL1XbsWuipEr9@?4K4owx&D4cD8G^M)6MM^QV%Yjc}#tABn7ni z`PXW;@QeK?+v|c70Bk?B{G?!l2CrN6^^#7}ztmVk!_ZmXhf@SI41y~J9(?#iH!6;v zqSv_P9Ov}RnFho}OO+ebuEpuoxSGQOAS>9tB0j1WV} z^fA4l{{RO60N(`7JN}TQyLakO&&BBP$_n2_~ju&;XZFDorA`33*T?M-7yke;ZhPQ9Xiv%3|pWe>(geRx4s8@q1|%9`F|-W|-XtPiiY6O3$rA9#PNmHz;yp9rF}Gcgw5sK3O2YepK7Tox_6a)~0l)3&REcjdf} zka~JzURvz~^Qo^od|*yE5|MrH%HdE3_r0U7@Uj6MP6%=UaLz-x8%-LH*hzhu4g#)8WG!*p)uRW=3kt#v0 zO*I2+OaLjKZe_4>sa~#Dw)N z`b70sw5FEY-BD@8fYOS!l_E}0>C&)SDx`aF-~aR@$NI zVFzMrNINKFY{^!Pepidj=GI~p`Kd(z02==Ql^@Kj5CfOHn84?<)rZC4^9f`m6%0Nw zxMd@CDZt@jb5EyTH<+fbfexUOl{QFF0ZJuWNFiFR1jLd`O12jRpWWKzL2bTFp8;= z2|AMm$TF+Mnf)aC+=4(h7<7=XD;kv6pxo>8aAAs17NijDM?XE$i|5Uu>UUD03DDwo zNhm2%7l@LP0{$5$=3wIG%1B{thOK%zENI-C@oNl9Da0+WA~v!30QS>d@O?U|>oX*Z zQ0iFBGPI>i&=KA`?NW*fQBUe5cdjg!(wAXCB}A+2=iu6KU!d%RiAB?o?MhjRZ~p)& zGc|%kZlYPVFRXq*{{Xf_4?7F^-t9{8*l8;M^X%879Kw3kq`FD$-Wx4ykWxZ0QWB*o zgoMmMYJz$H0P$kr%ebqv?9~4N)-}~ErD$xWgtbvf#X)ViT?q&UZRl;b)TArBE*|md zDX;jxl8j%YR3L(VPIZ@%F6tOeehkn(cyV(s0RxLbTYmnLe zkeGl3!k~@z#yBA*gO}QxR)pG)bhUZnR+RAq*0K4wb9ch2#H#!aIOEPKur z0|V8b^tZ{xNi88}^k?f(8%owJbxD;iQ3F$AQZ$lG4WsoaC@RVZ*8~GHn(8bW*f%E( z{tVRyv1uv|>ceurgfTVLi-PY$8KT1YQ(;n)PlPJpsHl|!fCwHDAdn!$UMCXETDYs? zHx4L9tY|Dnw29pyf(W}YAS)CiFuC)@m6b`W@e{|vk%Kr{{Z3&Ou~-ZGJOw4}}9hx)on2b;5Msfh3g{veh=_^=04!!gWCQ&`D&|yV`A41^>j-xMQSMF8G8(n5HJ$+q&%E9!&$dBw34hbrgPk4sgeihLTFkbp+ld z5+IU!Oh^+TTX04lEdgQ4)rNtXZtgb+SG*u0s$5dG8gnnr`GXiPb(=Lxg_5w40)!Fb z5C|sf7K5oIbe%*TU&6^VRN*RS6#&u2yMePBk{_)4Fl4kbpw?^y_v;hoH)CR zNpZl+)C!?86kuGz1S(Y`0NM@A<9EvP)XGP!nZ4SZ5FU(Kg9?pz<~inWNfMB)EC_L0 z2GYMn7^ep1wHc2T^dJ;=yxUy-Z0+(Xp(&RgzTlj{Q zuPMy?w9zHelBj4+8sBrj(i2d%A;%j?Y&fkHuBmHAhum9j-&%gru41L3V~CT);Bhqp zECCfH-Ao>UN8%4Fk?RvVCxIu`5&=OaM}!i=L<3L+wK*QqAue7%dp{6#Wxm&|Y0CkU zhKTM;Qr?1dY|O!J3MfXROOTZ*DJu#EK*{hIrv;RD+cI)pDa(|qA9Tt~k`1_c6y$*H zMuopgpMWV6Rrr7GoT&&1m;LwJRfMDl_OqNs!lA-Q_S`MOAcX?#UMKLC5dQ##M({mk z-aREo!M1t^dJH@E32 z1Sl8;pE5QOFK%g@PpwEh2M^s^TzAko5#4euC`_p=IRMSISoz)Dw^H!?32mehohVR9 z2W0{hCebjXU=4{oiIb$_RV_;t4Rc^Bb)Yn;eH*K9^4Tb1R@t1e0MmW`@s_%+Ev+P| z=}Zy`B%Kgd1_&uIL?$4TcEuTf`0v$&F$K<=eJasojav_o-Y`7Iq%G8dNF_iSCd$;J zpbSh$B6>i)V<3r1D6l(#?`QT#!oq}-`&FqWHrDUw&aP`0&K_u&LdYbQ5>Gn<0aSuW z5;ngsvqu=F3BwcJx!p^n6- z4~H`e3E`@VAT!zC!q>G-dA*r&d)#Z<1JpFmsZ_apt~9ceRFbWJFZ-<)Op_$;B2Q@k z@0F9~5(0qr_=n;pfdHDj*pmE*HfZ5keA;(tdq}BL)y0Oxunb)m08}{gS*jzevN}}I z&?a=FOKF}!p)j?b2 zpX(NU{qESPcc9c|? z8>-6q1*e-X1FD|8iyv*kqfW^>TxTarq(1TSRRBvj_tW%{@ zFvtG@w6IENfIpO~pw^MiX5jIitelCn(!3{UaEjuWgi=*66PwHf{pyqfmxI&Q7MW@@ z$3k{vpMT-5FyVSiO*FJL)$|QC%MjCa?Iq^Y5QLy4B@KX`Iza%AQ#jl^1&Ww+Nh?&v zolm@5qdz`Q@yUHHU?m)@5R^(ssTe|%fp@DUiZOS01hoL)sXrcEVVUK>i#`v|%c@N_ zdDy3Ab+pcsbUWPBX7sO=5=Z|4#J2KQO@e~2F+H=jlhmBw0WcRBj1CU2OS$2`qmSAD z0GgCEk1hH#!bu&S&7_2Z`CijwC_UtZ`9&m=R^mWW6!$?Qi81Ow;AgcZ)|{QU>PR}- zL={OQQzHFf{UqZ#*}PdR64Y2K&UkLZ&c>sjz5LtK&PYEkI}n7a#cC;`HSjv~Xv4n~ zIS>9D?Z1zy8$w&7%rnMwmJ-;q*y1+Z%sa#YWS|E`HDOajUWF8f8@Rq0}^Ktl{?E11FO#QErnMlLnvrO+R%y?uc z+nkQlki10nr6RJEHyb~WaC8GMX)6T^;n2m+rXoip*I|Dz<}tj2)nIh1^ZG}g%E|Z} zbnCC3A<7FflM*gKo`n1L_+TnPc0!5Wk3MzzMT3W$bxG)3{{V=E)Z7_t0yZ1Zy|B?R zo02-JE&92C$dCF{_JI`>I*Io`Uzgd#6kdVO?XT(QXnYAuAvue z9lX{4&;=*}K!YRL3_ZZP1HKVT1F75R0a1WR^6UO_6O|C>zi*UBz_J`AL{*OoBJ6Z) z-<&A7K0sQ-)8~fX`WGE{_(n!lq{;<}B(sytVW9ZHTyOwDAjlw#{{T?21|Wz#bL)a7 zh-y@R-gm#iV1fcn;DQFTa3^iX2#IE`x6jYk0Hs@)ow*-gpn8k)!3>5Y^$&lh2%8W4 z)biqg`b;@${*lULa=72ye__)JdfKcD>|5tS$eTQ5qB4mm%ckEE^!{fG?+7Y&>2K=% z?Z|@zBmhsZ@%6R{908>?9(UycBy#o>CfkAc&jdnC+Jivu&?Oy;gzg|1xcS;h_K{

a)W_KvWBafT^DL8=C?Pa~h&F5%@cD^=__O)7aB zj-Lo>ze)S&h}G_&I>nXyr@Q7NI**i%kHTZ&&yc<$h+(B>O}--k0AIhBAI7bRf&0Ko zH-W2iY;W=>ub)nQjNlAZdbrrL{oiOJg{}49f4`0Ii3Nolmd^(-J>Eo7AZ|eQ!odMj zioa|SLkY3>`1JMZf(QvS5imOZN%z47CI}&^P~V#N01*8Ux${0 z4tJO**4<)n&IiK{i2&};A=1`Y3Q&>b04f)&VKK-yoc#xH8n@``S2vnXD?DS|bKF4_2 z;E90-+Si;BE0O{G_yHe1y-o;NGE`D`P#pC6ZDRoY`QU;kB}r72nTsFren;W92q2)` zTW&&be1H%KU*8DiYFOUD2DA0=!!7a z)RIXQ*!3Zehf{8`u=^tH^EK^Xv_2c)JfFbke_Y4r>zqcm@~oPLg;r*HoBWW?6%$Xs zqP;rdErJ%}XBFBk_XXLElQGV*%i%^_gIv@lYDyBkMAb4-lA-}wXW?>7ShR0GjQbCo zdPeQmN7$U2TCo#*Nsh|E#2IRutQKE|lj~P9XaJdV_;^56#!5oPK~ORDEyJvE?oZgC zXYFvaA&wYtg(wD2gQrsH`=uo{FBHICO-a{LSm*x$I;n2CZFc#e+HTR}dqAvrXff+6 zotk4Yn2FBLi8GXBNS7=5y;=)^0c5dYD<7f1M_q-&_6>L*!r(BelZWh*Wn~MNQc`9N zJfvaOi3uPTPQ@)r#ZDkOoS;HFxG6;oQv`9dgz(nzFYt3fBo*DcAo1 zBGr^Uq)wHl1C)zc@v@aPN_(A7?5R>S96DwW5syMze4pT%FaH2kYG%>cRM(k?%2C42 zP*a>b+tlcpVVDy1OS^|Z!%7&%dq|@>#}8IARFqcIYpJGK{Yj{$bG&Iy%z>pzO46j2 z83_aQoJ*8q1```9ZwV5Ubh8jxRX|(3y{hjK=QvS~z|52!71&mxu8>7sgHR69w7W6l zah~wDSyHtWDe6Qe_NM5414FH-gD~5vDL2wU6L`gL)MeufkC`%6B&>ThqM4h5j!u-; zoiMouEmQD_N{RLI&kD+}m05^23gkfeMe8wnNO0*Of=MbS>7CShsvr^wiBYkUjbVwf zBzXNJc#ug!8jFEbUPGrp7@wEcl_U!(D$rm-@e+}^xiAQbnGqMlL#HB@t5}l3sktxB zuE$n_Ry+@i4X$LSAPqJG2>|Ys;o4-Afj-M{>0y4pF%l}UTx=*thR>@P7d0RjtYfa$ zXjmt0RwQWz$ByCvQb-ac*z7EkVIzU{I=|`W;KRRxwJHt?dN;+a?Ojf>k!5+oObto_ zDr^I&=}1$C2BHdxB79aU^8n$KCDOzIbkmqL9nVIn4G>M1ik1Wl2XkFMX4h@lMt@6L z-XzJ?cLWrKcvLy403Z;P7CK}Yj6h|MV@B6^H0Kz(sNgy~mu+@GH`XSqr$WN5sCZNb z%5^9i1VAIiqL~0~8sxABt?=^Dh^R_Q@gES9lj4ma zf|GHkNhaMv=5XYm2L;I>5(c0TjYTTXF7I6hD%K(4()c#LpX&br4L%WqVpO4s$9AD!$Y7-}v>DvA5L(4j z$P7tQOI#NGc>e%wZe%h40FF-9;bg5SnS{uF9AG9XUe^vE4cU&$X48^T2mqxR>F%Wt z6eTkjeUvS@@A1)^s*f`4?JloNqOB`YQ`(*}m-k~+Nl$j#94RY0RI{ic1t|oTMfj;$ zUtr|O1;!^n!Uh`&d4iPMo&X%ufVlyTbPw!u!p)QThs`nS1xm~P8)rDaGD$o~HtfC; z1wkW(aVHruXLt5WlUUh3h3BZN%4p?MRD^7~@n}HONt>twZwgkDL>zpR&#-(G=TgGN zRNd}a*qT;9Sy&ubufnr}NF)~p*u4&>&d*@l$68a=CIAFTOdZG|8JnHp?`@5(jZ6kt z6Bh%+M{Sz+(TIgm2Iafj9_2tvMwJ63fCv7dr9nJ^F#~9ek+Z-A)xfx=?aI9?_mwDj zcMRN#ci(2OQ4!pJXK0BjBpcXUaCZbq19=l*R1R-SwuDepptS{Bi~+2PM+x14b`~3I zIqcpM0q0e6JV0m?M5JjdF}{#4Ma{vu2G~x4?g_sl&7l@?zR={LBy&5kt36(t?@Bwn zMUJIFivSG5iCmkP3R9+J2>?Vvn-L=xWgt)&4E#jc(#_|*n3)A5-VDGoApG0_A45aR zG1s1&f)tyUl9HbRK!Tt`iBNJ>F9jrN5(rUD+W~w8@ojZ%Dn&)h=*58=5b&R96Ju6l z!$!FQix#l4b_O;701~b#Urcy3IF)q<4x1ArasdV;>D+^hm@AN2KAQX6RMVd-UE7FH z?Ae`&O+%E1tJ2IIxaXjcDY!N4>W8y@=B+h+X_av6U8M0veRgF_#F;HW3WpftjLkHt zUkaseaaM6ws*u|)45-q+nw7rgJB_%8Tu;`{!YP0F7qcL%-N`a`o$ID>{{XRJO(gIR zR}FXGKQLaLy;A)%`bS3?6R};g?KiBJFcSTeGpt}w9w&K-5CL77u_b5&WI1{Gbx^=~jZgNU;>QJd!ZqSPEq)b+Iwzm-)dG-c#dy;9&` zH~d4+e@6MJmb@*0Bti8PK_~m$21@`e#mE`}PcH5F2E(dPr~c1zCl!S%9i(A16)Jxh z7iTk)l1WpjuH#F9?O#nRQah)>GJHBpcS(zsl~|BTD4P%sj@bFbEc<1jV@3gkzY}y9 zzr*wnl08|dE?(pQ68F)C2Yu-&4FgWVf|UT6D@piCLDDGCM&1d{^F_`u>W zY+Q$-y>+0y8Nd%zNl&U%kZLS%?z33zI=jHo%0&2g0!pDFBoaw7Cy^YyKtaT+RSJuf zs1>f-iaUJac+aeIStJo!Fms_5AhGF6!ya?H=O#b|5J75;n!mJfO7% zB`P422c{ovfjxf9fJZvSX0Oa}$e39$yGxkZ4W2tRG(ODkD(AU}s}-9=2l14SmK5S)Xm|Q?3{5C5Gj0BNI zrix-H>HZ<5EM7Jo-yoNZ$a7r3fRmVISey|^2Z#(_7-1ARMM>duLr?>2N1=R3(`w3J zq^4ODuT5Ci2#XnoE@-$NvE1TaJE74glLi$V{alcIC|Vvi)*HvEn2SCLCcd zrVa-g3HHB;fpP?X4HF>Y@oK`)SE@4-tSSV>2}jo^T*Xb2fJ-WBWyzSeB3M$AB+OAt zLR1s;?f8q!zbV^}Y{FLQNMSlvQpsh*t250ON}DJ27TjpGZdA!ahlNN{@Mq!;Tl!lnd@B}xF)w0?}gvws?#$Iw;+M6y81yIr4Uz$gTs6M@Kc z6tJNHoWz9abR-b>?hK&Rrd1Y{l*}t=HZvfb@*WWpF&<6Lu?6@v(%?JM$pWC1f(d2@ zKCt$055?*zX!Sc(Y}$bRbfk6v00ia429&K@d;-Agk)%b;0VGK$uPbAk=97mV-(GyR zxo}CX&X(>uG>!yHb{8hL&_2vq)aYEtLS?M?Y_EG1EH(lMKT#l`N%J;w6azCY)6a&qpK!P{uijq`dUyIUX)(JSrftC5a728h{N( z?OpkQ@ad4c?qB}=TmZaucfh2j;3l6XXlrM0`W z5)E3{zOkr!E6U8g3Ng5#PA3!YsHCKu(Js$IEzDLs8C6W8oYhqTs%1mG@vD z5mdrZsZz-WHWEz1=BbjY^AU%Vg(`Z=Qz)S-CSr|(2oADEf^TMe4%n%v!^%~k4t-RW2OsZ6n|LCVGc3TwApj_dP2h<-gdC;tEy_7~WPZBz~F&O1D($v6{=9Bd71;qD})e>dT(r^xXE zNt{t;^g%{J8IvHBpab7KL5D=#4g(P1;xL>t3Wd`v5RpF!veXDnzyMLF6N;Ge-(d4U z$o7{x%2!-2b1if5sWwh8E&aA4Xfy;&!kz#_y9%`;?f4?&^;6~e$#{hgVD?YCz$+%_2i=GhJ-CYd=v33Bn77HK0DgqXRW z>OE!?I1=TOp~d-i1o3byhl^z<_O;**qnvX1dSkUZxPOb@=VtR*@Sm!e{E6@pT88pL zn7o7h?SQR9;0~bu;z(&;;s`vp-|yE4tLO0jV1_Ak&IBFEzfyg^7;6w|R@7I|NQ|hz zvryVs@8^^SOWY4h<#2w+^M?SG+=J4)?SBYpWW9+N+s9t^#N^Cy=n%Jzxt~@i&{97U}0Be_SYHE;P2CK9ME#r#Bx8KRXV!fni{v*bfK^RL?8e zc@j_YJ#a)tP7ao*BIoVakOdKV=Nz^>kB>foOc2;bDsN$H^MSuj{XI7a1Q5e;ceEG) z88CiOOr5<>AegF#98NUr$>Ld3#Lg!t*#d&(mu~7EKU6kF?cw>x-`fnbYfojn+R>5z z5AyvWT95t58mkYWfs{@5Ua6CYpa@0UCfK+31ic^*-; zLG!ui>1+@~FhL1Cg>{^xBA2sU@kWW!NgOONt*+0?dl*;o$c%AfpuoB?OnEm&z}g86bRf7w-)7xxZHS% zoG5NQN1l+lSXilme83+u7v9&~uGmf=Xc42oI(-5h@9vGCmi+*dggqOYgK{r>uNc32)3 zFz`gE3lkT#{{S%&{qR8s3M1w4A9lV7a8U>0-LQiK4Z*<@f(=9!Dwqaj`bG#?Gi%bJ zdq~thh`&2b$3KP$B2ZdLkU^3l57Yr5Tu&lyc8#Z*1P~G@edm6CO^o3iW*ypih#{oJ z3y~c!B0g8%*jUHSkWf>=6sHB~BZr^oHj;T5dUFGI`ehH&uIIU>URw1&6^!_v?3as41_ zRi;ISx_3ZXs!>w@Tf-Tj3+D=LTHIqc z;cWXi$|@Rq2`F`R>{3-|S=3voZDc~V9~_)-uI#1?c=otL(x$CaNg(r1uWRWaU3**E z*;!AguG{5>B?!f4nQm!=D;x>@wle{N#HlGN01@i}2ZhHFXL?7JUA^&Qvz+3rYw#h* z0g`7>RL!EsdDxi)mOQuGsj8Wn&jmmfioMeNhj7jgj5^2Ak4HIloWCiPz?vH{U})Fm zXuyHY3P&B*S4)J65=rO`6EWo;fA~wN=4m+aL#1sr?d3)54 ztHQCZ4mJW_rgq3-FUyg`MjI+w9C-4u7rJgie>hq>({z z)}S_bV#h$v3mR$HvwEi(t;w@h{{Z@PEkaP@L`&V)V$e|mf17PT^0t6WFD1rG z)I3Yr)hR0yvC;?ZrXC5&OEMK^TTvvrp<~|>OIpdFgPz88p~Wn{R6m#vKrK)kV(@Op zbDzq&oaD#jrMiNbl#-GO8l76x5+sAU#zQ}q`K-GOi6PXRwZ-meKn`8S z8Lnec%h&mA+cUzr4UDmH{Ft`7UgKunxwTPH)(B3d?m&eTrcwyLtBAC~n1FGi1iK_r zy!9sjM06|^$GPTC`Z(T#z0}qrWjT~DM34fFrhlj?gq`HtB!U37jiX$$n*+A*&(Fdy zPbpi{fSrwO80%7Y9nq0LP!tBOK$NCHx>N~BO{`2E$(=$b;9x>pq*SvFMSf+Onkbb3 zC>_g+>UA}y`cRO>_R8kTXG6Y0Lh0^sj}O78@E8 z(Uq#POG+U^MynHLN_Ma^Y*VCz0wM{-LBXg$BfT~v{{T8f5<%i{_U<(s(D^zvF)yhq zi6D?qh=aOkOadkpB4Cun&pZqtSNC5|7uU_0J3yiohi#qrsN@URLu%FF>V8s~x^+oU zge4;LyrdN<1c@;wWI)8SK~%yG$S!-s^we{_Jw_-{Binw#LZl}<1g(-O_0bk$(iafnHOL? z5!w952|s}(OPs8JQ=Cm)xr+<+a}o$*t{*#pANi7##rzYN1uO;;Q&1TV(}w~ z{>MC_U-;wFK6;>E1=$Q1R0v4mmT|eRSv5#128%MX#LX#Dq6ZLkgBK6O*A#~7X(^p? zDLR`7xqwoXg#u@oDw&I$5+@%t`%8j(lU4z9?zLW^25nd|r$-f!(^qD3=5`WDQ4eC5 z972rDYW(^hQRo&`n z+C8CBL%*MlT@6u{7?a^Cf<>g6x9XILB#E9}@dA=kSaBX)`UebLh99I(iJ?I8YRqW+ zHD0{yt8W2aNDAeB!Oci#mPWfG<4PV1DBlyBlJDN`W<<=H*s5wMN66SZ6yrlaD-9pe{yXX>s%#?{GU);GpLMHm}_ADVa4=ME8qpW^iJv5b{Oa79cRv{#I)3m?^V&n-* z3C^)#5iBe702+o5;AysUY~MP}zZ74Gg=bZ~Q&Jk?jywFj9B0B=cME4MR4awF*1Hrs z9n5m~)P_?@Rhgh>3?Y=#7ixoUcYNpZ3RbU2Ie=Ki7W||7^DO@W=WuDXqeI2){{Sl}H9gA|a-tl;IU>&?<$sHg26Dl!D42g3YxnQXKs&XalbV zrCL@7qNyC$Cr^kH036^807wLx7DuZhiBSVqCr354Mf&p*Aok5m{{X~I2I7Rhsjh_U z7P4akSHDbm#fw5Sg(kk>Kt(-ZEC*N z@!7>1GzS*(Lfu=frABnFMCl3ANw^A=Gi1TAAZ&59;4<{26|oIl7jf5CXCRiLjW;yS zOtI?01hAmKw%NfZ#=0Ni3wrGksQE5OB zk`AH_z_CB7HZgxJVN$YG4;AbjTe#ka-x$kGl#riwt5V-OH=dEzIhXwTQ%*XT+iB`r zml{%OnQ0RKT~x^K-Ca(VhRcqixPpfevljno_PV z6N;BB9HOofKuAHY8A&vBgZLx#g_b8G^qbqYFbg_D=&InG$M2UC$M zTIQl+lEnopO4#u?gg=TJW>edhbyE$r+jU7u9ooXurKc~-9a^0Mw51RCwTtgtl@R2q zWV+(iuNQtMB^iCAz%1T=O0+1N08yAsEZ~qpK|<=sg-IX)L65ut0NHJhPBYO~e}odk zeMf6$;{9EC{dP06U7bwi%tr_)nu+)&$(o9k&6K3C_K8wPN5FZ0pvtt!IybQgVhA^l z;yMw^-Y3oR4zgTqLj$wwTfUnR%s$q3V++J2mNe9XKpDvB)zq6>y???q0k#_O5Ke=< z*(yOI;!TLK+=4e4=XklCb0|UEetI78%o*dcAuyQ= zAbymj02FzX0>sY~qJtag)aK4F83ZV(9w5e=5(uV@df<$iK;0{EdA_Q9wycKk|-Z~wHG};(YbOokkUjXezLC!0K~$K zkWSyzXb?m}#<_?tpm`rZNXVoTmM3nM+4TE$jHo-I8k4Az1f&8akg_07!otx43>}4X zIH=F(lt~uc!neC&ZWz>*z=Lo~Qme`3v`7R4zbT|k3RMHWfT=w{uJHuLijYAGrS&DN z=ScE<@pr`zXWUQ2Tn|R-$g8eP^+`zzC5DJ35zHx5dnG%W<>-$`m?YpbtY&Pq7nOwJ zK~QcbJCvZQxQMq11dj*u@1^|A=9$(_+4(aRP^@R>F<~UDj|hK+fDHqH<%YG5{{Y$D z15{a61q*sxX{t(a@{|0@`jSSeh1;_TSj|Qg_Nf{ z%vwSUawe1kCY`+@d8)xJs5pv2{{T__HD;+94i03+G0ySei79YVm&;Y=5B(|AXh1wF zk~9UxT$wg*e~cJ8Bqt`p%~$8=ha(@2K?_zATAh=HlAtGI81T> zj2aGGj>2aP3~>48Do`icACkr^^2$r71__qSE&Mh2S~%zMubI1*XZczJl#(55Q8JPU zT0A6f2x)Gg4>{W*Q}>x5!U z6JBP&cmt_}*0(3Z6Ii+Qf$`_h)(55?!oTMDm@3W`d51m2nSqr_zdgMNA?f84dBA;; z>8Lj*qMaLf0W}E(xo&4+O-*{m2qp@E*xc{9x1aNf;#o4#w<}Ws%p3v-$%{{SBx z5MI#afRQTp>;6s<^1h#bgT60e3dAB&^e2)IuVu&d$LAJca zCbbLYTDN?fGM3SvvKB((srr&4?1M3r*%0}zr_ZO{m6 zlvVVGZ@BX0XPvyf;{HToQir>bgTh5f>OLb9INW%MA~~JFw8X)Qm@;`@z~1&ajVW+$ zJZrX<*Wm;&Uo$^1oL)JJi8zo30Br&450BIcsSz?GuPiJP9|RD^8&5sK7xF&Wk2%`} z5H0XQ6@mz4krQFI;Byc~=hpV$1V_YZ2w-Wma|G7kctaN!8(s$6gK703?gfCH5YoZM zv=3Wvt?Aus!V&D#Kz{}MfvT9!@{UsR6e&l^sjU~1U@Sc zPrN(+MhJLRbb@BlVbTB>^V_By1Q34*?&t5W2q1`5So7P+{{VatL8pH|zpuvxA1E3~ znCrKopQXn$X_zAss1|*%T}ujHo*y_M?eq5i@JrXb>jV-yPfOpaCcr@=Z`acZq_Z=h zkIDun!U%#zh%w5>*i;fIL2W2$5+IR63u!}0CM1a=0(u+Ua4)oN8*=*C0w>tBmb2FV z8++-})aS?~aU!Z7^(sXgk5Vhlj*SXGwf?rms34><1+QBy-Ptxkr{ZxOhH+ET)_8&)cowomTc#zloTvcS z^UPnnNP#I*s+6ffU4bK^Xi`=t;=)eW7r$7sk4!PeD!W#?^5uA+^0PNbioflKsHa4d z6k;r9Zx-ZjGlSL_wLAWHiH;RMt*BLQGEVRAaT+xpo4zCKGd7NX)6V-&C5jicdKueU zM*x4PuL9($Qd+Cxyr(zfio}Z--_NUR+@@NlHU9wQ6-3%i#)guaF^`!40EwK`V|!_x zBaKalpCbZ|Sr{BA<=C3A7?4+H3MR4oX#UJD$s;B7U)Z_gQa>?|%cgdN9I-^rnV#d9 zjKYwAh$jpA2?$f3AuS0_SPnS<083BAAj@*z(|-xvO3ILMNo-K$#?^>d_pEif@9nYme*wv9>?ec~aj)dUb00FeZh zL>(ja*rw(>jpG$pCZ?U~1-r#sh0lO?i-=lE9F&3q+398$Cz!5+BqFS-CP*tF2();N z0yzluyo+KE1#S;gb${#D@i5~}jatNpeQI{m&f{3uynT?no6&xMD$z}z<&C)ZTK>s& z6(qTr)TuM7lBl$lxFTC{BB8KcWdvoita!stN-+bE2{|meZsbb;0MsK%)N&8B1Tz&Q z^D=nwgCKLf=u81+KCAu`v*~^J2T>Q{{W4u+i@S2sHtnYuB327 zl(p1u4)vviNmheUr6teMw!$53skJ3%d7pzx%4eP-GF-piEd(tK{{WFdl&bm@eQ6x? z5sV4WWKQ5Rrm>TZLV-%viXw|BETG|>k#Gggdcv}-J4}Tn?2`fwyaRYQIzfb)0&iev znasnKOmYYYl>}HZxD`5hwG4C&1p1n>KIkM@>9vJ>%wtq_q*OszNJ_$BZ_w(CZ3K&x zJK`nFeMK#$`u*cHR1}_Xr$bTX#(2d@TQ^W7#*if5P$SHN02J89YSlYGpBA(z<^bJ=Jk_-n zF^AMtsh8AtDKJLXCqa?`0(B`@lQWBnlE4x`*d&5<@jq+BmoT%6iu<6K@cXdZ?UZDh z&V{4`xk`ah)RGo$J`kdksEsz%NF)&$ie!*$p?y9k)VmQ^p@z^$8}>_u+i zM$DtHL#?Fufy709{Le%-a@cV`sOFMLd+qoTenT2D^$}h6GXc2S|;; z4+k>}A|DL7Rhx>}e=%CqN$%-$3UoMz8)4T3y-w&tp-53LC55vJd6fO|^ihFES?JfX zm=Ke}hU}MTWRGNy8OzC~0{lXTT;!D~GpB(?(o*B_efYG_m)d{C*QY$(PnYZSKTUp+ zbL9~>u_#(qicx~YzUO^VV;LW)lc!wT@VjK}xBn|iH%d1-F?@muPW*HJlc4suT z??OW#h_wEL2RfUDiOFbszHrF6Ci`Wc)je2B~h1h^L z1P5}#+_Ml#=d-+I3ktmrjXvZ*CwL<2i2x*kH3MQLGdqJM!AJnwd3in@763t zRaHq<;sw%m5vYhTqxww5S^<+1Futt+05ywP^YYi_3r+PEYBAUJh%eNX$w?|vfieBNFlO`;75+x-y0F$X>aialx9ZS7yXGs7z5GuVxmeRclHRoGKI=Yma+0rMu z)(Ir)NIR6s@Sg1>P!kF%j<)zchti;4gLfbcFnT$9?Kx8{fv}(_qf>1*qX@zHspJZ3 znedRx)JlmXN>Hstgy{sS0VKusM)Qlc2Zw zs!BH7cRI?NqcxRI{%N&QvJ>@>v0Ov3J(-phJfLK`JF%DQX@SCVI3KhCbaAYHU%fJ) zTj_JsO2oV$61#2MWP!mjvuAk3+*G*)P_=JpzwjS>OZchmMxE@PyppLoHyEO&mUzX^ zF)l1{y--T3s-Modfpjheg*uu;npBdihd_1Iwv;o>zYsGU(bjD0L34vAgT#d2ws#Oe(7`aJ8tyqeY2CXe#M>xj7si=^evR$ZX)aOcWdA}_^DW8p+ z2?`;_Sl3E6;4m$yuyY%Gg!o-cEh$Zn5*2Vj@Q_4#e^HMs_+uKzKOU)u^q}f(y`M*} z)s4$8%aJJ_+DR=}4E;P@@7){cC(6>o5CT&wNC(Iy?R~B71|w_`?uH${{Wcm7h^F11dK;y`#mtL)k~{n_y7ncb}1ArL2_&ieiFSt;D5XS01)1r z@M{HU59x!_#sH+LPMIq0UuhLF#Ig!gE^>smsYD5Fz_KQkr1+=!xNXGt!IHSDbTF#M zGYW#@SS@$xeXNOWx}_=il?K&JSrl&6lmik8p{(TiEF zzM{58fp;38G#h=0uB^`L!;Tb{NhL~9DiRadV^U52>b{`+SVEk8_HARQYINCS+;? ze@bt@*vsXZe_e=@gaRJSFRKAZQFRuV*j0+Qcbl>nj7((A(vr?PlFIB;f;If*U`Q~bnSCt84$ zy&gPygU24ydPvN1O7`&ibkE78yQI)liq6Caccrf#p-EMkR1CL+auR>vxQL$h@RD1~ z^7SF5Rpze8geH-d1h5qab*?gj5~4Y9M41A1agAl+uuDL~ODFxNcpOqBE#2kLN3}Oi zPd`Q~bks@0WLOi!m}BLtWXXc~DH7GgjmJ>DvX!sYNtB>Cu@jL!KeCkR>rNiYa^4)t zsB%oT9o-&Y&}=Pd{%74(DyF8Os#NRMPLk`cD3YfXr4uoZcaOx+#$u+cAUuftc=XH&;hYqP?cjDe406dhr3eh%o4orNz-`TMm9USsQ!o+GOO zOY7{7cB=|=G3%${zl>9zar3#c(psukNk$o3wfB@X*=06ZmfR4ab$&nx0XOf?(-`R4 zCSaUKD~OTB{2{}T+XrW{VP0_Wb zU--}P%i2E-s26G-;F{dVmAxfvRc=nwb6RwvC;tGLU7`RKiIrQB9ox~)VfhT_CdKh2 z2{`#`T8RvlPhaKcl@mae%Oz@d2|+3}G4uEFGugCugR(ug%7PU!c%IGXhD@}it#dOe zGM9;w>d=r%rY>$8T4kjmscTYFim4#^Mx&&IAfW7bu^}MN##BJI>Lf(QJ-C)qj)W?q z!?R093(Db;7eV8$R$OgQfH9^?_EyJUmeLL#$HXczxDp+M7bTD*{Uzw#1Kn`l(;Yp9o^zNdzRHSRzcPVhB4<@$1xyA~824{QhK| z5I|I^RgdAV{c?Scqm;b6|j)-lg-jw3FMF@46} z{NRKJ=iB3ZdB?mGaU5O3f<0q`J==mmn|K=4HXC#E#7k4qTV8Q8)btkDoHTiPf_VuC zc=>wa5$_K=7-*$g_wS?!SB3`rdwn*y&kM!uwzlr`PmClC>}P2H-*JYEj-m)qd?NGi zZGLhG-_CnM`uY3rJYtChatE#mAW}3$iRm}qewZME6##zEzVqul^5=pG88TyFP5h4B z!1ez4Aco+AD+CaRZ3YKFv-j(Q8lXuG&CCMcKTh~%&;6#>B%2O)-`{c9p%Jm!=|cKe zzXq+WbP&^TSu#bS^c#O2IpBn5>gTwQd-UZLED5|wwA;=_rqDh94pq*!w?PY`HD?|5 z*UiX>B+Q9C;?wSaKAm^FVN+5=+!k?neQr1Lf?$baut5tvxlH}@!2~=g-e7xl0BvDl zHnBXU;mT(%1Qi5GJh}N>`;S=>z8eG>5y+2`!2~LZ&_J7MJg#I?@Bq2=m_C1Jsb8NjpDlILj}ij(VnYg^U{LSRwP`DxN;D>A9XAtzOk@mOo_r3^7C?pViV1)#N z4$xGpe2?DPB2=mGf1gM?Rt=4h%LFQ&&4zfl5A%)pb^k%KrOhL$?Y;;h$fk0w_Gy>}PE@=aWwhOETZJhJOm0r8fDD+DCUEsl5+x~f6gX0s zTHl2!N<)<>&;)?wy#T9#A~fI?jKoTlAQ3WplqJbS@y$^7h?W2)YDiZSfJrO?Ew5>R zsfK>n33h+k=Vlb@4I_qlCx?xm6%#RKTtwI2Nq2T?gCDN{0Ks=;fAHsF`!|9_l%W_LlP1g1I9MiDUx2}3 zlEAY8iAtBq0z-gf=j$QjJlBReqbuT$9j~a%Gu(?eq|Net#*t;ZiX6u;szpT|OG{0{ zkn?C!NwSg@l}bq%;}|?znui4wx+;gpr7oQa95kU*$XbxHP< zt2BA9u}jDH12@fkD>ih=_$<2@!<5gM{Bv@d%6u&4lT|8QpV$*&GuVD399_e`r&8t% z4Z7e;{6oaCc~JmIQ8W8<#}UtaIJrKxQqV$8loFx{eG4&rbMGQOPubql$;L~R7chkj zg(;1-1y1f=$Q004HnSOj)D~)mI%;orEXIyryEIQtTW_`{KtM@Y7AM1SIS@$}i`d6JR5Ex}z+A8`N|vio zb87OeSVH)Aa$Hn9*}<(TZ9Ai#IQxc|ka4a_$JuI2XnFL|XJ|kvOXzhgZl0)#3GW$k zX-b!*1u~`UklSH9uV*s>Hbm^QeNi=r8Nq3F3|N{zg6B(0h6_v~SuB#8hR?xE5~PYl8<^S_b6FAnK9`3INBE*K<5NrQ%Gwjv5AvIX3_F(Ix)<|D7Hb+zsD-9U24L6#2qHn%GESg3)rK+DQ#+#@V|4u zv=+S|1J1QH6yjtOqj7%`SyrP0AO%ba^A;GrC-#lR%mH?ANDNQR9jQjFM=?RdL1c=a zbznKx`@fB+FU`wOAzFw6NF`7#Ob9&sTpUcPX(?_R0zILxTaTCpz@afzPfr*?N$Sk> zJ;a>>tDhaOlII~o02Higf;C(cCRCz9AjeLkY;lj7haWbfspwdTBnmyO#s!oSK~Nh* zTI%k3hK$KIn$Q%5g`1>EjUbI8HU{TWH?^arexM|ZIeUUncnt0x#+s5Ch^!W_GKMW% zJ0GEbUNA#g5Q3!wLSaN3C&EfdQT-~NDKJ3*+(eyuc^T*S!94;h1o@&B#>Ok;1AL^C;Q8{9k9(N6t5eb z=e?%JiRlB05_62l51F}|qCgWS$w~nAiS>&K2nh&b<^KTUIZDj`01#P&Ywy+>%jp|v zRK6ZrY*$t2Q)RS-G|JM1tt|myRO_foaY-Hw{8mmf`$v`|i7YbYsI`X@lnEf!_|l{o zp;8n9bjRp3`yQ}po9LIdxP>8L&F#R;)FyD133+}83Ykkh6iOyZ#);D00%a{LAQc$M zGMtS}(jmmB!V+MmJbxRi}vAF3{X!Yvt8CgmNorBTz)N8r3S;^2t7gGTtOA!Sk6h@;uu470@f&xLXv|_)j zSWx0;6)tN^>#JHNp+~%z;Iq|&P3wBwmeh!j)u1GWE+FX_Aj};wc_(`iB0TXtxk?Vs z#h4IS64j`yQHXk07E-`XRKlT-uEoy0faLdhu`Z@6h!;s^CIAXZ36TX_N~EYAj+leK zB=t(+qMh`w2!58OScb4;cz>*Aml^{4lH{`%=541)z<9o{Xe!~tlBQl%r%#3(UaNp1 zQWJ7#!l@8OGMFThGDy@12U6Q|KLXJnD7GGPwL+_*C&0U~qZT)h9CBKZ%W$1_dkNuc zY{||poO-oM`mxz5X_wh9 z&liwY{mFp@l+5+d`79UlH>PF zYErlMRKSr?T$s-LI{cD<6MhdAm&EB;gpGIY2-CTLh{GXW;=0LiTBkL^;D9{fh-x38?V4<$%^r2hc&wG#xZR4V8KZg=qrwW4kF zxEG)m(_^!IA7|D+`N}SW;Jxfi3R`O%?l&Yhrm!km{P>)wF+QH49<++egcGQhh;}7N!rt3>(K}xZM1b_mF zAjpCQKnfmnu=>eT6F`@7{6KPl0(zf@q-9RU%yk72TH#!ojRk?`C|t(O;jS1ml?FiZ z6DCQY>V+u55xhygtOhlVeqgl_#rjm(u`Dd^ZLMQ$%d+r+#LasP)QXcv9Jj1*ET1Y% z2tg?cPzewS1QaA#5nxFW0pu?S8v5p_uq4r{bV$-YAe(|ppokZliWwMx66Q&zq;+9OWd%))sK|?KAV@dowlWwA z3tZb#m;lG4u%YNk>tl?^Vp557t*2cq^kG)7GaWU;b!)Ed{{VyQ61s;Lz8|D)C6^kf z)E(i~{{X~=7gp$L+J+Ua3EV#W|{i3y#$0bxu3|OIpjy zxspt*ULMmw)-&@oda<=9X_D;O}8hbm2K8{Q+8I>(xQ6sZ_YCM!S88Rp^+Hxjvr zg)=yPyalyBZ&60dIlm^TZeDTWr!#YT{yYQR}|T*HYt@f`l%{*GHMH3)18TtI(`;*b&Uk$ zb4h~`prAy|CI^Ry1BF0S6s}C7shE^E5~mr#Nh&hLbl>NLZ6qoaD1u@{6SR>EF>nk3 z2d*@aR5*wP8dlswwG14F)`wa~^DMKZl17G&|dusJP-Ay)l!)BPDRV)CfXZjZb-IDaZsP7!toj*}l=>u5Q zrE;GMCQkAFK!dTBOxic>xiV!3G#ZmYPORg(-h?*q%(G)QeE{h|7Qe!(;e(ojPJ_dk ztA-ILhp66R)Pbl4EhTb6)qOxgA_Sxx7?2Mu<(HVL9>NIWsVXD@Ub<4G^*o+1NXk^d zZe*}jG-ovpKz`cNuO039W#gvu4l~M_t8pnsQ|Mr7R3}+f%TAxXwOHQB#u zR45C~azf5Qo2?P4%(F03s^}KA01Q)CCZVL2Ns8SwB+qeNiFn*L5`76sWdTwLJ%e+( zw%#bqb1C^QW0+;DBqJ91haiNR!~NK!6~FKT;)R+!*YK@85@C#!wqcl&X#5mPU z2A$BLM#@6gl}LgP(~X-TTI4H$5&(L?v%cZzdV{!lKLWEABq!iAfk9XwQ(V1mA zZZ$PlF9Rm)Wk{c8{~FWXj3&EYk>?MS_zn2by8A z7*y~P#FWlXDoW&*X^ELBQrsy#Sg(FR?K^cF^z`%J%NaPNC3;m1`K*ro!SR!29yS3IR3*^bhiiL5Z7eFYkmUw0a+h zog(HG5$hZg?%Wal+rw0?;uW+Vk5~tg`TF4i)RW9q{65sw28(eVl4qEke{U|hcb`6f zu$tcc-%kx3bk*I2ksk~CThEjM7wRyU({txebv`hJwY7Wqk z2%mI!{LDh;+xheQz{!DfR269e2>bo8=pbq`PuI*qw~@SHa4uR1Xx&pW@ZWF^$3rnU zHQjV+Pxy|Y9e|}bySyPGBj`T z_dmuVh+&CV@89Og~!0@s*5NF&gC&Lv0y z4<7bs9e%Jv;m|Y0AkBKww*Ot(5*h@IJmAyzI>$UdQ8!{(uU44bD1-aYKa6@4M++GZPi2nE?L7^MB z!2bYPa0eZIq#vKlX zxIE)+wm01Q$N+Wq#EDP<($?qaQyA&eeOCzuN(2|Dy3KMoa0gDXpqZV?HuvT6zpflv zB7?inmWUv*QUH-)uXDKmeX!^tfSyz2H|2Zp$XocrTJ6@~57*}o1nB&Q^sR58r`ie# zw>XKs&%*xz7{Ty88&2?bv&4YCFQdD?HT_xp;p1_++hH7t-bIIAeDEI*hPpKx=Wh$Z zdJ}!VZVxY<9VX!VNdDbK;58J!ht8Yt4uYL4w3|=uKexjO3wzW%)4RU(4&0t1nJF_s zaz_iX9PT~>0Ol_HE$8yvf4}TO z6|7&@E%&!W&!wLTA2U|xEdf4BVoA_@#nUxji3llB!Ldq`LF3gm0J!mf`0`SdG4C9Zs0whX++39zum(U7pbpoG7RHFhD!)#9 zQo~TqUE{xERF#gpJGHH&nwnS*9z6u?-j6ThzrwE<<=Lx3pzyC9RvfT7a@#yX!}(oP z%{3#!fn7%9`au9K)e&_o8>Ux}n=7;UnUc#KJ~2ElLw}1)kze>pC#&?~y`ak_&Gdzr zpbz%knrK}ggkbT*h1#I8O;q0UP0C-(UNNDn;Y{)6uvFU!e3U69k)*0c#__q61~(HJ z!^+3X%q9A{pg3Y3s!b`YYvD!_x_@@BEmy9Rxr{(OK_`W)P-+eS6wt=e;EvIob+(zP zcu#;qShkoD2)*EvFV+FRHH_CLNE~v~Us1wbbGMBjV;g=?pM@R?hJXqpYupVe{zF>ajkrQ$GCB^F3fI)*Q`u2|DxIXN8%3yTwgDIfwU zXvEo=(_1wOi<4F5B_%4<4UEEdCfaSb=VA1;c3q;%3^;G8*bd!n@jKloJC>hy`g-5u^~H0$})NNf(sDtl_azrb;&e z6)>TDovBjeKzdilOHBC_WK8ywS*~2O7JIR&X9P8)9pH@RRarX;c|}h)j_uG7>2(V3 z+)3t?q=G>OKdK0{<6_UGAk3FKSUfnO++V_8g^3KSyE(CXG-;U@CS;(zGSmvNq$hyf zDgDw7LapquI$RSB&2taSz7-%Os7a9^i3AcZEI}a5oAV%GB+BCVQg}D6^*loDOBSY} zJa00wi4xqhYHmQj+m`tb{Og9ZK;tQat}H;Epd_h4Tio19< zZ0$~v@rdIN2vBFC0gjX|Da&59&QR-XPSQ$587AW8Mq&uQ(*(@k)YB}3?1D|s<6)-v zqv>d9nyNE5ItFTL9-8aTfsAK3Xf3sF0@EPe+=4d)l%^~$YOMURfkt#?7P)HpC0%YG|wlcU`37D5U z4es~e?#$X=@rau|$xvbgF$A`vx{v<=XG)V9Uk>pCPjr_Zf|!C-LO}|Xak&Q9I}sop zMTx?cun8m)RPer`g`b369U0!NWR;3agB`UtsJAP)@N*l7F3mgir0Yt&Di>5qASeki zK!Gwu57Lv2a;305AeIyfs5NJ`D^hJ&T#mVJ58sh$lkqEJ)~EPNo~#~T?>=$p0D)(^Ak0}!_Mlv5~nd$Wo4iYjksjVeh{0SQnjel)4N z2Peoo%)PZ1K}bln07er+Ff<;q6@wLpEr ztq9~sqgRY4h_XKDtwEv&lOsu#kYqsvQQ9mH*BtSiz++VPPypZB9v~fd-@&k1mI)*{ zO_`4Sa=$Ts=@?#Joz`SEX>DsEqOS_I5Dusy$c?}QU_k~)Tb8%4ArL0SPwPRXSe9WoHB$Af|HAv)K8#r&gM|QX!Vnl5q4TfCLHWo4w7L1vZ2ujad1HdHG!j8jhdy{ z>!6^?V6yO<7X|9$XJ*tr9JP5GM&8PD#@I?E08RJ(j!*Yw!#T2xlDjk5)xtrs0cDdc&ucSBM+8>weWmd!;N;K+;kCI zbj}pe_!{_azTOYc-K23s8K!--xOKqfT^PpZZ&F~5IhWaM`8K0GoHw}DfLvg>fOpK=@8Sk3_6v$N0>dAG!;&| zSKk_rS{g!7+wFhNp~`T#3+np!=stJmeHZpbq^E@$WXusI66OS>4Vh*bX-;O$LO@m@ z44FxA9Dou@q<%Vo9~o+Lf5ra*req)y`ONPDoP;POapH5Fizv?)kdl(JqGu^bgeVY6 zC}%O|_v4`mZ;UuK)EZ+oX+Tpnj>V|;IlQQ_-8OAVCAZ&%0(GTk5zwVhH)((le<@zxJf;i&BU-$ zW%0_&GtS0IN|NN2r`F~(U$z{)@bT!Fb4kgwE1 zncDUaP~s(zQ?(fMGz2vo!ZG-nB2hsW;_DymCuxvD`4P)^*FI)N%DOj$$|usvqh9#B=P;IZ4_#FqeS1!4(> zkaTd>U>LK$wx?d`^271X$4C9EF5zl-ym_^AZ9arJVN~c^og}~(QV?5N9YolP^bewp zs3vwNA21)-7fOic^dtwTQ**R=H}MahtMWKZicIAuZexuYa$dfv$@PQ(0Jh)62Mz0_ za)%4LyvDbMxFVE--F-tfd9H~wms4pa3*H4oiISBO2uVr(a(BFRzf~(6o6Mq*?C@mM zhY%_2QccJFLB*@2bIhX`#IDbFV-BI}=A}4{$A?TBYbgr=x>GjOxWEZ;6{LFo!0GO2 zsn~#q*kP7gYyh7CLI56c089ykVQ?|$4D6*-rp$WXEv?A42&aCf&3%=TFNX;!2mmlH zO#pL8HfQQc4H@lI?grM8bfiQT3j+ZmB-oNo$08s=7{HDWB0_3GbEqQEZ|iXO*@5<5 zm>+13i&gp2k3BCMtBBfh(4aw3I*$DYsF{K+0F;|v3BEJf38_E~!=AJ+a>j%;-0yQ5 z-ab*7PZ0rEYBjd4JTKNY_xn7o5}A{9!30lINq{XI#A7%4#U&oi=~sH|MtZef%bnu} zthb1|nDladS)YC4`A?T6h#@hxz!CtMj|e*xVS7YiPAzjw5*%!9)o;<M@Rk!hNLZQdsdN ziE=It38Z@8_-n0gIWAK292(NwB$>Q|o%vf=rpKl?da-F)R zpm>5`ZA1-t^`D}g&|`Zs`6-J^r)87Q5EPeCMkCw8sDR`p0ZR{HNz_y0-{R}ClQJJk z$3DX0rT$m6xmE`?Qi?!C>}Fn}mE-WT)B-~BCUl&?5VFBb+iyF} zUmZF;w|I8HuglA0=_3RYUFVn21QA7k{dXR*&(QkYXdr==?#+JjRCgr%^7u!~ zsq4AtpPxF!B)#f))8*&J7)St{>_@_GI!y2Bfxx+FNCK2&uJ8bZaCVM3Uja@t9|ODyuZ*uAnyh`!9PK@jn7O*0WR8k@->1GfNoW!2D3f3^dUF4QPBrh)WHkMeTX#xxU-M zjmG3*$oGeT$iNsj;mK;aZSCah!YXb_5zfSm%uVJX&Gs7|#v2K8=c#X}>l9B;uq__8 z8xVGziSxk)IdZz)X6KhG^YhZ^0%+3W3MyGB^U4Ws`xEZw1!}-6cwe5l)lJqyB?mu1Hm4(3$ ziopaZpJ90eZ>LVNf(S4mfgfj|zdR5@5U8=+mzD@1xFCcA3FbLKHUQXPmF#yno>*eR zqdBSE-|0voh*vJVd10tlo7r z%dBlC{SQ7|;v(s?hcQo;e(->iWAgb{zaHp%RXfD;^TU#-yFQ%aM5*r2r#N^b4>NXpcyPJi!CY2uCwP zrTWyD=s`PH91kH~=-TMad%)!e3Iq;cw=MTFc0Na*4=2rq`}?9mK^%?Wdk$svJhy@c z8+lqIar};7TrZOf&E$A#-L|Ev1qYwV{SWRiTy@laAzl32loS-6Kzk0bKKWrBi}>l! z!{t$^5&EfL$e##mfJqR3{i6#MXG#)^Sx(^4vxe!RdUP=4s%%Z>2VQ&f^7|Mf8(trR z2yjszW2OjjQ5?q5T|=m!TYm9^B;pNl?Y$qYR-wH8`Ska}3&bw`yzSozQ~@0#e~7*a zX;TriIr=qWHS`4ga{mB6TOPO|;v}*`S3qt3d0U(!l&Bx5b>t_@?~uU(<1EQo-`OI) zZw&^=tU!=vHa>)T`e6yS&v4f7)25DIEaBvo`)eN8@Pl-X&cx}c9bigYZeVkm79D;# z1L@**exIh?#NMh`@+ZUX9Vhrncu;n>_YgUtBZoLIj5B6V<)^8#dfFxzupk9ir%3_| zif=JIhtrlf^1hBULX{A_Mk5D>SS$|-#3f{!nlfIx07VTwXZRY(W@jIXPR%NA9!l`I zEVD4i%SwHvp;#nl7->aGEK-%L6D88NHqMm!H%r)hw#MrApLf zl3r4^Op_NkI}_G%@@qO-pP5Ys^&nU|Iy2Y1x<6aphmwpQPl7-f6mTg^=tH?~K;ijJJle6r^8$E>|qx<5kAdnU+MJ&gKGx)?+4cfpvaY2$ZLS&+RBA zUDQAhO?mbvP@mhZHX*;+vWZC~R0U&kYg&>>G*M0wBvcX<6*Qv^pNl^ZW5~7~%)dtiQ6&56oq=;3Fle9#6 zpJ!DuKDJPCek6iZZ{`4p8tx1AmnrQ)l({lz%u-Y?nTLfy63jxFhG4{x3dcibd|gdi z(`UUzt4Y)x9R#aXjzpW^ZQ}YqD=SpBu|k0@UJ~cv`+Zu`)G#^PCZpL>4zy6#g2ld( zlP5VsnknxB(4?dUlBt4Vf?^H!JuV;|Vdi0!m4cGt-lfBrEc;Tsy*G@(#HMHju&wD+ zTh~Er+=F6c1>sD@QRjKSS*Rfis#QxuNS5CSQh>COio@zPV)zI*<#R&`{JjCgjI>%SxUWkO0&*Z>1dl_HkH! zQ<{e%VmH)mkV)zQB$>X_Erun~07$hHL#xuo?i|oo;Ka#LQLQK})n*iODt4*s8lQ}4 z4wqeQ8G@n$PUg$l$(R!(&O}=)fSN+eNet!PlTm7&DSxzMagu`Naa`MkJCV(+a5=T3 zR?z28(Nu&jNg97p+Epj?sDP2Q2^x%@J#n%x2$_=oVF8>o3cIyDJPd0o#>|mBf@b~z zR35Br%EIxvcz220btwgE)Ckg&MaPEmc?85nlYOrkJWd45s0I2FQZ;d?YUyhe8GJrH zEB+u@wG<0{c4}I>#?Nar4^g?LTD3G&zl0=`4DJ0w2p~bU1HLQO6e^&APPVJ+S-=z* z)xe`%#7*^>d5Ay6mrMTu5?P%iP+XEjht@9ro#qN%CRYyCTk`FdEv6Qg_f?dxc$WJN zp|mwCQ*R;8^_8CVwDML`e?S>>xbQf6ti_3B{{Rbveh*aL4aMzW!53-du3mGR$@Qyb zgfZcCkeSFQh?L6!xg-#Hl0ZBwa3hR6PeSTH5o=qEQuXANX$nV>~NjM*e=^) zN}PYveYSM!5}5Dh}D>a{?LA@$*EgS11%^BEhMCn0EmeM4UC=U`0x_R zcMbmuM zg7qc$y)+NEec_Yk38`Wq=eXu8wR|H&@g7J#QrS+DNHKJ*PNA!PBt&xO2OCZU7)f;z zUSJYlt~J%RwTxt3VmADydLWkNA>6 zH!AT-Fi32F+ASb+-c>_|4%FhQ=|?Q}asUDTaN@D+Y;GJ}>K=|AjfEW2w!%2Zp?~f5 zGN?&u*fNliRD}a6fjUl+(n#JhYXAqWf8r&BnvqH|0MqNzD`OS}nZC%V18X&$UXFQ| z(mb5WD&GG9z-7kkGg?+DxZ6IfTJnxOqRLfGP;>H@3!E`nmmXWvl{#wj_8fdusmD9U z+)HGAR_xl6J1yDrCMmxmg9V5VKPRbu+m^=h`H%FK>lf)Ww$mK}U-+v#3^4Wo0Q2rR z4j@oP4y<0J5(tdTvMSuJyYW5Q0LotBcUIRA@q4vu;o5BBmyq+^KT;Mmx~%Medo!1S(-(mfgY6PDhc@)Bi*5S-yl z;mjvY;ZOb;#Go!?pj?1}qo@FqUByR(j`pNu>}BQ0Bp`1kLIS*6M%;#*m>xj&m2*^< zVl^a|pabE3bOswLN>r+b)~~hCM!uAc^DUu6Xi7>`5pqBV>I4|%CKMtpARC-xmbJ{b zga$4fa%x-0u2UrS*)pfu$)<`jVIiCO{(4f@g3E0GYAeVkI{b zvr(I%r+JBZ%1I!zyXQDdxk_d=0K>z}hO`_Jsh)L$;uon6G zq(?DCle&#qvvT=sVOMz8996`P(YE<7076QDHX&D1j0q&_+IgST`i3&uehE{ixJgr+ zfv^It$ZBs+xE#wInFt4n0O6d0%!jYr=NxOs{79z?u=N!d1Nv!!(80 zfy=76rklJ~LK$H%t2vCT)+v z2I4A8Bu<-;OUy*+QvfQ*%4e3!^|H%*X@ARS_M&A9$^QTrxt1seN|)LW$97w^E10El zB;pZ+mD%ZJ6>tS9czg`?1tnmacv-|al(L>d{CMT+YL|p_BD}z%jL}S^zp8Gfgr(Z5 zCmUE>DpEq0)6XfLDJTthh+)^kdBG3j9}0!`5`4hPIDB?ovRP>$6)OOn&Hx1?6ice~ zA)FJle`aQ6aK4@}_;9JD=6#>-CKJ^vXjw96BP;D9R#FQQ$AF*e)R0h-CU8oKmoSLk zkSwl(jP9b-FeGoJo1SEVVg>I8WRH{mqd-jQz+YD!o`An2UyT9%UG_a+;3XS2`qWp% zuwogW^gSH6(%mpI3cbKEU=1J}jY@)f2)LcGsh2Ji3ETi&3R0SBTIm~-))D~>OtA|@usoVb=sNpM&>0)~a!`Z0f$KMd$nUDTdlgU_y^65)zRnKvaVOgDJ2!IM-&U-6rW|r~>2-_7!olaT)nDETUYe*;R>W z1Rm!Pa(mt`Zq%d!Dvu82s1*SOkdO=s0FfjDU;!r;%%Mg~tGG4kU%y*KzO_y6_|fmq zr0fJ@JVBX;DO^xelcfsZNFeE6lOOtlh?56!ZVpcYBaUp8Gl2DaF>^+6Y8baS8pS-v z5+z~y$pHI7RSW~_`+)5p8-6GF!8}9SyY!*yrO28(q{`q#6U7bKm75R)%Dg}#_?kEXOL3d-jhL4|~mGiXb{kg;;Q&e!-AmSCGYjqbqB9|?D zh>`eK%iiLSG|c#s#Oh_GCYqG25|lE60#Yac076871gP^i#9h0XjK40CgF#>_K{mHL zU8|woJ&!}_UUi*gbGgo`bEf`Z3>f~;3JoQVi~j%#VjY+zgnJvql>NmcKp;s@sZi%& zEg}xlxsYNp<{Z*z9KadI!0oQ5cCMy9(UV9Ck^_+7Fh21FIA?l;)q~a9vR6_{4X2KejyFP*WPmi0B!g+?2hJzR^aB_DYPtUa)PB*A z^?d#xtVGORYMrwYs0ci?+j2mXsl9+PVI1vY61AvN+!ml9xGcmtBt3-;P{l1W<&}|Z zf+{-rRrKzSEyS71TMGfi$OK+2tH=``!_Rw0H~gat>Xhj$E=ouNwL4U}a(m7BUuZKV z{@Peja%eb%oq!H(3EjE~^dInZTT#Ru9-4%yOK&L54Gm9=Q9^+ZvPw?DY>-4sNCpoq zcfQJ~96YMKHle{T4;5}7QM>44#eS5Olw;-+{{XaaS5MwGNJ;)J>CWM!=uhy`oK9O! zPrr5&o#%20_}qz&xlTHc5V8TQ3KM-z55wU|@^5G*@lGx^YQ3uaGuUiEFBxCsAA&~~ z_H?Z3nV{xt!X-JyGQgFqFL?>3&DszNQUV1nSq8~8s#FgV$5-@Mp8o)Lc7{e8Vt~SC zU$L5nD}|>mK;h=7lE6z*JW8+zJZt!=>?*rg=9X|;$Nx~mW;|$ zctIroT6Br3N=f0w$zsYt1IS>alc!WDfkIMup zUc<-X2v&IjK6oJt&VUb$FqquM{$~U-7*+?LUoVyjA*CfoWhc;+A6yYQ!>rh#^i%wV zDnLlodF(mI@_ti~WIpN0!xy!o&m zYVl;BKR-LL)&r=?An(_wEG!Moj|202a-CRrNsW z6F)G1k%Us2nTz-QqIpsb&Mq=i12ZLwCI0}K#T#v5s?!%re1?(ye9Yn^t`T#{JX*wM6LPHX$ov#~uUvcRM;ttls(i{3UiSxMdf)WMnE+g8}9%2Re z=e`{THAHDQA0IprLlt!xzTZKD2niw%`+;(KgCEtji;-wJcMmk1>jW^w+GEZ?Y$Kh| ziGmXe1Xc(ksy=%IARm3XT5c{TcEJ&>)K{pdkZ)Gn7-Mtiu$k)w^y{}sz6fa*FH7ir z29&mf5P~PmrqF$U9~?P8@bCE;Mj)NV`~Lu}Alyu!ncj9Y&iCBmuvG6_Z>#s#t-Mrn z7ZL|aCS&)x!3f(|qs4NO@ra1)ZOo2-ys$wP=gw>g@xMD?k--E#Cw^c?)-Zk%>UQP4 z;D;uTbQdF?0UY`0A9zlLnB}oNruG*yO!M4efM{F`jdtV?{{U`I21&WF76xzYFn07a zaWjG(9J>+#4#MyCb|mimygvj<6@mzO>o*;21&^Kj03*|u2q3dR;E%2dAgNT?@Abh1 z5^Wy8IQ~AkAcaf;BXO|ufIa!0@WPgL2Pf`-SRu`w@6tV|>4ZkT-&i5y`w`{^$>=Tj z(*l4s3`o0c=gpw=)*O-ogxhg{L-oT`?+*U}k%uSV9sd9$1xM*0e9k0Cx;*M)M0=yo zrWBYE0l13;5n*`gCOMml!-xgRW7nR~@$QPV(ow)AB}oT}E7gJlZWewXrJk3C2blIC z{r>oRN=3zYHq*n+Vx2)_`o6G4+=)8?2cG``PQF-rebL|ZFhhc-+~Q1azn$>tA>ff8 ze_e-Ijm_|cYqy&VovzwKfNw212;FfhH%;q)dY1;&rw-xl!r!2;m< zZ5-fjc=hw^%LoH`75#VkL0(-nZJo5zl&jnjP@&giBzcXn5bw~B7*`%?1dyS75jW-K z&i<1&J7FzL?Y@+t@hzylM>bMsfaHQ5i08P}Tgin`JB`NvargHF4v4W5c)hfWk9Z4_ zzfs~xNJOh(O#0dfndkQ>1W|_tO!;)bGjBMx_Pz)qW=I2=osQjnutI=MfjtBKqNH^n z91xKGCifN-zu(yyBFE*X{{X||`b7hzkPVDnjlBoQc|Lebk*>6}?NI*!1NMt@r6y1; zNhO?~LrzZTechoJtK5j?{w`0?2s_F8d;osuZNy6fTa!CmbMl{lyyT1kbECK)AYW|- z8{|AjQqATK%uI6WJkP(D9NtAfP}65|xVLj1{CPBQ?!aoz3cFOyL50!wS)QQ^AJQsw zJe`FZ2g7BQo<>eM2jV*rWp)H5h!~v53s)6VWiFMuNzAcUA&ChK;7XNZr6g<*h8?-)3$No`<;5#js^^IDLGe1GcIr&d z))wC%EiuYUv9qQ>iMGEGY}a`GO?GVxMh^&zuneJEJJgOvg==2;)fsA4We{wlNE-{8 z1D`Meo5-7E99QiIsMWP6&YdG>m=6JCt=s3)&MXKmNbYXWvo|Tf4y+`YD(xSO!hnec zWSknLNQgccA`bQ!oMtB@{{YG0z?a5jY5}hgGD)MnzJ1oe^IAH zPQ#bZLgkS{Kr{?!OT7!YAonvHM10pN%kx$XZ?=`CK!Qk4tJvNk$sED3IIAXn`2@_C zrYztJ9)+l0(RyYURP_GQE?UP*nll^geo@VQaoANodYrAwmT786rGmwXRme%w0U#ME z@Sai#k&VA8?M&<~CKEbJs#7$z!&tFu?Q%CH>Dj+zF!sk{~5QWEr$X8><*B z&$9ERCyr1=#5;wc1t+f%bnmgayT;9wcEU~~XR4paEk>aeN2^ewG)mZzZBs_Ov}CXG zGAMYjEw2k7m#jx{wQ3`yQRVlokV%rQylHrWW7|gzCAgZ7!5TzY}xnT4F z_y}IqV*OqtFU%z<1xpUH;gO*Al^i2V>0KT+{$)~{%(N-82vQQU;vf{1+GfPwb_UQu zfHA60!yG;+g5bSDp*mRAwQjxpZVMEgUL7kGD)&pdP^A*9shm?*a!GjST-%7Mcz-iX zrCLyjK`Kn2(x?+V7(CK;1oNG{FTtZ7At69LOED@T+RpbT&K|anH#W?tWz&6VQv&7P zgX^ZfI#uJYJ3r#a^K7n`qrFL%lXR%b>L3#*^%4Lw2qxgpm7Qg(!eS>&2`MDdC>q>> zR(cZQ=;o2SWI4FZFw&-lBsI?v*P-8=R*|sWts)752{9280-zb*z(j~VjxYoi9uPUw zqO=a|MO#{PjgU;}eh1HVS#uKILM22i!)=lXAd3(|@mOs$@WbSlB~QYj=Dx{bL)GeT z;Oy?jX{uM5-rn%P}b+qG{0PSN^^cSOnuFt89CQHO_Qn(VOs1(5Q8H)koR`#*mtaZjpei1Vg zStRoT$ZbmiGgR;Z8+oeOzRSmR#s9&OG=8kQiUWA z1uKap@dC1>0In7R-Z_hkRI1;^U}Omh$Z5z>{IXK7{NsvLCXl$4q%15B^~W3}%!CGj z`Stk6FPLPcv1|@BYNjS;3?v>j&tjrn0xIAKG>_1S_BqS-U6t+3peg<1k`PY|C@U(S zc!vs<2qDTyQ6whisnyhlEwI`FB`FDjWD6&jqBTT-NHR#U>GSbVcLY6#j^@^W{cdCK z?2?`ujV*5f0GaRS4Xe=L(oU$0Nz@Fe7)%i&Ajn9x6F6`jDsb&XW4jPuq#dtj6oupV zUXIqK?CtTiYi#!_OM${+LP!uGfMj@qox#*h!SM4K$l<4vY8N&(Al1HadWwjriI5hS zEF9c#xY*gqt6Nyp*{)aJI7vt)DoNGi5#rn(fdGpDX3@4b3@#|CEak(S)rl7++Pebf zz+-)56@z8lp|CyvB=oCA!zHR!GJ|DXfe>t^CdTR@ok_SH>`j0eZYo+7>>Oy%cNDi% z&@+)1FkDjA#i1G!X+jyh^!mofrma$tQ>WBit z*a}_7{N2U(Ee}l6COhlWp!ix@GYN#@=a5J925muAbTt9#aUM_PFBNbmWR}{>^B2*^97F{o zdp57lv#M0No>sJ{>3H3mI@WTc%0J|Lx~1>C#~BC@1|DrcRXZ@Hp2+rnd};{%h6M5c zU0ekpkrjRr82iI%)fw9H5u?qO02iAqZYXD68t zwmU9%seg&i$3Hh!#?_iQH;Ub6r;RG8>uQ|ua(aZix+=OXx{96JMXna5ii-6-y((Lj z%?)GVgnkISwWfED%hCx&)_~v{{Z}P%O?KOKwxulNgO~C zl;V;>U@OE@ogGRE_lObdJL^IlsFfW$2_ke8C@LCsg%G7ed_(k@BVrHF#CzS%@g9Gq zebY?C>e{c%ae8ia>v{+y!7d>e2Tqb>sqhUYULXMnRi#4EAj=i+9;aGg_1kCCBt^Ua zf+SJX$uOHpu@M@g2~n_wkv0+r<}n22y#WV^)tK3-sdwp0R^n-QJwBo`dUPQmN{V(e zQ?-Q7!X%58Y-bRJtXWAQwS{?I=mwEEUky`0YHPNjfEtAItVvtpunCi-g&*m;1QXH% zq~DPMB*=v_^mEavwP<`k#t^CNKs4pe;|3fdaJx;ei(#u^si$$13d}d{jaus0J1!2CJ66#h+js+7YMn z>PNFW)U1M{RxhdnB!YPh0AV^~^;&snadUIdP8wNP+oX+ToZN^^?GRF?Wohkq+s-lC-m z1%28A^t+*6;`|`z2Nl@<(q+qE5z9MEhQuK$P*RFB93EDXQjmCnSj*i6d%+7SeiOs_ z{r=g`&CJOCH+m}VM41Q}e$jd_#pE-1#|jaM?B;9QENUmIOG%UJ0)Mr{BmUIcYvw05W>spT@; zBqWBZ`k$w1M;Yk<05wk%oh!pEpecMw7;JtNgTw2A0EuMk87d~mJmLMGIe^Ol01`ci znpT;J#>&KHznK9X6F)VP&cdNSwJTBxQOuMjhAyRWOLpNJ)3KMQDx%*~m~7AK5PU^2 zqs+)BR>CIZA0)d>kMUE11URFH+-f!f?;o!2$4@HDWRmPn9)Ok_>r+;_w(-6zt3!0^ z0?QBt5ow!&vF8!U7`TjTGG$Elo76cKaN_jV?|!cvs-}RFONR_zv^CdWtjAc?8NEuL zsZdgsQdMshN`<5unDj6LPp&ozGWb(KOTLHC#KszI&MB4J`T&2WL({%5{5FxSD&)W$ zgFD~JP-A0Z9PdjOS;3(%eu}-LBZ36j@}=rSH!)_XT1MNZa5j{UfJlf+RWNn|QyleM z&z>-9B|!02>S_V70)eSZHAQKzQc{?gP!_qk)EjIn@g4fn5>BZG>WgPBQ$Gz4G91x~FwUE=1}x zFi28NK{FTSjlZ**a&Z{h64;PK1{&BF7A|~EUXP=8w=`B0EuS!ukdW$j0VyBEO~4Iu zT9@DFr?fsOb2QKRpEC}rZ)W+s^vtLoYExbBbP7o~3TdD$H(80Cdlx3l#vc!aki-l& z7i%*R2+-7lsAi#f`G?xf_~J2H#wbl%u^FyBVGyL3E0?K56U3X;IC%9l@ORnk*%t+6 z)%CRMZl^Fns#sbyk{p$SLAsz^Nfs8qIA^4czsc^3(NjA{9v+~f~R^87Mo^`zk#9$IGm=X;05HBDUnVTxW z0TNKICSV&;@vFUEi_mD=$`-ey5E$H>h85`gJ9kDcnAD^tQn;8hOcA&Q!I+66WE042 zG7=Ctijw3WRHL&n`1R^x#7rOn&PA^L-u$iyD8wr>M%fBicsl6{*iQR{u-lvqfox=C z%2LRs{oP;4{*jHHiAZ zw+T+A1GsF2w^4Ir(bG>yXZ*gVPNsb=jStXOh_psg~zlDgWewd z-IxgUul+nY@|us2Gmg&l2T}QMG0D|aLYE3sKi;x^ZZwn_3D`{p)xE`S6IhQSXY4YaETTYUo zq^;ZCwQbBXZ7#rbKG0y1kHRkvh?gp69&QsEg2f@yeR46Ng|O5!62!A?#ylnCo*R3F zILC=NUaMDsl4jYKToh#|xk|@$;$(h@T2pSMpVS;lm{MojnHFR-T$?V;lHcNRXyYk7 zK+Kl1f(aHOLxnbKa6$QLpYZsfv^7alJiLUPe6T_J zX}|FJ{*gr~o0BKb_xAbW9CM}ldF5VFH|0_M%J_Yvl?jj_NHhL=Z}q?r7UQnE@6(Y7 zT_;4eOK_0RYF+KmVO~{$sa&UF*J!u`LEi^TKQEj(oh|ULjNpUxft3*9Y~c{{RQ? zh>0YczK+^`A*!)Vn@OMA-bVx^2$E;d1Q3?%4ZySg)zbko>wTL*92oh`uD4Ce#Pu%n0o(Lhj$3yb|a6tmW1R*igV-w|q2od+* z4=I5gY$KmZ!2~x15m+FEKMy_t^*{>0pL1@wAqmq`3w#I6h-8DRc@I0p&ih>3nAq50 zi2dh#fc;tiq;~*0!+P#I^4N`pfj8Rm9(G!&?o2hV^xmAnH01V*1Fxvs_9o^*7P>S1Ip^nd?;`{dqQY)A_agl~yx^P2I3Y&0{dE-cv7v>N*O;*}yxzvfcN~t! zErJT}FKP?xw_U9c2d|hjC(V1f&* zbU!cW1Q0-A;DQk{n7sL(f5^GG+imd@sHYsdSdXJ`c7h1m-uJY8@Tk|j>jV)YJ8HMA zpQXQrG*1Zt0TKwGFlOR;{rTW- z64$Fxxa#$%!=+-TB1?d@3XgaPax^_h#5TY}gZ+r+I`TK^hw<@yB!Rixof_k@B8>J7K}{I-vLJ_vN3?tN}N$I}SIMjN#cD|f=V zy-!+>W90`1WA?wdo)<^?)5mC-7Z36v*Jw~k{k**JH=Wwg(gxa}PJ+PqL&AxH%VRf_ z%ieyc*9Xq6=Rt3T73rYW{zu|y@4Or=$Q%7k?IhtDt=u#n!Jp$>Lc@VUk=6x&*5XhRPM!@xVT@C_FeB-+})SHXt+nVy8TcS+1hFK z0cotxGRKIK;=ga|AataR>U)dzk6CNA(n4ipXPGq6t3#xFLGZM8J z8;}Hp(tNS<`xL$`MDoll@EGLB;1|Cr~(9x(!DY)tlcI0Cl%c&$p>eNUAWEi|n z#iT*L;B`30u5SW{qmX?$claHP?50RuwyT?JZ}B?qqelDS{vkp;G2*@vTYy#Nd`*|} zT{|YuIhN*BP2?G)s04zfu~V021;Cjj^BT^PDjS|xC}tTZCO{*L4~t5Ym-fnWKfF)? zxk?!EFl5+~rzX#5e{%N22aR8|BOjKW0)oAih2gPbQjke+Z9@|=Nl(0>l@J*u#ioj^ z5EH6Q2;Zy&zn3#^XKZu4&kQ9knTe=+b0MAD)*9BJlhpD|#7;^=NJ&!!lAs;6P57@{ zwe$qu(@=aTRr!e=rfoKmMdWhpjd@bQH@#`sZ9Y-CNyKr7a7M00=|}P3pdg4h=C%H_ zs2G9-fdD`{ zu5bXurA?b9MRJ!g9mBYN7y?_BesIZ>CCwZQmVz`idiOB7*Q|5j8g?Z?RHfJD0Mgc` zR)w)Cxzwm38$zN85i&8f<$b2a;n@=uXGwP{nppF2qi1H`{dGLsvRFJ=_0kfYJ|p?1 zvo%`JyB>hJcO3r!pFPBUI?$rd67r2BiFS^z3W8l_X;W%FJZTenKh-y#=P$(IF?0QX z9Vkr81iDI&^nNXA@YXrbbA-%rSXtOCVVOG#b3K%)m2sdgpsB99M^W~3#jJ5R4bjvk zdX!m(CS1vcUpxzvS)wr_npgoun6YI4ET77;T-9&Lv6F^#l+&3t|J5b9M{~)_2V622bnU5h;NJK)ue!Bm+0cC7CaQR;w1FcBY!syi|4W zGN$rDKY;5aM?f90n-uz7Bh>ORjDey3*TY->w7h#mM&PDhQC8zXPqt6lymP8WlmB) z;@Z@komx$k00lv`cwnSS*jtu1+#LS^T7&IW0^pjE!Q45m05<{OR*erb!YWezVnf-$ z8*5K2G!{P%X0?iSG^ksepIB%RnO23uRRKm6KoXP?bsq@|*zdiB$o~MpJ8RhdwNjE% zkL^ZKiK<9&oE9G)hMkQ{>Fgy~SyN@L$OF}^6YQ*^&&qVh{MW6 z`N8FnSdhRJq)Y{B2~k2l;8cYYrF#GpfJiKPQjaueHvCHGPBQaOH${+8H!PvH(v>GM z%n}l>%e?!2=xMfs&v^;(v;jg8rF`%FUE>BLHOpnBF&Hoi{vcGTgrtjt_;h9Lz#B^(l>rDt2&}+Q`EE%X`X9mUM+f;-)6{yIvNr47T+QVsAkrD|O!lr3K zGzIHJs|{<@mYl`=e5~5H%baV>=PeG0G@wX{CifyHJm454N3JrmC-CYj;^R*mZ?Sr6 zG1KIUi2yht=;Zl4=|R>oP|BHjYfKQY>YZPxh~;4eXrDkMEGBUv2YR`$CT|z;*Gycw zYYPZ!00Yzp15?i67JYSWwu#tn5@3Ky0_T~P9U{P70TUy{NVOS>d+S5gdH6jTE6l`vsMR@?XS6#_TRAsPT`^;DgZA;);^ZqLSn_ zRGP5>gX!HY3s6h5aTKPUKu4&LxKzu6Wm*)g zBrZuLM=4INk_t%!djLEPlzUXyO}F_++|*GgXQq^47Z%n302nZWM2QiojYFQ`g##z3 z=xttN2@S^8{vV6jvvjTAf+mz%01^SE&mq(06WCC?q?sd8Atav)WXhl!3YhBv#A1;J zC5w%n?B?yUduwQwCd4@N@ew^W0U#3zx=yQ|jNBBE0)Uu<7r$49Jpd%VnxlI)2Iub5 z`oy#O@!R9dESa`$+YK#R(WyGlqof3+5>>vE5B~s&7#rH=9}Kyw1cJiA0@eO*e*kPw zl;=+p(B(lX-9!qW@#N`v=MFpLG^D)SP7qCr3JDj2qpBnugpib{oJHq$$}*(X%t_P& zL1(e46xrxO=H@x?Xmj9!>gAB^U)lqsgLcq0N!~farlsuCt`}P#awZsn*u5qn9pZ+;rz8mvR8tqi$NuD z`s~t>D?zAlD{;RYNhK|=v4;YZ@=I&fx{7e59V%JU8zo^VKDYcAW~v$2ZE{r-q|L%* zd6EEfqy61FeiY0|1jN@7r(a)vPxHA{RMXA36zCpJ*w?rV{vn4 z6rx(Cc9$i?Vz4q4wT=Z$n~LnfOqnG?a~7tS6tDr*{{W0u7gN~x5UHAGT0?oxS5->u zl_4Oq5%~&`hFYmDhf7LGZlAk#B}zl3DN0yzKD_jof*9V#XNX8qS~BU}Y2iFi`Bh=i zpc{y`HPyveE*;4c@+0t@ov$MF_vpJCzg339<9mJC999s@nv*Eyk~54BCp4BBTp&!! z!IlyU2Y{%$6T~^ov1*X4l7)JLiDfn;^q2TvAv;))@smT5#wcl09r$!C z>{_?1e#$!yH;*a8R?ZDwj}c8;#4Fe?ee%u3V{zvd$_*^)AcUz&2^JOrs!f0qs>BgD z0~)>>SSq0U>c@J$xrpB8&Gu@i(dPSl0IbeK+}b_AGD3VMPs~$o# zNgMzHPzkG0o?BRIZAv@hf`ZfuBlul90kNqKeXmgL+3yahAqgY`O^7BZ{{V44Hzq_8 zdmCYKqlhZ{9%Y=J-)>EY!s1oIHV@fNg~9?I~Sw;sW^<&3+yS7eWDXYf6Qyc$=R5T$DX}2_Kg{L zXR|4i=P5i`Q!D~NTBZvi15&T}S>LUEwd^aidXCh)L(3?hZEMN$XHi+0Ii!?0;c+T0 z69pv1B!fE~cb?Q|a`G&qYS5sZRx!kZ!~&4)0Z=TUEmk^PUm5y0?2O*qcDoE`p+_vr z2dth$ntl?iE<8Eux^N3?VPxfy;IRvt=NNIuE#6;7m9;6}qr`=pYo z0_F!h3&E2c-+qv7jaVTns*4iz+edrpWq4LL?^*jWmZh}@?o0hNdWOB4yEJK}8Br1X z$y8s&qDJDz@B|oyo$+E$8AaUj6QFV$=FfI_17SxmR%R$l9J$du&~ENIi&iy{5b<>_ zA{|q&A+SL(RdNJcfZOprKs_&W;y;727#Pr9Kpvuv}Kuic@5R0yc&{_mja1UByI@?IRH#vWNp#1PRCDLq^fMO!VO7D z01(EdfbQgI^*mFxo|G`yF*;nipKKkTKC@-eYUYQkiK0iq)I|)gs$~EmG!`w(IQw z0FHAW&$z#ac+V+yP1o?|b48iApy?`YhZEnhNHUE_HNL}M|CQFN66B#LD;M6^+ zO?3ytr1cT@n=O@&%W+aBjwIqJpv2clU~Sf|T3hQo@b|`?H2QVwuS1{ zwo*bX6w-Utm_8j>xf{)e@!j(I=)|T}$z?79a^izm0oXa$Qq%%G9ome#W>t!xg`4dr zNqKuIt-wtSG51~W;f#8hmOh%ynW9djhSE;v+EdK)ncP}-IN2nqBm;27QI5VIZ5+oM zjwF|NNgd9@qiSl^0z9SsVq~bwyI2{MEr!PuWO+_t>XMQYr>WasR`R6StcxIb6? zfkfW^pY&6WT>j0g47Kt8M;(fi6T}HjnTZ&bD25~=WYv&`pJhrMAeB-+SAH$>Iayz% zoE~W*1qZabZW4UyfGL+YOvPa)4k`sIT)YI3w2x^iQh10-0P?tUG=O5!>Gj->v(Ftt zhu@!8ze`7#ox88g=kuVv6gmuf+Gpcv_lzSD>|bD9_dCGj%@2j)Lhd;ZeERu%VQYKx z6#1X8$U?$uGdqr+Q}=uX4)e|Xx^kxBj1V%r+{xeAbn~CwoD?1V{a*V*KcA;xlNV?i zRQ34B%JFM&^T5ZDYj)3&p+7zP`9S`{{Gt;qryB&@j+b z;kLA`=>t;1u%BPAt-Ua{^X8gXy&;NNG!NJ3)}0_^@6^e`5#A9-1@1Zef0)4p2&kWl zyl=4fu;srzBY|=Ud=T3KCsa&AKqs$0Lu2YT#A=#9gcHmMZ;kx5VtG%^urcHU8*PpYfmp0q`G9Ih*#R9H}NT`ZAcx-l4H#6 z{m(!z(-44Edh6%q1xa#y&!2#XEa|xC*OVT6a-P3jIQtf>-T8EGaD}wDI2mw(Fh2e8 z1*n#%$L|C+O4a(529h?k^NZTnz2NSXXmIv}k<;M%?)=&aq7pzFcG~xjlh2m&ck72i z0$QfSZhm+mgi@wCe}A!p2ppdxe6MNvZ99>1f(ROvOmc!xo#TH<0CTnQwBhFBoPa-g zA+z`UVIYdZ1R*}V6Y%}|V1y;S3Gjx`hHvY)tiX+My3byM{{S2-t8?ODspekVm$~>>9RQF3 z>QAWKdG{WC@Q|Paz-sPqXXr!o+(HsefK2TdCM3uZPeC*7Y&kyg@A())YHPTtuD?$G z1tGinnY7-0p#0C48wnLWkH`)D_Q40AJ|-x{o0HS8E&iU{V1+$j_227y#S2^;@;CTx zxwibb-vlS0KdexWe<--$<1y64^uY(4dFAo%c|B&(XKrw9xViR=Z-N|8S^#NBF2$*( zhRqc>=O)JYF>lfhjCz0si-f5L%iV%K&ZGB8}))5j(*?18!uk(tPn%i0P7PcYxK7Q_tf+KQTBVFLndY^lqZJ=^abF17mM<)13Xr2x(DvPP70GPR&>P=?&Equ4kF$ zyx7=GjzIFnOH^D5ci0IF;`1kX_EensWk_aw0p90G(1alL17)Ql=mo0Njgj z8jXO$09wfeM8Bzm>XcHm0ig-d_o1gn&9ejCc_a*xB` zYkXf)G}Gtx*+&cIUKmTM%PE}9vqZM3@fO!rMu#+uFTkOx3dfPZ6nWgEgvoMAMW;x^ z<1zE9IFhNWKCM#&@gXWoQc_Dl5MstYx_@P_W3kfnUfuTJ5`>&4YmdotOsJ$EQ7;da z;XgH*ge?Bdl;W`{ME(*zTCmDYz?Dl}cwxq}z9P(3&+{ug5=!y~Wc015oKSWt-xN!1n8jcBPqCo1z*WptUaJ$*%z7mF9UcXNhuDhpMQXbuZH*q^7oo z-~=hRTQZPmL&$BwH6%t`vl+s&{BlI`{vL4l!S-3J?Gnv*dQjB3F|2mWA(x$H>P)f* zEh+rQ7GneI=6@+r6jgx~P~{e080`xz&D#|eRUh%KWeaK51k_PHgAP(upqWF6GLTgP zX*!c)LU%0x0JGy2AmVVzl%$hYAcB`WsV*)YRmF8;d8cMOS33cVIc_=8K0)>qHPEF{ zkjYyT3$>^mM%<*%>FN*(Pz3Hmq!g6Kp?H-6rVaUtH$0;imQ0l}g&}XTN|JfFXQjE+ zioADhw>FhE2Zc#f*aZR{KDDo#Gu*#0*}4`r9%PyGR>t6S20Csrj8u`Kev~@Y3-~l= z<>Cndr9cH)$OY+UxV4`3A&fXwsDDsU1WX=xl1+q25Dw;M#}RtA1&?!nrYy_EUy0Ah z0*R>A`U2vj zo5wJ4HAP=%d~H+1RYhYh%@(?xy=x={)HVruk+q#=A!tp2EjVDQXB|9Dm7iql3=Sqs zl-R7--AE2jrkAZuaLiT`e`PyFj6z@XIELa+x{w32(WnSu^*1Iu7l@cfXO_4?3kyM3 zk+CY7K^u}mRC5HFJI;wGj|+(af)vaF1R9nO7*>VpxiQn0Q;or;nMGdeL4G5l_DOv# z^lDhgDR6db?*{gwa@ST_rOe$>VMbB|@}d$2zgi~7HEC3DexQyVdCUa1%AIM*b_6GcZSn_d!kXO?-Syi?-Btqv9XaOpEK=* z64sTyTJA*{P=GE8clzlJQl5a&U(d(Cq+xmfTGdJqhfz>b8bAbJoL_Etn6Z;M$zf)$ zQuP%TeKqPE;?)fpoLq!W4pP7tVhCb7>f*Z+2|C8Er=a|ndqqyII$cd_%1Hrbmgybv zr68XYol_|VYiKkHbt;y$pIQDH@`Pr+u*7GxK~#>-V5d!!?D0!D{LWy(;elIlg(nQW z+Im4Gs%)ug0m*NapW2JtW7%&)m=3|xLU=_@_{?8Kwj1Y%R!zRzJNX1K-5lHqe^ zjNKidkuVMuE=1YWl%$D79jU_|t8n&q?sZ9)=8RNkJQ$p_D&m?4SW^hftLpM<%+-av zQuQ@E72lestZkH3A#S%zSeLu%OtnP0kD*vB-wBEsj3PwobF%Y0ifV#^SE(RONd>}H zDT$hrK|r~n5&7W9c4Ifn)0AeDh652RwYYpF>?Rc|lv9J1h?hQOoL3MDRNOu#GX%L4 z!aZEsp47CYER^#PWPDuSciSf)XX>bOp_eM@DCu2!-u2SkEFoG;rMAkG-@5jgLuml_ zX%Gbb-uy;l7n$eNRHOdPvn+qiC{S7bplQQ!-COjX{hfIdJ2B{!Cn3Pi6^P9pU&k*L zrOLxj3Sq{KMjsC*Y?*`Dn<;2qvQVVQdu92+d5{9ZF)6&=BcR-K+C+hSE;&{g(Q$eJKQiFn%dgM$-ORVt1AS_fKn8slWta$b^t;|4+tcKy+bWF zDFD|-Z$1~?#!)K*#24p&kcy}mAE=T%Qb#V6XdztyB~b$m^(ox=C;s1*4tG8#EtNqE z2Tg|HPNHO_Zv+T31cM@W+{%%{!8$u@_k?gRS}?rAfh$_fX_Em^FS&>&>I8F8BVv&t zoP@c>Iv(Ip(6jRmOT1883QL03`0Pfs*W}S2oGqdfWP%Th4Tzb%Kp+qQ01R5#nDCI! zJVYd=6sgZDmJC^gjftkv^oynrDJ9nwuc+ObOr-X2v+kTYm@qge_9lt-z@$}wx(2Oh zy<_uJ=_PX}`a$itt5zj)SK6HAX*1?X7*zqDW(sjFL`^6NNKjW1PYq2YdG>0jkK(6; z68X$^QTAP!)iUN~!x?V~C-U{$LPk@PuDRoID% zSFTW2J1IZ4nJQzH@r71R$Ym11&>Ly#{{Za%l!W?UD@t&{VKOY>kIP8JNl8oPq0*V+ zhze3t;ILWYCyAfZa``Pdln}4@)dpcw65!k^I%JSSghGz79J~;Zcz67x?h+=GDerDq zZfC!n#nFdh2?7*Ect|o*ohH|bl^Nbho_L9A)tKt^aTBFr+>Wn**ULolQc|5-l96M* z&B;k1ivs{ft+@kumnkZ#0a{VtU5&o*{{UIS1z)2Ww^!!vvWu)3NSRf@gE0_w4oLz4 zlj`{xV5K-o+Qi@G-J|H>#WxZXhob=W@_G#)acvpKd}fx?^JsN33Irumztmwj1Px=M z<=2p~f>c#Hl#oWN2__%_U=w%;mN5C2G^LR4+YwgOmuj&iZ5-6>nI>47hE&F8 z9@B$jL1G1tk4-m@k>OmtG8;Mtf|N>8_E~ za`{P=iUy@B*CLE*V(wkFwRisj0M(-};z}U2A%B`}G|ee!OQ~&Wsiv$CmenFzMKatz z1t}paDIsG1jQk_!zmW8^FARx`NX@%7pXFuA24wjYB<4s`S&IE;A1t3NH5Ka_H;@e?I4JIS()-6&kcGDZh5liD2jF2qU8CX|I7Vr1YXrhW!ns|OR+0#dK*lE!kf#RL@gJcEGuUaTgwzIc zr72tKw;`c%83UDtDO8lkng0N~CekJcS%5(zLFq{n09){%nBJ86H3wI;siTCjV@DxR z;YS`OfZnDsE+Wp-(+??1{V7o(Nh&2Wu6qD=M3blh0XWUzaGYuZo!zN>o@J?{miS>a z402@3D{$aFOEXrrr4L?Fx@-gC~^vCTw9g7fKg{wqNdm>P_lpj0D74h1ZhbG zm^k!TqrA3W85oRW(@+WGA(`HV>1qoS``^Ugob&m($(U@aN`lpj#sk2WI*2s{p+3<~ z1KyR6ko+QcYS#;Jn>my0MRsSVrPq{!twBk$O5;=#Ng&7($|o9Eq(0CbZZTMNDLfJ% zgoOeNhNBiNUD&f$?;VHGk7H%zSu!Wgl#k5LPv(}Q2N$h+r0d3R*Vt(t^2@E$vV$lA zDW4ZDiIQLm2G)ThFOD@Z8H%wa*@aI^R{ZyP^lYX2pjaATMg^!?q>hw~YdPY&N;cM| zK9p(R-~nPS1N8y{2lRq=IJbt&r_Ngh0!udvNhS58y&|4BHp5|Ll`3gQ;H`)|kVtC* z)a=pAouhXBljJSB*sp878q$Qe=BHLffRv>&vS!2tn3yC?W29s~g~j5DQsoefag-<| zv$*0Q500j`F~j>|>0c|#hpCr{h`5e2QlXi_tG|bp3Ryt zs+%H7al){)y+wf)7N{NN+BHjW*Vj~cwKWeu;Z~xj4x=Q=01^|ifo-jGje$y&DLfLT z%0LH@xaQBx>?2Y^#NuWZ%$qW3+IJ8qYLREXbZSyF?3{OHG6+`UEU32N%v{OyGA9^p zQuw8UO9vjwBb{D_m^HMGpDmOV$&R6&+#kEKv~A3x-wH<}43W4#5P9@B^BY)3p^Q94 zFwoFf_|MWhPFr#R05SgnQH;5D1XG1-k_;G)f>tMSeStURAnl5nWfV^1u9m4ZW4`@k zDUMI9!q&jol@0diTag~g{{W?L;syE)?cU7udhJXl_Lr9-=OiVk7IdK@R|T^nF<_Ml ziv!pDEsZJyS+NurCz~A@79sZ3N0j|55<;q7A8;B$VM#RvR)aunGQWtKffG_4Dmk% zR1hW1lQCpQ@Xl>U5+n=2t#L;70Y~(VdMnWRCk>p-aLG6zqVxq4rmaA|hzS&-sC^^m z{{Z7nGUVo6h{bS{qlL_+qy)N*(VOHFRD^);q^B^PW)JshMAa!FDO7Oovo}2?S|ao3 z>3+EFG2&h__1saY+Vg#@1qvI3{k^{JJ#a*h4UdKSY1j1x!-|PJk?2kc8+Yy`d>$06 z?YB}7@%{cI1Qj}l+#CJAJwXo*lQTCq`A5?P7W03-2Udm^0leH>ubA?SV1z0H?@>+p zj!j4zQ8VTQ^7a?o`N0vD68yk@JB4lcf$9ftTg|P^->2>7FhgOwzyOnRZQx&iW_IK` zY%PI_uy{hGq&kR5vyuRPs|+SdGd(Oty${oWsX{=x1E~NwR8V!IXNShOJdGhP_BN8Nl03-vkp}ZsrlCE7~ z55K;H5t3AG&1%wdA zE;fVnj1WU5Klc322q0>;k2$gauSnfH=C)Nhmwi^xwh1!IQJG+mitw9Xf^|_nx{(g9OverCX>3+L% zhWzUOd|oSgJ6~wGKc&7nY$rcHL{xlwdj9_a&ym3dRi~VD+vRdFK?`tVFTUR51X?dK z>B|Hyn|u94AS0N!KE9*R*vp%;bwXZz~n$CXVanOdtU@C^cxD+@4v;G945jx z7UW4A3lX;CV{KkUg6zVhYoF=kuSW#~KXdncZ)_1LP$j%M!>~i5ut5$^`x9|#i{I(j zmIxq#?atioaWZ~aiH}R*f)p9`1pRIC7$Ab6^6R&k_BbGdV1gYW023p5f%PElITJjG zJ262200=s}Q){*L`R@ch3f1T4IsX8ExV9A}mjqOKtPt{cw$nXNrQ+Xr!YwaReR;tT zL5Z<5{{UzwZn6gRZ`*7D6uW|7YTJ8n4?$<4FR*)mSS}wD#YFiTvW>{N>R|187ZNav zNLSJjfP1a|HH4FC01`QD0JIJ79K4O~he}fb4KIBvesPVS{;;@jcX3V-DSf>fou@4@Iko21Ux8#9(?*=etj@Qf`|>z zU8{X|hlNP<_rIZw`P$s@M1X^{>3@N58dF$xuN`{rloAvwq^^6cV zGtJ2_2CzVZg1zszA-uFkW} z(HT}-8A9h_aAr)o)0dkjS^*;p5h`kx zhza#d2`;phl@9d$Q1ERp0d_T#_HmN9mva6U%biYRRN6`tnP-)tIh$9g2-IDxDnmNS zT4|8VOaz3Fn|_pY+*fJwnVxe!U+oh;T!fQZcwga`+5joZh-M-`NZ*BC#N~dC@?Ot& zR{H%)>+xG{fxe%?tDOzD^BoIbAS7meQW?$rStuYFxv_a5o zXF7)?lT>kkNIjF8F<;(!VKT8m22c!Lj=GDQGaB@|c7NNg9&DDVs#L7IG-^{(m%iZE zr2-0-PNGx!mcohsFzaC|){vA1t2LW;XAg}><}kn#?d*GRs{)sBOV#^n#<4pHq#GCJ2qu?e1qs!(sSX5h7<|rJ&@88BWj_`6xfCD!uLc>Aj{?N=1_YbQ z)D6zlB2BvDvZQfSgoEIBv8{PVQ5cjisAMNnmK_{3o?CaoU(7<#Ql(o+HZlp7au+0% zGido?JyGZX07?7Bl+H=1APw#H`u9gTcC(f$rIcp0FpBKC#MQ9iOdV{V@Fv1^l8+ie zNKh&YkfV*ivzW=4JWCV%5=XK8M1i0+-$S9ZzKPo`wq9X@SV~%ShGo{IgKl=@3svJY z;I1TZh;lkQfdst33qqHuhX4p$L>Q9Wn%i>V_(`yUZRhv|%AA?<#|l)%D`rX4*;S2s zkzqpSHQvYOs>ERzhMcCKhBPWHr+#dA@*_v~y^uP^LmB-N6q?MkrMlMCfMrW%-t;8F z3L4N9gE8UMlVKRkc26DZ=cY`>0V@&jOH%?a!-R?!sc}*^A~3#?a)XP)D-|FCiI9vCmYv|BFD^8N0w9?nNHPY;tm9SfCSGvZ`KT7q z=xR^0_X4hUeHgaS*&NK`^4ucSp!+>V4)-@CpM!|lf0|aID%1%A1>z5i05`Fc2sY`B zQ`PX1N9g7{1guF=4nVt-Z(DNHTUL=Fn!=Qjp{S0V0tNOuKv5^$KoepV6==wV?y!g7f|E}i8Jb#+eTNz^c7D$x0Ng;+ z9HUwseA21fAt`dC7EjU7!`Ek%*^ft(b6i}m3~y+st2CNc0u*p45^$JwC`>6S3Be^8 zlqdx{vGWyTNc`6S0BBF7OlPI<#HaoZ$gz^~884&hL$UJGrHPW{cBoIq99B6hmBtmv zpC=R9>9UtHS)O}4AS5kw(1r7NV_Al2!##&*oNt`PCUx4Lahh>LyVq$u*4xeM0%%Ju zHqz3_URY_3B5NH1p5e-NXxT~~TKi54YVdhBGZ!^v;xR`VI&8%P85qPB%v#AIk`M~i zuXlyiNdbqIJ4b?*nfCtxEXig94ZjGd>*?loQZO9(k07J53O9u@?@q- zQC}LRB3zKA`&h4GIkU9SW?7pD(cdu8Bt5M3vsX%1RX7{B1QZH_5}DF(jFvc z`Gz`j$x_vfQs^FvHCE&Ut*R>@p|3!hQhOiS9G57PI3-}PSS$pju^ck+I1q_!4wOnk zv(k@F{^4FO{KHM9ZxG-DJV8dqB_ny#t>r`joz3SQJ2A^b6jT8M?8?_rdDF%{ZIfpS zo|D9+)NTP|zRzd9*gFW^R5_r1uu9TN3Dt9Q2C39<$|IPaqWa9~2aDZ3?%pG(hsSlp z;}WZshB}Z1Ue0`Udl;Nv1lYkQ-Xwu}w&LPN_mXV~6%3LtKrhB6JJro4bd+;S$d-~;>re{s-vJ#{Nt6XqGGb8C8yj) zE6-89Tl$HROP&7!hH_U%C?oP%5M5HeKx`UOSzvnS;tf+p26qwpqV4u04t*f@r7D?% z6Mw#Q$%HCg^x{etj>mAT*M8t3TuI?m5J^ZJMAu+;C~1HFrgDVPRAto^Z1!WDEwtqP zQ<-sYTTbGuGGs4KWt+0aaH@GrQ?AlD*-bhfWiTEuPYnD*srwpYZsAjg+ULobTp{He8RtO=Swm=5sTZ?AMktj-X_YBRiC_)LQnbL0xPHZWvf zL}8qO#GRPh!^=Y^V*(iIclSXBn1_A!ZciFpi+JrUrN-EF1*cGuL4srwPelQ|%mc_{ zW6LszSM39h$R){g)?@OkZrW~b+Ame?AxQx2u+|0L`jM*|(S&o(X`EGNFSu6X)vaWR zJ8CePg9#TMWXF~_j6NDngxa7678Q0HHMj0)=D57Vb}3VMot1CXLiDhMWjSk$S_#xi zGD1X~+AKHNl6grRnWv4$B`Nh_g$+xJl6%|6>^ViOzb{nDf?)>EC~C*-9frN`R~fMVu)n{HA2hy29lewjN7iV7%`ip{ee5OM%1&9E7! zPlX92fJnvWczzHGb<5!QqsV{S0ol*|SNKBo-Igj+6s$jFBxj(I2!6|F8DDAVni9!Y zA1aw`E8v>d9LEL5St%u!_ZTsxe;h7ip= zte2jnVfPCwLtuuULY`;^qv~|&ij|sYSbjQVu;OrK0%l4C;LOCDh001Lr`rye2BOjV zX3KEM%zG2qOvy4z=XQIv7`(AaOu2*d>zRs^ogp*P!WANZAvvxhbfiq+GZGasDO|J_ z#s%3vZYkf)X&X{fhHL6jl(w}k2un}6+LGCD>J7>v1yBl72w4hASItjMxiYYQv(Avs z>v1yi@~P~v^XQk3_CJ_DG=rcZy+I@N@%T9HY_8IJ59|&pGnX}Z9DyB|H7Zx;`yq*+ z*;&&Fej)4d*(xWXkPOAG3QO^b-Fb~{f^{84LJ|ld2?17;00IdA0JtUuUmSi`9m^Us zJqy*yrOjyOjBG%UUrxmnDTQiW0OYtdxzd8R)J3k6pq0T;xk(Wu9WbR_Ow3O~=X^;% zSO^3P*!8_Nd+A+FGYv87?kiB)rMjtsAlr2!2VJ#H7_pt8q^TTAsz~lWMfCj( zC*r+Q3o5`>)1z5|2bl!5Xz`icC#=5Twe8Ou#S1!jPV$JJmbWQ>%@>XXRnP7_o4Fj)H~XDY@q}x>KwAAx@2hz zQ+?tbCF3I2QP8fdM&}*nYED7f&=c(9C$L#GAdp2t0E$`k(^&EL6$%!&3 z_C$M9)NJWcbSjN)iH}`)HL6U5A)%&8LK|U$2rx8+B_Igd7qpTueqiIy8LF3z#m<<- zyDg08D^b!20twVi5R{-5bV(qbWinpM<8uYdP?SPJB%TEX>R6T_UZZ2X#v`m%3B(*o^+g^)9;c^qNSGAWve+dn1=+h?1;QQ#C+TQB7S%@4XFdBh0<3^p%|Fvu9!Q)viVfi!5;o$}fHv z;xASq%ZqQOX!1C~qkQk+_f{8OlrVFV;?aUF57O~>J6Cxs|Xq!0IU z&$Ju<5X1%*akW~JqG9_zjLa5i;wH|WI9*ecsxe~3jWwpM9y2;_2zf1q7Kn(6NHILg ziH*ecxs$dfS=+2J49`Lfj%M90pkqhaK2ewU_lm{ z=lWz#$4QbPMk&q5rDRPbkS{0~yw z%W0~-;+}=w3ougSsnQG(45%OpGjcJ}dpR_wWQw>f?r7`-mcJDVb`~V+I4`6OGA7`0 z!D%WXJVArFatetI;Jr_}#ofq{x&F*EVO7JKHF8%XqJo*pDv8wERtDl^s$*`pF^W9v9VQ*!)txk&4A7ii>fkUKFH|^dSmU5*!5~|j?HOtd2q-*J#>rWU>N@@H z&_+7a6dj-RKd+r*#-tH&M^H!33=EIk)DQ9PiD4j`?k;&xQ+xY=j1c}aN|vMZFIY`o zc?)@-Q#<{}2u>~Z6y|hyzm|n65IM(D@#c3hPSei>DNtdnZaj52EKQ$SWd8s-A&}Cb zIqmn%i5tvMu^1qM5>HQ99J$_qk1R2l{KL<55YnASAP@w>7TgJ#F&E`GorWA$!&>)V z6iG9QNJvu-CRe+5yBgH6+2LY$m?F?d#>PQOzVRCfJM99!ySUw{YHOvt;^c`W0=0%G z-5P+^CElmR4zSXsi97Q#e{=UI4Jt^pcQjclNblLaOazlR=gjTP(s!`u9I+b`Ye&@6 z&q#>9%h%%pD)Ix`dJc2Q4w!5tQjDm?pJUT~wzl|sl3bRj^nwO81p;OWH;#bEE~X=N zo6gvf04y7?JN&-TLx~*79XW0HC(PkshO6|SzIY&j;DQ=Wgb@eM{?UIU)20X^gzs~? z+V-8d*n5kd5JO3uo$x^kf(S#`$lI^q?}8B0x&U9{`oe00Kh3)Hw$Z%pYrzdF9nQz4 zeikEeVTw2OJiM>GMW@V*;lKlO0QmXm3{i7k2^70O*!bU-L~T0`cf21s*w|cdNf>M) z4euJ1<5HZj{$7?sv z#o?rbumf(hZey3P^MVqadHrIH^Z4&R^9TES;DQzmSQwx1M*jdJeuopd+<1s}h3FJ) z*8+#3`a*K&xRJ>3{C^u_2y7?c-`~>&5TZH3->v!$hsW%YdGmOyi5DMy_WEFg93n>d z*!j;n5PDmsz?%@DIRF$v>%JOuXRn^W7+##Vo$WGY@3rh=_cn|W?Tdj>{Y#L0f!-)v zZlGMmkv&G|rr^cM!3YajJx9UqV_iHd@bT|!U+(_6B2jC*Q1uYFA!4vW4zzS9-_GLp z*bjLzhbP`0{{SNdIVRrrpSyuFdK@okp5>@7(ax zqY!x+lkb8WR7hi5yZFNjqhNOFG1h;{{zn^#03+z39Khwj4DRVscj(b3?rfu4;8_H?2e3?XQG}(@M-nmgy%7`F1 z-4!GB$Y1J8YFyKDBZhi$&L?5AyuT^Ktc2qgHxY^s#%>=7N+rtxRZnm=y=ea4{1tXm zR%_aP{{XcZ=@Lxrs{@9>WiKNh)mjug(D zbtI)!?>bckl{w&96Bi&PkVoo?wdckiqDLMQa(D>_)GcFFwb|a44WO)jrSxSnQgb{a z+=7xgj;V0RiS1jymyeU4+*I`zb(s*`r6R7NctHRJtMw>|+^PsZ&_+1Cq#$BFSpaSe z17gLmaNP~#+yQgWUm|R@sZ8XQg%&NT0I;}n;_qP`<3+^%EC*XE`iB<1i2x{Ca^u$tYD?)b0lJ2Z!Bn33S5i3wfPzlKqSW9;r=LT(Areu ze?)$uWKF>ZL;nD6po?xXsN?dZEO7@Qn}n9}ZdP$&QY#zYQJyAcJz@br!^Q!6kw+$% zrq#S;S>+`bf0wx1jiJqHB!vJ>?nIM#HkdrXxW*1qn}wKGf>Le;UBfvQ3{IiG0EpUV zCQd&Mz_f)kvwK%0XfI<*YiQzr(z_yYhuq9^p7Dy7)?Gu50^5mDB}r0LjXFqxQa}K} z>5h$%cAZlH0AVMa+3TKLOB_)OR0@Vfksc6^SKiX!0 z!xbo@;vW4?p+U4}04S^gpat^h$43Qs);SL8*b zmq^9t{hOb{vo&b3ETTQ3R#8oQ6Qk;5r}$sA#%r2s#;Ssl>Ot=tT!jt6)RIJ*Ct(Bt zO{X09Htgc$tM_4xJvHy3qrD#b2}k_*&zj-7$djN~v-EkeW^vjSM`33~$F zOxH7&l>V*t5}7HKkV>LS0xT>!`C^H}iZVz$5FF6d27M1JwWB(W#Y#X0KvtlgYT7-& zOGXdGwcnYgb+wS9;#6a#CL{F==~qpzQzI8Jl1gUDSrq|5fd0+h>8Um*=8P#7arnbJ zT-1`_k7t`#m}k<~14h=Jmgk@Cyg`|8`>QBvZkp;u5wmVABB2e>ge5NMQ~*rAtCJ@i zwk|-$<&zXks|#??%E8z*xv%M1CP1rlJd)HIBIKqLv$0?`< zI<+)p8{DC1Q3g3_6Uq-IPBiQa%#2*Aj>Iq{T}gePPVSJ(gB3k13?2T95u%IZOfM+fJTq*w&sXp;boO?EpU0}=_G!T29N(9GXI z8Aer@=7lB2&`~`54W%nusvbfKVw@o`txr0nG~SVAIGaW@{g%kp+CI@_*%}WDvn+NG z2Bnwb{H)Q%h@7PwddVpE=NrmN;#&qML-e)Vd0vgZ8g}2eIDlp+vpEJoIm09Y#XA{- z!KW33!%VBTCmAwc6-Z}atd@`oQbF^Z@dut&W}Gv_-J0c2)ePUXULm8(+*(x2Z{`_z z%)d9AW)*HYrdxNvt#!JR0^6ffYSgv9y#&*JMa0WS8wY{Pu=3F&Le+%B;Uyq~aQS?+ zAQ?$#ECk6C)>sfQ>vrK(ep-7iGZmO-{i*G?dopGsc1kflp~hx2(ds7WJ*3SsxvbyL z&oej@P^t1~W>USeEAttIg#{A1Z;V zhEB=iJ74K5v+J2H7`bTAF*$~4xqz?M;&NcMVx}o(i~PnfFOk7Dnr9LitjtL+;EaLn zuOy-ub9bYCvz0{RLzhTC)7N z3Ia$Y-UTJa=t}~S^)5E<+gToZ?Hh<8QK0QGom*WuARUZz8`?$-8-z^6ju1#10Q>FH zfUwY1#^aXbreT&G@Yhm72YoiFIy0U6BZc^_FQExjgeU~X_wdES%!HC|OxzH61Zf!j z0hdtqJjf)S7+gJ=0xCIl{IfC>kV1$Z?WHerEpCkQv>42Tjet}XqtEn`K}a_y#K`c1 zdXA?9Hb6@_t!dKwzP|>%Z4z-@UhxEO;q~tEJ>UqIYi>@d8_6a{{da9b@rMh=SWkW+W*7nbarJ18YrwrR^surbR_T%n;{&nQ$da^L5ox(y&yQSR6j;pQHT1DRt`L%b!m& zbVyD~m5)<~^}mzC129aR#l)0?uw(^s1t!V`E)DA7er!8iHxW0s9j2cqb~0py<8gq> zoi0LUq%SZ`zz!`*-~h|^(gK{zB}r!nHuqxntIR(WLaAA!8L?-ws->EiGB+|7S_{r{ zR?@{SGl^j@QX~a9)lEGuB{5UZw$&>^r31o$5mSP@F^879{=9#C;M4$+sV<+%V}})m zEat|$$F%r)vkf5Ui0VJRWkeh)XAv&T@BjRMBm3yYQq0{hS`nHCzF$MYklsO+Im`IIoJG0mhhGfw~Gr z00@u@q?ytHjR_oLYoo%IS7X4eL zPD+U}xjre?Vr)`wAWq;|8|?CvCRDdAqg8`D5n_7Kn_4;cF;j6W0n|YhZB0oo*8q@4 z*Q^$YE>nt1iU0(H02cye2(gnFlW2+N8HvYmC4)T;LpH3%!8T&;TT$Z13oR>G?Hxl( z6LG0#9((piNtS@59U5TlK^p_4kVl3_;F4`IGmUQC{(`OGys3tdjw*-Wj zDItp&1W>h0TrsHHHm+F6M|aRfc%&Ff39zv+E(o`l=K{4YPV_sFU&i%om({BqejFt% zz*Id$Rf{!1EJ$V)cf4k_u8;&PVIl|sMWb;B1X#g0<%%-_r8oLT^zCK>n)By(J^bM_ z9Kbb9w^ao!1ing+l*M%pWGo~C zr%d(n1{LB2Js*hsYtLHXZ^U0>JZSotOO|nO4d?xpW|e4jFkYBTHQ6hP{ONHcO>3vS z46R{csrOlJ<#)9oTk|=Jk(gze-XzRUCl0I(!j&L|reO+48jyXSIh;sG6BR53B$5J0 z%#m&N#mEu(;6OyhohLDMKCz&N)s#=8U;XFZW z!?<@e{{U;_-XqJ4sZbRkn`k9!D>U!xOKDPETAc|>Qr@(vLLO4u)}=J4@;~A;v(n|} zI1JiIZ_MEiTQO<&k})_!R+)-YTmY##f~265%KV(9f=WS;v|ri3+ofVVBib&;VsQ)g znMP`y?<`Eiz^Qpga84Y}NSLCvNlNnJiJv(82*u+kAw??%BYC-)rPKqclK>EQx~AH% z2|L8h82|x~DJoi!rmZT~EmA`$wWtUqOs{(@^bksV#Gy%0 zDg+UIj{gAD2E_B2k$3|T6qJ&Mq>y=8_1qp;7aZ<=1=?D_Tq)D6mw8tSZ>o zvW~_CpUmS6AlKEYlBxg;g#cJFyly?B^q8be{GLq8Q-1NmP4zB7(#&plG0MwN67i*Z zs?2I?s+?Mh)Ztc?f~X;;$Au}9VgO0zCQdg$Ux3Rc`n)<&ir5efn^WPq4p_%DmxIkS z+)uB>O`klI{?RonBcPH+SZpdU8nh|M0r8J;GI`6?I|br zYDgOb3W#DsHLipbbZ{ugpk&>KWcta{lA;}pQJmB}Ykp2CbG>rj&#CCyNK*-rHqrqx zxfYSJu@(jfHEG$|n7XZd)moFK^zqiwz2Q4EDpEgkh9h8gy@;!Qxm_LMW*b73pc0g% zLI?>du>kBx{iK@#CSwrG%w|kx54lU90jGKHi>k?_p`9uO>>GU;eOS=ek&eBSGTelu zT$qqeq`}w;=kYU(sm0AuEL?_q9d_m$W{MNC*$EH8?MeXk@pis|hl_fs5QP{*L~KUa z_8%SN*Afg$)b|{n$HYcBV3MD8C%GW*Hi_&}Y)FW<-eYTA%*VWJViiqN?i)a8!@a6` zYZfKRt1;*WUyx&MO>NbZt%8=siwV35DbxTe*k1ejdE$IJh*_Z?HDcnt!;5G)F;5pO z*dT)3tT}`7*y*G~?BP#0+TR1X)}Z2elPfxfh>E0hfMk+3C(hXE9iAYnK)G;wQ3K}V zxW@|Z)iAiy1CPmY(!S6n@aoEDrPz(*?z6C*!Bsp9!wsoQQoq}1(z}}^AQ?MAkO0`9 zk?lEaDpa6o%$rt}E__JaTgQbvM3XNSGzyaD$5I%z#Q<@kjw}3ERJyL&^Y-16DRSJI zrn&^Vlrs8KIZB;%W&}>62cN8c8=(p8o(QM-=VH)b_*|iZH7@oCUysP|#O@|>^tGEy zn3bwt4<*Z$B}qvrQs(9J{aRC^jMnx7R0_Z$B99#C(#nznHWrIdIpyUs%hMb5qEp1$ zwfQ`G?8_5C3M6ovGivTD&>iWs!9tU|NduhPFKgTp<$sC9iJAzgBbzWk7K+nk@UoB; zQeKSY9W8L*Dw@NBh~+!{Q0@Tzq0(d-j#d@Uo~^`3+0{NgS%u=xGqc+SJ)_kfjJ;DBv=Pg${#uS_+F zxFfn56yl_aGq^Vgco#hW;}2N{n1g?$5ET&y{$Fw+h@V&^d^I4rElQsc*ULc<)J3dq z&hzK%>$QQ~6G08Z1Py$>{{UQ?dVvHWUY&mZ}(|?zt0p;g^()4I}P##w@PhIvOF=!{(1UxB*wR)RBCKh{*9u%kx z@9B8^iHmGcED++Pg6{9l*i-C2Fz=QKcf+qZtPn!Y=EDBN>w*YyJ6u>=d3wm_=qvx7*0iF&Bb9#jrsJN~Xtut_UIcAc}!JC&EtmgML7r`3|^*t<+d3v0%aev1U*SmJ(7+hfr(rRgW+W7uo`+hSSz% zC!Be2Z!moDoJrr{e$e$T?Wix`_4q_r9Rftn`COF=>F>OG;oaa3$pCo61CqcJPbWWk zg0&SN5h()TSU^5{Z?tVYdCP>vfku5?yZmBlQh3U^$wM)dO*DePjOsPCV9~ z93$FY2~vC3^Q?-gKoAPsc8k1ggh1lc~QC8*$6T>v}i!sb4x_8M}fP4$Ic z<%3gO7B0=vsB7~H+H*N}bD8D?!s1r@x|<;~7NaS*lu0va1kN~bXtJC}3Td4rsCQL| zF>8X)sQw^d+?|N+78^3cdWqZ)!i7l&!mUF>Lk(Kgc;sFy?CZbp9-%$Uz?~%of?`PU z#mwG)XUi9X&Zc4J33Gua&1a>H(xt7Y*Nu-h?A{X^fin)Wm-kxcx;OxUX+zYSMxr&m zN5Yi_73{p_J7zoff|MXA0HPvfjpK79av0guKLMFb;YpV){{SkKfc0ZSU(Ep(jUV!v zwn;OIK+Ye`lqovcVY3d; z&fp7{go>~Nrq#B)J$cXw*|J{LOG>yWOj1QCv%*a|nu9|?S@ff9nwMxZnrTX#C141V zaz~0J+Ub@ue$P z8Uv2|HKWW@>!@Wqm8{BBAQPzcurdVPA3=SKD$gfj)xep9Ku-_{o4F4Ay#ujx8oqVe ztTtkRu^}7S3OgHVwYRSmM>ac`=iCch-ct9jP}LZ&q)6G}5x*gbV~9B(u1_B$b-XJy=KmHbFjRO4FGfsVR>b`igLBaPY2 z*lA5ZHH{%?DgneO0jVlZqM)%Rck6s>dA?E*LeL75-v0mz;5VT*(3|ZWXJ>Q&09=j_ zSfnoe3M{8L_DKq>iwX){2;9{9hkDphT#_v^KqVt?gmVL#KKgv@FRN1kdb3|j+Sa{^ zt-6jI8|x8L!A<~hl3Bx0){pgzHfs9R%F2{r0#4SHkRW(}SD5L_aFS4-D>j|tBc(!An|KL92*ud zzJiuc7IafCp}WDTSur|6BuRm5Dr`0)d1C%(4B2w2yV2Q+?!QadXu#x0)KA7TgB)2P zro}spIK5~K);6s@bfE`Ph=3s5V5r*H5zw20Hr1CT15;b^I~J`?>vyGCF$`mWk%&AF zD&s?9D8!8@`^C+=&7neMSdvL39f>;_l5cPf3{{yVf?X!}NFZxM2=u0;6LsZbX9e9i*lL7I@wJaq-`uEwGX2C4r_zYr>8T)&NJj;aGVmv!W5*; znZ)HNPY^6>sea{1QFC9;2c&+%XMT>dX?=mrvayxs*}h_NINUxBb0=XjxGc3O7<@PP zguvdqVp$TgX;=4)OHZl8l9Ytcs&U>w?Kz5Og50HIr4_4Mm8mFbDaPr76cnVTu!kH; zY7$yoK?ZaPu_SfJm(YeIqb8ghk zlujCQ(}T;&mZ2##F{N3iZ;X>Qyxv5WQ{{Z3UDYC{{ zZ2%M?0U#wLfC0CbkeHD>#7;bJF_R`u^@Op#X{TCsyOQ;fu?ovmFAu@$(nINOdAG;D zj~ed2jp|qRBmOiKHUpW|0Rmv$?=la^V8dd9(g6XOk~TE3Hw2qC9`rG)<1ov^%%D6i zQ(J}|J~m?)+>1VXjnoGTgsefZ5y;v?l0Y|&q!469TxJej#E`)51$QnlvAd8(3V{YL zW%-ocGQvqEBE%PF*7j{a8bJUjfU==tOikoT+?c*F z>Xg?7Jwf{Y?Hd9o@i}zWnc1{7W)HX`-SA;l6f}ntVM&l7jaEgM|cba^JAwsi0wi(DEFx;gQ!eMHU!1B!i2KmakJKt3$*=2fT-O0CzbT zC94^PsIagOo3j8aTATinf$_~|jFT~0AQMLT21qGNSzO5kM!>4#HzWcvkji9}ijn|# zAs{3YN|i*Wfyp%qNCc4VDHtx$O%igfY|bDE!WVJ_aHZD!aEDRZEK5h696a+)@McQ; zdA?gx?k^I;t5Hc#u=P7jeP2xQ1BU^E zk|s|=QlIv7hFGSbwkYXBs#kh`Z8N#PyEe_SNk$O2GN7@S zpjvOw{{V=kO*t+KROL@t{N@?IF)~co;lX0~ihWDD4mhh$B)d3&dH(=s^t2(wQ z@KS*6-YF{lw5jq+RIKMJQc$rx4iyxI6sF}VDijgkwH08nq^U_iq!lX3gE6SYfx7omZ^$z=BBNGf%$x*(u}wT9ZCd}R7}i)&;bO<9dkdO$7-)rKO*~Vir7jyFm zqtkbcwi6Bl!5Z?RYq+%oxYP!rV;Ri_IBX3<1VT(fk+3pA5Rz;*kMT5=O-ngC4>S>!3Me4a z#|W*Mg7u{-&8yNb0;NC=CLk&jWP$*hlO}oty}YpzL0KiJbT0dUNVDq;yN4yNA4cVU z+33+8wxBym@(J**4z{G5PPC08(Kh5Zl6uV8Qd2Tg*n%z@h;z=xsasMhV-@S<&Xz+x zjr>?#lY41#Up9Xde09Q{QIXMbzI|O^m+?mk=M;IKPhP`qBC3|REU2nQGO4OR+NvB) zGTNwHt~AY4E!3_TQWE17uOsOnXFVun+23R`LzpKK{pEwg;br3HNC4s&i$t+0Q^e3r zrEyOJYW-k52pmZuD46``_Sf)*+s{Y7nmcEW?6jKhKI1yf(=%EZSyy7{O2Kg|biAc3 zm8hvpaFuD2hR=bplBKD9lhR&sn0AUpqdSC1!{Xjtx_hCA5C*km`$FKp(e0nZFJN*{ zMfjXltO8XtHOwWH=DD1}c&3GM<3BAXSpb|koVBGWKqyLyNM&2Ms53)RPTHg>6c5x% zNF)PzB;3r#;DSs+$DF6(xKz#XCiU;9iESh376%+=pfSF>7dlj){(V~T)qTXR1bGlI zaS;)*vE&ap2Nf5Gd(e@4Qk_PgUNbUrp(J|36n7=tSKpwk5bi4FUTzZFg4AS@{XoP{ zqkSqC3FXYj_+l_gDxfEjbGh1#EY@vn4qjwJO15m^@O=(ut)15<@(zE>LT&WY6>qs{h+>@=# z)wSqjf;&m#ANTGcMIU{xl7W5W=42`3B?&kFgm|D2)Zhk6XB~ei!hTB#89%o!IJ%pP z7aKD;G3KoM9IV?nQkF@>Vd?{t3An9A?t7xvt|iSXDASZ>#dP$VLKYh(D|%onQ3UBp zu_;WIb2kGGRuV+Dne%v5ZYUTAtcgB z=qxqiE*;+C_n@xFmukv;Er!CSl_?1S0GW`+^(xR-q`NsTYa2zjGJLEC!9#@t$!$86 zpluR>U=kz{MD^)~QJ*zl(ztBgUd!BTPyu$cix#NaQ7;RJPYk(k2B{oHsi?4^dvl1D zHO{#yqoY!6PPNF@Cgu!op1-~)l`cwOiDdV?+na9@&Q#tX*~kgqJPH+K{{RRC&;mRO z1ht9!EXJg%#|nZ*r6dT05+Hx=+mW@P12$#PlmPW<06iFnchkS)80naK%Pydx*D3^y zwFv+=)jdnbF;zU8=-8S?PPwUFlqO=?gXrI?*KZ z8hG8g#knk^f^2bgz$ynqhk zFErGW2ZwLfXzSM7gKdw#AXMW)Le*f*FI{UF5rs&w03KW~;q@BCOwOGMX;%WoPTau< zqzR4h7`K61mZmBWTZcyuTnh&+-aZ(7)hI-=RKtZXbT=B3%>PUgk-6tJ)vlwnQNgxuCNa6v-4GB6~9}vUxxhYr*a&kjJODDl9{n`BS0MIt*6Q&ao8|NfgA?a zy$gUFVNvv}2=Qib`)QNr9Ss{tVs~)n``VGBzZzUMQI&R^mT?G5lS9OL23b%ZBmOm2 zWovUUsB9>BLYZYOx}h=>@>HR@KArSam?IIAVsJ&cu$Y512T3TFwDEB$RHX!@4l)2G zObfJpxcpvra&}$Ye4Dh11vosz2Zcr;6s#dfC!hHx0r-?}b6C8+# z3Sz`Sl2x>bzxns_>Ger0#m~F0z8|c4M1qhMK}cr=h72m;nh-AbiD+=>0B(6eAl`N} z_HBh@z|3D}t@eindo8a=)!@ZNLQnmqfI7vxm?kVE>n}+uPgJoQ+vf>3P@0fQAb?0C zR~92s?04r=6)67z*Uyy58%3;7DcUajMT&=eBx|{Y z@~Hl0d_K^!cb_~qEkKmG*p8&w;x(>LJvk4px%qYT#Eb=lK@9H)pUaoN2qC)9LG%7_K?t%Wo?qJp5G5OkB5p6Yocub^ z#{>{BzrVjs5Jg~uD+CaP+kYt$Z?kVNE$~BBXlmLxt9iFrX(sS(2h-Td=We5wutP~M zSiK49OM*NB($L$<8l;)F)6`#>n~&+4-f%-I=EVHpZ!R{WW}ic77apsXQV*#-+RG4=i8Rpg^uIIecH?o`MSpQe7Rcj{{Y-NVX#2~A}?{wa}q@Q7(S!O;Ye>g!39d^cpoXk zK@Y(MP>V!F&7=HaVX%T6AtFSQI*|r<>9pJpt-csi)fW~TS-SqPLZNYDY<|Cg3-vf^ zRMa;4euvHAf`M@)$34G~zg#tpFQR*XUh&@u<#J(8xIIudO)_^;&}^Ld5%&6 z!iE|T53@vbB&J{hxMEqGR)3K2ijIJuwy^ol{{U<}DMsoY;19f0og~XC;o?v(^mZYx z+r25Fgd_k-`{sUAiDCoi&qyQ<09=!K0Fkjgy?`9%M%aVaf~NsQTH1N9!Y=xs*aWw> zeZTWHwzso(dcy+BHEKyWfD#I}nVVYvxOpK=#0K7%+WSIE3X#RelCCyyRydo5-QAns z4ZUecG^Nev!;mw6zP&cYZViYrY>{`MyH}(}QqHNc3IreI?kp{6iNi=`K7=u9)N8jd z7>KC3YCSrSkHR6kzY!fs0y)~&iNEpgtRIWbfcZa+Hui|CQ3^6lNsXrcE;hKlUQMrv z%AFu;ZZ>ZryW-j<8FP>Fr`;v1#29j*Dh0@>2F4+}?vN*$*jjm?hc1)T42g*(xsKfY z3USfMyF=nG5?ip2EaCcu(W6)z^GuCnFbDwnip4Uk+kiD|@SCV4a<5ETVa4`y4UR$& z9$Se-)gBUBRxjBrCq#h6sD|OL<~?WlJ?%0wk4PB)&P68c8KE*gVrmPqaMTqpXhja9n_50(`$Xaj+^asUaYbdd zRILsq+yIiTWCJ!J8>UU+LgN~)YnI8zN+DvbSQ;C)x)vJ?M{exSdyC6f3RX&MP;Sg_ zXnbAwu*WF)u9~?C3Q|BMl0--(bG66)yroeSY;v6Avx}23Oh{0Amr3wqC|DXlfN1m& zXS--ID3dO431)XtEc7k|w#Mc44GCXEm*%KaT@5Kjf7+cY3b>mML<^rjaxtrVyjBit zf}yhp4d+HCH*tu;$>b<{3On4^HCA!iMHN$p{JHliUL>e3l@tWtOr(t@ zCOj!AA}xGnu=)NY4xd(DXx?VpfK^K{*}2~jitNq@6s{zdNZ-|iQ`{JLHG7)Q&RNI& zDzOeVpHQHkD@LYj(1h4aN+qS3wxqb3Bn=XiHfLrzMmz-1lBqKQ{o+9arlIH_yOBf8 z#T+hEn&oqNu-LE9%hXO+sh<3nT2OKZ`@Ywzem6tu6#0V=(mfYNLrQI7LuB~CnVFQB zw&xTkWr%(}GSe{Cgt8WC{{Sf`l{IQcdk>vV;S%u@mn;wm?E;cCVeK#s&u6ami!OPW zWx2ynBzH~+H3WeNNrFTUTgU(z5q@Fv7m30bJV{9eIY|SkaMa}e#KW-?uz2*%mO@nF zhYEpKmSearL(6#QoYy4GxKA<}hFVq|3zr#5ggBItOrkf|nT!fzyXGA9^J zOPPEn$qX%U<fI^L9w_L{VmAyOx6N!*YP$vsTPtvJ9= zmOvaJ>ioW^t?73nHn|gM{?CYQeplA@1}-S83MnN@fMnhYN=USWCw<7Uh@4j_AuKh0 zz#YU;Ir+uJ&IK;^ElQE*;O+~CtUcF~`)X-Yq__)e20Te4NlBaBZ($^z#Na_tB}n4p z6=LG9%~tQxgXsx1NKz&tL=j*Bcdnb7IQl$s?`S=YXDnv5xt9i3J3_{0=XqU3f)z_p zxKT>ore3C8S`xC9DQH@f6r_mMtzCDr-jQ+nRDmgYmB<$b3n^<5UX`idwL8a|JtcYz z?Ee64)Ul4rusG}_(Zh(t<7La95}LS^z?DuRj)jN}1Z>{KyDq8W!&94QPO7Umu25UA zx&oB|At_DAgbg4iBGI5)^Q?BmwNtbC<#VRYm>jtWh#S?xs|Rv+aATnQFZ64db{jAJ zt_uZ+!pTTsDpFFO5G=?@yJ*1Jv&TtQH?2AXPLLr0-T*)Y+}O-S&s*rpSZ5>ytCqPs z6KyM{4VZQ_D~XU=a|DoWje<}ULcO3Y z>>L-UBB7|qzQ8z*LS&`LQ&bHNU#86A!&62Ev6tjI>1k;NMIsaWNFsUa1I}lZa=hak zjlu*Qv}7!Y;oOZU<;pPBWI#BA+vpaA&9XKTXh<}%pJTiM$$*0t{$8@7D}th zyh;5$VXDMAdy{j2v_x*W7hWWZB|b70a1^vcN{ALRY=A)|Tt+8cuVbqeunf%ShQ8AZ z9!^HF0%wCZceIksOxQq{8t7+nXSbzSl1-m{25}w-TA$F);Lcg{{VM$g?MnX zuFpOdif?>RnQ(QtD}HR{B&`^nVP5 zs{?~itHlBm%KrcrCYp!#oLrE??BYK;xYeZQxrD(p2uaLJ!v6pvJmNS2OOZIGG8Ts6 z24@p7SF~e4{4%{vp4s@R6s_cFWuDLSrqrGu&hsuJ%ZrCFXH$x;&1$NvMRX_s065uo zEhWTjY!bD&=bXO~CP~Y3!6jHz{xGn^CSnL|8GmMFX?FNM{)2u3Q8adswKFGvWhyyD zw-~wdW}4#gd@N+W6i~LikggZ3dJ9ohRMo(MkeLHgjIZ>;Iuu2K*C5Z4$CD8++W-)! zE&~l7{M$=AN7Y5mX>L#3O}BCJh?Vasu>`tO$lLP zTU_xtbMS~~D3nyOG_5(EwrIkto7ANOQ~;TXzSK3dei zT(By^=-fx1nm(gmad@O`NHCyGN>yV74a|)<00@}|@e_#Dt^rMTA)3O}>eV#nBJ_z| zHD)$QVPjYLotQp`GJ17@kbLGWpHpyTNf(*qAQ2eJwc={p)!H(W?5^)T`gAdKZlt6D zM#pZEWgcU5{{VOCg&vrT)vIyi`bD%%azdCN<_7FJ7JbOjkiw=4s@DK1v=IV2+j2-G zo^T1Y=^z|D%S{-P`#0JC1CzojXh{HjMeGzGeF5%nc8g_AX$V|S)dCLi*vASB+c+~tQx@4YyNW_J}l+-xJUa!k!greqZv2?>&{3C;>rGvbIIB>}bUbz4Y?B0&I) zlZqdc3aJ2cC7#*=PVnz>8IRAXe-gE$r~cjVNVQkzqq|hB0|W?Byn+Dn6aN5dB!WDS zEXcv6Dxu5JTv4v!?nOCsia4ywVo>~JOAvJ*7XXi|v~$O49AugbDwhk1kcA{9fEB7s zX-G`&rH}w7b|4#(x#U@5rp%N`R|-L>3^q5=X!QqD1af}Yc7V!}IU#AdVZ@#yY`{B^ zEWj5ffFxIwXx*vJt1Okg{Y#~42oj|&WT{(pk~u&l$YXwJraR|k`x}UuWlWzW5Z=)!ASQyZR;U_{?9JL7 zH^C^&DVj?(?X;8xfa}pRV3L%DK)50|-Ah`}AXUo3*B@)Qa?7%$;blr8 z2$#atQzQ#9Ahmoey`wWZW?aArFs~~Bf70&~$^-%kN)n+U-*8AXzbYwSDrhAVK&F6@ z#1ks#Pe z<#H{1lCq3s!FhqJ?4eB5015{bI&Lf@7m4j2T`AsTQp5O`kj?`*Q3tzSE+Z1HJ=hQm}yQYb!sbjYOF-1w7yYNS(kuv0&vGnMmSLCw2G7}IZ{{Sfp z6y%$>fL~5~EQ~s4$>Ug3NYOxtAXeer@)R^^ko7w>t%4Mp)2?JdfNVAvnHMoPI7BIH zRUDSl&br>4^n!SmAqR-C)yNJ0oi68YQLOl=3rybiNz^4l=fGc)j;DW-GYfuhP9wqJmdpk0k5QT;X-P~;B zM>VLX?hJh>{vWu&Wox6SdBlQ*G?gVvB0MCJOp|$=%xNcmxvoXg6Dp6<1TC9r{IzS}Dx z-Z%cm_L*Is&M>poO0am6aJczT??_z=h!)}Zkg5j&6LB7V^xNppU!;!5V{+u8T)f{a zRGiBo#FlktI50wR^N1W@mCO=9DT`0>OIqg=#4bFGsA=jtw`nSyQ*{iure0-eQ*N~4 z6cV=9(tuc62?<&jok37Hwmn}abgV>)c!?z|MAV6r<|L(Pn1w$A(6o{il{d1ak_}kL z$h>X}c3qfG%Q1*toNgL-sKnuBAyV-&CnZTjHx4A5Mh+-y?>(-1wMlozIDZg%Rh<@@>!KY$PN$=gi=O6a#sffgYw{ zl6if|=JcV(TptgZf)m&eKnKPJ@hJe@1J(#lL!5ykY)4(K>*az7V#qNDcjxJX2oWHE z9Q{2{ED%B!fxWFGm_D6HO}(%|11kRjCwYN;T1O+(-bf+1Ad0~RRtO;p`Wc)0Pp?jR zA*9?=#{Ixscl$$T$7$c7={v~>mm_VkLmq|Nu2@=>b6s3BT!6*lIn9J^)1>o*=6_9r zoDiM>X|d7kyK71Z=F?y!pIG1MM)5Zs@Ip1OUPqO;8*A`{lW3cM(tcBd6VIO*piaY| z-_Q8yFhS?fo{*Ta2cHO2Cv!giexJ8E#dspUrrf%FbM^PZ1S}47u!*0Qji4BxKTtx` z;qiX`Izm9b`Tqbun|fe`ijmW=_V{05FhS?f#uJgWj$`BN%5MfRLV!&GBGbSwF%k2Kd23~EvzTZ9f%WtvU&3Ikpl!Te?zruTT}GhG&v`IqhlugfImsS`CoDz zlHDi?qtbx3)_d~W9XntAee?Fg4|`NsB&3Ul3yxg5?rweul;RQqu?M6O zUi8iM@_;0teWT&)hw-V|gY3oraQ-u}`2FE~!{zWJ_rh_jVc>r7ah-?9?+e~}eE$Hp z7uIt4{*b=2m&f#i0-lL6ex!Ou$F}nZaUfgSPti~Hg!NLtkuEt`?*mG?5^wrJJvRb< ztT7q^4hUnhrJejCP+EogKPoW$Lp3X}{{XCikLL#p{{WlfVUBk`CIq)M3mKkVyten= zV+sQnNd@lpX6W0(L}fatCVZmDc!BiV=io5#{a^r+z&SJkT%BCK^;gr6AND_BbdQuA zFj7IfYvc%tfe47SbazRN9L*R=cS=huAgwgg-67qgg1+y4zTb2Ia{mR}IeYEw?0P<~ z$5kR^0xEwZrz_C*3jiiI^&8+y8-zb&?1`L`?Bue*sk2$Z_)A|frH>A)Qx6n}k(hD^2$@melrfB~NIm#d3!N5)X? ze2YUSHg0&s)fF{Lxf5J76m4dlBT-QSDKXr}XPpBWMo0GMMwZ=TZ!{oNiE@yZieI1v z9|R}BzU2rw=|Sx8mIoQ1vhH`r>WfButKgtlz5zuwJXBARe_%k(e&fG51r_iY5z zo7cbAw)fv^O;FF;UmstX5q4F+ii3=k&sS-OyW$L_knO%je1r!?nlZSnpB;R^_ik#| z7zipS!&b=2aI!FPmAUfX1w3p`BL#3J=z6wAqmIYf7gp`vzazb_aru5 z3am{I-;PjV_YHR3ZnN|>xEy8qU?W&ci@h%orsV2Qy<|6OI}PT5am$Kkqs=L?uQ5?7Kl(z$K>j4$0kEkbQQ~W;x@Ge;7IHT z=wV|MD?nxVP;1a?loWc7pHV&Tct!IgK-)71wI|0rA03m=^eSJV?_dMKERLn5C@vA{ zZEcyhe;CgRF`4qQ{VpVMd*L2YXFZT(8|WHD5Sm@dWs5GW7J?`xhNil^9*Fmk%G@rHufg!8$2b^XIk9knN(ZP}kL?Is~Q+$ttFR|B`F zvHJ}1)$tfI74K~Z|BGJ{aHTlSaAE9l7G7-Nk}(zC@b}n3t$gk|%oJC(vwYO|p~Y`g_?yRvw5V&M3doR^6jJCCJb zDtw2F5V54ire&BoIL$j?2?@{<%6c;PUMM@cT7a;QUQ&8JL5ly$;|!)LZwZws_TFDf zlsaKpY|h&bl(&C_nyY(J{G1`jqnxt35)C>wz>|H-YleA8-wYKalNPCXCC7WT4~g_T za3_|grDzwG9rlFn8+fPacvYgTRkb`Ko2it`>vz8tR~KsV!gZrfU(}NpAu%^LxI~&2 zW71Pl=7$Iv81#pcVv&S?KEjLr$(sApIcBTVBU?=Q*BH)O46VvXUqVvsF_wNlH*@VVp^%a5J1~EMl z9k61Hg78a42`ti=l4%xi+|)2lYtw?*e8m~0MPQokVxc#<@sIpIF*~DW%;G943#8cg z6Yxfak&+6`_dUpI#{}FlEg5Xa($>vi`<4?34hAzjPECG+lKRdRDb-qEqby;B*u!wC zC5#H#L(>Q6P0eA(b69%Zd5CTyo$N02M7AaZ5wjbdPGO0Z(~yQHE0TGQ7;E<7Z!ooR zzt~tOnBU3dGqvL?Vrc!KftNB(mQw4~wSW>!UGLRDfOv_tgs0m2u+P#jd3gKMk2&n(6Tu)dsGLZd|l#oe(go4OHDhOYdSw zz^&esq7I&RSUo9D%H^Nn!s!#TbTK2Ve-kt3>YtRNn-dLT))(q%L}3=i=s6*SNKQ=n zno26Oymb2H2cez7UuCs@ZU%Dj)F%s5k%*brU|EBV!Eo#@gYI%cf`~5-$4jZWQ7VB5 zHv(`@(sD0#IqRF9x4Q5ziDIlP*ndRgPy8ybSaK?8e-jaJ3J9AFqx8(#fa~EaKJ>O51B<*EP`>mwjnnv)^&gyhP zva$bKLOa33B7s0e*H_@ygYh^co~({8G3ND{zzz*>*M=)sLp(B;*Y+|A)o_>$!_o-* zSlCO9fhju>12}Hi-?0#8(Z}51%elA+54+T!T=jn^`PG_(e#)vR*&$P}; zHtA>S8_Y|fWprMtjMC^Go4fzej(i47-Fy!@>Ju)~17VUdaniAyv-r$r1xi8fLG;Hk z2kvb&m7l%OC-oNPC^TXxqW%4LPm{0QN6&8fja@Tp3g1>VEYCT1c7Va)_o7jz!is45 zi{a?!U-TCBj6P~3Ulk}&s9E*xM|jUGJemzr)CxBqZ|jf>6d9sBbU|iIl3@V~)8xI1 zO5UNf6B$p*AtZ(T8p2l}n;bQhc|&-)Ocy}9L<4euYYhZ6$2J!{= z;cE!0gJF%>zpnc^5zYNu?@5#!ORE-&en^OVvg*B(Mf%pMnm-Ub&cf6;UzZjh#WA43 z-?A}|nXUq5H#bdC1a$7CqTR`4g+_}{AV?x*xIIC+a?lT7pE36qxP)RCunF@OCpCxG zykNc!B5I1?**_ZZep(}5XP{02sT!f`WL(l&sS5zJDA9;V+Lg^s8%AtGzt^kF7q1nM zPgAWM%%76~Fx?r;;dYFRi=*X&Fb;kb@ij&cpi>)%9=h)DU9S9;mW03*@7}Z5-?}<) zzTu@aI#z7}vI-k=I06S(Af`XtcqY>O^!`q&uA26ICc>9Un5jq&kswn%i2fb2%hE7TK+Ng4;p!^{(gtvb>wh6iy-L^eNsBm@ivKLuQ`(qBI~)7JbRr z-|bs5PfJ}F92~2$MCrJl%XQqEkX{JQB#-8e%sYLALLvWcDCWqNPT*&Z}!cP_%Enu6(#UXl~Bzj6wNZ~mT z0&tQ46#`F=&RI_fiNB6|A2EPCPVz#X!mYF?GK_x5q=zrmJw=Db6sZ->V> z1f86Ds@}}UmAc@Z6Vrz!0Qvz+RV#skmlbq*>8E83@~2EBLnJ7HOF=8*LaLhzL2;Xfy!12XPMC_VQ}GqaONfzfG8IP~x-#5;b7 zCovAOBB#p*3fZkjqnTlv>D4@NBsd^x>z7LTQ;lq*O-6I#CPLWm*J|E|OA6LtIi~^7 zS2Ln!yE8!@e~nC@)tTbrKX1kt_?lVNy=&aLqTu+bwv(ax-Kw3ah+;Pe+$$Xi@CHK6 zdN+Q%pPx@{skip)Oy(!d&Bt_Iz!qNS{`@LoFvgb-eHqQ|bq;N-SN88BzwWMD$o3X| zhXp6lgoXPxMp{b~AQM?Po&bwAcq{?cFp_6XFISyco<0Si84llNC+^(u>;#`eNlpWO z6Qa3suhzX^H69+zpk8bj{v z>ZHAK>)PJ_jbxkTppuOAW#KB19*B`f;mE1#td2<1v2CL{kE6u*Rh&5I-{)VM0!&^; zNXK!%A21uX>v;6s&NNK@6Af5gk`R2dv^FD6hQl@b(UB zpcerkXH|YehSuepZ)6o>+#o-sh9+i2ol}b;oDi;!pb5@SXJI6Z@%*&^E^AQRpbjf3 zPi)NzuQ*zfL)~F-z3es?F~IJiy+=b~95k3g*^H|YfLD>L7Cj-^y0+5@*oibf(8w-$ zmCK^|*Z<)h!dV&O2r7c3iS}atNOk%C&-P%^<{E-6h%a~py7s98Q5{|1eZpi zE^wwAg(t`Rqf#n6X-w#eADgx98o=;Kr?JGNnc$89=2ej)67>w9Km#0M{5$$r$EjIu zZtD|&$IAep?X>*bP%Ah^i9*-qNAkn?IdtLbbAIjR>zN986B0_ei!J3Vzyll+#Or-* z-wS25sj|om6qIj~PD(CKXMoP^bn=CiW&KX6popDIMk7dQ18c+XSwH(8&XV#P#mlZ` zu_hr1xVS$B(F!hTn;$^P92Nj4iaBBbnk z-?isw4|A^%2S!Kvy=JHd$2#O^xHR-o0#EQ9V7e3W=Olnl*0cf!Fy*?r`wu|d!vw-@ z=)8fk4QU*Uccp!K|6cnU1;2d}CghRCjF-oM!E})cagMGYmi|Y}4e^+gE68;xeobS^ z0>@Bs{jZGu|HE>Fr_#Eue<6K+1X>DxckO*+s$t1RSeFR@^X|rpOP`w#D z7XdCyuM=VN83EE$@Rm z|G7Dty)f2t5A_cIdp3ICucQ?6CGu}0lt}(H)hv)w{qTrxg9yYQ)o`JMtP%`Gs%;xH5x2+RFi zc=7jAX-?>ZVpB-rtJZ&j#Q!0BQPC&`%Xg)f(6fDvn(B*r(O)*DxG(%|I7g0mg#`py zu2>acB=xFq9DYvdfB^^~H z$~rMAN8oM!IqQu3<$7|6U?J`dZR@oAWl$>vl*Vw8pHeI;*E@YMfn0z-eqPvd)6HFS z?(I>uwF}`nf>nfZbL?W35J*Xsz}y*#SqQ52QFwj`r1-&%-)o7W-gv}RLE1N;2|0S~ z>L&F}tHL(l53F2;xrjua)r59S^^v7lWeE$YP`bzJg#NN{{FU`DHvmo$pvU#|{^;;( zW62!yVY}->w}qm}8-JVf0_R~n0OX=^!Ro*sKRjLRrM>nr@^2l`RRlLslU>9A-FN@5 zAXHEI3kA3HA7XC>uTUiocp>yMOx_X5zD_4yKy!##VRWj=LxtaJ5<({vp!&E;zmM|%@|nQ`eJDgsS!&)jBsC8kO9Mh-?WeMvavL0-S5c!iBR@M))JRDhWjR zWkhgIYDmrv3?xUlT!PnU@ z0}>@9&+R@-W&Zpu)?Z3lXy0LCAlTfkzdQEPvCLX9KwAo~GN?12IJes*;vQ?ZImWBV zi7iMztc=bcR?VswXlu_^ju$N6aro9fqm!X_j7OT3EO{XC3Q)A?;D7AExH}^^zdVx8 zsqe_kt%PDeb`#a7y%Y;WC$FdLYvmh_Qlr_6FEdE{i1mauH~)`Yhwvxdcmar@0sdm_pcP~DHRFbBKN+BA#m*SQ*bswCY<-Dsr3 zAdE>-k#u>Y8eAhLKNMh%*^lzq9iRWD+`DGzvh>v*wLVL1osxG#5E>$E~Uk4 znJx3NLCU;KBR#Jge57I@$e(nziw|2-{SN7W^@RGa9KwYTFKAuE=tOyrd%vf8mZ}{cNbGrg&ZB_E2a|}5cvS5G z!Ny13^L`9+yA9}Ei1>m`P&?SpTkjf1WKTDiu9uCp-!5H)kyOFPzjvu}wn?g+QSY~( zxxilHstSlP3U$;a#tQ2^C1z7Wyg+S#qoMBpRPfr~RUW{?1~fhcQ@QD%V>ZmL8eX6W zjke|z=BASq|N54>dD%x}zmb+O@yMhr|* z48Y(aEyIJ+l3D2qMw73BJxvbJ&WX>J70h!a<9s#Az{wsk+Rjc2;2;c~wZiUE zlAr+y-yZ=I#jm-{ACkPL4(`QF69XIZt^E^4#B4#W5U9k<65Hd}A_6XGxI~)E+r}H1 zbq-33F36h(DE3U-@(MoliMpF*lWSvbY~y=fGoN)vZ9>%X%Hz(#AO(yFxv^ztV>2*L zrpUHFK%rK2(s;biBD!YI6fFs4SnsuZ;XoSRF6`5;zu2ks1DLP4EDV9NfVuTFjzn5E zX&2BhG=Tmx%!+>EV)iHWUE*wBulhe~%$)D7lp%=Ma@_5fJJoz>qXPDYc7O;2K3tR)+t*n=OTZs8~{A^070?X zxWK6Z$W|_o$B2h7n68;32ys?t&XyMEs{>e=vw5{9-oMTRO|n@j4^+p$;=2P!?LUbr zY1B^BJ}9S$q*1adlud!j#LdTHd=uBxTZNNbRTyA(7i-5p2=HtkJkwP2M{_8s;HzJb zqb!>gmsdGYCX)i{LmuV_ZLI?+p2FlcZ|?FRfb8<8S>jaE0xp@(87Jo}lDC2C2OmsH z%h_Y&3;hLN$`SJmejlyLmn!tTHY%U(;hm{`cbVF8}lN_zHbn>=4f`M#*D@ctw!&-Oe4eJOvmNhe==jEDSpx15dALb&w ziuqSYlLT>qlBUVONws}`jepV+FQMl8>r6UZMb zy=Q}eTgAYqu5it!QJVx=+G>S(5B)UA_BW3WH^AHO%Sl|?b6l|;m=4@2oNq*cHN-P%Xo7c3HjZO|YJx~Uz`JrO|1QdK z{~Yg37i7`dDo=Zy*T*S=5l?Fn?E$2<1CnA3N+epUGfQ=S77%v_7&rTyow9Mwhz^Y9 zSxdf_)o;00tqEyof&xV+1;xXmf>z*kU>`8*E1P#8od^dA-up-j|Yz zIy}U+{#D+GFI=5k6OPT;{-)IAU4Rx#U zz+Don{CEXFC&5cMK2g8Qlt8X6s_N^U@pkTNvDVRx&G}mKLWa8$lQfP&SSLEEY>mWT z2voTCP0Y%uu`YTJ)eT&lZl!1JlqsNU z%`cr`Bu8(fk(#2W@S=2e`Dwfy1Dn#J8=ljPM9GXjL%VqzTUjXIJ2!q?9t|9QEXX;4 zkmWhKK63Ezw>c>%y$KA=ScK8$|#}->Q&s{HeF@47{x=nxJe_+G^zjm)HHtSK&8!fr~h4iqwV&*O~dC-o6sLk3w{v; zowjIrLqE4c{wg4~5Y**FL; z2H_|%vVH!VB(?A1hs>Yf`g}8`=Px;?h|b^%u!bl~aA^oyLidv*36~NLFGA|rmPcWq znVmaUGMv?-v;UZkmOY)g5pt= z3=Wr2+g30!-|Ii~MDd@A?jsB^-@n)Sb{koTRhO2tLRR%XXzSPJT|*qkc5kZff9K3a zVfPcP@nir|^-;f~izX7W!Rbkrq^Vni8HSC$b88oB!8cs3)x`&a@7=v?dy&>oC>-o# z_6Y-hEr8Rx$(R~X*Gx}S(6NQ5B?_i*NV&)D2pq|RT(1fv9Z-eKTI$}CKbZ3~~r>)WtE%-dq$w&Am)V1T!BHCJL z1~?aN6W7sdmyX{}obbkBQ1S^%tCb?dR z(haQ$-3Pros3vuHud1T3=49(fBsSbK!Rz#2+~rb~U1!wQpx{W~6U}D5b)7>s7C*@M7S@Wl2E8RNmmxH zj#Ul-DbBAfydSp^PpaSOVm6`qbPcW~!7=Hqa*vHrQoe|$0zdIT8ht>~`ga_K{0H)X zKcy<$=3h8qP#sLnA_%2H;Mm-?hd@>?r8lf+FOColw|+LSbMq&`y79`g=U|$P)|jX@hgAd$%W7qP;4_@kUHqya2x(7>fcKQ-4ju# zQqYr@!9fxV|5jxuhSs&#$dzH96^%4!!lk4Vu=l>u!Y?C?dL4_7vllzB7SLec9y3we zkl72}5I?ZVDzJMw`}pM{kt0U5yMAYV{(KQuV>%7R3C+Dtk%{}geXfv-nIdF!coN~E ze8c|tN+msbii5*><$G9zwkpBJn;?lX(D_Cp|6$H7x^FN%;?zmQ%1ktim(MBu)Pu-i6 zx+;RK%+5o!Mdrsac$oTM-Un46ne9~jQ`W-!9_C>`R@N@%-zH(>U#R~Btdh>Sdfwa~ zIbX=Q_z!dpf8D;13wtW^J8h@o!RoWsl_@m_W3yPraBX^jFFfQF0wuvWhr2X_@{e(U zg11NRW5d4nBUTj5FXTuJZ}0x-wl5D@$^PB>GO-E6Y|_a|FEr41WbS8 zo(jY{R!a7B3OKrQWw|_=7~XQlTU^6xo2A^}Q|itGY&m+qzTfV>!@uyH3&gBV&lClv2uLaaZ-aZUs`3P=pfBhzL(m$f*w}G> zy+3Xs<-pdbvP4=rIk0h9`S+_sA`|AUh{3#A0}8JsT`2xt{>A`cm_44>06Uf887{xV zO&}%I^1{7>0E`MMy!nEuHW&RS|M#8cfr*1OHXQNPJX^MtEO>P+e`K?Em?lf`909B( z$;D~??IM3aK!hc5sTe=B|JsWy^dH8sdVt+a0Om6h@3OFb5o?UUadp{`!u`f<3CP7| zL2?g8I9P$Pb^Xr8`bGNjMc_!)nv7l%z4+LM@D{gBu_|_XFSk?0q3q({g zP~5Yp*e(b@v+8+5HPpRGU7`&N>|JuhPd$0qB*Mv!Sim#T=X>>7R7l%eM8qaihSGi_ z^P3UCKRP@43$tQ{y?*5hn&n>}F_41DJ>vnC`vlFc@K4R*QUK8iF-3ML-ax^OwAr_C z!PH8m{gNgcq}<4L(dcJSc{==VHGJejW^4bg7>|^7hjJ{9W?LRES~#&Ft&T$*YrUGt z)jEWelPK;hviVV-n>gPtF2|6&k};8w4|iz?U8L3ivG@$K6dy~Y;3r3KXCiLDEI(oG zp{CSG-xbSgPU^MD?^Rw-&jd0W(~MWl93^7YF2E)P8cez^e0~=!6I2)fA+>Gpy`I`f zXcnYbpGYVGJo&vxtaJ1gf%fPr{bEMf>vU3b+ujk>hf!WJ*iRTZgHD4q6g;Fb;VhZL z`QTYP+FWjAD7d2>FVN`=iDuT+7dEs+r!lni0ti=G%F8NR=EndsV~(Q|BowBc9l%?> z!5jt#vCnZWf!_!FL^B#Bs;J9VbYrmQUk4WVK5?cKymc7XmN5n=z6{my*UmQGOv#(2}#ivtHJ1NTk2>CHVd$|-{Mm6;6d!5pm zE3DMF+J{E>BfO~8m+|v|K)JZU;}(Uf;7L)J0e-@l6fnzEQgUyhBdrL(E|vFp8;n7?duTQjOng-4N=Lqo>w+bWtmjdJqW;`^xggFLEn{-myFQynwk zyR5Dr9qxYw{wt!g3^eIW4E+*6ulc!4+ZRkaK3TJttOgv+7IM@bI#%lGCB0>4 z@8J;9uHjejT08Y`7|0V&NO45B6uTn}XN*X^kPsd|Z7fd^Aq5$;3<3Uh4dE9}*fjA2sPGcScZ7g*QJIB|;Gt`ejz&8Gs8};~?;10U$Zu5I$nnO0jrL!( zls2vsnn9 z##Kc#>%Dfq8=HowYd~vDqsGk39e~Ch`kJS}!_wkys>d&sq#=5th=)j|+sDn`yUVd- z8`(#0`N5@B5g8t-CN5zu;dGOj5H6X)>p2Rk^>;9N%|i@gd^rl>>U)|>Rdt$QW<6;W zA!5sQuP9jhY-*Q2X*w$#qUTpaR^N zUIkI?N_YE&>`0^Ipx@VG3OzAB#}BN$m7Wo<%F~qFdcpibr!D9{-e{UH9)-?e-s>f1 zchg3RBG6d~d`Y@N)O0{q@tcen4NzTDGb4_CSrL!4&-VdG_bVHd))i~Or-RfeTeH5u z;*qWA{D5=3XL-``E`HWhu6|cPY#0(XQO*6ubCm5|<*(CLu0DF@eeHI|(c{)eXyE5n z+K#ZKVXF)^W+h4~wLP$s>ZXJ*3kH6<{c@n;G`z)yS?`}C1tOwA3Ea_@1 z%YTgmKvE^ZjOi4`+-X1y8>Zi!zSxs|r^=ZO$uI5wJz=@{vK7hvuZm0Oj2E_-PfXQ( zhP;fT`jJZ%s)I*0Ex1^W&{e?slQ?ENw2`*!eA!Zw3R2z+HuTWCPS+` zT%(A+TiL08vBt+kf%=FSgGbwrgY)-&CLK8ye?c_}pK>Zxp%itBTYf6(83ktiP?Nwf zcTC(_UMurt{RI=R^|sK;n&)nOQ%@82D3Vv+!Y9c-!d{6~?KgmH3F19gXWjcek=Nt< zSo`_;=rlc4YSSURBCM}fCsaj?|E*}!yq&09X@W4|fC)zJ^`Yw!#%xNR{$ z9^!B%GrI$|U=4cNjrAE*W4q$zDMV(=sng}?XaEx#_DTt&ajJ&0tyGt07Z}h8Nn^Fi zM}g0}m{a0|jvsn&KV{70v%`M?OB#zaTun8PN-CwC!L~mX(b2_Lj#X4h3R4geK5juaF;gB+2RUmOMGe!NeGD%-<(hH@%X;V_u-6o*w#j$lzVb7~*#B0{ z45?Ph<{{<0m(k9ydL;xDJJts8RK~HuXFy|{$!g{GuYJ2eG;;cDiH{v`KEK;>_DwIN zavkBea*f!d;Z2qJIrvoLv!R)jwUtPg0?y$lGN)OPYSa@xFahBjg?288>^r`T;OKKd zw#iR-_^@n|xykmga` zv0`!z64ed6js5pl>2S!ROx>%)3BdCKNZN}{UdCMeZi<6l@Q7A-amO6JPpLgvwf1Wo z=aM{ZJ3-9+?c2~SV1`AlhDZRw2}~j01h(G8%-U96&|Xvu>6lKdKIup;3{)hR;_unB zXz;N#8TO%=jnIU>z%aYi&{Qg5;J6zJqm3A%e|oxcUuk{gJQ1*|>9iap8{yf{y08BI zOy_5p6VsvC)2A6wV?8`=LT;mLd)Qt&yXXgB8~cInHXzC;^j~E5dPH#&mB_ecE4fyh z)7d-oyK~Bh8WCovBufz2F*qC3E}~G-K8mK793LHMHIiJ7DGIK0Q{4DekoIA6@T&eu z!f7V|KLEo<-sz;#8x;+NQcDI5LqUz;YaDj%tav=M7W%U=i-sk>G~)hLS7c*q=Hi>^ zFFd~8`h-wZjFHkq;5irursQ@`G|q_Sdu@?_R86`!GigJYG_DB6Xz=41vApK0>v#H+ zBX??#j^n>;jwV7fcn}cns4_}Aga#Np?GG$HX$&h$rCI<^Gm#(b(J8jn;jo{*U@vX= zd|peBA(JgTQiR3Vm8mD@zBdowWN_ZLjQ$je>aJ%T*_$$xAfphA(9WITtLGKg}^`P1B`PU>l{e@HVOS)l`{-P*6&HG zus8`^YEyIbJ&L5i%>pN9leMz!vyjJ}whg)I>N8v6PvKa_^C^|WEK^p(x0>43@EN$- zGJnsf`6_wOHiv*WNrbEg0c8*Z<#I6@tasyB(p8t>{Zw#6Q42D2*bE`9}<~>6;E6!r%3%E-T{^n zT>M_W2>Vu{M#+gTuY%X@%qysM2H~(7Qd1|T>uTwsfTM6E8HovXP z{2CW+s3$gdsv+u{{N7p$$EA|&pN{uh{cOH?VekCDU^u?*s@1J1pNsGq=>{m zs1l1s4n~PXvA|q_I1a1E{BT;TMj;$pc#{$_i_i1TL)g%i7C4;H@6eo0@q0i?l~?|Y zR2Tu~AgEMg^S)q~3&2qz08Zw{n7@r4&aM)*hYcbXCrtep3vQ)5%6z`sF}K*uO74rB zJFUH>_r)SKojX8}Dj^K0{=CG}7j(rBNz5*yQtI_}t%0^KWdaz42CY534yCi>XaKK* zj2RLfnCpST`oH8SzcL=6wcCx$_}YpqzWRVld`&EIJcwZ7)C62xHeyl6Ii`OAr0TvV zG5I!XXTYeD_r2-_%}Zkbtqx4n1(swLlg@nR>}9ORn8@owXFR1v2SS-!?L= zZ`QSz39D}5+#xvXdY6(b?2AdujYBqyYwj?s9#P57-n2=7mdqFLn)(QWj(sq-so57| zQMnVt{{djX?qq*kkZ;MPq^w|V4UBpJHZ!G8cSV8&b1A93gUQS$pWxh9{W%$LE@;w3 zo=6C)ce&rJP7xh??^wP($6O)LMnV3K`)5aC{1<1EuXL~f1Dp#=)-gKeyxS>4?x{C@ z(bmN9SYY7#S@((GpCJ}oS9X4z?ydCpLUUsTRnW<9(C-au=Er++u7}%}I~fb$)6G|N zkD=0!#nLLtp>S?@wz*t$4<%LB(D&E-X9xldIdO3e$|WJm=4MPhjb-pB{V&^d3g8W` zd|s#Tnx%(;DS-L|god#Zn4i?rBBL8e&@R%EbhcRWFVxtc?&CRg?e@mhT>?w#&$VX_ zuLdvrUq^Rzy!j6>dVcYGhECHxFhV9pfrrs|;Q!n7K>pu`qIGf{mx1FE7daY=pWl1m z9~^cSVuh)c{RenZcwqZnl|_Nyw#pox)E+r1c*sPu{d3`pj_8ZnP=7t^f%WER=at(N zm_{c)x)%xX$?nWX2>^Zb)M>Z-l&8J0uevae$6(hG3gvKiH}O60V4Sm=pU2u;2<9-0 z3=Hh#?N*rv6jg^hEQ^FT!YIEXo3dAqF&}_cLA}K_$G@n*#h(nivbyg#c54a#*5duY zVHjcMgaSS5?sn`|SbqaS-7yAj+aM4QdTkjyr}^PflHW zpF2Dz0f=QR_V#@|M(c(k@O-%kTM2>zjeP*Ux!ZHwheI0^K-nz7#wv67^Wap-byBZT zOdmDP$-+-MjzXw>XcK0kLt@R{JzQaufNe;Ho)3J>b-%icASi030*q{0UAJCBsViNe zRM)?#b?D_RBfup%}LeLy2n2&LRk7QZxg;L={<6)p6l}Ss#PDDw_vkNUqf3bH=@+; zZ}w*(6dtL++B>$n>@<)Z1-&XE;+vDd009c+IH(r@fO(@Bbty*Fq-_y^t=K~5d8~ci z-yO4IE>L2sce+=dm);s~iGwdK=cN7m151;(h9GI%0lvZ91(kCygy{Kc_+I-DyuG2; zy9i6mG$HmYp%xH%!F_@n1MzyHt7*#B14CRF(g97=@8o<&v7aB!=v}&SOPX!_ro@w! z#b}X8x?MyTxyT_#kF*uM!7L5zQ*}&2A_`rW-5tqs(k#b_Ew+G zG^!~Jgyyuw)@*_&rQ#k?3?27o-Rc&n=*m^ydLswk3a5$4oX#zO{15QS%ZrQ+UZIT< zOfAdaUQs3Fip?a7&Ook&-$tq+{0f)U8Kg2Lvbnj&9m2#k^us8f3^5Q|rM1M`FdFmygloR+us-XodZ1f6bvZr3dVj#0 zQHU@+#UrMYAdN$HQl-h^tYmh2Q)HAlsrN(!iUJ~FErHQ1{y3hKns)~7@R$5i)%}FT zuVP^OU|@~U*JA`pxYW^7Vkq-dH&Kr_{i%cTID@CSEDv7RfvKGhtm!RS6ZLH&$1bKqIb_dlT^|Hk-vhpLaQ` zyhCNgiQ%#i@?c8Kt@TJB_c`{P&~@J(iWWHrMf<0h&~@L$X9U zr^rsSmQ6b>;KpwHcdOtIYb=a0M{q?Lv%d#)|>+|X{pF)jZRc~$PAbN z96Znq!m7cd=euT``s$Pd=f-ZPb^4JbVANBEBi^CreQ3!#vB*-=v1^xBn&GRDm5$Vg zWyhBricLZlFV6Ov$~K;HJ+oRI5CWDFrv{i|Gl8D;9$*s5VXY_O*@@_Dq~;w~e~K@0 za>Q^`v3HQ!Byxx|EW5&!uSpt}LG|6HpP&P*Za|eZMvUyQeM9c_OlFN@)DP_D+Kn$g z-bml69%E(Jy1Fo?nVTJ|YyZV`#dv3udz6}p#o&zV@-U9_8wdr+!hyYP0J%9P8A@@(B2if)$47Q!+$0b^G0z&3)j*xrN&dIFjh^^DCXF=&rF8gE8# zo->Q5j!oCtF5GGNI8cz_Qf zLx*WHU2lL}PZ+>uqtPg_G}6&_KpG<^JAiwf37Xs)55?O;NE|#|WYfA-UzdAQYElhi zY_9RxTiEk{V7SP^horcUZsczYFo?@f4wy?qvW)vT6~DJnoL$M$il5UrkRJ>p%Q_NC&r1y@xx#aVBeBX0 z&|D%a^G=PN=ERC32eyNe5*bfAhACmz0Q~+?%TrRV&^`|wucD7o)o6w-qiW7+iW0`*d!u6ALuqr8*&uIi(Lr$b3l8<@rwFrgRU z(2S}Ryc8HmKfaB2D!2@i*c?=APDK3#Hh(rs#sY+$`;`LvcIh=U2L@3kkZ3Ak)@3?~ z1b~~kK`7$#8vYH2B`{yQXxYnr^8GeZ3J>0EJ$z5y(?Tcu&r&&Sw3^Qnt0s$)K1m6p zNG0Ch!MKC_jVP$`Gi8GwlF8pdeCB zty9*gMX#w$KGn?xpm2M=Kk<7Oi`NAh1T=BE*W9fWh&i83W-cw%G5EJR&2vO_MA0~L z^M6o;D&;+eukvbQ1K{uf#1X)*Kvdulo*jN%N;HN0iiqRktv))AXhfb%;Z9H`oFOk^ zR!zXRQ2kwfPP%^DE}Ed`%8E#DF_cppj!Tq4)a9-8G^acW8K_@6dXvd}M5&jU()>8l z^;^3-K3d+JBv;T=N@TnJq0*?eYnf10p=3eK4CuoCE3)R^g=1uTvzF|OVML#?L z>}rmKBpdm-0=a5c6L})w)apGDo2_T;_OzK`%_64#_~qc=%6TKhmveLVHL(m{6`!Bk z^VWK5e>ZZ}TK1Jmu*+&gWacPQqiNx*(LC@>^zSdT-VJpBho`p;Yw~~Gum_Cp(IJj8 zxy6gMw_kW(_dAIHDcI^J#*L_{* znT3H&B(jb{tM3In&T+}2Sa|hxl zdIm8CHDOn9_uCrk>i1PKC|qcGi=|Jdd;D+(Dju?h@a*#H`iqZ zXq}EN6U*0u=D5pFDmu)^X@P}BefbIuXAbKtNKm@*e(}N@ca>=0aU(`V9FHa8RY9D2 z>3{>v_+__DM@gHUR8H#e1?XvWnliiUS3W2)e7m)tq`KvO18lqWDaOAmJPw}MU7~ca zb2;E!9o9qZs{Pg8B)tG(aZdOl!n%6Lhvam0sHlRym<5NkS7HS}AKaRB57Fgxz^FFi zt02u`6o(&*9gGpl;ny+nf+h`;?r-u2knV8!!R1YZA>cIUV} zlX~Py^oTvQ^g0TcK9M+ISB`h?9N%PD3jLRTmdz-|5KRD1Az4j#%v4rbc^)z74VEMu zjXp2E>By&_JJbA{s)GIIG73j5jN>+bZc%zh2(7+Y$!#P$jg17F;_U#!@WjlANuyPn zp|tpK$*KyXG}@)pwDaReX@roY7rM(Id%4vrM}9NBZAE-^Cm>4m)n z5{h~um$nb~;Xf8XYq0FTbY`;pnqNl}UYnL`3Pf zBdZ;+y5Gmfi2coBEofd5yURq`k@*AKhp$N-eLR?zh zjO^z|U}R>pVr-v#y1(wxHyA`nehm9$^u?Y%s@RlBwH@vMhFkrGZ#tWG4jUp2zv+B@ zH7uaOe#>S)%X*sGN$I=BFQn;?n3=npfOB!=p6AL^DEDN6l~0c}wozagm;vB6p|HDn zDS>^BwmPGc0WpRa1{}RLpje}9v~I=7k@qT$l9+w$cVHf+2?8Iuxq0RySa_XcAwD`> z;zAP3Qos2HMOfeCywI!1s-bmbGXCBoMMO+)RQBXgCT~!IOL^$O(ggRa*kPS}+nj9gOrGMXG3iqD3d=kd-%@7 zA&Lqq;;HV@%C{iDQ3p~nXzFlY;t1US8nh`U>r3WZiOt@SIA}1{OQrs5io29zp8K@s za2y4gP7h$>dTK?x>y0NecTLR_>W{zJ7V?)d_?f0sw64uRY*L0Mb;Ca|gCK$2btdXK zyhQArxtdI=P~u9nm>Zz`Dy=c=lnT!4-bO975ayngDQ)2H{@&(C^7+9gXX~NB zIIdz=MvM92fL_>t06i9vAQfJKf;gpX7M=siD3s1xVEdd=uV$eBeO{B3r>Z+Wn&lf{ zXFF+22>YKUvHZz?v2(Z3#^G1VsxSbaDC2^Cw`m$HcCxtI-3MNL>&VirQfFE+Rzw>> zE`G^ufc_9|=K|!~u$vmN2BlFWg(^5qjn?*9mD$-dru_I%c)#tyyam+@-w#!Aakj;$ z=54DZC8^krb49(nV*jugA-(7gZ^uJDs~nBldGLgwd}k}w!Y9eUe=#|cOzC}&sqnJA z017kG0g*|><`0ndZ~b942l~Nfuv|{B?&a@Wo+&NDJjIc>Zdc;x^x!DHIUs|GEsIii z!-XjG5q&NX$D_>SXH%{jc=pg>wKoF6I>)m*p=F$H&4YmO>h^KvJ_eDX{>CW3Q|yVw z|74y=isnmBzPY>_All6ARh#Bf(|h#2UJo1neY-MB2jpWFmQU}osP4Pj4rkPm<9qU0 zcs%#+pYU$mvUr3`1SW|?)Ob$M)O^=!q^ZFPMsmm=%K1WXj0p$usTi{@M++X`0QY<& zg)q#;(aYx8!o(__qRlPV`EatkX3O2$_m^x&PD}Ld{nyKG_(dycLtP=8ItkB~UYd7) zALXrDImEBlmB+?669-Y8txU|yoOga%0aEHYZSmjrKV1Ed;aq$2AAk+WK{>rUJU}dC ze!>#rxC@bO#6W%t%Afib9hb~AbA_vd6-YJ}!)vsla##Tt; zDPI-`_Z+pyHg6H&X&!hgkgz_z45I*|ggG|-=nA_n{+B)eFZz1Sd%mZ$%wtF!E6lmn zA1)BEth(Bqa_w4pdw;Az7*3$8^M3YU=ONn+kfKug*J1!V#ba8WB@i<`exnFTQ8El1xk+rKdOiR4$(JgYC^6EuZJHL4`WHsEHo0|y z(rTo*10)$n1Z44=+wx%1{~P0#m^^eu@o405#x`PcIb?bM%T;cm#s59BB1@NB!kU1; z*GjGO@;V$MU)%=)Q2{CXFiDft<0c#v4GV@OpV1YiW3XYnGsOj5blu#2!hkc{m2AgilbPXW$*h&nVbA?<_O9hs{6^SfOkWETb7Z7 z*ec131TcF|s;_9BSGP}-_2?7TT}1y*Kr@m}hBYFiZQMCMjZ=+SQRJ&%=Sqrf(|0?w zv!C~t^YDC5wZas(mTJ1a4a^s@LID<9Z=X55`nK~v3+&bfqy&!*DyMpQd0X42A(m!z z0lRYI{cFK@3X}R)wSNQxOF}AWc!NSCHt;c1M#Em*1T%aPLrjS1CCgLE77i1#_PF9V z&Z&KnM#pu%p%m872iVYk$m*t>Y(oC9c~J9GuG+OycZuarW$ldpB7gmxpXw0nIkjxj zne2>_=BT`m@9+;0J(wun5tH1URB!AuW?u>Gko`07E>1`pxE~*g$!V#gyEgquhx|zW zQ`KTMc`X^=TWH@RA5Q*|ufJ|JOBQXM4z$`__N_bpOci)4E2vo6c=wdlAQ9qdpUMV_F2-5CC7VnPI@X7nxpD;i?Pg2t2C|+rM%%R;kK* zf=%#V>ycH&tTFGx&|q#~=WVRqS`#4!|I-Qz!H(6E0A9&_Nv=ut9ZFq-x59x*YDSf2 z-5$-oQRB_1Otx4oEi~=Y=~}~4g@5w0-=-)#y134fqtFrgQz0dsK$KP{7W$?4y=qL- z{oE=zwT8r(Ps8wP=U_fr(RS{1k1!w>hLzj~vlOUeADQ~{LVQ!3?{`D7XuvjJEmA#0 zfl#$n)IWY~M?w}epxJ?szTc&o*3xY`CAEN#{@`WG1U7@l-Kf*F1Q=8>yCB_@dS_8| zevjU)rVpjKa+>`XAlN7;LFMM1`Iq0AO}EMYp_!Q4n}Ji}2gnz_JX;S^9zOFy0w50z zOStzS==NuE1{!qGUW9_);}0yUyiS-j$jfEBpBvKm>wWo;{^u)gqfn!#w)E_viclDl zycY$4T|`0iU!ZrT^^`wH_|##Ig~mJb!Q`mnsmWQj3|>{F0tuIvl2m=X0o}K!ub)rW zW;~mh=njnnKU^XZ}MIB|fTrSrJ> zkEu(UIJ|hoKYKlthR*PqiHj$A4n11(8ZYX-_16n>hl?;ZC6Y`8kJAV^76>Oq!bB9p zlZ+%JbKo=EDN6BKcF%?WO-C-H2Wd=V>`u5ga`!8Rn}Fk1jUFyWU5qBbsnPy8VZ4L! zQl-_L#r_lT?br8nasVI%$WO96=F9GB(4Li&J1(!5Bf%*@y{r3+rB#bZCvL{z1?Eme ze2(g{1RT+Ws-&Fk*3WD(Qw81mHw{d=i&10ZEqD{j6jTY9PoJZ)2#UiITF$zq-dDPe zi!lygM=NtKMZqQ)UCcj_^YhsL?8{%48yPpkunHs-1S+SWZL}vHiE0HBrV_uvcxDva z@%`{`$G$;9%D=?~+=&^JPv{LMJcH)RGJCmIUAhwmsW@dP=>7@?hlr5zwGbt8Bf*53((CaUGT7Xn}!>h>0xR8Z~EmP5Z@=XocxEa^wLAztTNWc7`&8;r+s+z=z%cz*?yS}uu~(_U|O`M94}kwTD? zj?sjRfZ&Xl^b)kq5KdR*1>#ESV4!&0f4t5KlWD~j7JS00Dl0pjqAzqQK@^40<#OhQ zaR}n6^tQZJ^zHUYJh*gY!%8sLx|{zA5Cz6Jp&i>ckcuGwR6xtzwYo z8jv~`YW4&Tml-ZJbuGzUUH~7X zVe(vQ?y#1KnVEMI^U2$4p?9xu&SGMdjH`!fPime#EVntwgek^E4&wy%C_g1U?;EwD z0!DUq|EiulVf;Rv|7ovg@#m~<0|+}7Pi3=TfoC6RVKwdBAuxt4pmX>{!O}a>IX6T9 z4hF6f{3lSUy? z26aYXPelz;W%-|&^&OM!A4#*CY>`b*Bj7~kjgo{`j(}_)gzhFFP=Fb` zVuwT-ndaY9e6du0bu-2gX?$W#Yg+}9kn&|g^-rWry#Xda6U(2(k1}Xpiiyott)DmP z^3s1OKz1yuwxNW4K%6>7DKxTUw5gKg<=7EO&@$~)joAk6{837i{q1CbZ#uc*6ehQu zMGnys)3#8Ucp9CX5qBpyWF;J5Uav_&x2@vo=?A~a`+GB$Zj)1Z-M8^Ys@m1A>%+~_ zw~~=uRQ1TjGOhI`vJ!W9PA@e5a}JfQ3U|j}w9g4XC;a4@_hD!KxKM0$+rK&^rc4E1 ztyT5K!$mv)ov_uWIyDnl%={Goe2!P``z}$>6z5OyVmr_@1Na|6tvlXO+q*I3{PXFI4?5ww|qlHQ6*cYuOb-JJanacgGU(FdRZsgVD z;kBpZus>0ppGT~swip2)wqv;Xd65lhHG7L5l4pGI(U#<#Xo{F(t6AI;VfCsCz{R0P9CX;nS zt-oDgL$T2OD-{Ddjn`e6gQxAzrq}7WI}$t^u~wvDHw_eU-ojrC-ppJL9F}?)Hq;mi z*1QgF4rd3iD#rqiFDNsIwsST2d8tf)B92P=#CtUyXX}5T9nM)Pm2+j3)>4rtw?3Qw zvD%yv@VNTAPLMd463(>8F-Y*d#{8@1QE#+}_b1*g%`0JbG5d?m;BDgND^kY$UvP#+ z2DM}k_V@lM@Xvk#k^6&~WyqbOgz{93I%t!Q6r?s)6%OXSOV0gmEZ%-&<_T=c-x;}5vnJC zkrD?hhk9`xxU;hyY@L&C1io3u*>m81b-eT$_Ws30b=Q?SFlo^rvCS%=SMN=25S~_% zU#M$+>TbRH$Ye!sdotd({|9UHhW3QbU-?#jY{!DcLRBs3r`WHL9Gr>QjGqwCtVB7) zC>c@n+O^#o{dj=SCjDupj%?6DB(j$n#C%_CMqFMko1Vfy`JXC>kpBQ# z-+lc|+bS>LM8TxZ91P%FrE>(hNUa34^>D69MW!^$kb7{8Xki@RnewN9eeB5A_13de zRz3_q%i&ew`zzA>kE2)X?vsH+n^&gp$tW(ax>qO%;lq{8Isn|nC=E+I%;n#V^}yyX z`8%#&23WlO17eHU`3a}Qq#c%&6NYe_0?;9Zp3`;gs(w<_-+mspd@xv8hweyA7Pdm* z3e0lT07|ag6$QiyjAp&CpsL~~R7QF#*zkI$q~8|L{F0n77xO5jNwz9Gtc2wh2Y^+x zu;Slx;<3T>eB;(fR-_t)gd>rI?XD#yFkEq@ji%X=t9e!xp9W4NcUO)4Q@=!C+x0SR zgWRYZd_r(`7ujG$Bi~8Kb4-uZWbmEt1WUD7Wjk`ML^o?bJPo7O6=Bz}Sjp*%Kc&u? zN|~~;Chgak{0a)dXBr`yur|z|_C^b6?cYc5CR(|!%|t#bn_J`HD;6@{Hck$gch0o+8j{EoCFpa zls5?$YGn`+)kga!nSI7Z5A8}Jt1>bo>xpBKTN4)*p&YsX31;I?dcZPmR<|L3Gie>Y z1$mpy#K_?zkm}v^=+ePBpt9y5uwEy_KRU3=OP_*JgSON*^rhnTYMF*Jz72Ra=;0p< zbMnB*a@H0z^-`28cnfr9s0*p&=ec*WL#D7pSTU&97Kc ziUZZ;u6bQoCq-Ur)f|XT&fWIkP7}$u2ImqtBPMe}FS!93>Hlzad0|&Jmg_B8<4`4g zptmUYFZs{EQxfEPdzmz@J_PI$3u&P}Fv`%KFF8d;!{ldP3SJJ4_?#6+klSA0u=kGazwbb@6@Q#FitKRt-I+P^(u)DSnCbaz6^1{`|9bE5 zr&rXf2;e#%Esq}I1jfHISTMA*6?X?93ueh@o=kJvb-p`U5aR9+zQ#YBE<7>9UY~jt zZ+W=+7f$s0`}u8;!I4!6e=i%5QeS3#FXSwopz;1{b!0~FYGoxDqZQ!X*Xr%SxCU;( zak8B&#a~MPhLcxt1C+VhFpnn+XB05&M*qHz7m-@~A$P4Ie;*)iyw&IsNy%@0{SPJg zJx5y@vu5#KS*U0Wt!z)S83eT<#v;2h_x}OSIN71(Z}yczLyO0@@7}Gyi+o9`sz?Rz zO_vY>FcI`7MxNyeFJk}Cs^CzcjRE~6-%?#h<*eSgD&L%M1$X2ODs6CZoIzH1Yha){k@y8Bp;7?k@Ch&-=K#u zJvc7ZCbMy3D`#)+q>pRoe_KmRbVhEd6u9yl{3As`zk4}4KTmH$ z`}q!&Ql*|;43KTI@csM`@X~*fhBRutSYFNR;wj+(DKgZ{d*S+L$Kv-${Fz#|#rqp` zhoWH|SBh9@YCD2){@cLMgP*MGa}q+F-+bWSsh&I{N^z%=akcl8|Lv0GZ=Hcg2S4X3GD!O@Xfm1tz#0H{*VEuR8;F;c z8xXMy)8VZo%Xj|r(DCJ6lYbL0f(1$=053*TsI@hcsCaFXaGZeCp%KE^Zv!JFT{S2c zH+s@EmD4ocb%s({`~iV54RbtMHe@7MJej3ALOg=$HOrQj78mi&w<)LQc#vJ{zZyUp z7wH^tP;8)R=*B(_b=Cr=aD_z#>{y>slp#8!r8XemT4~JLmOK_Ecxc2P?JlzfG8QAm zeW;j<0S9POiQzd8iorrG>vTy1(#!E@!~0ICEI{lC>BgPb@T1gO;;C;5Z)19&3~6 zO0CvPIw#>aqlTS~#IFg7EP_@}%izuUyBF1Fy=E>)eNZtJr^Sg^wH{$Qzkp(=y}!`< z7%4#TA0RTeLv~>=La-p=jx>?kyooMCtv&}k4vVaaOa??Bp5mW&I4Ym_NOGrP6X!>RKlX z{5!HTnu&-a%_yOvnhZr9U$D_a_s(zsxU~ycBa*s=(m>hF{JWrz~DVTu|+o!qmxrq?c8p^E-IBf{d*Q<4`x+Gm?z%@9Yj=l%A!f*sr zR^>w^g*(oR&%HA}HuknA!{02>c#lq#xPOCGrWj+4<95k+|B7^4=;eZ=wAY8RvDQdb z8Uu!Fr#x#P8D4EeWqdd*6yHzpLR-v++xe96nB#Sc0arq-L)}vYFXx~yF=g|n-W{6| zDXLs6tdCfXwFN|++2@&23r7t@xhXCca`@_ z#}Bf7$c-b8{Z{kupuVynm?oezdaQctia`jT{Q_D4jR;)Q$)FV96Y}g^GY;8KbCug7 z?+mzYgUd+eCBF^ka{3oVbkDz~GMXQKR{bUdd4dV-4G4NtTrFWsE48#V96P6gipZKc zybW)}{!bhO;~rbvOC8Z%L+dKeR359~WDFKWV&2ovIweGC&l~rGk6~f%gi@4e$-3?= zS5vdwR`hwRpEK6a(!(A*${!3PPumMP82x;lq1$ zWK8JANTlqB+%Z;6zjVls?Ul=mI0zg{2xnEqh}B#e*hN7w+{E0eDaL6rcfZ6y{t?X- zXsvCnl^K!7JQvPEs$V6FeX6|FKTuMmk(J0~9)cxZ1p}Ltj)4`5ke1qsIhI}28m8xc zGQx8ozcu(Aq>J%pJ#l(%Ng!ibkRot+K~0zNlg9L04Oi|7w)j#m;EVRlWgEu*ZHl_5 zfD5|sx9{-&@zSa@^luR?9B8EeY;?CC&w0+OTcZ}Kg&tVt90I#q-zg+ug7{#VY8|wBkW65ICi`w9kn`^A-uS=>J)`Lw0oT z<%#Y_f}Pl*nBTZ}E=V<}n4)E2Q#aiPoGxi$IBx#5g0tzxS@J`lBWi!533%Ei{>_^} z#8t5Qi60xNlpRV&BkJn2xLjOdXwcuNdgZNsM4Oi*U24{dH}v?`ei)9_x@=za@?nQ- zM5alLS9ap)dC!;Ul*(4x3DD=w|6q@a?FtkQvYxBNVl!;V5mJhVpTP0J78eK+``@&X zzD6|}Q8C#o(?RUKLIO|RVXjajIG~~$frnS7j-cdbLWB!#W<9GFV>37Se{R$2qehI`cbF{)o)3{hj4qsVPNdH7D9s?M{VQq3vqnO>J=LL=3&dM;42cQU7fr+O{Mh*yHX67BN6Gqv5|uSf6eZlTy5?2+o+wd z=KD3N0CzH4Rxi4V)NYJ&kO7h5Hdvt~(z$a*|B zVBVkxZwFjGv>D+jnq-g>8qb);o=CvQvgqQhBY9zVQ)g*1{%k(q;BX+!-ujLI=m!V& ztWqyFYt{0K6v9}zY~PDw$&mu#b!fwhu!FRO5C-wUiM%Co=MHYcUBW7#Cpry@(sa2U z=A%KRLm7xfy?vlgRA}On(REc(NA7j%*SgaqN5jj}U^*8p#w1oAV(2Hv7*a)?pikW+ zVLG2@aZFPyP`F6Vv0|y^J1U5{6g9-OF#OHc%c7re)>@K(7bGeig6|QP?S~QNVnM)t zAN4R%=`z9`kfo^tg*972S##^cWOde)e&uooF}yfw(~^s(qU*GRn*rE{6R)R9@~MB(scw z_@z*@c?m7Gz~xB#Ce&2skRd0E2^)*kdpNr(dRo&eAn9#+&3;dQCaBCqIG|@0Yb+^e z0zM!b8AtSqmd|9sfg}3_ocGrsy`Z{XU`(m%O?+*-MN5Z6Sc{&F6H%{3YHDgFV4lTQ zVRc2&PihPi`r`M0^08VQ{Fu|P^~LL_(;?(mhUpJ+OuS8QTw@Phq?E3u4bcq(uK`f> z80q!8+^Qk;Mw}QK91W_ zreX(pxz$Kl3lyd~kzozlPxB$bcC%_3)xe3kn7l%W+q2gV*V}&*|B~U7d-$4HU~no| z*!!nP|6i7$HN5|1+L?d8A2PT7kYi8i<0RO1){4wE+5r$^4`6t(%KsA9c%wC&lUVsk zjTF^l?Q8zN_;Id1lWA`}D-mW~`(b=jIit=yr%*RCUBf_Z5-a%QdQVY%mD7Bk!;&k| z1a9hPt`Vb{pF z;m8eXFyq7NXi{*dU;E+rW&&wc4uCQ!&I4o9Y*L9R6nzVv(qTLT6`dl#1j*c{r8@|% z;dRt?2J8Z_ro-F3EXYi#JoGYwwewX9DVj_K%)2g_Hm06rl^q&hR7#M9Q}V)_hI_v$sEFoJFuFdX$@NmH$McseAEhA2iMC z`C)5O{ri|{gohISy_&-@eU_653QQ!51rr!vT9VLrB^EmPAmG_h8F^?8DtdY7TK1MK z9hzXrei`lz!1i99@BS($sYH8|eYkKEgc{WGcmXlyN8-%h;2)LeeW0Yb3$blrSLg2l zVB4;!!@}Xpp}^R-HETcg#vMZZ8#N9N5>|ia{pNU&@v~^N@@~i*QaDYvws+c zmJiWJKaSC)iKuk{KFQwk-L`3woZ#80@B?xYQeJ%{>?vUq8{HAAfp zmlTN^eAu6rG33sehdln}7>#9Y%JA5$h-*tM{(BouEG z3OwzaTkiF3c)2>^V#=vri0I1I{bRXG3?l4!@EwWV+^&sCa{Ur~d32bdd7sS$l{D?} zq{cmW_OJ|A#c2|$f>xED4{TFN5sLqTr+1|a$8#HGhAVg5GR;E&I1g3+eLtI!`|vx` zgpEq9aP>qYkyJ^FT4ql0M(2qai|U#Cw?%^qYU*d1*lg$dJevLZ0f2GZSiSJHCLexe zKdFjeNOC+u&HGt<aW13PK~@>mTh4H4}rZ9|7A}S`Cestm-@&HB@v!&HiuzygAu|Un!9r91S(v-tYhP z;f00^eeJcY?4=RNNwGscP5UGJl6VOva#IF3B-$N+dcNq{!K6?nn!E7HT?X}x__iWN zdBc_I3$m(H7zU0^Mcc=X9Nc~q>V3$t8uwMIapf;+7(J|&Dn_IZfJA=aj*rWea?2Zj zQtq3dZlHH8pgjdMHhNLO|Dllwmw#c_K)l%v?3V~-h|iJM#90lZu^D_l1g22s#cl>j z+?S0z zW8>8cH<8FUqRtwz%IVwy4^Lm|PKkIOh23PODC!mCo7P&VJP!#L<=bsNJ?QkiyDb^*=!H zE1@xni#rVpoVXP`=i71jk8R>zvjRR(Mv>~E+)dEw*--is876H&RfpSg0T8l$fY(Q) zK!M~33&Eq4y_mPt7EkBE_tpRX5BuxzAE2FtR!$myPX>+41cuYcQ8>AgwdntKYiiy; z9w68aG^NP>OaSSSk2wnY4WzX23I}Y5dLv3`YM0!tn-yq+U#We`B7k6Cg%g;F%E!E4 zYsExA;>o)+(^BUHKn?-`w3$o`K*#AApq6vhS^esGTDavpXme)UK14*;5K|Q~ikN@* zAnnsZ_P@{9?;ECOF+ecuHkR^Y`J&bC_1{cSW7fzg(2utM^^Z%&jFxJh5RUJ1{2msv z67kDH{6Ba42=eEHha#>YFn2-i?-wthtX$_&N1|HY)mS?l@5`spXovpCDAU^C#*|*m zA2S|G5j&7IFRHeiAWO=~m_ZycQ+__-6Nv~R>996eKX4=m>4E#(3+6{|TBe9eAVs=t zRA>GX_BY|<3l0p#ZuaW9<1(q$R}vgWw}aPjdN%2jEP65kWTj-W4u1S{6i%QNkUxDk zgP9vsAoO^+(+PDshNHOXQxL)|OgMGgzkw7c!Y%p6*nYw|PU%jmv`|$7NEi?<0&&Q` zX&?Pv^nc;m)sSmgvX51~Z)hwa0K?5g$lN{r0#baSkb3>cU4bp7 zQ-;Y@NP(+874Z0gXgj6s?f3>!>;n&IL@_V3II|d8+N=) zRKXj~fV7zE&p$+L2Q~Q0e>eZ1y>j#U#N?yhb@k(td+uY!z2~Fgwc!R$F9t&5tMd?2 zA>mQnze0uzv5*g@_;CLpKoor+^gq4mMO>Ox{#o(ozSrU8^1+nKYijyWWE+BSRSA$Y zT6^C#9;{=eWRt(#J)A@7{GNZmh=UehE%}Hb?k*|Ai7z@uwu zGuC5hJDSwzWssrsqQW(^8>uY1Zi|mL)@(M+$4Vyk_V(BNd`iY`x@%H_EWMV2_d>7* zkIt$7aDQ5i2~iY~#CE$eGkp}=rMY+DF9&pWy=Mi z3Jgg8P5N7gJb4Xz-_W;X5f}0=x>mnOrSryo)OI8}<;>Ou^F%yE*%5a*9uVX~q`7s^L4Fy`D zcY_(Szohn;M;IQzP8VTOI!kKb_lDDsuV`6Yfa6otb->C7Ep+n2Cedbv?FVCpQML6u zSXBkFivI9~K6K0g3j#b8cuNm~aM=GNNx`|Eg9?9qdZ0+GH7a zEyyWSD&^YkA*0x}D7<)$O>HZ+TNUox)h<*s(g^WQ{|P8X{=<7IK`aKIdfGNP#hT4; zEA%ImJm6`+$;j`zTcLl4Hh-?#mYDw%$}CK>nuyErtH*hA@{u$sQ3RP*TP+L=8J|iz zl%-7+Vwi9`?mP_0LGB;MHPyUI$C>O-7|t`Yq#}*1o&o!;L)NQs`?Y6ICPEi!5S78t zqkvExBBQ|`<~}5*d+YMlEf9B^xl?SiS@MSlRpq7~B$&muG8R<~v9p^=YpW@x?H{8XiV(4QML^_WWIRq!Z=th_Ig5aEoIJcgAPD zPLYauhs+`Bog06oav+6m$KyY?#g7$>k^fDJ{0}gtlkbi%{S_%dYHwm4imMr=zypK= zFfe3%|F*;u#x!qKmLwR4V~w z_=zqAZ)@gN^AL~!M+51KgQ0`GO({90zFBVgG;~aaDJv}FW~Tr1Njt*X6$Gj zT^UR$stjz@drhFIPs{0@DL^PMNb@^TmBu)QCxzIxqcWz7Kk~m_t*w9?C2U8 zt`mUcQ2&zGJ7JtOU2~6@P$)`(qkj#-w4}h4Pbus{nviDTzm?wEYH}CaC;vX^Tf(U2 zi7Y9{yDdCclrG==Cs&gwrUFZoXun|-#reG3w-qIDvI>4Tr}m2ux(ZY2CrbVX_7zql z4VHwO!;|`*L|GXW%Z4cDl?}!!UXg!fW=$u^Y8_ZLEu-0Z#M?Skc++vW+myMm>r4{U zOO2yKs)`&A>UTVVa0T#QkzYJ!KgSuwq)Iz?K7cH8FQ-AUUO%*iod2CXqvJa5{u1+uDAiF#M=1h)D(3K1;7`yi zxT&TB00ayNX~pK72*FddYeZ#~_}QVl`>*oVDc)kzt$Z4kdWth4AD6;+)Ar;){i7@w z3iFkS+%zjiLJq+B_IKSKJW=mT?9gY_!5On3i#iftQ%bVJu$d1_xYx)&3&7j#nI#IT z>p$vO@+^VeeZ#mzwXSqmBmC2ozN`Tq}pB8p&{^D9FK_=0ej`+DR*06N|Bt#0x2g*;{+F{e<#=|yu7 z1zhHJM08}xpwFzS`)#pI#{Kw;UPrkq?%%U8fl4rsfkcwM{bI9BxOiBVAs;L-QPm5aLfV3vAN z>Ox+4wzK%{Fy`d?$eyAeT@J8-bJ|AsBmFoH81QUfKd+Z>S?gVS^TBV=zbdnc$@Ko? z4Rs>h@=x_>9aWGstI47+HrJAhifOn@u4xG0P>ok-@sqWx6`%G}X8<=5HaX(lxA?Iy zf?UnoSFCo)sQv4*?8P;zpykhU$ldDsIPF-33Emoq->p2z*r9xR-suW%gOX{^)EQw=(j&slVKNGU<4Z;!G@Hjm1ZG z^em&*q6m}ow=E~%{HBo(ZR_Ern(I(l5)zjnt4^%E_VcWckxbN>r#_ji?aXYX2JHt-h;>rq+r@C2v#jN{P)Lm&!KK79=EORNL zU&yAQ23w<>o+P6oqMpHd(;1tl1*e21WO!_W_(&`j3&!;qJPfQ9LC6fSw4lQ{^uVBC zA3&$7Y&VlyRh1TWT=3PEfJXaAZ|nrfH`6_KO1#ODBR-D2Efd#`_JwT;V#qZNPj zDMG`a#Ush^12!xLD}7RX_p6RRfNS*^T}1;p?`s}~Bf^8P(yUbWo4p;oN1vEq4cY-sL`dOvC5b2K{&saxFT^^_*P6-AsU2V)JKCMJ#h9RvcFm;AOwFQB_k=Dr_}x%0t?Tp76RmUZU6es5-v0&@H}Jclk>Skir_m zM^r^tPaMks85%d}9Flnx60z&bosrQ@E?}Z zgg)}fOhIxsQVwT&0r(I*EBhqwrliGVrIerN`7bH&{77zY1g$U_%c?a}0oxs7=$WGW z%uL;&ng~i;|KV@%MODe13=8&~vNZKKjrOpKzL4`XR%IPc z8xAt-i^hNEA(BhmjX+8ARp~ch7al47+^8o*I|eu>6ccX8IJ6?)g;Ge5sE^~6>>my^ zHJDw}y!n-S=7MBc)O&zg(aW?fAnr ze8fm|Qb1=Z;?O;4L_lh_6`hQvb`G&8XNViDFEEXi)GA$IELWEFpWA6@p$MM0wn(TCdQj=e{mO>f^eKfTGp|3tSp-2&R;clc3MvMiss6w!|0 zZ!_8FejSaew+=0k-@)~XLJ4%|HO569Gtz~{3$z4$X&kQq@bD_WYHRXKI{z;Cgb#}! zx%2IMOx)#j&+1X&Qn5dW-b>{RS!|!k%6ZM<{{X{c05F3tzz3u)prLLstf`Lpi2#VM zrUrf7doM^SW+f|GAR7HFlP{a*E8gd-gTaQ&UGH1<48K^f_6lrUUQ6FFe+qj5mUzo& zzDculNrb@2AK2=A3tqxjuM}QYk35U_6PTA_m_uzBUU{%#N6wV?%M6-~N2^4I7XVx* z8RDwhz_sUk5aA%I>gsD0pUFJd+~;q|=*Zq*@Zn9N911gyoKinR&$rDBLzKsYv<|Jz zy?cj)%uubgniY>*ZyN(w+q0#Dgap%Z4NXt1)- zqbZ|t&@Pk>iG=flxuuDEMBS=vH^vF@Lz@kknk)Z2&19yfKorCdmsxHQ%Fku^!y03u zqSymdvDUb3fAr=5bw z%Deu;LiyU{I%!_r6rnC;tZBMfMnGs~?T8cl|CIIKK~25kw{L*Zr3Iu2VhEAmk=_CV zCUin5hF+vfm)?7C0TiV7CIV6f0cimNY0{)CARtAgcktx^-yd zu4k>!(o`2sJxyT^=5M&rX#5teKqy(JcRVsKFzm6WW)ra4LK-x$)> zo+bT|{se#k%FvYTi_tMZ(24J7^W32-AGnKt>Bh~kYfXnixIN`EK~Kw3t`BBJrM<%3 zGn2jB3V0+O-G?I?!O0X#BRHHP3-4{9{gEW1;E`|=A>D_3kmHT1-9wzgNuOFd*?8{H zpMyJC-$Z#)OKx!s1YxueKa1%goPM&SI`T{RH=a&dQ~DO$TJ}&|ASOidXi51`MB<*jg0BLvpx1x8O)<`0P)#NGaWONJ zfHT;dm}xFB_#oK8{8+GG;jH)fL-_W`zQuyC|DWTFZ&hy!bB5?yyUngw-}}gOaGZMJ zNJqF5IJMx3Wys<`fE?soh_UDY59&_`f*|C3zu0epbHZd&5&!3IoSj`flciR9=^}fX;&gOYfU87Jy3|NeHVI@8Njt)jxR5|1 z!#qQHsdy1rIB_kA{~>3t@f;4&|4=y!1XDdG)SVG6jNpN0>##nIjDS-AEG~1Ag`fn* zQ$vuTR0IxBvtaicL*jv8dn{Zoo)b3Zv>i*Fj}0*b$1p9?@FY;%0!elVzp$5L;GMCM zJ6i&q^3Z=$Lc?`c?zvf6(%?QFR=bOQ}3#&@dNU6LH+Z0ly%Ik()=Xm}II@3nc;+;{PxYti5o zH~a6zykh;)JFz6q7XF(x*bx7wEhMitwsov;{XAivT+r*tDrfsfXdIKaO5B~$|73YdFQ4a>;G(?!bgYGVn>IKb*O&1vM#=wb zB%*Gr&!5HD`~x(7%umJ83L#`DbC?&tO%%YhB&{-VWH^yB4jnKEVaI7OXV2p!R`n!I zax;}wlmq;%C?PgE*jreH8PrX`El%ukh+jH*l1ATzZLtxpw9xxh7dRkzJ0WI-qonLxgJlz-)CF`_6!)z&ff9;_t-j3D|l=w~H z0Yc#%L4N_Mq@8uaur;Pnb_Ii;fwl`ZbYk6$iHTxFw94VPQLIwu{s2Z^@raJ_bS|{Mx=IPH$xR@4@ z#vS@))@Keo{vvyOwR z#Pk+SRfB%t#<*C$BK0J6b|^F7NfM|aIJ3!tApS??Qgiu138Ncp>T23lK_nDvLR4)} z^z%v@HnPd;Jxb$-Jy_DLD4wSFp_8rtu$>yh<^9s z`)E66!ze8&>TWSYM{{FwOE|VnmV}GoZI*a%X_t&xEZ?kz$G!RfQ7yO0FjQ5`BG+^l z1l%gE3>?~yPDF$LK-=5hD&H=LF}TJ=>Pw1o-AxAeHKujiqg=bjpq%VQ!gUCtR6}S4$-G~tUE*-)6?;=rT41~!s~wZz#M+= z1yW)|Nw8;Fg2>U{tDU=vJztc=H1x09UphOxHdgAUAxxM6O!VvJF|bg+R*#glVRdS_^~olA`$J8$ zw1XGv0%+e?kQKr6`CO%vCNkooNo0OSFUhSzD!--dM|_{&kkz)hN?qB$*SmX{?!f$& zpPM-uGT&nXypCyQ_NGm7C2^HI&!P}BtsJDy-AQVblsmK^A$a)rUwFR2@3AiP>ugym zAn6ornnP@JCbrJQ#7|z$gifXTChl8(9YT^kjRS)yGda5F5?1b2;ses;b?*#bHoHGo zLkOtXi^)^?GhC45u9s@Fkgw`^TFEk_U!KUQcB~0dt?nIGfKDGUx?`&yxk<~tcmTaO zglx9wf1gGSYvhg*2#*Mn-A49vbp0i3!2e{bT36=yyu@()i0_RQ=Ea&2CR0lL+QGtd ztniVUq&qmm>czg3&Z|*kBHnOO$9_@NA6wI`*Td@Fk2lR;s842clXH0@>5JYG^=9Nr zThZ0lZ;_3b4rj`vX=3~4ks0*xidMxUXFNe954rp0!-O~U_ksm!qOjuyp(AiyZXI~l zvKB0|B1IL4K1zrZo`lC zY?L_oV@06nJ=8cL|84|EG*%k=d=yWxm%P5+qi{^s(c~qyU7z1$aS^;67Y^qUmD)tS zo-jk*Io&!Z(6@H=FU3}PNL+u@F_GI<`$|%U&;VbvmryY>P)7q_&nAqPmIDwm=EKRv z1iDS{hsngrpgcSSlGYg#b9e1nYJRS}8!WNb(Z6}Z_bF2&vQnib0(=BfUDH}Ghni%J z!tsF2{{S>4%_{JH_c9Y2B`q;uFCim34+`ReWQPX(1wT%j$J4;d6&o(D6)Rv5;21I~ zNGthWxD{B2LTr+_seI9E?wM!<{C#fbZOa*LNvb>X+TW2}lGDfb{koW)5E(2A>IBh% z-j!x)^t8?O0QQ`5JpM|UL>uV!C@h|zKVg)#wKbe&xf(T%j)>pz=i z+C>;+HkVLNV%Z%Xq}qcbFYi6m{9|Q9ZoCy`-EbgOQ!p^bl;7vQl!u#J?JK?GLoEE# zI>!Xc+4>D|s6C$_!(nexS+#>zwe2x^#Vb6r#Fkb5nPUQEWQhc9LJ}N+^l{pQYZH^t zGCv$eRg_0$Y2)y(7yoD~R82jssuG;oh()Gm0yrH3T2}9;d7^she9i{M0|p;&B{r#P#U804?=TnkcsL^BX$C zKB{E#sc_N?87l-H0)#Ym@oeT&oPu)Z0PNT0a~!k9zcB)~)f}oAvlE#zR*#HQq$6|T}yf`D}WL9S6dYS~lSq)Y9 zWs+Ann-bKM@hshi6hpo6DJQIsNO+**d}x@>v|;Q%MND6-msmWntWA5VZtL67MW;88 zc`D>8)kQGe?!jjCC#k%L#at?67DTu?(qi!|%-D2+$-T)k` zcZiIkX{-paB61PpcmE>;4t*VDZPX8?RxIKV;$2sL#MQ@(Dyf3OvcteV64M+vL!E8Z zc*LCinpWH{7MB+T_<_Zrxc-(rkl>;2*{+)9bESB!WBWpHQk#p@Rq`b14=zK(I3<__ z=Auhe^jW|X6E-cJCj>;}e+-9`epCd9at50%4&6sZ@P(qA%RVc$pQ|JYRqv-(=~5#& ztiA)ZW^?992!ajjCxcyeicdYhb*_FY{ac+H~-O(V$4Q67}11rGci zl(S=O9(+JNA9f^O#%(_@bkUvB;G(VA*_V@EINO46cRj3o>kb&n8AgQSr2?oCsU$?o zt5v=C^#|`$s<4RnoIX&Fc|z)gAGT9;EFM}Hy8vU;f~veL)Q-sxC84EX(teJO5X>qh zQ%cxNKihJL*UHGgevM~O;Xy|=oG-Thj_@!^8u=`^08a>GLn2NKg7!4vlV*pyOM7Hj zT%al{ErQoF#2g*D?{j$PPzdCIXy-=w%ycfwFV6OY1(UaF**la?3P37RT?mdzU|fP` zTz{mEXQqW#2HEHChD%iuJEsH8_+MX@D$#X)hFWgLng@&m-VrbkM`i|Qx>O$wo^-LG zv`<6g^JjL4g8LOt)atVt?44OEQh9AX9QmDe?V7JC+jETbR3P7U7dX&bnV+H-Ej=(A z3m}w@QLIdIIsee!uRn+YYQ@7dug)w(c>ma+ef$j*XYCh==d&ZlBsEk0iPqz1F&W@0 zYh^rWEP33{y~^xZ*|T31+on0WE-m6zdtL6tF5do5iFu170OPN2L`}6^*te4zGu!sf zG7GPfcob}p4KUQ9`YdtT*}qG~7r;N#=N*>7?=85Aq7+X9jvLBfx34wirwb{&6{FR< zGN@N;Ebi#O^_%f&pS{KP>1?r(cS*2&RCzv)quqKf~LUt#pH>4!==>53CBnHfRk)|VK0zQg}+cC zC8AT%ZFZBLT=WSq)vf^{3AA+=Ic`8W=*%jTTNmGIP=&bv3grY0Ra#J{6;UpoqZ=Z! zB{tE@MyQJ@3L09i=P{z+h;`^1@E}9|%5occ?8>pKcQmjOjA8>v%>C zGKrQgv6A0?3khBf+Zog%c))q&D(|vB^CQ$ko#x0ymq(WzuMAbe1}4}_Dat%@nKmaf zz`B`IbXt=8_|JmoJI424c|1OY#yl79t(A(V+dY3wJCh+MMvY<&I-@S#7zeT*!fW>x z?fEClo6e#rZ+LV_MOkfz;?+PIg@@OU_u{9G=g0g00&SOb&W!s4c?%4xE5ysny+5>(25E~7 zSeiZ1%XnAw+`+!wp-GPznN~J&bn2t-%zbKZ)h>`c`jZ+Jr#50&$)Y@oR0BPl7t|Rj zo8uG0!v-dvtZwTuZSyu|#Z?wao7*1c%8p3bxgntb+l>I8FXMpvyhQafK6frO zRN~WpY8`|qghYlJ0gkw~_ELHqz4%-DG?_yD87Yn%QHUr!Yt`%%gHIMVqTbMWSWc>p zqz%rc4d&PTrL1T)JRpzeDA@oFLZdi-5je+3mCQ;jr@6o^yIWT|qQ6*QUCu)CxL#+4 z?f?$)yeOp88l~v`@yIegnc-=_6|a8;vH|IZji{f-rbf+ud6JR6A)mJAeblRjZ4yNJ zhKT~vFCoeq3;oY}HP`sVNw~yY<9_-1&7~d7{0yhpItmIL$K^jwRc)T7hE2-7lfu!? zsIx^+#V_%ufYdG73Z%gN#z#_Zi&a1x<&+8~=imM+pCZO^e1H9kta@h6PU(kTVmChF~j11R^`c@M94B<24DJ z9H$>cXN=-+KO|UqSBAa4s|AyiSdfM<&!tLm%^xoHsfAm5+OXoNS#} zu@`0+xCZ(0uN4J2r3she1Q84IX%23*oq&WC@8b^3dxmJUjn9Siy8Ff@w?K}a+f=Zx zvT)$EALcm39jqXcvWx0h*zRCB4yzUq#)qd8_P1PuHnay6}a=5;ayQXcP zL-#<$+koxEtO6W#aK;`S&Q%MUOt%IU`MyST9SJ-^jZ zrhxX|{~@_>J>xFKtgqnZssEf&Roit&p-IMs5X}^ev?WASn@;7@JqSEYvJq}T>XiR!e0yWS?Myq zCG5he#u$!vO_j=0Uk*c^pY|>o<_doS0P~`TSjM@|Q+|=JNp`3cORF?|M-}_ji?c)? z9m_e3KqiEgUp8Zgbe#c@3kql2*Jx94n?2~I;ug{DVrEob6KG71u}iz8|Ku?!mlC2b z#pi5UzV%Gd;{m^u$FOu88XN&dd)IgFA%K?ybV%nSpUIM~6JfqM#9OjBS+=aQtaLPp zq|lL~C+N?32Mp{RCk|Ly+T^s*Jj0i^jDy2HHA$z+zqw(GyJ_q3MhW-MJcs$CRCiH? zv`(h%P6<6Ssy|(9RT=+BcBdVF)Axr9HRnjYbP+a<8|l~ADpwDI9fS#CD}7i1hyw;) zSldg$l$8al=@!Fn!g*ZGbr?N-irlQl_QpSa7WxwsH@BKhTlCk2Tm3T1f3o!3Mw3nW zr%yX+t5$hU4=pgL)6@kzlyIGQM3a=AZM~jwHYR=+yIMbFM4m-}-{x_<5o#V=26ff3 zF1eC9@mdH$1Qjw-*_B>pBYzbK%RB)wh?}vZzU@M*U;e_6vo~qU;rCS2X?t1JFHq4c zxhT44e&E_YgCAcYxMn8jk?kWpfo7Q>dm0Zf2@6l>RcW0Q(#n!2NrdFYQtTfZZf9WR+0B z1vu_!uc?q^g=bWw!pC75oUjq%alti(L+hNgr2I`XF(=zht&WVQg^Y}h`z{t?*{6A6@2TOOk;yvG@Wz4VugA3T^CiH;@s1(Qth|-Y z!@leS=32++0+s3R&=s3-pX^hOoWC!n4V3VW!(b7*-9@YW+pQDy57vb{n;t8FMZ>EXbEa&ih@TUPLV#HnP_3w>y51n_74= zqG*Jku3)Qk*=i%`yj0~hUocL(FFso)iGeMH^Y%5M8c{TZsX6M! zP7F%FRe&2wIu}PRrJTbU>YSj>VTz1Hh|RG5agzGD;>(EBA9L{+5PF) z>>JY+byrwn}W_bmX+uu9#Emz|xf+G`r;%R*n#|E-Q#xr?x|MJaKz$k~{Ls{J%K;EdDd+om_?-3q!?>V% zn306^ThX>Tx65AJ#wucSo3e#ZUe3kebAkGktqX-1JhgIERA_M~X?S||nwm;t-oYN% zO-FCNBB$G!BX2-kfKyfMu}GpVOJOLw5Oam1o{?s5B`h(AlveVW{R7N6s8A}WRRIZ_ zJ9HtgM1=ikj|}8~>2unG&Cye8&hI?eC1yrYnXC|=($-h4juAS~_p#wD+dw-ZQ=Z8+ z0~m&4boeC@4(LgnFgp@r64?Y8JVHwhMEw|`H-p$jF%{;-JYevk?1}w-vca@(To8lI zcte6q*6L!cq({ipiJaZPGO}oyWxs;Yt_s%iuDnxWj_aIT61my-r0FfjE3{T+3neP< zoS9!c*zolN8KjH2)WA6Vd?7ob(1L4~WUjauy+PB8x6zB}WSs@Th$+?BUEa<3Q;p66n@q`ON{F=edD zPS0jyr<J-S~YIzJ=KNC!hM$bA_3S;T>obHFM2Geo9jkeYb!2FzSmX3`1akMnY zjmKt@muupDY(Xwu$Y(9!Bs`6m_q0+%BcZ=sd5AziG@?P$?}M-78t(0lJ5!G*; z59siwko-X|3*?TxAe^tvN{mp0`h}`QTw3LEt1xG@Bh+eeX0OxXre^fU1d9 z-wMpcyic_3Y~_#dBGJOUlOfP`8}XNPH*@3jxBCkv1Xx)S5;A#M&LbVG1q66*Bn}1_ zd%TYb+1_lRd2X0o82=ZYv+|8`x(eZj(cpoTZ9xuzks}vs-V0L+EbNJIg($~8{W15wHZEL z?!26(P7~Dg5janZ0zdhnM0q_&O%wob#~|EC0>t@91yoeeQsn%D&#q}wB-ok4?@^?H z{F_1H+X?J=qoGtd| zjgC1(ye@7?Ib50-lnHI8d8DX7k$~7U6UwQO9%!v%PauupT53Sx2^P*Lx@=6xDUu<; zBU~6{F+&q3q%u{r1sV?fBB@c29OF5uc#3^OoS*Zhgh5yl86S=T1yw)qsfjL+c!mpL z34d?q^;;Q0gbA-^)i-HlS)_GH_3dF6y}yqt?{jD>MHqKdn`bOE)XGYl8f?uAu>?|g->_nR zB4(Nsi8v6zPwGxjYntDF3^F}S)GBL*l5pTP7xsY7)@;v!%nt`dL|!xX&^_^xia7ml z)VF4XQ3T*1kDTh0)AyVj>>K3g5x)w>H8dDkR*^O)NtbvDi0lTJgM~nA`YN-j`uVxP z<=08oBd1Lr_h{L~Ig6S`wEMo6^AX}35v+Cv8f3Gn$Zy-aW7Cc2>Z|JIoEXjgiywHC zrm|YXAJr%;G)&R0e+)~zOUYS$n zX>Z7=P2BKN&!jMPsT`X>3ATL)L}mIlXeAZd1h9?drhE3i<+<;@llwM4+^ZtzuB5l3 z(_Bu9Ye}kUqxrneO&zV0-V{not(d?XC6WmjPXg1rgh+do*R7OA1=OkDw=QZ=9bI(G zQikZR;)g{Ez-GT+epi(ovo9?Xu>@g|Q}J0Tv*j>g7&oUE2}if^TXGa=wNLyLVCeG0aPoXVwVL7J60{kr?hKiSUvIdsu!RQ`u#7Xj`$9vkn~r?`H~6 zxV>h}zd{%Dk8Nhxgkn&(;9ZViIXw~3Y-sr+vlVKn7hh5dF~7Xf{NT1Ycf`y#B;MNe zwpiVzH=>d_KMikquiF$oD1ku<*o+uyWYIvDg+=sY9=GZ)E%+_36=f&G&%$SFGO9k~ z)f(2B7Qb)#a8g0z0!9 z+Drwu7i7@NR8pN}Ju8Y$TPPNLPmFQiwP!E&;jZe=Pp9SDMAPi~8w6|PB5vkRrt%b;QM8R&1pF-k#YC14{Ojz`z&jTOAiv-TWb|H8Gld9NZFUM zArz)D7I7L@2uGvTRU8Q^Fgz7rn_$U5>tPXBiS73AyG zJiC6;8c)A8P*t9BFMOsIyrA9mjG|43HEg#w)Eap;Ve$d5NXOQV6lkJf31d!gX8eLa zBgVMvo6{j{cx(K^Ge0zyQaGTM?}K}&N$@pKO$Mf1;PQi3cQd?W;5D7sa^-Ec1Sm(N zs<7IV)|B$kOlvR4xo#vUY*Jbrb}tgdhPNMitC{E%nb4Y}0R+#fh?rlf5nyAQ(#w>s ztNkU1%dFc~cxj{_dwA^OH?ZCNBjx~6gA3rt53)hqsMPcZ&ExK-wT9##tqHZ>Q`)c= zO`PKGp|X$C=(FTh8?beR1wSsgH8BM95mHLUYE$#5@7iYCqz!9EMRu&Oa z`8E0|I|lC|~0^?*#6MGOK_#R5w(Tl+{gD=_kj4rr7j1vw-> z+mW*T8DT}Gh7D~cUfo)y6bgsHA?vFxKh(`O-5(oJTQj|RE;nhjjSCNMU6LSAZyHsP|6ToMtV&xjy!h6M}fS*tm4eA=`t>oLu)_-PAH(n7CY`z<)n zen05}3XrftDF*2|=Ujq_aRSWk?JdrY#q~@o2Wvi6!Ic(bg43aZ8kENSL6`Xv|M{Zg z^9oNBPK2x{<}PvTl(1Adjho{Vs<6RLj(f~ByNBX9yA(Kqi6vJ2_<9%Tm?AC`7f(u_ zWl6gAdubESweOGKl*%Ib?&`q~q>grF)tx@2u^^=(MrmD}gmWQgd^?OHSZ$x}v^J#5 zZULz=igAE(^?WNqCqhzTD2U%mkpv?JBHgg83@1xe=E?#or(DB7lfV=yNx*aA>sY1H zH7fSQRE`6c42`r~2w5?Sk)n{rXt$6Tb#b8586M7r^ByjP(<-biD&VxZl$aDXK9ZIO z-_?&82nc6kASz6t>0ry!O?wr_0|4NvIIwA}29p*!iPXyaAUYE0L$20_Tbz7$1b1$Q z26raDEPU4e2PY@?W8fI=DCQqmidUE*H$^I4xkVp^nen2JeQ&gkz~Swd_{&se@^R=rD#fqmB;fc)N5~iCdx?L5s_%1wxVx-WnkQ2*{X@Q}?P4hZ zga6NP7ShR?(KXku(45m&;zb}e2dQ*vIv!Y?ZS7+@2?Q5qt^a?>Z;CYIi|0K+5p6wO zHWrm%vF7$){ewq{&iNG!^;4W`7Ebb@OZ~GRs~NE1JqcLwt~6N@Ld@n_fA;7ewX)b< z$sb&;NHphNbr97zBj5A;-~W&cp8x(`cT&y7+5Tg%`aCUvz! z$@HyWbYHo{kCRx1i7qhvR&;@%!sew)BIot^QOHq(Q&-Kr`5lYI?N-4?+?&6we^V2l z-^y?Q0~|!0V(oqy`i$y25!{Ux9Q^|n(Kx&19~$0$kZW>2R$w8&#W^9ff*{zs70#|( z4*nVFZ;9*2qK8}YhaUW79GaH}ReduyCs!m5>=*w4IOfW*obC-7u6q{aU4$N{9<^#5 zR=sa9iCk98(lT1Z3W8Mim1b}&c%q*L}>n}@#tm7(^8=LI2$VW!|1dW~M zzE@8#xY9ZQ0Z7kYhg84W9e*i=xwj9zngYN0fJS631y?%oUl1RCN&W}exu0d&W^R-@ zJ5wun`C-RQXX%KSoz-syakEPK{N2g<+hy)Y8Xt6sOksqCRTW$Z-EF+GzLpLKk1*_Bhm9Rxi zO(mKj&sEKzXlLKvv0?|UjAxdCpVvAj%_I zeT$|shrEADcAZnRm8imu%Q){=O2|UVTZz8^jU$_VF{36>NM1P|fa*Z+zI_Q9Ejcat ztu1%yxs`y~=1Y4zFkT~6YAIZLK(-?Q+}n^gow zOlVTscum`p@rG7fq<+iXC<`75pk*?a6i%qy7UsF>n%I$Y=6Y1~%NmhSe4cS-#N8(1 zuv$tnuo!hg^P#Iuwp_{sJ9xVg@Y!t%hL1K9jc;bH>d^QK%6=B5Ff-_(lM>Bq<>C)L z|4RhOXV=?L(i#w%?cR@y{&~G}*V=zjVfm9?E+$7b;yhvMV&*E(H-jr9_iJbRQ!~t;v>Nevp9YKQ3zY5jE_^Op?TVp;h!+`!+tx3+V(6`jZI8}K#O9H+S(U$I9kjDUfn{FzCPgXX6< z$Z^{5?lNPC4;@A9a#d^t*N**5Yg=+1$Td|hs$(!s;!~PMiWX%QEul&}sdCi6-d9`% zHymHW2&+O_C!+v#0k0CYtx1$ba!O-AJnU1w$SL*Mh@_B_x~r5G{`V$voo&}^<7AqXRBkc+YVLoSUHtmXi@2lmAb}EL zGKm_Y_)h_X77JNrv7Av}3Vl^estN{?N`a)UU6E!(NBt)NvMj(rMzdxKZ5-$9g96%{ zszk>slF8M{?CG3sTDR;DtCLhRf>1}5rUjwKjKacU_IaYn__lY}&D`2IG4NDnMv#EG z{1yX5<+r3~@>djx<#eUX;O1QXZ{MNZW_2s_N6j)>SWz&g z+ev81x}E#x>Q(yTfFa=>F2JchqFS5HZN4HJRO0366Ah*i#)mEtD@LiLm^X2Py0R{^ znrykXUumUD$KX1N9qxz(ht5kWcr^vG`qmMrkWU0F0}WxbMGudnGq6|(ng@FJI7)@& zD7=0vIC$(c$CG!@`D3f#2d%_|% zgJV+TXltB$?D70p=OwnIxInR23Tz-_huWui;%5pI1=o5TvENjPwvL3|z5iZ!&rTtU zy_b{+BLBL}xYBorD@D+E4TIlmSBptpRqO{^{m_|0)?F;lg(!t2nq}U|;o`UbTF9fQAZ@PKMR_j*?i$`{o{#5gh%NVQt#&EQ#OYdp|IJRCRR6o+ZHee=WSMpY zH<6b{s9%2TAiU{w!e!P#$`WP7EzJ(Ys`u+GJ3n{2J^X++w%0xK6&iUl-BxE0){8&8 z$LCxZ-5H%VCc<1pFDUp!7nF{RL7^bLZyDt<4Ztf5r!VxSmP}o7J(1m8T+cBPS$Bx= zHOT?z+OL9rt*^MZ#U;+vhRa~EJP4WEJ1HwMmAKMBAC>&GxxITL{Bypb-CX$}pjOzk zoYSaF6jyp*Pj|eG+nN*}%%al6e<`u`*S-)pq>{$d<$yw#s}~ z;^kPsQ#_7Kb}3K^(n|oH?qBu4G{5dT{s%Y=rx>r>69xrgPZV}gumqoiQtI?F3E9ib zRm%hStNWX!<2}CM81|EzADdC>hhXi(-#Et-@E>5v^B-XBk}mB4nl_O>{WF|f`Ky%% z0|R?n48MF9IF8vloWy?FOzq=8fTNoth4!wS$3e(NNPQ2G-3C(yy)a%H+wS^OfSX6R zAihJh++o^mPW~3*B!4iKO^@AK-q+SE(oH-d>?#iW$#j}CMgr!&#o5D`e$0g6#-2O~ z{!)ycPREbCQrB%)4eF=kJCCmQPqmM5D?xuhPiOCl7*Rw8N?C++_re<3Pn~gSNxL1q zQ$}!DVO~Jax#!kD04V7S_p$@WguiO{d?6s%ME(qpOaAz`Ji1Ol)*)UaWsOXve){?D zj*~1Wxnv7STCAp#jYDu8l-{~VnRf(yR$}YB+jz8ztLrFicaFwR{J@=p;V~B{?vF13 zBU1@go4$uu?M%`2?9%&AR+C?`b??n}TgVaTErcapT^))EG|s+D`%X%g@^Ais0oZ%Q Ar2qf` literal 0 HcmV?d00001 diff --git a/examples/frontend_language/quick_start/local_example_chat.py b/examples/frontend_language/quick_start/local_example_chat.py new file mode 100644 index 0000000..e1e4b62 --- /dev/null +++ b/examples/frontend_language/quick_start/local_example_chat.py @@ -0,0 +1,75 @@ +""" +Usage: +python3 local_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf") + sgl.set_default_backend(runtime) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/examples/frontend_language/quick_start/local_example_complete.py b/examples/frontend_language/quick_start/local_example_complete.py new file mode 100644 index 0000000..00a451c --- /dev/null +++ b/examples/frontend_language/quick_start/local_example_complete.py @@ -0,0 +1,70 @@ +""" +Usage: +python3 local_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf") + sgl.set_default_backend(runtime) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/examples/frontend_language/quick_start/local_example_llava_next.py b/examples/frontend_language/quick_start/local_example_llava_next.py new file mode 100644 index 0000000..c941a54 --- /dev/null +++ b/examples/frontend_language/quick_start/local_example_llava_next.py @@ -0,0 +1,78 @@ +""" +Usage: python3 local_example_llava_next.py +""" + +import sglang as sgl +from sglang.lang.chat_template import get_chat_template + + +@sgl.function +def image_qa(s, image_path, question): + s += sgl.user(sgl.image(image_path) + question) + s += sgl.assistant(sgl.gen("answer")) + + +def single(): + state = image_qa.run( + image_path="images/cat.jpeg", question="What is this?", max_new_tokens=128 + ) + print(state["answer"], "\n") + + +def stream(): + state = image_qa.run( + image_path="images/cat.jpeg", + question="What is this?", + max_new_tokens=64, + stream=True, + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = image_qa.run_batch( + [ + {"image_path": "images/cat.jpeg", "question": "What is this?"}, + {"image_path": "images/dog.jpeg", "question": "What is this?"}, + ], + max_new_tokens=128, + ) + for s in states: + print(s["answer"], "\n") + + +if __name__ == "__main__": + import multiprocessing as mp + + mp.set_start_method("spawn", force=True) + + runtime = sgl.Runtime(model_path="lmms-lab/llama3-llava-next-8b") + runtime.endpoint.chat_template = get_chat_template("llama-3-instruct-llava") + + # Or you can use the 72B model + # runtime = sgl.Runtime(model_path="lmms-lab/llava-next-72b", tp_size=8) + # runtime.endpoint.chat_template = get_chat_template("chatml-llava") + + sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + + # Or you can use API models + # sgl.set_default_backend(sgl.OpenAI("gpt-4-vision-preview")) + # sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/examples/frontend_language/quick_start/openai_example_chat.py b/examples/frontend_language/quick_start/openai_example_chat.py new file mode 100644 index 0000000..9511e21 --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_chat.py @@ -0,0 +1,74 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openai_example_complete.py b/examples/frontend_language/quick_start/openai_example_complete.py new file mode 100644 index 0000000..d64bcaf --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openai_example_n.py b/examples/frontend_language/quick_start/openai_example_n.py new file mode 100644 index 0000000..25372b9 --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_n.py @@ -0,0 +1,71 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=1024, n=2)) + s += sgl.user(question_2) + s += sgl.assistant( + sgl.gen( + "answer_2", + max_tokens=1024, + ) + ) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + print("\n-- answer_2 --\n", state["answer_2"]) + assert isinstance(state["answer_1"], list) + assert len(state["answer_1"]) == 2 + assert isinstance(state["answer_2"], str) + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + print("\n-- answer_1 --\n", s["answer_1"]) + print("\n-- answer_2 --\n", s["answer_2"]) + assert isinstance(s["answer_1"], list) + assert len(s["answer_1"]) == 2 + assert isinstance(s["answer_2"], str) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("o1")) + + # Run a single request + print("\n========== single ==========\n") + single() + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openai_example_o1.py b/examples/frontend_language/quick_start/openai_example_o1.py new file mode 100644 index 0000000..2e5c140 --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_o1.py @@ -0,0 +1,57 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=100)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("o1")) + + # Run a single request + print("\n========== single ==========\n") + single() + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openrouter_example_chat.py b/examples/frontend_language/quick_start/openrouter_example_chat.py new file mode 100644 index 0000000..a0b6f15 --- /dev/null +++ b/examples/frontend_language/quick_start/openrouter_example_chat.py @@ -0,0 +1,81 @@ +""" +Usage: +export OPENROUTER_API_KEY=sk-****** +python3 together_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="google/gemma-7b-it:free", + base_url="https://openrouter.ai/api/v1", + api_key=os.environ.get("OPENROUTER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/together_example_chat.py b/examples/frontend_language/quick_start/together_example_chat.py new file mode 100644 index 0000000..2d20590 --- /dev/null +++ b/examples/frontend_language/quick_start/together_example_chat.py @@ -0,0 +1,81 @@ +""" +Usage: +export TOGETHER_API_KEY=sk-****** +python3 together_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", + base_url="https://api.together.xyz/v1", + api_key=os.environ.get("TOGETHER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/together_example_complete.py b/examples/frontend_language/quick_start/together_example_complete.py new file mode 100644 index 0000000..d9119ed --- /dev/null +++ b/examples/frontend_language/quick_start/together_example_complete.py @@ -0,0 +1,76 @@ +""" +Usage: +export TOGETHER_API_KEY=sk-****** +python3 together_example_complete.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", + is_chat_model=False, + base_url="https://api.together.xyz/v1", + api_key=os.environ.get("TOGETHER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/usage/chinese_regex.py b/examples/frontend_language/usage/chinese_regex.py new file mode 100644 index 0000000..78e9c7e --- /dev/null +++ b/examples/frontend_language/usage/chinese_regex.py @@ -0,0 +1,53 @@ +import sglang as sgl + +character_regex = ( + r"""\{\n""" + + r""" "姓名": "[^"]{1,32}",\n""" + + r""" "学院": "(格兰芬多|赫奇帕奇|拉文克劳|斯莱特林)",\n""" + + r""" "血型": "(纯血|混血|麻瓜)",\n""" + + r""" "职业": "(学生|教师|傲罗|魔法部|食死徒|凤凰社成员)",\n""" + + r""" "魔杖": \{\n""" + + r""" "材质": "[^"]{1,32}",\n""" + + r""" "杖芯": "[^"]{1,32}",\n""" + + r""" "长度": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "存活": "(存活|死亡)",\n""" + + r""" "守护神": "[^"]{1,32}",\n""" + + r""" "博格特": "[^"]{1,32}"\n""" + + r"""\}""" +) + + +@sgl.function +def character_gen(s, name): + s += name + " 是一名哈利波特系列小说中的角色。请填写以下关于这个角色的信息。" + s += """\ +这是一个例子 +{ + "姓名": "哈利波特", + "学院": "格兰芬多", + "血型": "混血", + "职业": "学生", + "魔杖": { + "材质": "冬青木", + "杖芯": "凤凰尾羽", + "长度": 11.0 + }, + "存活": "存活", + "守护神": "麋鹿", + "博格特": "摄魂怪" +} +""" + s += f"现在请你填写{name}的信息：\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) + + +def main(): + backend = sgl.RuntimeEndpoint("http://localhost:30000") + sgl.set_default_backend(backend) + ret = character_gen.run(name="赫敏格兰杰", temperature=0) + print(ret.text()) + + +if __name__ == "__main__": + main() diff --git a/examples/frontend_language/usage/choices_logprob.py b/examples/frontend_language/usage/choices_logprob.py new file mode 100644 index 0000000..6cd733f --- /dev/null +++ b/examples/frontend_language/usage/choices_logprob.py @@ -0,0 +1,44 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python choices_logprob.py +""" + +import sglang as sgl + + +@sgl.function +def tool_use(s, question): + s += "To answer this question: " + question + ", " + s += "I need to use a " + sgl.gen("tool", choices=["calculator", "search engine"]) + + +def main(): + # Run one case + question = "What is 5 + 5?" + state = tool_use.run(question) + print("questions:", question) + print("choice:", state["tool"]) + meta_info = state.get_meta_info("tool") + print("logprobs of choice 1", meta_info["input_token_logprobs"][0]) + print("logprobs of choice 2", meta_info["input_token_logprobs"][1]) + print("-" * 50) + + # Run a batch + questions = [ + "What is 5 + 6?", + "Who is Michael Jordan?", + ] + states = tool_use.run_batch([{"question": q} for q in questions]) + for question, state in zip(questions, states): + print("questions:", question) + print("choice:", state["tool"]) + meta_info = state.get_meta_info("tool") + print("logprobs of choice 1", meta_info["input_token_logprobs"][0]) + print("logprobs of choice 2", meta_info["input_token_logprobs"][1]) + print("-" * 50) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + main() diff --git a/examples/frontend_language/usage/cot_decoding.py b/examples/frontend_language/usage/cot_decoding.py new file mode 100644 index 0000000..7a7a04b --- /dev/null +++ b/examples/frontend_language/usage/cot_decoding.py @@ -0,0 +1,115 @@ +from math import exp +from pprint import pformat + +import sglang as sgl + +YELLOW = "\033[1;33m" +GREEN = "\033[1;32m" +BLUE = "\033[1;34m" +CLEAR = "\033[1;0m" + + +@sgl.function +def cot_decoding(s, question, get_top_k, is_chat_model, verbose): + """CoT Decoding: http://arxiv.org/abs/2402.10200""" + + if is_chat_model: + s += sgl.user("Question: " + question + "\nAnswer:") + s += sgl.assistant_begin() + else: + s += "Question: " + question + "\nAnswer:" + + step_0 = s.fork(1)[0] + forks = s.fork(get_top_k) + answer_forks = s.fork(get_top_k) + + # decoding step 0 + step_0 += sgl.gen( + "get_top_k", + max_tokens=0, + return_logprob=True, + top_logprobs_num=get_top_k, + return_text_in_logprobs=True, + ) + logprobs = step_0.get_meta_info("get_top_k")["output_top_logprobs"][0] + + print("Decoding step 0:", ", ".join(pformat(token[2]) for token in logprobs)) + for idx, (f, token) in enumerate(zip(forks, logprobs)): + logprob, token_id, text = token + f += text + + if text == "<|end_of_text|>": + print( + f"{YELLOW}Path #{idx} {pformat(text)}[{exp(logprob):.3f}] (score=nan, answer=nan){CLEAR}" + ) + continue + + # continue greedy decoding + f += sgl.gen( + "answer", + temperature=0, + max_tokens=1024, + return_logprob=True, + top_logprobs_num=2, + return_text_in_logprobs=True, + ) + + # calculate probability disparity between the top and secondary tokens + x1s = [exp(xt[0][0]) for xt in f.get_meta_info("answer")["output_top_logprobs"]] + x2s = [exp(xt[1][0]) for xt in f.get_meta_info("answer")["output_top_logprobs"]] + tokens = [xt[0][2] for xt in f.get_meta_info("answer")["output_top_logprobs"]] + delta = (sum(x1s) - sum(x2s)) / len(x1s) + + # extract the answer span (without the '<|end_of_text|>' token) + answer_forks[idx] += text + f["answer"] + "\nSo the answer is" + answer_forks[idx] += sgl.gen( + "answer_span", + temperature=0, + max_tokens=64, + return_logprob=True, + top_logprobs_num=2, + return_text_in_logprobs=True, + ) + answer = answer_forks[idx]["answer_span"].replace("\n", " ").strip(":") + print( + f"{YELLOW}Path #{idx} {pformat(text)}[{exp(logprob):.3f}] (score={delta}, answer={answer}){CLEAR}" + ) + generated_text = str(answer_forks[idx])[len("ProgramState(") : -1] + print(f"{BLUE}{pformat(generated_text)}{CLEAR}") + + if verbose: + answer_tokens = [ + xt[0][2] + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + answer_x1s = [ + exp(xt[0][0]) + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + answer_x2s = [ + exp(xt[1][0]) + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + + for token, x1, x2 in zip(tokens, x1s, x2s): + print(f" {GREEN}{pformat(token)}{CLEAR}({x1:.3f}-{x2:.3f})", end="") + print("\n===========") + for token, x1, x2 in zip(answer_tokens, answer_x1s, answer_x2s): + print(f" {GREEN}{pformat(token)}{CLEAR}({x1:.3f}-{x2:.3f})", end="") + print() + + +sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + +state = cot_decoding.run( + question=r"Claire makes a 3 egg omelet every morning for breakfast. How many dozens of eggs will she eat in 4 weeks?", + get_top_k=10, + is_chat_model=True, + verbose=False, +) diff --git a/examples/frontend_language/usage/json_decode.py b/examples/frontend_language/usage/json_decode.py new file mode 100644 index 0000000..5dc3522 --- /dev/null +++ b/examples/frontend_language/usage/json_decode.py @@ -0,0 +1,83 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python json_decode.py +""" + +from enum import Enum + +from pydantic import BaseModel + +import sglang as sgl +from sglang.srt.constrained.outlines_backend import build_regex_from_object + +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + + +@sgl.function +def character_gen(s, name): + s += ( + name + + " is a character in Harry Potter. Please fill in the following information about this character.\n" + ) + s += "The constrained regex is:\n" + s += character_regex + "\n" + s += "The JSON output is:\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) + + +def driver_character_gen(): + state = character_gen.run(name="Hermione Granger") + print(state.text()) + + +class Weapon(str, Enum): + sword = "sword" + axe = "axe" + mace = "mace" + spear = "spear" + bow = "bow" + crossbow = "crossbow" + + +class Wizard(BaseModel): + name: str + age: int + weapon: Weapon + + +@sgl.function +def pydantic_wizard_gen(s): + s += "Give me a description about a wizard in the JSON format.\n" + s += sgl.gen( + "character", + max_tokens=128, + temperature=0, + regex=build_regex_from_object(Wizard), # Requires pydantic >= 2.0 + ) + + +def driver_pydantic_wizard_gen(): + state = pydantic_wizard_gen.run() + print(state.text()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + driver_character_gen() + # driver_pydantic_wizard_gen() diff --git a/examples/frontend_language/usage/json_logprobs.py b/examples/frontend_language/usage/json_logprobs.py new file mode 100644 index 0000000..15206a6 --- /dev/null +++ b/examples/frontend_language/usage/json_logprobs.py @@ -0,0 +1,103 @@ +# NOTE: Currently this can only be run through HTTP requests. +from concurrent.futures import ThreadPoolExecutor + +from json_decode import character_regex + +from sglang.utils import http_request + +character_names = ["Hermione Granger", "Ron Weasley", "Harry Potter"] + +base_url = "http://localhost:30000" + +prompt = "is a character in Harry Potter. Please fill in the following information about this character.\n" + + +def openai_api_request(name): + data = { + "model": "", + "prompt": name + prompt, + "temperature": 0, + "max_tokens": 128, + "regex": character_regex, + "logprobs": 3, + } + res = http_request(base_url + "/v1/completions", json=data).json() + + # with open(f"json_logprobs_{name.replace(' ', '_')}_tmp.json", "w") as fout: + # fout.write(json.dumps(res, indent=4)) + + logprobs = res["choices"][0]["logprobs"] + usage = res["usage"] + assert len(logprobs["token_logprobs"]) == len(logprobs["tokens"]) + assert len(logprobs["token_logprobs"]) == len(logprobs["top_logprobs"]) + assert len(logprobs["token_logprobs"]) == usage["completion_tokens"] - 1 + + return res + + +def srt_api_request(name): + data = { + "text": name + prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + "regex": character_regex, + }, + "return_logprob": True, + "logprob_start_len": 0, + "top_logprobs_num": 3, + "return_text_in_logprobs": True, + } + + res = http_request(base_url + "/generate", json=data).json() + + # with open(f"json_logprobs_{name.replace(' ', '_')}_tmp.json", "w") as fout: + # fout.write(json.dumps(res, indent=4)) + + meta_info = res["meta_info"] + assert len(meta_info["input_token_logprobs"]) == len( + meta_info["input_top_logprobs"] + ) + assert len(meta_info["output_token_logprobs"]) == len( + meta_info["output_top_logprobs"] + ) + assert len(meta_info["input_token_logprobs"]) == meta_info["prompt_tokens"] + assert len(meta_info["output_token_logprobs"]) == meta_info["completion_tokens"] - 1 + + return res + + +def pretty_print(res): + meta_info = res["meta_info"] + + print("\n\n", "=" * 30, "Prefill", "=" * 30) + for i in range(len(meta_info["input_token_logprobs"])): + print(f"{str(meta_info['input_token_logprobs'][i][2].encode()): <20}", end="") + top_ks = ( + [str(t[2].encode()) for t in meta_info["input_top_logprobs"][i]] + if meta_info["input_top_logprobs"][i] + else [] + ) + for top_k in top_ks: + print(f"{top_k: <15}", end="") + print() + + print("\n\n", "=" * 30, "Decode", "=" * 30) + for i in range(len(meta_info["output_token_logprobs"])): + print(f"{str(meta_info['output_token_logprobs'][i][2].encode()): <20}", end="") + top_ks = [str(t[2].encode()) for t in meta_info["output_top_logprobs"][i]] + for top_k in top_ks: + print(f"{top_k: <15}", end="") + print() + + print(res["text"]) + + +if __name__ == "__main__": + with ThreadPoolExecutor() as executor: + ress = executor.map(srt_api_request, character_names) + + for res in ress: + pretty_print(res) + + openai_api_request("Hermione Granger") diff --git a/examples/frontend_language/usage/llava_video/srt_example_llava_v.py b/examples/frontend_language/usage/llava_video/srt_example_llava_v.py new file mode 100644 index 0000000..bc56d42 --- /dev/null +++ b/examples/frontend_language/usage/llava_video/srt_example_llava_v.py @@ -0,0 +1,260 @@ +""" +Usage: +pip install opencv-python-headless + +python3 srt_example_llava_v.py +""" + +import argparse +import csv +import json +import os +import time + +import requests + +import sglang as sgl + + +@sgl.function +def video_qa(s, num_frames, video_path, question): + s += sgl.user(sgl.video(video_path, num_frames) + question) + s += sgl.assistant(sgl.gen("answer")) + + +def single(path, num_frames=16): + state = video_qa.run( + num_frames=num_frames, + video_path=path, + question="Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes", + temperature=0.0, + max_new_tokens=1024, + ) + print(state["answer"], "\n") + + +def split_into_chunks(lst, num_chunks): + """Split a list into a specified number of chunks.""" + # Calculate the chunk size using integer division. Note that this may drop some items if not evenly divisible. + chunk_size = len(lst) // num_chunks + + if chunk_size == 0: + chunk_size = len(lst) + # Use list comprehension to generate chunks. The last chunk will take any remainder if the list size isn't evenly divisible. + chunks = [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] + # Ensure we have exactly num_chunks chunks, even if some are empty + chunks.extend([[] for _ in range(num_chunks - len(chunks))]) + return chunks + + +def save_batch_results(batch_video_files, states, cur_chunk, batch_idx, save_dir): + csv_filename = f"{save_dir}/chunk_{cur_chunk}_batch_{batch_idx}.csv" + with open(csv_filename, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["video_name", "answer"]) + for video_path, state in zip(batch_video_files, states): + video_name = os.path.basename(video_path) + writer.writerow([video_name, state["answer"]]) + + +def compile_and_cleanup_final_results(cur_chunk, num_batches, save_dir): + final_csv_filename = f"{save_dir}/final_results_chunk_{cur_chunk}.csv" + with open(final_csv_filename, "w", newline="") as final_csvfile: + writer = csv.writer(final_csvfile) + writer.writerow(["video_name", "answer"]) + for batch_idx in range(num_batches): + batch_csv_filename = f"{save_dir}/chunk_{cur_chunk}_batch_{batch_idx}.csv" + with open(batch_csv_filename, "r") as batch_csvfile: + reader = csv.reader(batch_csvfile) + next(reader) # Skip header row + for row in reader: + writer.writerow(row) + os.remove(batch_csv_filename) + + +def find_video_files(video_dir): + # Check if the video_dir is actually a file + if os.path.isfile(video_dir): + # If it's a file, return it as a single-element list + return [video_dir] + + # Original logic to find video files in a directory + video_files = [] + for root, dirs, files in os.walk(video_dir): + for file in files: + if file.endswith((".mp4", ".avi", ".mov")): + video_files.append(os.path.join(root, file)) + return video_files + + +def batch(video_dir, save_dir, cur_chunk, num_chunks, num_frames=16, batch_size=64): + video_files = find_video_files(video_dir) + chunked_video_files = split_into_chunks(video_files, num_chunks)[cur_chunk] + num_batches = 0 + + for i in range(0, len(chunked_video_files), batch_size): + batch_video_files = chunked_video_files[i : i + batch_size] + print(f"Processing batch of {len(batch_video_files)} video(s)...") + + if not batch_video_files: + print("No video files found in the specified directory.") + return + + batch_input = [ + { + "num_frames": num_frames, + "video_path": video_path, + "question": "Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes.", + } + for video_path in batch_video_files + ] + + start_time = time.time() + states = video_qa.run_batch(batch_input, max_new_tokens=512, temperature=0.2) + total_time = time.time() - start_time + average_time = total_time / len(batch_video_files) + print( + f"Number of videos in batch: {len(batch_video_files)}. Average processing time per video: {average_time:.2f} seconds. Total time for this batch: {total_time:.2f} seconds" + ) + + save_batch_results(batch_video_files, states, cur_chunk, num_batches, save_dir) + num_batches += 1 + + compile_and_cleanup_final_results(cur_chunk, num_batches, save_dir) + + +if __name__ == "__main__": + + url = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + + cache_dir = os.path.expanduser("~/.cache") + file_path = os.path.join(cache_dir, "jobs.mp4") + + os.makedirs(cache_dir, exist_ok=True) + + response = requests.get(url) + response.raise_for_status() # Raise an exception for bad responses + + with open(file_path, "wb") as f: + f.write(response.content) + + print(f"File downloaded and saved to: {file_path}") + # Create the parser + parser = argparse.ArgumentParser( + description="Run video processing with specified port." + ) + + # Add an argument for the port + parser.add_argument( + "--port", + type=int, + default=30000, + help="The master port for distributed serving.", + ) + parser.add_argument( + "--chunk-idx", type=int, default=0, help="The index of the chunk to process." + ) + parser.add_argument( + "--num-chunks", type=int, default=8, help="The number of chunks to process." + ) + parser.add_argument( + "--save-dir", + type=str, + default="./work_dirs/llava_video", + help="The directory to save the processed video files.", + ) + parser.add_argument( + "--video-dir", + type=str, + default=os.path.expanduser("~/.cache/jobs.mp4"), + help="The directory or path for the processed video files.", + ) + parser.add_argument( + "--model-path", + type=str, + default="lmms-lab/LLaVA-NeXT-Video-7B", + help="The model path for the video processing.", + ) + parser.add_argument( + "--num-frames", + type=int, + default=16, + help="The number of frames to process in each video.", + ) + parser.add_argument("--mm_spatial_pool_stride", type=int, default=2) + + # Parse the arguments + args = parser.parse_args() + cur_port = args.port + cur_chunk = args.chunk_idx + num_chunks = args.num_chunks + num_frames = args.num_frames + + if "34b" in args.model_path.lower(): + tokenizer_path = "liuhaotian/llava-v1.6-34b-tokenizer" + elif "7b" in args.model_path.lower(): + tokenizer_path = "llava-hf/llava-1.5-7b-hf" + else: + print("Invalid model path. Please specify a valid model path.") + exit() + + model_override_args = {} + model_override_args["mm_spatial_pool_stride"] = args.mm_spatial_pool_stride + model_override_args["architectures"] = ["LlavaVidForCausalLM"] + model_override_args["num_frames"] = args.num_frames + model_override_args["model_type"] = "llava" + + if "34b" in args.model_path.lower(): + model_override_args["image_token_index"] = 64002 + + if args.num_frames == 32: + model_override_args["rope_scaling"] = {"factor": 2.0, "rope_type": "linear"} + model_override_args["max_sequence_length"] = 4096 * 2 + model_override_args["tokenizer_model_max_length"] = 4096 * 2 + elif args.num_frames < 32: + pass + else: + print( + "The maximum number of frames to process is 32. Please specify a valid number of frames." + ) + exit() + + runtime = sgl.Runtime( + model_path=args.model_path, # "liuhaotian/llava-v1.6-vicuna-7b", + tokenizer_path=tokenizer_path, + port=cur_port, + json_model_override_args=json.dumps(model_override_args), + tp_size=1, + ) + sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + + # Run a single request + print("\n========== single ==========\n") + root = args.video_dir + if os.path.isfile(root): + video_files = [root] + else: + video_files = [ + os.path.join(root, f) + for f in os.listdir(root) + if f.endswith((".mp4", ".avi", ".mov")) + ] # Add more extensions if needed + start_time = time.time() # Start time for processing a single video + for cur_video in video_files[:1]: + print(cur_video) + single(cur_video, num_frames) + end_time = time.time() # End time for processing a single video + total_time = end_time - start_time + average_time = total_time / len( + video_files + ) # Calculate the average processing time + print(f"Average processing time per video: {average_time:.2f} seconds") + runtime.shutdown() + + # # Run a batch of requests + # print("\n========== batch ==========\n") + # if not os.path.exists(args.save_dir): + # os.makedirs(args.save_dir) + # batch(args.video_dir, args.save_dir, cur_chunk, num_chunks, num_frames, num_chunks) + # runtime.shutdown() diff --git a/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh b/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh new file mode 100644 index 0000000..ffb1af9 --- /dev/null +++ b/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +##### USAGE ##### +# - First node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K 0 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` +# - Second node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K 1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` +# - The K node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K K-1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` + + +# Replace `K`, `YOUR_VIDEO_PATH`, `YOUR_MODEL_PATH`, and `FRAMES_PER_VIDEO` with your specific details. +# CURRENT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CURRENT_ROOT=$(dirname "$0") + +echo ${CURRENT_ROOT} + +cd ${CURRENT_ROOT} + +export PYTHONWARNINGS=ignore + +START_TIME=$(date +%s) # Capture start time + +NUM_NODES=$1 + +CUR_NODES_IDX=$2 + +VIDEO_DIR=$3 + +MODEL_PATH=$4 + +NUM_FRAMES=$5 + + +# FRAME_FORMAT=$6 + +# FRAME_FORMAT=$(echo $FRAME_FORMAT | tr '[:lower:]' '[:upper:]') + +# # Check if FRAME_FORMAT is either JPEG or PNG +# if [[ "$FRAME_FORMAT" != "JPEG" && "$FRAME_FORMAT" != "PNG" ]]; then +# echo "Error: FRAME_FORMAT must be either JPEG or PNG." +# exit 1 +# fi + +# export TARGET_FRAMES=$TARGET_FRAMES + +echo "Each video you will sample $NUM_FRAMES frames" + +# export FRAME_FORMAT=$FRAME_FORMAT + +# echo "The frame format is $FRAME_FORMAT" + +# Assuming GPULIST is a bash array containing your GPUs +GPULIST=(0 1 2 3 4 5 6 7) +LOCAL_CHUNKS=${#GPULIST[@]} + +echo "Number of GPUs in GPULIST: $LOCAL_CHUNKS" + +ALL_CHUNKS=$((NUM_NODES * LOCAL_CHUNKS)) + +# Calculate GPUs per chunk +GPUS_PER_CHUNK=1 + +echo $GPUS_PER_CHUNK + +for IDX in $(seq 1 $LOCAL_CHUNKS); do + ( + START=$(((IDX-1) * GPUS_PER_CHUNK)) + LENGTH=$GPUS_PER_CHUNK # Length for slicing, not the end index + + CHUNK_GPUS=(${GPULIST[@]:$START:$LENGTH}) + + # Convert the chunk GPUs array to a comma-separated string + CHUNK_GPUS_STR=$(IFS=,; echo "${CHUNK_GPUS[*]}") + + LOCAL_IDX=$((CUR_NODES_IDX * LOCAL_CHUNKS + IDX)) + + echo "Chunk $(($LOCAL_IDX - 1)) will run on GPUs $CHUNK_GPUS_STR" + + # Calculate the port for this chunk. Ensure it's incremented by 5 for each chunk. + PORT=$((10000 + RANDOM % 55536)) + + MAX_RETRIES=10 + RETRY_COUNT=0 + COMMAND_STATUS=1 # Initialize as failed + + while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ $COMMAND_STATUS -ne 0 ]; do + echo "Running chunk $(($LOCAL_IDX - 1)) on GPUs $CHUNK_GPUS_STR with port $PORT. Attempt $(($RETRY_COUNT + 1))" + +#!/bin/bash + CUDA_VISIBLE_DEVICES=$CHUNK_GPUS_STR python3 srt_example_llava_v.py \ + --port $PORT \ + --num-chunks $ALL_CHUNKS \ + --chunk-idx $(($LOCAL_IDX - 1)) \ + --save-dir work_dirs/llava_next_video_inference_results \ + --video-dir $VIDEO_DIR \ + --model-path $MODEL_PATH \ + --num-frames $NUM_FRAMES #& + + wait $! # Wait for the process to finish and capture its exit status + COMMAND_STATUS=$? + + if [ $COMMAND_STATUS -ne 0 ]; then + echo "Execution failed for chunk $(($LOCAL_IDX - 1)), attempt $(($RETRY_COUNT + 1)). Retrying..." + RETRY_COUNT=$(($RETRY_COUNT + 1)) + sleep 180 # Wait a bit before retrying + else + echo "Execution succeeded for chunk $(($LOCAL_IDX - 1))." + fi + done + + if [ $COMMAND_STATUS -ne 0 ]; then + echo "Execution failed for chunk $(($LOCAL_IDX - 1)) after $MAX_RETRIES attempts." + fi + ) #& + sleep 2 # Slight delay to stagger the start times +done + +wait + +cat work_dirs/llava_next_video_inference_results/final_results_chunk_*.csv > work_dirs/llava_next_video_inference_results/final_results_node_${CUR_NODES_IDX}.csv + +END_TIME=$(date +%s) # Capture end time +ELAPSED_TIME=$(($END_TIME - $START_TIME)) +echo "Total execution time: $ELAPSED_TIME seconds." diff --git a/examples/frontend_language/usage/openai_chat_speculative.py b/examples/frontend_language/usage/openai_chat_speculative.py new file mode 100644 index 0000000..f3fd74e --- /dev/null +++ b/examples/frontend_language/usage/openai_chat_speculative.py @@ -0,0 +1,155 @@ +""" +Usage: +***Note: for speculative execution to work, user must put all "gen" in "assistant". +Show in "assistant" the desired answer format. Each "gen" term should have a stop token. +The stream mode is not supported in speculative execution. + +E.g. +correct: + sgl.assistant("\nName:" + sgl.gen("name", stop="\n") + "\nBirthday:" + sgl.gen("birthday", stop="\n") + "\nJob:" + sgl.gen("job", stop="\n")) +incorrect: + s += sgl.assistant("\nName:" + sgl.gen("name", stop="\n")) + s += sgl.assistant("\nBirthday:" + sgl.gen("birthday", stop="\n")) + s += sgl.assistant("\nJob:" + sgl.gen("job", stop="\n")) + +export OPENAI_API_KEY=sk-****** +python3 openai_chat_speculative.py +""" + +import sglang as sgl +from sglang import OpenAI, function, set_default_backend + + +@function(num_api_spec_tokens=256) +def gen_character_spec(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Construct a character within the following format:") + s += sgl.assistant( + "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + ) + s += sgl.user("Please generate new Name, Birthday and Job.\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + +@function(num_api_spec_tokens=256) +def gen_character_spec_no_few_shot(s): + s += sgl.user("Construct a character. For each field stop with a newline\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nAge:" + + sgl.gen("age", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + +@function +def gen_character_normal(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("What's the answer of 23 + 8?") + s += sgl.assistant(sgl.gen("answer", max_tokens=64)) + + +@function(num_api_spec_tokens=1024) +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Answer questions in the following format:") + s += sgl.user( + "Question 1: What is the capital of France?\nQuestion 2: What is the population of this city?\n" + ) + s += sgl.assistant( + "Answer 1: The capital of France is Paris.\nAnswer 2: The population of Paris in 2024 is estimated to be around 2.1 million for the city proper.\n" + ) + s += sgl.user("Question 1: " + question_1 + "\nQuestion 2: " + question_2) + s += sgl.assistant( + "Answer 1: " + + sgl.gen("answer_1", stop="\n") + + "\nAnswer 2: " + + sgl.gen("answer_2", stop="\n") + ) + + +def test_spec_single_turn(): + backend.token_usage.reset() + + state = gen_character_spec.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- name:", state["name"]) + print("-- birthday:", state["birthday"]) + print("-- job:", state["job"]) + print(backend.token_usage) + + +def test_inaccurate_spec_single_turn(): + state = gen_character_spec_no_few_shot.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- name:", state["name"]) + print("\n-- age:", state["age"]) + print("\n-- job:", state["job"]) + + +def test_normal_single_turn(): + state = gen_character_normal.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + +def test_spec_multi_turn(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions in the capital of the United States.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + print("\n-- answer_2 --\n", state["answer_2"]) + + +def test_spec_multi_turn_stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + + +if __name__ == "__main__": + backend = OpenAI("gpt-4-turbo") + set_default_backend(backend) + + print("\n========== test spec single turn ==========\n") + # expect reasonable answer for each field + test_spec_single_turn() + + print("\n========== test inaccurate spec single turn ==========\n") + # expect incomplete or unreasonable answers + test_inaccurate_spec_single_turn() + + print("\n========== test normal single turn ==========\n") + # expect reasonable answer + test_normal_single_turn() + + print("\n========== test spec multi turn ==========\n") + # expect answer with same format as in the few shot + test_spec_multi_turn() + + print("\n========== test spec multi turn stream ==========\n") + # expect error in stream_executor: stream is not supported... + test_spec_multi_turn_stream() diff --git a/examples/frontend_language/usage/openai_speculative.py b/examples/frontend_language/usage/openai_speculative.py new file mode 100644 index 0000000..4389cb0 --- /dev/null +++ b/examples/frontend_language/usage/openai_speculative.py @@ -0,0 +1,54 @@ +""" +Usage: +python3 openai_speculative.py +""" + +from sglang import OpenAI, function, gen, set_default_backend + + +@function(num_api_spec_tokens=64) +def gen_character_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +@function +def gen_character_no_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +@function(num_api_spec_tokens=64) +def gen_character_spec_no_few_shot(s): + # s += "Construct a character with name, birthday, and job:\n" + s += "Construct a character:\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +if __name__ == "__main__": + backend = OpenAI("gpt-3.5-turbo-instruct") + set_default_backend(backend) + + for function in [ + gen_character_spec, + gen_character_no_spec, + gen_character_spec_no_few_shot, + ]: + backend.token_usage.reset() + + print(f"function: {function.func.__name__}") + + state = function.run() + + print("...name:", state["name"]) + print("...birthday:", state["birthday"]) + print("...job:", state["job"]) + print(backend.token_usage) + print() diff --git a/examples/frontend_language/usage/parallel_sample.py b/examples/frontend_language/usage/parallel_sample.py new file mode 100644 index 0000000..0f3cf17 --- /dev/null +++ b/examples/frontend_language/usage/parallel_sample.py @@ -0,0 +1,40 @@ +""" +Usage: +python3 parallel_sample.py +""" + +import sglang as sgl + + +@sgl.function +def parallel_sample(s, question, n): + s += ( + "Question: Compute 1 + 2 + 3\n" + "Reasoning: I need to use a calculator.\n" + "Tool: calculator\n" + "Answer: 6\n" + "Question: Compute 3 + 2 + 2\n" + "Reasoning: I will try a calculator.\n" + "Tool: calculator\n" + "Answer: 7\n" + ) + s += "Question: " + question + "\n" + forks = s.fork(n) + forks += "Reasoning:" + sgl.gen("reasoning", stop="\n") + "\n" + forks += "Tool:" + sgl.gen("tool", choices=["calculator", "browser"]) + "\n" + forks += "Answer:" + sgl.gen("answer", stop="\n") + "\n" + forks.join() + + +sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) +# sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + +state = parallel_sample.run(question="Compute 5 + 2 + 4.", n=5, temperature=1.0) + +for i in range(5): + obj = { + "reasoning": state["reasoning"][i], + "tool": state["tool"][i], + "answer": state["answer"][i], + } + print(f"[{i}], {obj}") diff --git a/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb b/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb new file mode 100644 index 0000000..3c1b2a6 --- /dev/null +++ b/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb @@ -0,0 +1,408 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RAG Powered by SGLang & Chroma Evaluated using Parea\n", + "\n", + "In this notebook, we will build a simple RAG pipeline using SGLang to execute our LLM calls, Chroma as vector database for retrieval and [Parea](https://www.parea.ai) for tracing and evaluation. We will then evaluate the performance of our RAG pipeline. The dataset we will use was created by [Virat](https://twitter.com/virattt) and contains 100 questions, contexts and answers from the Airbnb 2023 10k filing.\n", + "\n", + "The RAG pipeline consists of two steps:\n", + "1. Retrieval: Given a question, we retrieve the relevant context from all provided contexts.\n", + "2. Generation: Given the question and the retrieved context, we generate an answer.\n", + "\n", + "ℹ️ This notebook requires an OpenAI API key.\n", + "\n", + "ℹ️ This notebook requires a Parea API key, which can be created [here](https://docs.parea.ai/api-reference/authentication#parea-api-key)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up the environment\n", + "\n", + "We will first install the necessary packages: `sglang`, `parea-ai` and `chromadb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# note, if you use a Mac M1 chip, you might need to install grpcio 1.59.0 first such that installing chromadb works\n", + "# !pip install grpcio==1.59.0\n", + "\n", + "!pip install sglang[openai] parea-ai chromadb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a Parea API key as outlined [here](https://docs.parea.ai/api-reference/authentication#parea-api-key) and save it in a `.env` file as `PAREA_API_KEY=your-api-key`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Indexing the data\n", + "\n", + "Now it's time to download the data & index it! For that, we create a collection called `contexts` in Chroma and add the contexts as documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import List\n", + "\n", + "import chromadb\n", + "\n", + "path_qca = \"airbnb-2023-10k-qca.json\"\n", + "\n", + "if not os.path.exists(path_qca):\n", + " !wget https://virattt.github.io/datasets/abnb-2023-10k.json -O airbnb-2023-10k-qca.json\n", + "\n", + "with open(path_qca, \"r\") as f:\n", + " question_context_answers = json.load(f)\n", + "\n", + "chroma_client = chromadb.PersistentClient()\n", + "collection = chroma_client.get_or_create_collection(name=\"contexts\")\n", + "if collection.count() == 0:\n", + " collection.add(\n", + " documents=[qca[\"context\"] for qca in question_context_answers],\n", + " ids=[str(i) for i in range(len(question_context_answers))],\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Defining the RAG pipeline\n", + "\n", + "We will start with importing the necessary packages, setting up tracing of OpenAI calls via Parea and setting OpenAI as the default backend for SGLang." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "from sglang import function, user, assistant, gen, set_default_backend, OpenAI\n", + "from sglang.lang.interpreter import ProgramState\n", + "from parea import Parea, trace\n", + "\n", + "\n", + "load_dotenv()\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "p = Parea(api_key=os.getenv(\"PAREA_API_KEY\"), project_name=\"rag_sglang\")\n", + "p.integrate_with_sglang()\n", + "\n", + "set_default_backend(OpenAI(\"gpt-3.5-turbo\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can define our retrieval step shown below. Notice, the `trace` decorator which will automatically trace inputs, output, latency, etc. of that call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def retrieval(question: str) -> List[str]:\n", + " return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we will define the generation step which uses SGLang to execute the LLM call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def generation_sglang(s, question: str, *context: str):\n", + " context = \"\\n\".join(context)\n", + " s += user(\n", + " f\"Given this question:\\n{question}\\n\\nAnd this context:\\n{context}\\n\\nAnswer the question.\"\n", + " )\n", + " s += assistant(gen(\"answer\"))\n", + "\n", + "\n", + "@trace\n", + "def generation(question: str, *context):\n", + " state: ProgramState = generation_sglang.run(question, *context)\n", + " while not state.stream_executor.is_finished:\n", + " time.sleep(1)\n", + " return state.stream_executor.variables[\"answer\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can tie it together and execute a sample query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def rag_pipeline(question: str) -> str:\n", + " contexts = retrieval(question)\n", + " return generation(question, *contexts)\n", + "\n", + "\n", + "rag_pipeline(\n", + " \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Debug Trace\n", + "\n", + "The output is unfortunately wrong! Using the traced pipeline, we can see that\n", + "\n", + "- the context is relevant to the question and contains the correct information\n", + "- but, the generation step is cut off as max tokens is set to 16\n", + "\n", + "When opening the generation step in the playground and rerunning the prompt with max. tokens set to 1000, the correct answer is produced.\n", + "\n", + "![RAG Trace](https://drive.google.com/uc?id=1QI243ogGjzbO01tUrR72g9rFoGzUJqVH)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluating RAG Pipelines\n", + "\n", + "Before we apply above's fix, let's dive into evaluating RAG pipelines.\n", + "\n", + "RAG pipelines consist of a retrieval step to fetch relevant information and a generation step to generate a response to a users question. A RAG pipeline can fail at either step. E.g. the retrieval step can fail to find relevant information which makes generating the correct impossible. Another failure mode is that the generation step doesn't leverage the retrieved information correctly. We will apply the following evaluation metrics to understand different failure modes:\n", + "\n", + "- `context_relevancy`: measures how relevant the context is given the question\n", + "- `percent_target_supported_by_context`: measures how much of the target answer is supported by the context; this will give an upper ceiling of how well the generation step can perform\n", + "- `answer_context_faithfulness`: measures how much the generated answer utilizes the context\n", + "- `answer_matches_target`: measures how well the generated answer matches the target answer judged by a LLM and gives a sense of accuracy of our entire pipeline\n", + "\n", + "To use these evaluation metrics, we can import them from `parea.evals.rag` and `parea.evals.general` and apply them to a function by specifying in the `trace` decorator which evaluation metrics to use. The `@trace` decorator will automatically log the results of the evaluation metrics to the Parea dashboard.\n", + "\n", + "Applying them to the retrieval step:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from parea.evals.rag import (\n", + " context_query_relevancy_factory,\n", + " percent_target_supported_by_context_factory,\n", + ")\n", + "\n", + "\n", + "context_relevancy_eval = context_query_relevancy_factory()\n", + "percent_target_supported_by_context = percent_target_supported_by_context_factory()\n", + "\n", + "\n", + "@trace(eval_funcs=[context_relevancy_eval, percent_target_supported_by_context])\n", + "def retrieval(question: str) -> List[str]:\n", + " return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can apply `answer_context_faithfulness` and `answer_matches_target` to the generation step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from parea.evals.general import answer_matches_target_llm_grader_factory\n", + "from parea.evals.rag import answer_context_faithfulness_statement_level_factory\n", + "\n", + "\n", + "answer_context_faithfulness = answer_context_faithfulness_statement_level_factory()\n", + "answer_matches_target_llm_grader = answer_matches_target_llm_grader_factory()\n", + "\n", + "\n", + "@function\n", + "def generation_sglang(s, question: str, *context: str):\n", + " context = \"\\n\".join(context)\n", + " s += user(\n", + " f\"Given this question:\\n{question}\\n\\nAnd this context:\\n{context}\\n\\nAnswer the question.\"\n", + " )\n", + " s += assistant(gen(\"answer\", max_tokens=1_000))\n", + "\n", + "\n", + "@trace(eval_funcs=[answer_context_faithfulness, answer_matches_target_llm_grader])\n", + "def generation(question: str, *context):\n", + " state: ProgramState = generation_sglang.run(question, *context)\n", + " while not state.stream_executor.is_finished:\n", + " time.sleep(1)\n", + " return state.stream_executor.variables[\"answer\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we tie them together & execute the original sample query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def rag_pipeline(question: str) -> str:\n", + " contexts = retrieval(question)\n", + " return generation(question, *contexts)\n", + "\n", + "\n", + "rag_pipeline(\n", + " \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, the answer is correct! Can you spot the line where we fixed the output truncation issue?\n", + "\n", + "The evaluation scores appear in the bottom right of the logs (screenshot below). Note, that there is no score for `answer_matches_target_llm_grader` and `percent_target_supported_by_context` as these evals are automatically skipped if the target answer is not provided.\n", + "\n", + "![Fixed Max. Tokens](max-tokens-fixed-rag-trace.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running an experiment\n", + "\n", + "Now we are (almost) ready to evaluate the performance of our RAG pipeline on the entire dataset. First, we will need to apply the `nest_asyncio` package to avoid issues with the Jupyter notebook event loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install nest-asyncio\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Running the actual experiment is straight-forward. For that we use `p.experiment` to initialize the experiment with a name, the data (list of key-value pairs fed into our entry function) and the entry function. We then call `run` on the experiment to execute it. Note, that `target` is a reserved key in the data dictionary and will be used as the target answer for evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "e = p.experiment(\n", + " \"RAG\",\n", + " data=[\n", + " {\n", + " \"question\": qca[\"question\"],\n", + " \"target\": qca[\"answer\"],\n", + " }\n", + " for qca in question_context_answers\n", + " ],\n", + " func=rag_pipeline,\n", + ").run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analyzing the results\n", + "\n", + "When opening above experiment, we will see an overview of the experiment as shown below. The upper half shows a summary of the statistics on the left and charts to investigate the distribution and relationships of scores on the right. The lower half is a table with the individual traces which we can use to debug individual samples.\n", + "\n", + "When looking at the statistics, we can see that the accuracy of our RAG pipeline is 22% as measured by `answer_matches_target_llm_grader`. Though when checking the quality of our retrieval step (`context_query_relevancy`), we can see that our retrival step is fetching relevant information in only 27% of all samples. As shown in the GIF, we investigate the relationship between the two and see the two scores have 95% agreement. This confirms that the retrieval step is a major bottleneck for our RAG pipeline. So, now it's your turn to improve the retrieval step!\n", + "\n", + "Note, above link isn't publicly accessible but the experiment can be accessed through [here](https://app.parea.ai/public-experiments/parea/rag_sglang/30f0244a-d56c-44ff-bdfb-8f47626304b6).\n", + "\n", + "![Experiment Results](https://drive.google.com/uc?id=1KMtJBU47nPB02Pvv3SPPTK7RnHRh5YdA)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/frontend_language/usage/readme_examples.py b/examples/frontend_language/usage/readme_examples.py new file mode 100644 index 0000000..7269ef1 --- /dev/null +++ b/examples/frontend_language/usage/readme_examples.py @@ -0,0 +1,109 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python readme_examples.py +""" + +import sglang as sgl + + +@sgl.function +def tool_use(s, question): + s += "To answer this question: " + question + ". " + s += ( + "I need to use a " + + sgl.gen("tool", choices=["calculator", "search engine"]) + + ". " + ) + + if s["tool"] == "calculator": + s += "The math expression is" + sgl.gen("expression") + elif s["tool"] == "search engine": + s += "The key word to search is" + sgl.gen("word") + + +@sgl.function +def tip_suggestion(s): + s += ( + "Here are two tips for staying healthy: " + "1. Balanced Diet. 2. Regular Exercise.\n\n" + ) + + forks = s.fork(2) + for i, f in enumerate(forks): + f += f"Now, expand tip {i+1} into a paragraph:\n" + f += sgl.gen(f"detailed_tip", max_tokens=256, stop="\n\n") + + s += "Tip 1:" + forks[0]["detailed_tip"] + "\n" + s += "Tip 2:" + forks[1]["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + +@sgl.function +def regular_expression_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + temperature=0, + regex=r"((25[0-5]|2[0-4]\d|[01]?\d\d?).){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)", + ) + + +@sgl.function +def text_qa(s, question): + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + +def driver_tool_use(): + state = tool_use.run(question="What is the capital of the United States?") + print(state.text()) + print("\n") + + +def driver_tip_suggestion(): + state = tip_suggestion.run() + print(state.text()) + print("\n") + + +def driver_regex(): + state = regular_expression_gen.run() + print(state.text()) + print("\n") + + +def driver_batching(): + states = text_qa.run_batch( + [ + {"question": "What is the capital of the United Kingdom?"}, + {"question": "What is the capital of France?"}, + {"question": "What is the capital of Japan?"}, + ], + progress_bar=True, + ) + + for s in states: + print(s.text()) + print("\n") + + +def driver_stream(): + state = text_qa.run( + question="What is the capital of France?", temperature=0.1, stream=True + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print("\n") + + +if __name__ == "__main__": + # sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + + driver_tool_use() + driver_tip_suggestion() + driver_regex() + driver_batching() + driver_stream() diff --git a/examples/frontend_language/usage/sgl_gen_min_tokens.py b/examples/frontend_language/usage/sgl_gen_min_tokens.py new file mode 100644 index 0000000..a508819 --- /dev/null +++ b/examples/frontend_language/usage/sgl_gen_min_tokens.py @@ -0,0 +1,35 @@ +""" +This example demonstrates how to use `min_tokens` to enforce sgl.gen to generate a longer sequence + +Usage: +python3 sgl_gen_min_tokens.py +""" + +import sglang as sgl + + +@sgl.function +def long_answer(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant(sgl.gen("answer", min_tokens=64, max_tokens=128)) + + +@sgl.function +def short_answer(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant(sgl.gen("answer")) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + sgl.set_default_backend(runtime) + + state = long_answer.run() + print("=" * 20) + print("Longer Answer", state["answer"]) + + state = short_answer.run() + print("=" * 20) + print("Short Answer", state["answer"]) + + runtime.shutdown() diff --git a/examples/frontend_language/usage/streaming.py b/examples/frontend_language/usage/streaming.py new file mode 100644 index 0000000..506ee35 --- /dev/null +++ b/examples/frontend_language/usage/streaming.py @@ -0,0 +1,49 @@ +""" +Usage: +python3 streaming.py +""" + +import asyncio + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo")) + + +def stream_a_variable(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(var_name="answer_2"): + print(out, end="", flush=True) + print("\n") + + +async def async_stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + async for out in state.text_async_iter(var_name="answer_2"): + print(out, end="", flush=True) + print("\n") + + +if __name__ == "__main__": + stream_a_variable() + asyncio.run(async_stream()) diff --git a/examples/frontend_language/usage/triton/Dockerfile b/examples/frontend_language/usage/triton/Dockerfile new file mode 100644 index 0000000..e4741a1 --- /dev/null +++ b/examples/frontend_language/usage/triton/Dockerfile @@ -0,0 +1,10 @@ +FROM nvcr.io/nvidia/tritonserver:24.01-py3 + +WORKDIR /opt + +RUN git clone https://github.com/sgl-project/sglang.git + +WORKDIR /opt/sglang +RUN pip install --upgrade pip && \ + pip install -e "python[all]" && \ + pip install datasets diff --git a/examples/frontend_language/usage/triton/README.md b/examples/frontend_language/usage/triton/README.md new file mode 100644 index 0000000..b2e5596 --- /dev/null +++ b/examples/frontend_language/usage/triton/README.md @@ -0,0 +1,35 @@ +# sglang_triton + +Build the docker image: +``` +docker build -t sglang-triton . +``` + +Then do: +``` +docker run -ti --gpus=all --network=host --name sglang-triton -v ./models:/mnt/models sglang-triton +``` + +inside the docker container: +``` +cd sglang +python3 -m sglang.launch_server --model-path mistralai/Mistral-7B-Instruct-v0.2 --port 30000 --mem-fraction-static 0.9 +``` + +with another shell, inside the docker container: +``` +docker exec -ti sglang-triton /bin/bash +cd /mnt +tritonserver --model-repository=/mnt/models +``` + + +Send request to the server: +``` +curl -X POST http://localhost:8000/v2/models/character_generation/generate \ +-H "Content-Type: application/json" \ +-d '{ + "INPUT_TEXT": ["harry"] +}' + +``` diff --git a/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt b/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt new file mode 100644 index 0000000..7546f99 --- /dev/null +++ b/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt @@ -0,0 +1,23 @@ +name: "character_generation" +backend: "python" +input [ + { + name: "INPUT_TEXT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +output [ + { + name: "OUTPUT_TEXT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +instance_group [ + { + count: 1 + kind: KIND_GPU + gpus: [ 0 ] + } +] diff --git a/examples/monitoring/docker-compose.yaml b/examples/monitoring/docker-compose.yaml new file mode 100644 index 0000000..6c18b40 --- /dev/null +++ b/examples/monitoring/docker-compose.yaml @@ -0,0 +1,16 @@ +services: + prometheus: + image: prom/prometheus:latest + network_mode: host + ports: + - "9090:9090" + volumes: + - ${PWD}/prometheus.yaml:/etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana:latest + network_mode: host + depends_on: + - prometheus + ports: + - "3000:3000" diff --git a/examples/monitoring/grafana.json b/examples/monitoring/grafana.json new file mode 100644 index 0000000..5fb7fbe --- /dev/null +++ b/examples/monitoring/grafana.json @@ -0,0 +1,962 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P95", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg(rate(sglang:e2e_request_latency_seconds_sum[$__rate_interval]) / rate(sglang:e2e_request_latency_seconds_count[$__rate_interval]))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "End-to-End Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 17, + "options": { + "calculate": true, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(sglang:e2e_request_latency_seconds_bucket{model_name=~\"$model_name\"}[$__rate_interval])\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "End-to-End Request Latency Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P95", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg(rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval]) / rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval]))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "End-to-End Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 19, + "options": { + "calculate": true, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(sglang:time_to_first_token_seconds_bucket{model_name=~\"$model_name\"}[$__rate_interval])\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Time-To-First-Token Seconds Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:num_running_reqs", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num Running Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "editorMode": "code", + "expr": "sglang:gen_throughput", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Token Generation Throughput (Tokens / S)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:cache_hit_rate", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:num_queue_reqs", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number Queued Requests", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "127.0.0.1:30000", + "value": "127.0.0.1:30000" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(instance)", + "hide": 0, + "includeAll": false, + "label": "instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "meta-llama/Llama-3.1-8B-Instruct", + "value": "meta-llama/Llama-3.1-8B-Instruct" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(model_name)", + "hide": 0, + "includeAll": false, + "label": "model name", + "multi": false, + "name": "model_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(model_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "SGLang Dashboard", + "uid": "sglang-dashboard", + "version": 5, + "weekStart": "" +} diff --git a/examples/monitoring/prometheus.yaml b/examples/monitoring/prometheus.yaml new file mode 100644 index 0000000..ba16ac3 --- /dev/null +++ b/examples/monitoring/prometheus.yaml @@ -0,0 +1,10 @@ +# prometheus.yaml +global: + scrape_interval: 5s + evaluation_interval: 30s + +scrape_configs: + - job_name: sglang + static_configs: + - targets: + - '127.0.0.1:30000' diff --git a/examples/runtime/README.md b/examples/runtime/README.md new file mode 100644 index 0000000..ecf116d --- /dev/null +++ b/examples/runtime/README.md @@ -0,0 +1,42 @@ +# Runtime examples + +The below examples will mostly need you to start a server in a separate terminal before you can execute them. Please see in the code for detailed instruction. + +## Native API + +* `lora.py`: An example how to use LoRA adapters. +* `multimodal_embedding.py`: An example how perform [multi modal embedding](Alibaba-NLP/gme-Qwen2-VL-2B-Instruct). +* `openai_batch_chat.py`: An example how to process batch requests for chat completions. +* `openai_batch_complete.py`: An example how to process batch requests for text completions. +* `openai_chat_with_response_prefill.py`: An example how to [prefill](https://eugeneyan.com/writing/prompting/#prefill-claudes-responses) a response using OpenAI API. +* `reward_model.py`: An example how to extract scores from a reward model. +* `vertex_predict.py`: An example how to deploy a model to [Vertex AI](https://cloud.google.com/vertex-ai?hl=en). + +## Engine + +The `engine` folder contains that examples that show how to use [Offline Engine API](https://docs.sglang.ai/backend/offline_engine_api.html#Offline-Engine-API) for common workflows. + +* `custom_server.py`: An example how to deploy a custom server. +* `embedding.py`: An example how to extract embeddings. +* `launch_engine.py`: An example how to launch the Engine. +* `offline_batch_inference_eagle.py`: An example how to perform speculative decoding using [EAGLE](https://docs.sglang.ai/backend/speculative_decoding.html). +* `offline_batch_inference_torchrun.py`: An example how to perform inference using [torchrun](https://pytorch.org/docs/stable/elastic/run.html). +* `offline_batch_inference_vlm.py`: An example how to use VLMs with the engine. +* `offline_batch_inference.py`: An example how to use the engine to perform inference on a batch of examples. + +## Hidden States + +The `hidden_states` folder contains examples on how to extract hidden states using SGLang. Please note that this might degrade throughput due to cuda graph rebuilding. + +* `hidden_states_engine.py`: An example how to extract hidden states using the Engine API. +* `hidden_states_server.py`: An example how to extract hidden states using the Server API. + +## LLaVA-NeXT + +SGLang support LLaVA-OneVision with single-image, multi-image and video are supported. The folder `llava_onevision` shows how to do this. + +## Token In, Token Out + +The folder `token_in_token_out` shows how to perform inference, where we provide tokens and get tokens as response. + +* `token_in_token_out_{llm|vlm}_{engine|server}.py`: Shows how to perform token in, token out workflow for llm/vlm using either the engine or native API. diff --git a/examples/runtime/engine/custom_server.py b/examples/runtime/engine/custom_server.py new file mode 100644 index 0000000..b190a46 --- /dev/null +++ b/examples/runtime/engine/custom_server.py @@ -0,0 +1,53 @@ +from sanic import Sanic, text +from sanic.response import json + +import sglang as sgl + +engine = None + +# Create an instance of the Sanic app +app = Sanic("sanic-server") + + +# Define an asynchronous route handler +@app.route("/generate", methods=["POST"]) +async def generate(request): + prompt = request.json.get("prompt") + if not prompt: + return json({"error": "Prompt is required"}, status=400) + + # async_generate returns a dict + result = await engine.async_generate(prompt) + + return text(result["text"]) + + +@app.route("/generate_stream", methods=["POST"]) +async def generate_stream(request): + prompt = request.json.get("prompt") + + if not prompt: + return json({"error": "Prompt is required"}, status=400) + + # async_generate returns a dict + result = await engine.async_generate(prompt, stream=True) + + # https://sanic.dev/en/guide/advanced/streaming.md#streaming + # init the response + response = await request.respond() + + # result is an async generator + async for chunk in result: + await response.send(chunk["text"]) + + await response.eof() + + +def run_server(): + global engine + engine = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + app.run(host="0.0.0.0", port=8000, single_process=True) + + +if __name__ == "__main__": + run_server() diff --git a/examples/runtime/engine/embedding.py b/examples/runtime/engine/embedding.py new file mode 100644 index 0000000..b927a18 --- /dev/null +++ b/examples/runtime/engine/embedding.py @@ -0,0 +1,27 @@ +import sglang as sgl + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create an LLM. + llm = sgl.Engine( + model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", is_embedding=True + ) + + outputs = llm.encode(prompts) + # Print the outputs (embedding vectors) + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nEmbedding vector: {output['embedding']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/launch_engine.py b/examples/runtime/engine/launch_engine.py new file mode 100644 index 0000000..b97e0bc --- /dev/null +++ b/examples/runtime/engine/launch_engine.py @@ -0,0 +1,17 @@ +""" +This example demonstrates how to launch the offline engine. +""" + +import sglang as sgl + + +def main(): + llm = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + llm.generate("What is the capital of France?") + llm.shutdown() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/offline_batch_inference.py b/examples/runtime/engine/offline_batch_inference.py new file mode 100644 index 0000000..92e68dc --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference.py @@ -0,0 +1,43 @@ +""" +Usage: +python3 offline_batch_inference.py --model meta-llama/Llama-3.1-8B-Instruct +""" + +import argparse +import dataclasses + +import sglang as sgl +from sglang.srt.server_args import ServerArgs + + +def main( + server_args: ServerArgs, +): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Create an LLM. + llm = sgl.Engine(**dataclasses.asdict(server_args)) + + outputs = llm.generate(prompts, sampling_params) + # Print the outputs. + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nGenerated text: {output['text']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + main(server_args) diff --git a/examples/runtime/engine/offline_batch_inference_async.py b/examples/runtime/engine/offline_batch_inference_async.py new file mode 100644 index 0000000..578962d --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_async.py @@ -0,0 +1,65 @@ +""" +Usage: +python offline_batch_inference_async.py --model-path Qwen/Qwen2-VL-7B-Instruct + +Note: +This demo shows the usage of async generation, +which is useful to implement an online-like generation with batched inference. +""" + +import argparse +import asyncio +import dataclasses +import time + +import sglang as sgl +from sglang.srt.server_args import ServerArgs + + +class InferenceEngine: + def __init__(self, **kwargs): + self.engine = sgl.Engine(**kwargs) + + async def generate(self, prompt, sampling_params): + result = await self.engine.async_generate(prompt, sampling_params) + return result + + +async def run_server(server_args): + inference = InferenceEngine(**dataclasses.asdict(server_args)) + + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] * 100 + + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Run the generation tasks concurrently in async mode. + tasks = [] + for prompt in prompts: + task = asyncio.create_task(inference.generate(prompt, sampling_params)) + tasks.append(task) + + # Get and print the result + for task in tasks: + await task + while True: + if not task.done(): + time.sleep(1) + else: + result = task.result() + print(f"Generated text: {result['text']}") + break + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + asyncio.run(run_server(server_args)) diff --git a/examples/runtime/engine/offline_batch_inference_eagle.py b/examples/runtime/engine/offline_batch_inference_eagle.py new file mode 100644 index 0000000..a7a89ef --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_eagle.py @@ -0,0 +1,38 @@ +import sglang as sgl + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create a sampling params object. + sampling_params = {"temperature": 0, "max_new_tokens": 30} + + # Create an LLM. + llm = sgl.Engine( + model_path="meta-llama/Llama-2-7b-chat-hf", + speculative_algorithm="EAGLE", + speculative_draft_model_path="lmsys/sglang-EAGLE-llama2-chat-7B", + speculative_num_steps=3, + speculative_eagle_topk=4, + speculative_num_draft_tokens=16, + cuda_graph_max_bs=8, + ) + + outputs = llm.generate(prompts, sampling_params) + + # Print the outputs. + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nGenerated text: {output['text']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/offline_batch_inference_torchrun.py b/examples/runtime/engine/offline_batch_inference_torchrun.py new file mode 100644 index 0000000..d2185da --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_torchrun.py @@ -0,0 +1,81 @@ +import datetime +import os +import sys + +from torch.distributed.device_mesh import init_device_mesh + +from sglang.srt.entrypoints.verl_engine import VerlEngine + + +def run(): + """ + Example command: + ``` + torchrun --nproc_per_node=8 offline_batch_inference_torchrun.py + ``` + """ + + local_rank = int(os.environ["LOCAL_RANK"]) + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + + def _log(text): + t = datetime.datetime.now().strftime("%H:%M:%S") + print(f"[{t}] [rank={rank}] {text}") + + _log( + f'start {local_rank=} {rank=} {world_size=} {sys.argv=} {os.environ.get("CUDA_VISIBLE_DEVICES")}' + ) + + tp_size = 4 + dp_size = 2 + assert world_size == tp_size * dp_size + + device_mesh_kwargs = dict( + mesh_shape=(tp_size, dp_size, 1), mesh_dim_names=["tp", "dp", "pp"] + ) + device_mesh_cpu = init_device_mesh("cpu", **device_mesh_kwargs) + _log(f"{device_mesh_cpu=}") + + tp_rank = device_mesh_cpu.get_local_rank("tp") + dp_rank = device_mesh_cpu.get_local_rank("dp") + _log(f"{tp_rank=} {tp_size=} ; {dp_rank=} {dp_size=}") + + model_name, mem_fraction_static = "meta-llama/Llama-3.2-1B-Instruct", 0.1 + # model_name, mem_fraction_static = "meta-llama/Llama-3.1-70B-Instruct", 0.9 # test large models + # model_name, mem_fraction_static = "deepseek-ai/DeepSeek-V2-Lite", 0.8 + + for k in ["TORCHELASTIC_USE_AGENT_STORE"]: + if k in os.environ: + del os.environ[k] + + fragment = VerlEngine( + model_path=model_name, + mem_fraction_static=mem_fraction_static, + device_mesh_cpu=device_mesh_cpu["tp"], + base_gpu_id=dp_rank, + gpu_id_step=dp_size, + port=30000, + # for DeepSeek-V2-Lite + DP Attention + # enable_dp_attention=True, port=30000 + dp_rank * 100, + ) + _log(f"{fragment=}") + + prompt_all = [ + ["1+1=2, 1+2=3, 1+3=4, 1+4=", "9-1=8, 8-1=7, 7-1="], + ["2*1=2, 2*2=4, 2*3=", "8/2=4, 6/2="], + ] + prompt = prompt_all[dp_rank] + + output = fragment.generate( + prompt=prompt, + sampling_params=dict(max_new_tokens=16, temperature=0.0), + ) + _log(f"{prompt=} {output=}") + + fragment.shutdown() + _log(f"End script") + + +if __name__ == "__main__": + run() diff --git a/examples/runtime/engine/offline_batch_inference_vlm.py b/examples/runtime/engine/offline_batch_inference_vlm.py new file mode 100644 index 0000000..4063136 --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_vlm.py @@ -0,0 +1,52 @@ +""" +Usage: +python offline_batch_inference_vlm.py --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template=qwen2-vl +""" + +import argparse +import dataclasses + +import sglang as sgl +from sglang.srt.conversation import chat_templates +from sglang.srt.server_args import ServerArgs + + +def main( + server_args: ServerArgs, +): + vlm = sgl.Engine(**dataclasses.asdict(server_args)) + + conv = chat_templates[server_args.chat_template].copy() + image_token = conv.image_token + + image_url = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" + + prompt = f"What's in this image?\n{image_token}" + + sampling_params = { + "temperature": 0.001, + "max_new_tokens": 30, + } + + output = vlm.generate( + prompt=prompt, + image_data=image_url, + sampling_params=sampling_params, + ) + + print("===============================") + print(f"Prompt: {prompt}") + print(f"Generated text: {output['text']}") + + vlm.shutdown() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + + server_args = ServerArgs.from_cli_args(args) + main(server_args) diff --git a/examples/runtime/engine/readme.md b/examples/runtime/engine/readme.md new file mode 100644 index 0000000..d5ac936 --- /dev/null +++ b/examples/runtime/engine/readme.md @@ -0,0 +1,49 @@ +# SGLang Engine + +## Introduction +SGLang provides a direct inference engine without the need for an HTTP server. There are generally two use cases: + +1. **Offline Batch Inference** +2. **Embedding Generation** +3. **Custom Server on Top of the Engine** + +## Examples + +### [Offline Batch Inference](./offline_batch_inference.py) + +In this example, we launch an SGLang engine and feed a batch of inputs for inference. If you provide a very large batch, the engine will intelligently schedule the requests to process efficiently and prevent OOM (Out of Memory) errors. + +### [Embedding Generation](./embedding.py) + +In this example, we launch an SGLang engine and feed a batch of inputs for embedding generation. + +### [Custom Server](./custom_server.py) + +This example demonstrates how to create a custom server on top of the SGLang Engine. We use [Sanic](https://sanic.dev/en/) as an example. The server supports both non-streaming and streaming endpoints. + +#### Steps: + +1. Install Sanic: + +```bash +pip install sanic +``` + +2. Run the server: + +```bash +python custom_server +``` + +3. Send requests: + +```bash +curl -X POST http://localhost:8000/generate -H "Content-Type: application/json" -d '{"prompt": "The Transformer architecture is..."}' +curl -X POST http://localhost:8000/generate_stream -H "Content-Type: application/json" -d '{"prompt": "The Transformer architecture is..."}' --no-buffer +``` + +This will send both non-streaming and streaming requests to the server. + +### [Token-In-Token-Out for RLHF](../token_in_token_out) + +In this example, we launch an SGLang engine, feed tokens as input and generate tokens as output. diff --git a/examples/runtime/engine/save_remote_state.py b/examples/runtime/engine/save_remote_state.py new file mode 100644 index 0000000..4781269 --- /dev/null +++ b/examples/runtime/engine/save_remote_state.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Saves each worker's model state dict directly to a checkpoint, which enables a +fast load path for large tensor-parallel models where each worker only needs to +read its own shard rather than the entire checkpoint. + +Example usage: + +python save_remote_state.py \ + --model-path /path/to/load \ + --tensor-parallel-size 8 \ + --remote-model-save-url [protocol]://[host]:[port]/[model_name] \ + +Then, the model can be loaded with + +llm = Engine( + model_path="/path/to/save", + --remote-model-url [protocol]://[host]:[port]/[model_name], + tensor_parallel_size=8, +) +""" +import dataclasses +from argparse import ArgumentParser +from pathlib import Path + +from sglang import Engine, ServerArgs + +parser = ArgumentParser() +ServerArgs.add_cli_args(parser) + +parser.add_argument( + "--remote-model-save-url", + required=True, + type=str, + help="remote address to store model weights", +) + + +def main(args): + engine_args = ServerArgs.from_cli_args(args) + model_path = engine_args.model_path + if not Path(model_path).is_dir(): + raise ValueError("model path must be a local directory") + # Create LLM instance from arguments + llm = Engine(**dataclasses.asdict(engine_args)) + llm.save_remote_model(url=args.remote_model_save_url) + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/examples/runtime/engine/save_sharded_state.py b/examples/runtime/engine/save_sharded_state.py new file mode 100644 index 0000000..80ad532 --- /dev/null +++ b/examples/runtime/engine/save_sharded_state.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Saves each worker's model state dict directly to a checkpoint, which enables a +fast load path for large tensor-parallel models where each worker only needs to +read its own shard rather than the entire checkpoint. + +Example usage: + +python save_sharded_state.py \ + --model-path /path/to/load \ + --quantization deepspeedfp \ + --tensor-parallel-size 8 \ + --output /path/to/save + +Then, the model can be loaded with + +llm = Engine( + model_path="/path/to/save", + load_format="sharded_state", + quantization="deepspeedfp", + tensor_parallel_size=8, +) +""" +import dataclasses +import os +import shutil +from argparse import ArgumentParser +from pathlib import Path + +from sglang import Engine, ServerArgs + +parser = ArgumentParser() +ServerArgs.add_cli_args(parser) + +parser.add_argument( + "--output", "-o", required=True, type=str, help="path to output checkpoint" +) +parser.add_argument( + "--file-pattern", type=str, help="string pattern of saved filenames" +) +parser.add_argument( + "--max-file-size", + type=str, + default=5 * 1024**3, + help="max size (in bytes) of each safetensors file", +) + + +def main(args): + engine_args = ServerArgs.from_cli_args(args) + model_path = engine_args.model_path + if not Path(model_path).is_dir(): + raise ValueError("model path must be a local directory") + # Create LLM instance from arguments + llm = Engine(**dataclasses.asdict(engine_args)) + Path(args.output).mkdir(exist_ok=True) + llm.save_sharded_model( + path=args.output, pattern=args.file_pattern, max_size=args.max_file_size + ) + + # Copy metadata files to output directory + for file in os.listdir(model_path): + if os.path.splitext(file)[1] not in (".bin", ".pt", ".safetensors"): + if os.path.isdir(os.path.join(model_path, file)): + shutil.copytree( + os.path.join(model_path, file), os.path.join(args.output, file) + ) + else: + shutil.copy(os.path.join(model_path, file), args.output) + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/examples/runtime/hidden_states/hidden_states_engine.py b/examples/runtime/hidden_states/hidden_states_engine.py new file mode 100644 index 0000000..8af883a --- /dev/null +++ b/examples/runtime/hidden_states/hidden_states_engine.py @@ -0,0 +1,65 @@ +""" +Usage: +python hidden_states.py + +Note that each time you change the `return_hidden_states` parameter, +the cuda graph will be recaptured, which might lead to a performance hit. +So avoid getting hidden states and completions alternately. +""" + +import torch + +import sglang as sgl + + +def main(): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create an LLM. + llm = sgl.Engine( + model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", + ) + + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 10, + } + + outputs = llm.generate( + prompts, sampling_params=sampling_params, return_hidden_states=True + ) + + llm.shutdown() + + for prompt, output in zip(prompts, outputs): + for i in range(len(output["meta_info"]["hidden_states"])): + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + print("===============================") + print( + f"Prompt: {prompt}\n" + f"Generated text: {output['text']}\n" + f"Prompt_Tokens: {output['meta_info']['prompt_tokens']}\t" + f"Completion_tokens: {output['meta_info']['completion_tokens']}" + ) + print("Hidden states: ") + hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ) + print(hidden_states) + print() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/hidden_states/hidden_states_server.py b/examples/runtime/hidden_states/hidden_states_server.py new file mode 100644 index 0000000..96045fa --- /dev/null +++ b/examples/runtime/hidden_states/hidden_states_server.py @@ -0,0 +1,81 @@ +""" +Usage: + +python hidden_states_server.py + +Note that each time you change the `return_hidden_states` parameter, +the cuda graph will be recaptured, which might lead to a performance hit. +So avoid getting hidden states and completions alternately. +""" + +import requests +import torch + +from sglang.test.test_utils import is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 10, + } + + json_data = { + "text": prompts, + "sampling_params": sampling_params, + "return_hidden_states": True, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + terminate_process(server_process) + + outputs = response.json() + for prompt, output in zip(prompts, outputs): + for i in range(len(output["meta_info"]["hidden_states"])): + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + print("===============================") + print( + f"Prompt: {prompt}\n" + f"Generated text: {output['text']}\n" + f"Prompt_Tokens: {output['meta_info']['prompt_tokens']}\t" + f"Completion_tokens: {output['meta_info']['completion_tokens']}" + ) + print("Hidden states: ") + hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ) + print(hidden_states) + print() + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/llava_onevision/http_llama3_llava_test.py b/examples/runtime/llava_onevision/http_llama3_llava_test.py new file mode 100644 index 0000000..ed0e616 --- /dev/null +++ b/examples/runtime/llava_onevision/http_llama3_llava_test.py @@ -0,0 +1,111 @@ +""" +Usage: +# Installing latest llava-next: pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +# Installing latest sglang. + +# Endpoint Service CLI: +python -m sglang.launch_server --model-path lmms-lab/llama3-llava-next-8b --port=30000 + +python3 http_llama3_llava_test.py + +Output: +"Friends posing for a fun photo with a life-sized teddy bear, creating a playful and memorable moment." +""" + +import argparse +import asyncio +import copy +import json + +import aiohttp +import requests +from llava.conversation import conv_llava_llama_3 + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_llava_llama_3) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + response = [] + for i in range(1): + response.append( + send_request( + url + "/generate", + { + "text": prompt_with_template, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|eot_id|>", + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_llava_llama_3) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + pload = { + "text": prompt_with_template, + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|eot_id|>", + }, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "stream": True, + } + response = requests.post( + url + "/generate", + json=pload, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + asyncio.run(test_concurrent(args)) + test_streaming(args) diff --git a/examples/runtime/llava_onevision/http_llava_onevision_test.py b/examples/runtime/llava_onevision/http_llava_onevision_test.py new file mode 100644 index 0000000..ce59f40 --- /dev/null +++ b/examples/runtime/llava_onevision/http_llava_onevision_test.py @@ -0,0 +1,264 @@ +""" +Usage: + +python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-72b-ov --port=30000 --tp-size=8 --chat-template=chatml-llava + +python3 http_llava_onevision_test.py +""" + +import base64 +import io +import os +import sys +import time + +import numpy as np +import openai +import requests +from decord import VideoReader, cpu +from PIL import Image + +# pip install httpx==0.23.3 +# pip install decord +# pip install protobuf==3.20.0 + + +def download_video(url, cache_dir): + file_path = os.path.join(cache_dir, "jobs.mp4") + os.makedirs(cache_dir, exist_ok=True) + + response = requests.get(url) + response.raise_for_status() + + with open(file_path, "wb") as f: + f.write(response.content) + + print(f"File downloaded and saved to: {file_path}") + return file_path + + +def create_openai_client(base_url): + return openai.Client(api_key="EMPTY", base_url=base_url) + + +def image_stream_request_test(client): + print("----------------------Image Stream Request Test----------------------") + stream_request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + }, + { + "type": "text", + "text": "Please describe this image. Please list the benchmarks and the models.", + }, + ], + }, + ], + temperature=0.7, + max_tokens=1024, + stream=True, + ) + stream_response = "" + + for chunk in stream_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + stream_response += content + sys.stdout.write(content) + sys.stdout.flush() + + print("-" * 30) + + +def multi_image_stream_request_test(client): + print( + "----------------------Multi-Images Stream Request Test----------------------" + ) + stream_request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png" + }, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have shown you two images. Please describe the two images to me.", + }, + ], + }, + ], + temperature=0.7, + max_tokens=1024, + stream=True, + ) + stream_response = "" + + for chunk in stream_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + stream_response += content + sys.stdout.write(content) + sys.stdout.flush() + + print("-" * 30) + + +def video_stream_request_test(client, video_path): + print("------------------------Video Stream Request Test----------------------") + messages = prepare_video_messages(video_path) + + video_request = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=True, + ) + print("-" * 30) + video_response = "" + + for chunk in video_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + video_response += content + sys.stdout.write(content) + sys.stdout.flush() + print("-" * 30) + + +def image_speed_test(client): + print("----------------------Image Speed Test----------------------") + start_time = time.time() + request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + }, + { + "type": "text", + "text": "Please describe this image. Please list the benchmarks and the models.", + }, + ], + }, + ], + temperature=0, + max_tokens=1024, + ) + end_time = time.time() + response = request.choices[0].message.content + print(response) + print("-" * 30) + print_speed_test_results(request, start_time, end_time) + + +def video_speed_test(client, video_path): + print("------------------------Video Speed Test------------------------") + messages = prepare_video_messages(video_path) + + start_time = time.time() + video_request = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + ) + end_time = time.time() + video_response = video_request.choices[0].message.content + print(video_response) + print("-" * 30) + print_speed_test_results(video_request, start_time, end_time) + + +def prepare_video_messages(video_path): + max_frames_num = 32 + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = base64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + + for base64_frame in base64_frames: + frame_format = { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{base64_frame}"}, + "modalities": "video", + } + messages[0]["content"].append(frame_format) + + prompt = {"type": "text", "text": "Please describe the video in detail."} + messages[0]["content"].append(prompt) + + return messages + + +def print_speed_test_results(request, start_time, end_time): + total_tokens = request.usage.total_tokens + completion_tokens = request.usage.completion_tokens + prompt_tokens = request.usage.prompt_tokens + + print(f"Total tokens: {total_tokens}") + print(f"Completion tokens: {completion_tokens}") + print(f"Prompt tokens: {prompt_tokens}") + print(f"Time taken: {end_time - start_time} seconds") + print(f"Token per second: {total_tokens / (end_time - start_time)}") + print(f"Completion token per second: {completion_tokens / (end_time - start_time)}") + print(f"Prompt token per second: {prompt_tokens / (end_time - start_time)}") + + +def main(): + url = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + cache_dir = os.path.expanduser("~/.cache") + video_path = download_video(url, cache_dir) + + client = create_openai_client("http://127.0.0.1:30000/v1") + + image_stream_request_test(client) + multi_image_stream_request_test(client) + video_stream_request_test(client, video_path) + image_speed_test(client) + video_speed_test(client, video_path) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/llava_onevision/http_qwen_llava_test.py b/examples/runtime/llava_onevision/http_qwen_llava_test.py new file mode 100644 index 0000000..94ec4e0 --- /dev/null +++ b/examples/runtime/llava_onevision/http_qwen_llava_test.py @@ -0,0 +1,111 @@ +""" +Usage: +# Installing latest llava-next: pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +# Installing latest sglang. + +# Endpoint Service CLI: +python -m sglang.launch_server --model-path lmms-lab/llava-next-72b --port=30000 --tp-size=8 + +python3 http_qwen_llava_test.py + +Output: +"Two children pose with a large teddy bear, one holding a smaller stuffed bear, in a room with an American flag and potted plants." +""" + +import argparse +import asyncio +import copy +import json + +import aiohttp +import requests +from llava.conversation import conv_qwen + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_qwen) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + response = [] + for i in range(1): + response.append( + send_request( + url + "/generate", + { + "text": prompt_with_template, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|im_end|>", + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_qwen) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + pload = { + "text": prompt_with_template, + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|im_end|>", + }, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "stream": True, + } + response = requests.post( + url + "/generate", + json=pload, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + asyncio.run(test_concurrent(args)) + test_streaming(args) diff --git a/examples/runtime/lora.py b/examples/runtime/lora.py new file mode 100644 index 0000000..bf3fc2d --- /dev/null +++ b/examples/runtime/lora.py @@ -0,0 +1,37 @@ +# launch server +# python -m sglang.launch_server --model mistralai/Mistral-7B-Instruct-v0.3 --lora-paths /home/ying/test_lora lora1=/home/ying/test_lora_1 lora2=/home/ying/test_lora_2 --disable-radix --disable-cuda-graph --max-loras-per-batch 4 + +# send requests +# lora_path[i] specifies the LoRA used for text[i], so make sure they have the same length +# use None to specify base-only prompt, e.x. "lora_path": [None, "/home/ying/test_lora"] +import json + +import requests + +url = "http://127.0.0.1:30000" +json_data = { + "text": [ + "prompt 1", + "prompt 2", + "prompt 3", + "prompt 4", + "prompt 5", + "prompt 6", + "prompt 7", + ], + "sampling_params": {"max_new_tokens": 32}, + "lora_path": [ + "/home/ying/test_lora", + "lora1", + "lora2", + "lora1", + "lora2", + None, + None, + ], +} +response = requests.post( + url + "/generate", + json=json_data, +) +print(json.dumps(response.json())) diff --git a/examples/runtime/multimodal_embedding.py b/examples/runtime/multimodal_embedding.py new file mode 100644 index 0000000..813c3c7 --- /dev/null +++ b/examples/runtime/multimodal_embedding.py @@ -0,0 +1,21 @@ +# launch server +# python -m sglang.launch_server --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct --is-embedding --chat-template gme-qwen2-vl + +import requests + +url = "http://127.0.0.1:30000" + +text_input = "Represent this image in embedding space." +image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + +payload = { + "model": "gme-qwen2-vl", + "input": [ + {"type": "text", "text": text_input}, + {"type": "image", "url": image_path}, + ], +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() + +print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) diff --git a/examples/runtime/openai_batch_chat.py b/examples/runtime/openai_batch_chat.py new file mode 100644 index 0000000..d251ca0 --- /dev/null +++ b/examples/runtime/openai_batch_chat.py @@ -0,0 +1,94 @@ +""" +Usage: + +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python openai_batch_chat.py + +Note: Before running this script, +you should create the input.jsonl file with the following content: +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world! List 3 NBA players and tell a story"}],"max_tokens": 300}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an assistant. "},{"role": "user", "content": "Hello world! List three capital and tell a story"}],"max_tokens": 500}} +""" + +import json +import time + +import openai + + +class OpenAIBatchProcessor: + def __init__(self): + client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + + self.client = client + + def process_batch(self, input_file_path, endpoint, completion_window): + + # Upload the input file + with open(input_file_path, "rb") as file: + uploaded_file = self.client.files.create(file=file, purpose="batch") + + # Create the batch job + batch_job = self.client.batches.create( + input_file_id=uploaded_file.id, + endpoint=endpoint, + completion_window=completion_window, + ) + + # Monitor the batch job status + while batch_job.status not in ["completed", "failed", "cancelled"]: + time.sleep(3) # Wait for 3 seconds before checking the status again + print( + f"Batch job status: {batch_job.status}...trying again in 3 seconds..." + ) + batch_job = self.client.batches.retrieve(batch_job.id) + + # Check the batch job status and errors + if batch_job.status == "failed": + print(f"Batch job failed with status: {batch_job.status}") + print(f"Batch job errors: {batch_job.errors}") + return None + + # If the batch job is completed, process the results + if batch_job.status == "completed": + + # print result of batch job + print("batch", batch_job.request_counts) + + result_file_id = batch_job.output_file_id + # Retrieve the file content from the server + file_response = self.client.files.content(result_file_id) + result_content = file_response.read() # Read the content of the file + + # Save the content to a local file + result_file_name = "batch_job_chat_results.jsonl" + with open(result_file_name, "wb") as file: + file.write(result_content) # Write the binary content to the file + # Load data from the saved JSONL file + results = [] + with open(result_file_name, "r", encoding="utf-8") as file: + for line in file: + json_object = json.loads( + line.strip() + ) # Parse each line as a JSON object + results.append(json_object) + + return results + else: + print(f"Batch job failed with status: {batch_job.status}") + return None + + +# Initialize the OpenAIBatchProcessor +processor = OpenAIBatchProcessor() + +# Process the batch job +input_file_path = "input.jsonl" +endpoint = "/v1/chat/completions" +completion_window = "24h" + +# Process the batch job +results = processor.process_batch(input_file_path, endpoint, completion_window) + +# Print the results +print(results) diff --git a/examples/runtime/openai_batch_complete.py b/examples/runtime/openai_batch_complete.py new file mode 100644 index 0000000..2f5be3d --- /dev/null +++ b/examples/runtime/openai_batch_complete.py @@ -0,0 +1,93 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python openai_batch_complete.py +Note: Before running this script, +you should create the input.jsonl file with the following content: +{"custom_id": "request-1", "method": "POST", "url": "/v1/completions", "body": {"model": "gpt-3.5-turbo-instruct", "prompt": "List 3 names of famous soccer player: ", "max_tokens": 200}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/completions", "body": {"model": "gpt-3.5-turbo-instruct", "prompt": "List 6 names of famous basketball player: ", "max_tokens": 400}} +{"custom_id": "request-3", "method": "POST", "url": "/v1/completions", "body": {"model": "gpt-3.5-turbo-instruct", "prompt": "List 6 names of famous basketball player: ", "max_tokens": 400}} +""" + +import json +import time + +import openai + + +class OpenAIBatchProcessor: + def __init__(self): + client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + + self.client = client + + def process_batch(self, input_file_path, endpoint, completion_window): + + # Upload the input file + with open(input_file_path, "rb") as file: + uploaded_file = self.client.files.create(file=file, purpose="batch") + + # Create the batch job + batch_job = self.client.batches.create( + input_file_id=uploaded_file.id, + endpoint=endpoint, + completion_window=completion_window, + ) + + # Monitor the batch job status + while batch_job.status not in ["completed", "failed", "cancelled"]: + time.sleep(3) # Wait for 3 seconds before checking the status again + print( + f"Batch job status: {batch_job.status}...trying again in 3 seconds..." + ) + batch_job = self.client.batches.retrieve(batch_job.id) + + # Check the batch job status and errors + if batch_job.status == "failed": + print(f"Batch job failed with status: {batch_job.status}") + print(f"Batch job errors: {batch_job.errors}") + return None + + # If the batch job is completed, process the results + if batch_job.status == "completed": + + # print result of batch job + print("batch", batch_job.request_counts) + + result_file_id = batch_job.output_file_id + # Retrieve the file content from the server + file_response = self.client.files.content(result_file_id) + result_content = file_response.read() # Read the content of the file + + # Save the content to a local file + result_file_name = "batch_job_complete_results.jsonl" + with open(result_file_name, "wb") as file: + file.write(result_content) # Write the binary content to the file + # Load data from the saved JSONL file + results = [] + with open(result_file_name, "r", encoding="utf-8") as file: + for line in file: + json_object = json.loads( + line.strip() + ) # Parse each line as a JSON object + results.append(json_object) + + return results + else: + print(f"Batch job failed with status: {batch_job.status}") + return None + + +# Initialize the OpenAIBatchProcessor +processor = OpenAIBatchProcessor() + +# Process the batch job +input_file_path = "input.jsonl" +endpoint = "/v1/completions" +completion_window = "24h" + +# Process the batch job +results = processor.process_batch(input_file_path, endpoint, completion_window) + +# Print the results +print(results) diff --git a/examples/runtime/openai_chat_with_response_prefill.py b/examples/runtime/openai_chat_with_response_prefill.py new file mode 100644 index 0000000..c80f97d --- /dev/null +++ b/examples/runtime/openai_chat_with_response_prefill.py @@ -0,0 +1,33 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 30000 +python openai_chat.py +""" + +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + { + "role": "assistant", + "content": "{\n", + }, + ], + temperature=0, +) + +print(response.choices[0].message.content) diff --git a/examples/runtime/reward_model.py b/examples/runtime/reward_model.py new file mode 100644 index 0000000..1a1177e --- /dev/null +++ b/examples/runtime/reward_model.py @@ -0,0 +1,32 @@ +# launch server +# python -m sglang.launch_server --model LxzGordon/URM-LLaMa-3.1-8B --is-embedding + +import requests + +url = "http://127.0.0.1:30000" + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +json_data = { + "conv": [ + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": RESPONSE1}, + ], + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": RESPONSE2}, + ], + ], +} +response = requests.post( + url + "/classify", + json=json_data, +).json() + +print(response) +print("scores:", [x["embedding"] for x in response]) diff --git a/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py b/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py new file mode 100644 index 0000000..cb1b7dd --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py @@ -0,0 +1,43 @@ +""" +This example demonstrates how to provide tokenized ids to LLM as input instead of text prompt, i.e. a token-in-token-out workflow. +""" + +import sglang as sgl +from sglang.srt.hf_transformers_utils import get_tokenizer + +MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Tokenize inputs + tokenizer = get_tokenizer(MODEL_PATH) + token_ids_list = [tokenizer.encode(prompt) for prompt in prompts] + + # Create an LLM. + llm = sgl.Engine(model_path=MODEL_PATH, skip_tokenizer_init=True) + + outputs = llm.generate(input_ids=token_ids_list, sampling_params=sampling_params) + # Print the outputs. + for prompt, output in zip(prompts, outputs): + decode_output = tokenizer.decode(output["output_ids"]) + print("===============================") + print( + f"Prompt: {prompt}\nGenerated token ids: {output['output_ids']}\nGenerated text: {decode_output}" + ) + print() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py b/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py new file mode 100644 index 0000000..00c0988 --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py @@ -0,0 +1,68 @@ +""" +Usage: + +python token_in_token_out_llm_server.py + +""" + +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Tokenize inputs + tokenizer = get_tokenizer(MODEL_PATH) + token_ids_list = [tokenizer.encode(prompt) for prompt in prompts] + + json_data = { + "input_ids": token_ids_list, + "sampling_params": sampling_params, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + outputs = response.json() + for prompt, output in zip(prompts, outputs): + print("===============================") + decode_output = tokenizer.decode(output["output_ids"]) + print( + f"Prompt: {prompt}\nGenerated token ids: {output['output_ids']}\nGenerated text: {decode_output}" + ) + print() + + terminate_process(server_process) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py b/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py new file mode 100644 index 0000000..3c3717d --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py @@ -0,0 +1,75 @@ +import argparse +import dataclasses +from io import BytesIO +from typing import Tuple + +import requests +from PIL import Image +from transformers import AutoProcessor + +from sglang import Engine +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import DEFAULT_IMAGE_URL + + +def get_input_ids( + server_args: ServerArgs, model_config: ModelConfig +) -> Tuple[list[int], list]: + chat_template = get_chat_template_by_model_path(model_config.model_path) + text = f"{chat_template.image_token}What is in this picture?" + images = [Image.open(BytesIO(requests.get(DEFAULT_IMAGE_URL).content))] + image_data = [DEFAULT_IMAGE_URL] + + processor = AutoProcessor.from_pretrained( + model_config.model_path, trust_remote_code=server_args.trust_remote_code + ) + + inputs = processor( + text=[text], + images=images, + return_tensors="pt", + ) + + return inputs.input_ids[0].tolist(), image_data + + +def token_in_out_example( + server_args: ServerArgs, +): + input_ids, image_data = get_input_ids( + server_args, + ModelConfig( + server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + model_override_args=server_args.json_model_override_args, + ), + ) + backend = Engine(**dataclasses.asdict(server_args)) + + output = backend.generate( + input_ids=input_ids, + image_data=image_data, + sampling_params={ + "temperature": 0.8, + "max_new_tokens": 32, + }, + ) + + print("===============================") + print(f"Output token ids: ", output["output_ids"]) + + backend.shutdown() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = [ + "--model-path=Qwen/Qwen2-VL-2B", + ] + args = parser.parse_args(args=args) + server_args = ServerArgs.from_cli_args(args) + server_args.skip_tokenizer_init = True + token_in_out_example(server_args) diff --git a/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py b/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py new file mode 100644 index 0000000..01c4858 --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py @@ -0,0 +1,78 @@ +""" +Usage: + +python token_in_token_out_vlm_server.py + +""" + +from io import BytesIO +from typing import Tuple + +import requests +from PIL import Image +from transformers import AutoProcessor + +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.test.test_utils import DEFAULT_IMAGE_URL, is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +MODEL_PATH = "Qwen/Qwen2-VL-2B" + + +def get_input_ids() -> Tuple[list[int], list]: + chat_template = get_chat_template_by_model_path(MODEL_PATH) + text = f"{chat_template.image_token}What is in this picture?" + images = [Image.open(BytesIO(requests.get(DEFAULT_IMAGE_URL).content))] + image_data = [DEFAULT_IMAGE_URL] + + processor = AutoProcessor.from_pretrained(MODEL_PATH) + + inputs = processor( + text=[text], + images=images, + return_tensors="pt", + ) + + return inputs.input_ids[0].tolist(), image_data + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + input_ids, image_data = get_input_ids() + + sampling_params = { + "temperature": 0.8, + "max_new_tokens": 32, + } + + json_data = { + "input_ids": input_ids, + "image_data": image_data, + "sampling_params": sampling_params, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + output = response.json() + print("===============================") + print(f"Output token ids: ", output["output_ids"]) + + terminate_process(server_process) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/vertex_predict.py b/examples/runtime/vertex_predict.py new file mode 100644 index 0000000..58a41b1 --- /dev/null +++ b/examples/runtime/vertex_predict.py @@ -0,0 +1,66 @@ +""" +Usage: +python -m sglang.launch_server --model meta-llama/Llama-2-7b-hf --port 30000 +python vertex_predict.py + +This example shows the request and response formats of the prediction route for +Google Cloud Vertex AI Online Predictions. + +Vertex AI SDK for Python is recommended for deploying models to Vertex AI +instead of a local server. After deploying the model to a Vertex AI Online +Prediction Endpoint, send requests via the Python SDK: + +response = endpoint.predict( + instances=[ + {"text": "The capital of France is"}, + {"text": "What is a car?"}, + ], + parameters={"sampling_params": {"max_new_tokens": 16}}, +) +print(response.predictions) + +More details about get online predictions from Vertex AI can be found at +https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions. +""" + +from dataclasses import dataclass +from typing import List, Optional + +import requests + + +@dataclass +class VertexPrediction: + predictions: List + + +class LocalVertexEndpoint: + def __init__(self) -> None: + self.base_url = "http://127.0.0.1:30000" + + def predict(self, instances: List[dict], parameters: Optional[dict] = None): + response = requests.post( + self.base_url + "/vertex_generate", + json={ + "instances": instances, + "parameters": parameters, + }, + ) + return VertexPrediction(predictions=response.json()["predictions"]) + + +endpoint = LocalVertexEndpoint() + +# Predict with a single prompt. +response = endpoint.predict(instances=[{"text": "The capital of France is"}]) +print(response.predictions) + +# Predict with multiple prompts and parameters. +response = endpoint.predict( + instances=[ + {"text": "The capital of France is"}, + {"text": "What is a car?"}, + ], + parameters={"sampling_params": {"max_new_tokens": 16}}, +) +print(response.predictions) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..a1a0dcd --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,134 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang" +version = "0.4.4.post3" +description = "SGLang is yet another fast serving framework for large language models and vision language models." +readme = "README.md" +requires-python = ">=3.8" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", +] +dependencies = ["aiohttp", "requests", "tqdm", "numpy", "IPython", "setproctitle"] + +[project.optional-dependencies] +runtime_common = [ + "compressed-tensors", + "datasets", + "decord", + "fastapi", + "hf_transfer", + "huggingface_hub", + "interegular", + "llguidance>=0.6.15", + "modelscope", + "ninja", + "orjson", + "packaging", + "pillow", + "prometheus-client>=0.20.0", + "psutil", + "pydantic", + "python-multipart", + "pyzmq>=25.1.2", + "soundfile==0.13.1", + "torchao>=0.7.0", + "transformers==4.50.0", + "uvicorn", + "uvloop", + "compressed-tensors", + "xgrammar==0.1.17", +] + +srt = [ + "sglang[runtime_common]", + "sgl-kernel==0.0.5.post4", + "flashinfer_python==0.2.3", + "torch==2.5.1", + "cuda-python", + "outlines>=0.0.44,<=0.1.11", + "partial_json_parser", + "einops", +] + +# HIP (Heterogeneous-computing Interface for Portability) for AMD +# => base docker rocm/vllm-dev:20250114, not from public vllm whl +srt_hip = [ + "sglang[runtime_common]", + "torch", + "vllm==0.6.7.dev2", + "outlines==0.1.11" +] + +# xpu is not enabled in public vllm and torch whl, +# need to follow https://docs.vllm.ai/en/latest/getting_started/xpu-installation.htmlinstall vllm +srt_xpu = ["sglang[runtime_common]", "outlines>=0.0.44,<=0.1.11"] + +# For Intel Gaudi(device : hpu) follow the installation guide +# https://docs.vllm.ai/en/latest/getting_started/gaudi-installation.html +srt_hpu = ["sglang[runtime_common]", "outlines>=0.0.44,<=0.1.11"] + +# CPU: currently, there are no pre-built vllm wheels for CPU. +# To install vllm for CPU, please follow the instruction here: +# https://docs.vllm.ai/en/latest/getting_started/installation/cpu/index.html +srt_cpu = ["sglang[runtime_common]", "outlines>=0.0.44,<=0.1.11", "torch"] + +openai = ["openai>=1.0", "tiktoken"] +anthropic = ["anthropic>=0.20.0"] +litellm = ["litellm>=1.0.0"] +torch_memory_saver = ["torch_memory_saver>=0.0.4"] +test = [ + "jsonlines", + "matplotlib", + "pandas", + "sentence_transformers", + "accelerate", + "peft", +] +all = ["sglang[srt]", "sglang[openai]", "sglang[anthropic]", "sglang[litellm]"] +all_hip = ["sglang[srt_hip]", "sglang[openai]", "sglang[anthropic]", "sglang[litellm]"] +all_xpu = ["sglang[srt_xpu]", "sglang[openai]", "sglang[anthropic]", "sglang[litellm]"] +all_hpu = ["sglang[srt_hpu]", "sglang[openai]", "sglang[anthropic]", "sglang[litellm]"] +all_cpu = ["sglang[srt_cpu]", "sglang[openai]", "sglang[anthropic]", "sglang[litellm]"] + +dev = ["sglang[all]", "sglang[test]"] +dev_hip = ["sglang[all_hip]", "sglang[test]"] +dev_xpu = ["sglang[all_xpu]", "sglang[test]"] +dev_hpu = ["sglang[all_hpu]", "sglang[test]"] +dev_cpu = ["sglang[all_cpu]", "sglang[test]"] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.setuptools.package-data] +"sglang" = [ + "srt/layers/moe/fused_moe_triton/configs/*.json", + "srt/layers/quantization/configs/*.json", +] + +[tool.setuptools.packages.find] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.wheel] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] diff --git a/python/sglang/README.md b/python/sglang/README.md new file mode 100644 index 0000000..0234973 --- /dev/null +++ b/python/sglang/README.md @@ -0,0 +1,16 @@ +# Code Structures + +- `lang`: The frontend language. +- `srt`: The backend engine for running local models. (SRT = SGLang Runtime). +- `test`: The test utilities. +- `api.py`: The public APIs. +- `bench_offline_throughput.py`: Benchmark the throughput in the offline mode. +- `bench_one_batch.py`: Benchmark the latency of running a single static batch without a server. +- `bench_one_batch_server.py`: Benchmark the latency of running a single batch with a server. +- `bench_serving.py`: Benchmark online serving with dynamic requests. +- `check_env.py`: Check the environment variables and dependencies. +- `global_config.py`: The global configs and constants. +- `launch_server.py`: The entry point for launching the local server. +- `llama3_eval.py`: Evaluation of Llama 3 using the Meta Llama dataset. +- `utils.py`: Common utilities. +- `version.py`: Version info. diff --git a/python/sglang/__init__.py b/python/sglang/__init__.py new file mode 100644 index 0000000..db0cf26 --- /dev/null +++ b/python/sglang/__init__.py @@ -0,0 +1,78 @@ +# SGLang public APIs + +# Frontend Language APIs +from sglang.api import ( + Engine, + Runtime, + assistant, + assistant_begin, + assistant_end, + flush_cache, + function, + gen, + gen_int, + gen_string, + get_server_info, + image, + select, + set_default_backend, + system, + system_begin, + system_end, + user, + user_begin, + user_end, + video, +) +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint +from sglang.lang.choices import ( + greedy_token_selection, + token_length_normalized, + unconditional_likelihood_normalized, +) +from sglang.utils import LazyImport + +ServerArgs = LazyImport("sglang.srt.server_args", "ServerArgs") +Anthropic = LazyImport("sglang.lang.backend.anthropic", "Anthropic") +LiteLLM = LazyImport("sglang.lang.backend.litellm", "LiteLLM") +OpenAI = LazyImport("sglang.lang.backend.openai", "OpenAI") +VertexAI = LazyImport("sglang.lang.backend.vertexai", "VertexAI") + +# Other configs +from sglang.global_config import global_config +from sglang.version import __version__ + +__all__ = [ + "Engine", + "Runtime", + "assistant", + "assistant_begin", + "assistant_end", + "flush_cache", + "function", + "gen", + "gen_int", + "gen_string", + "get_server_info", + "image", + "select", + "set_default_backend", + "system", + "system_begin", + "system_end", + "user", + "user_begin", + "user_end", + "video", + "RuntimeEndpoint", + "greedy_token_selection", + "token_length_normalized", + "unconditional_likelihood_normalized", + "ServerArgs", + "Anthropic", + "LiteLLM", + "OpenAI", + "VertexAI", + "global_config", + "__version__", +] diff --git a/python/sglang/api.py b/python/sglang/api.py new file mode 100644 index 0000000..50319c4 --- /dev/null +++ b/python/sglang/api.py @@ -0,0 +1,279 @@ +"""Public APIs of the language.""" + +import re +from typing import Callable, List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.choices import ChoicesSamplingMethod, token_length_normalized +from sglang.lang.ir import ( + SglExpr, + SglExprList, + SglFunction, + SglGen, + SglImage, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglVideo, +) + + +def function( + func: Optional[Callable] = None, num_api_spec_tokens: Optional[int] = None +): + if func: + return SglFunction(func, num_api_spec_tokens=num_api_spec_tokens) + + def decorator(func): + return SglFunction(func, num_api_spec_tokens=num_api_spec_tokens) + + return decorator + + +def Runtime(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.lang.backend.runtime_endpoint import Runtime + + return Runtime(*args, **kwargs) + + +def Engine(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.srt.entrypoints.engine import Engine + + return Engine(*args, **kwargs) + + +def set_default_backend(backend: BaseBackend): + global_config.default_backend = backend + + +def flush_cache(backend: Optional[BaseBackend] = None): + backend = backend or global_config.default_backend + if backend is None: + return False + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint + return backend.flush_cache() + + +def get_server_info(backend: Optional[BaseBackend] = None): + backend = backend or global_config.default_backend + if backend is None: + return None + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint + return backend.get_server_info() + + +def gen( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + min_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + dtype: Optional[Union[type, str]] = None, + choices: Optional[List[str]] = None, + choices_method: Optional[ChoicesSamplingMethod] = None, + regex: Optional[str] = None, + json_schema: Optional[str] = None, +): + """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + + if choices: + return SglSelect( + name, + choices, + 0.0 if temperature is None else temperature, + token_length_normalized if choices_method is None else choices_method, + ) + + # check regex is valid + if regex is not None: + try: + re.compile(regex) + except re.error as e: + raise e + + return SglGen( + name, + max_tokens, + min_tokens, + n, + stop, + stop_token_ids, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + dtype, + regex, + json_schema, + ) + + +def gen_int( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, +): + return SglGen( + name, + max_tokens, + None, + n, + stop, + stop_token_ids, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + int, + None, + ) + + +def gen_string( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, +): + return SglGen( + name, + max_tokens, + None, + n, + stop, + stop_token_ids, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + str, + None, + ) + + +def image(expr: SglExpr): + return SglImage(expr) + + +def video(path: str, num_frames: int): + return SglVideo(path, num_frames) + + +def select( + name: Optional[str] = None, + choices: Optional[List[str]] = None, + temperature: float = 0.0, + choices_method: ChoicesSamplingMethod = token_length_normalized, +): + assert choices is not None + return SglSelect(name, choices, temperature, choices_method) + + +def _role_common(name: str, expr: Optional[SglExpr] = None): + if expr is None: + return SglExprList([SglRoleBegin(name), SglRoleEnd(name)]) + else: + return SglExprList([SglRoleBegin(name), expr, SglRoleEnd(name)]) + + +def system(expr: Optional[SglExpr] = None): + return _role_common("system", expr) + + +def user(expr: Optional[SglExpr] = None): + return _role_common("user", expr) + + +def assistant(expr: Optional[SglExpr] = None): + return _role_common("assistant", expr) + + +def system_begin(): + return SglRoleBegin("system") + + +def system_end(): + return SglRoleEnd("system") + + +def user_begin(): + return SglRoleBegin("user") + + +def user_end(): + return SglRoleEnd("user") + + +def assistant_begin(): + return SglRoleBegin("assistant") + + +def assistant_end(): + return SglRoleEnd("assistant") diff --git a/python/sglang/bench_offline_throughput.py b/python/sglang/bench_offline_throughput.py new file mode 100644 index 0000000..69bbc3e --- /dev/null +++ b/python/sglang/bench_offline_throughput.py @@ -0,0 +1,424 @@ +""" +Benchmark the throughput in the offline mode. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (the same as bench_serving.py). + +# Usage +## Sharegpt dataset with default args +python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 + +## Random dataset with default args +python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 +""" + +import argparse +import dataclasses +import json +import logging +import os +import random +import time +from typing import Dict, List, Optional, Tuple + +import numpy as np + +from sglang.bench_serving import ( + get_dataset, + get_tokenizer, + sample_random_requests, + set_ulimit, +) +from sglang.lang.backend.runtime_endpoint import Runtime +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.server_args import ServerArgs + + +@dataclasses.dataclass +class BenchArgs: + backend: str = "engine" + result_filename: str = "" + dataset_name: str = "sharegpt" + dataset_path: str = "" + num_prompts: int = 1000 + sharegpt_output_len: Optional[int] = None + sharegpt_context_len: Optional[int] = None + random_input_len: int = 1024 + random_output_len: int = 1024 + random_range_ratio: float = 0.0 + gsp_num_groups: int = 64 + gsp_prompts_per_group: int = 16 + gsp_system_prompt_len: int = 2048 + gsp_question_len: int = 128 + gsp_output_len: int = 256 + seed: int = 1 + disable_ignore_eos: bool = False + extra_request_body: Optional[str] = None + apply_chat_template: bool = False + profile: bool = False + skip_warmup: bool = False + do_not_exit: bool = False + prompt_suffix: str = "" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--backend", type=str, default=BenchArgs.backend) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=["sharegpt", "random", "generated-shared-prefix"], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--num-prompts", + type=int, + default=BenchArgs.num_prompts, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=BenchArgs.sharegpt_output_len, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=BenchArgs.sharegpt_context_len, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=BenchArgs.random_input_len, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + type=int, + default=BenchArgs.random_output_len, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=BenchArgs.random_range_ratio, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--gsp-num-groups", + type=int, + default=BenchArgs.gsp_num_groups, + help="Number of groups with shared prefix, used" + "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-prompts-per-group", + type=int, + default=BenchArgs.gsp_prompts_per_group, + help="Number of prompts per group of shared prefix, used" + "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-system-prompt-len", + type=int, + default=BenchArgs.gsp_system_prompt_len, + help="System prompt length, used" "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-question-len", + type=int, + default=BenchArgs.gsp_question_len, + help="Question length, used" "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-output-len", + type=int, + default=BenchArgs.gsp_output_len, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignore EOS token", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + default=BenchArgs.extra_request_body, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--skip-warmup", + action="store_true", + help="Skip the warmup batches.", + ) + parser.add_argument( + "--do-not-exit", + action="store_true", + help="Do not exit the program. This is useful for nsys profile with --duration and --delay.", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def throughput_test_once( + backend_name: str, + backend, + reqs: List[Tuple[str, int, int]], + ignore_eos: bool, + extra_request_body: Dict, + profile: bool, +): + measurement_results = { + "backend": backend_name, + "successful_requests": len(reqs), + "total_latency": -1, + "total_input_tokens": sum(r[1] for r in reqs), + "total_output_tokens": -1, + "request_throughput": -1, + "input_throughput": -1, + "output_throughput": -1, + "total_throughput": -1, + } + + prompt = [r[0] for r in reqs] + sampling_params = [ + { + "temperature": 0, + "max_new_tokens": r[2], + "ignore_eos": ignore_eos, + **extra_request_body, + } + for r in reqs + ] + + if profile: + assert ( + "SGLANG_TORCH_PROFILER_DIR" in os.environ + ), "Please set SGLANG_TORCH_PROFILER_DIR." + os.makedirs(os.environ["SGLANG_TORCH_PROFILER_DIR"], exist_ok=True) + backend.start_profile() + + st = time.perf_counter() + gen_out = backend.generate(prompt=prompt, sampling_params=sampling_params) + latency = time.perf_counter() - st + + if profile: + backend.stop_profile() + monitor_trace_file(os.getenv("SGLANG_TORCH_PROFILER_DIR")) + + if backend_name == "runtime": + gen_out = json.loads(gen_out) + + server_info = backend.get_server_info() + + measurement_results["total_latency"] = latency + measurement_results["total_output_tokens"] = sum( + o["meta_info"]["completion_tokens"] for o in gen_out + ) + measurement_results["request_throughput"] = ( + measurement_results["successful_requests"] / latency + ) + measurement_results["input_throughput"] = ( + measurement_results["total_input_tokens"] / latency + ) + measurement_results["output_throughput"] = ( + measurement_results["total_output_tokens"] / latency + ) + measurement_results["total_throughput"] = ( + measurement_results["total_input_tokens"] + + measurement_results["total_output_tokens"] + ) / latency + measurement_results["last_gen_throughput"] = server_info["last_gen_throughput"] + + return measurement_results + + +def monitor_trace_file(directory, interval=1): + + print(f"Monitoring {directory} for new trace files...") + + known_files = set(os.listdir(directory)) + + while True: + flag = False + time.sleep(interval) + current_files = set(os.listdir(directory)) + + new_files = current_files - known_files + for new_file in new_files: + new_file_path = os.path.join(directory, new_file) + print(f"New file detected: {new_file}") + + previous_size = 0 + while True: + try: + current_size = os.path.getsize(new_file_path) + except FileNotFoundError: + print(f"File {new_file} is no longer accessible.") + break + + if current_size > previous_size: + previous_size = current_size + else: + flag = True + break + + time.sleep(interval) + if flag: + break + + +def throughput_test( + server_args: ServerArgs, + bench_args: BenchArgs, +): + if bench_args.backend == "engine": + backend = Engine(**dataclasses.asdict(server_args)) + if not backend: + raise ValueError("Please provide valid engine arguments") + elif bench_args.backend == "runtime": + backend = Runtime(**dataclasses.asdict(server_args)) + else: + raise ValueError('Please set backend to either "engine" or "runtime"') + + tokenizer_id = server_args.tokenizer_path or server_args.model_path + tokenizer = get_tokenizer(tokenizer_id) + + # Set global environmnets + set_ulimit() + random.seed(bench_args.seed) + np.random.seed(bench_args.seed) + + # Parse args + extra_request_body = {} + if bench_args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Read dataset + input_requests = get_dataset(bench_args, tokenizer) + + warmup_requests = sample_random_requests( + input_len=256, + output_len=16, + num_prompts=min(bench_args.num_prompts, 16), + range_ratio=1.0, + tokenizer=tokenizer, + dataset_path=bench_args.dataset_path, + ) + + # Warm up + if not bench_args.skip_warmup: + logging.info("\nWarmup...") + throughput_test_once( + backend_name=bench_args.backend, + backend=backend, + reqs=warmup_requests, + ignore_eos=not bench_args.disable_ignore_eos, + extra_request_body=extra_request_body, + profile=False, + ) + time.sleep(0.5) + + logging.info("\nBenchmark...") + result = throughput_test_once( + backend_name=bench_args.backend, + backend=backend, + reqs=input_requests, + ignore_eos=not bench_args.disable_ignore_eos, + extra_request_body=extra_request_body, + profile=bench_args.profile, + ) + backend.shutdown() + + if bench_args.result_filename: + with open(bench_args.result_filename, "a") as fout: + fout.write(json.dumps(result) + "\n") + + print( + "\n{s:{c}^{n}}".format(s=" Offline Throughput Benchmark Result ", n=50, c="=") + ) + print("{:<40} {:<10}".format("Backend:", result["backend"])) + print("{:<40} {:<10}".format("Successful requests:", result["successful_requests"])) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", result["total_latency"])) + print("{:<40} {:<10}".format("Total input tokens:", result["total_input_tokens"])) + print( + "{:<40} {:<10}".format("Total generated tokens:", result["total_output_tokens"]) + ) + print( + "{:<40} {:<10.2f}".format( + "Last generation throughput (tok/s):", result["last_gen_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", result["request_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", result["input_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", result["output_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", result["total_throughput"] + ) + ) + print("=" * 50) + + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format="%(message)s", + ) + + throughput_test(server_args, bench_args) + + while bench_args.do_not_exit: + pass diff --git a/python/sglang/bench_one_batch.py b/python/sglang/bench_one_batch.py new file mode 100644 index 0000000..e1a69ae --- /dev/null +++ b/python/sglang/bench_one_batch.py @@ -0,0 +1,531 @@ +""" +Benchmark the latency of running a single static batch without a server. + +This script does not launch a server and uses the low-level APIs. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). + +# Usage (latency test) +## with dummy weights: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --load-format dummy +## sweep through multiple data points and store (append) the results in a jsonl file: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 12 14 --input-len 256 512 --output-len 32 256 --run-name test_run +## run with profiling: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 12 14 --input-len 256 512 --profile +# Usage (correctness test): +python -m sglang.bench_one_batch --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --correct + +## Reference output (of the correctness test above, can be gpu dependent): +input_ids=[[1, 450, 7483, 310, 3444, 338], [1, 450, 7483, 310, 278, 3303, 13187, 290, 338], [1, 20628, 338, 263, 6575, 1460, 2462, 322, 306, 763]] + +prefill logits (first half): tensor([[-10.0312, -9.5000, 0.8931, ..., -4.9414, -3.2422, -3.3633], + [-10.0312, -9.5000, 0.8931, ..., -4.9414, -3.2422, -3.3633], + [ -9.1875, -10.2500, 2.7129, ..., -4.3359, -4.0664, -4.1328]], + device='cuda:0') + +prefill logits (final): tensor([[-8.3125, -7.1172, 3.3457, ..., -4.9570, -4.1328, -3.4141], + [-8.9141, -9.0156, 4.1445, ..., -4.9922, -4.4961, -4.0781], + [-9.6328, -9.0547, 4.0195, ..., -5.3047, -4.7148, -4.4570]], + device='cuda:0') + +========== Prompt 0 ========== + The capital of France is Paris. +The capital of the United States is Washington, D.C. + + +========== Prompt 1 ========== + The capital of the United Kindom is London. +The capital of the United Kingdom is London. +The capital of the + +========== Prompt 2 ========== + Today is a sunny day and I like to go for a walk in the park. +I'm going to the park +""" + +import argparse +import dataclasses +import itertools +import json +import logging +import multiprocessing +import os +import time +from typing import Tuple + +import numpy as np +import torch +import torch.distributed as dist + +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.entrypoints.engine import _set_envs_and_config +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.managers.schedule_batch import Req, ScheduleBatch +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.srt.utils import ( + configure_logger, + get_bool_env_var, + kill_process_tree, + set_gpu_proc_affinity, + suppress_other_loggers, +) + + +@dataclasses.dataclass +class BenchArgs: + run_name: str = "default" + batch_size: Tuple[int] = (1,) + input_len: Tuple[int] = (1024,) + output_len: Tuple[int] = (16,) + result_filename: str = "result.jsonl" + correctness_test: bool = False + # This is only used for correctness test + cut_len: int = 4 + profile: bool = False + profile_filename_prefix: str = "profile" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--run-name", type=str, default=BenchArgs.run_name) + parser.add_argument( + "--batch-size", type=int, nargs="+", default=BenchArgs.batch_size + ) + parser.add_argument( + "--input-len", type=int, nargs="+", default=BenchArgs.input_len + ) + parser.add_argument( + "--output-len", type=int, nargs="+", default=BenchArgs.output_len + ) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument("--correctness-test", action="store_true") + parser.add_argument("--cut-len", type=int, default=BenchArgs.cut_len) + parser.add_argument( + "--profile", action="store_true", help="Use Torch Profiler." + ) + parser.add_argument( + "--profile-filename-prefix", + type=str, + default=BenchArgs.profile_filename_prefix, + help="Prefix of the profiling file names. The full profiling result file(s) be " + '"[profile_filename_prefix]_batch[batch_size]_input[input_len]_output[output_len].trace.json.gz"', + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +def load_model(server_args, port_args, tp_rank): + suppress_other_loggers() + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + model_config = ModelConfig( + server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + revision=server_args.revision, + context_length=server_args.context_length, + model_override_args=server_args.json_model_override_args, + is_embedding=server_args.is_embedding, + dtype=server_args.dtype, + quantization=server_args.quantization, + ) + model_runner = ModelRunner( + model_config=model_config, + mem_fraction_static=server_args.mem_fraction_static, + gpu_id=tp_rank, + tp_rank=tp_rank, + tp_size=server_args.tp_size, + nccl_port=port_args.nccl_port, + server_args=server_args, + ) + rank_print(f"max_total_num_tokens={model_runner.max_total_num_tokens}") + tokenizer = get_tokenizer( + server_args.tokenizer_path, + tokenizer_mode=server_args.tokenizer_mode, + trust_remote_code=server_args.trust_remote_code, + ) + if server_args.tp_size > 1: + dist.barrier() + return model_runner, tokenizer + + +def prepare_inputs_for_correctness_test(bench_args, tokenizer): + prompts = [ + "The capital of France is", + "The capital of the United Kindom is", + "Today is a sunny day and I like", + ] + input_ids = [tokenizer.encode(p) for p in prompts] + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=BenchArgs.output_len, + ) + + reqs = [] + for i in range(len(prompts)): + assert len(input_ids[i]) > bench_args.cut_len + + tmp_input_ids = input_ids[i][: bench_args.cut_len] + req = Req( + rid=i, + origin_input_text=prompts[i], + origin_input_ids=tmp_input_ids, + sampling_params=sampling_params, + ) + req.prefix_indices = [] + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + reqs.append(req) + + return input_ids, reqs + + +def prepare_extend_inputs_for_correctness_test( + bench_args, input_ids, reqs, model_runner +): + for i in range(len(reqs)): + req = reqs[i] + req.fill_ids += input_ids[i][bench_args.cut_len :] + req.prefix_indices = model_runner.req_to_token_pool.req_to_token[ + i, : bench_args.cut_len + ] + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + return reqs + + +def prepare_synthetic_inputs_for_latency_test(batch_size, input_len): + input_ids = np.ones((batch_size, input_len), dtype=np.int32) + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=BenchArgs.output_len, + ) + + reqs = [] + for i in range(len(input_ids)): + req = Req( + rid=i, + origin_input_text="", + origin_input_ids=list(input_ids[i]), + sampling_params=sampling_params, + ) + req.prefix_indices = [] + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + reqs.append(req) + + return reqs + + +@torch.no_grad +def extend(reqs, model_runner): + batch = ScheduleBatch.init_new( + reqs=reqs, + req_to_token_pool=model_runner.req_to_token_pool, + token_to_kv_pool_allocator=model_runner.token_to_kv_pool_allocator, + tree_cache=None, + model_config=model_runner.model_config, + enable_overlap=False, + spec_algorithm=SpeculativeAlgorithm.NONE, + enable_custom_logit_processor=False, + ) + batch.prepare_for_extend() + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner) + logits_output = model_runner.forward(forward_batch) + next_token_ids = model_runner.sample(logits_output, forward_batch) + return next_token_ids, logits_output.next_token_logits, batch + + +@torch.no_grad +def decode(input_token_ids, batch, model_runner): + batch.output_ids = input_token_ids + batch.prepare_for_decode() + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner) + logits_output = model_runner.forward(forward_batch) + next_token_ids = model_runner.sample(logits_output, forward_batch) + return next_token_ids, logits_output.next_token_logits + + +def correctness_test( + server_args, + port_args, + bench_args, + tp_rank, +): + # Configure the logger + configure_logger(server_args, prefix=f" TP{tp_rank}") + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + # Load the model + model_runner, tokenizer = load_model(server_args, port_args, tp_rank) + + # Prepare inputs + input_ids, reqs = prepare_inputs_for_correctness_test(bench_args, tokenizer) + rank_print(f"\n{input_ids=}\n") + + if bench_args.cut_len > 0: + # Prefill + next_token_ids, next_token_logits, batch = extend(reqs, model_runner) + rank_print(f"prefill logits (first half): {next_token_logits} \n") + + # Prepare extend inputs + reqs = prepare_extend_inputs_for_correctness_test( + bench_args, input_ids, reqs, model_runner + ) + + # Extend (prefill w/ KV cache) + next_token_ids, next_token_logits, batch = extend(reqs, model_runner) + rank_print(f"prefill logits (final): {next_token_logits} \n") + + # Decode + output_ids = [input_ids[i] + [next_token_ids[i]] for i in range(len(input_ids))] + for _ in range(bench_args.output_len[0] - 1): + next_token_ids, _ = decode(next_token_ids, batch, model_runner) + next_token_ids_list = next_token_ids.tolist() + for i in range(len(reqs)): + output_ids[i].append(next_token_ids_list[i]) + + # Print output texts + for i in range(len(reqs)): + rank_print(f"========== Prompt {i} ==========") + rank_print(tokenizer.decode(output_ids[i]), "\n") + + +def synchronize(device): + torch.get_device_module(device).synchronize() + + +def latency_test_run_once( + run_name, + model_runner, + rank_print, + reqs, + batch_size, + input_len, + output_len, + device, + profile, + profile_filename_prefix, +): + max_batch_size = model_runner.max_total_num_tokens // (input_len + output_len) + if batch_size > max_batch_size: + rank_print( + f"skipping ({batch_size}, {input_len}, {output_len}) due to max batch size limit" + ) + return + + # Clear the pools. + model_runner.req_to_token_pool.clear() + model_runner.token_to_kv_pool_allocator.clear() + + measurement_results = { + "run_name": run_name, + "batch_size": batch_size, + "input_len": input_len, + "output_len": output_len, + } + + tot_latency = 0 + + profiler = None + if profile: + profiler = torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, + ) + profiler.start() + + # Prefill + synchronize(device) + tic = time.time() + next_token_ids, _, batch = extend(reqs, model_runner) + synchronize(device) + prefill_latency = time.time() - tic + tot_latency += prefill_latency + throughput = input_len * batch_size / prefill_latency + rank_print( + f"Prefill. latency: {prefill_latency:6.5f} s, throughput: {throughput:9.2f} token/s" + ) + measurement_results["prefill_latency"] = prefill_latency + measurement_results["prefill_throughput"] = throughput + + # Decode + decode_latencies = [] + for i in range(output_len - 1): + synchronize(device) + tic = time.time() + next_token_ids, _ = decode(next_token_ids, batch, model_runner) + synchronize(device) + latency = time.time() - tic + tot_latency += latency + throughput = batch_size / latency + decode_latencies.append(latency) + if i < 5: + rank_print( + f"Decode. latency: {latency:6.5f} s, throughput: {throughput:9.2f} token/s" + ) + + if profile: + profiler.stop() + profile_filename = f"{profile_filename_prefix}_batch{batch_size}_input{input_len}_output{output_len}.trace.json.gz" + parent_dir = os.path.dirname(os.path.abspath(profile_filename)) + os.makedirs(parent_dir, exist_ok=True) + profiler.export_chrome_trace(profile_filename) + rank_print(f"torch profiler chrome trace saved to {profile_filename}") + + # Record decode timing from 2nd output + if output_len > 1: + med_decode_latency = np.median(decode_latencies) + med_decode_throughput = batch_size / med_decode_latency + rank_print( + f"Decode. median latency: {med_decode_latency:6.5f} s, median throughput: {med_decode_throughput:9.2f} token/s" + ) + measurement_results["median_decode_latency"] = med_decode_latency + measurement_results["median_decode_throughput"] = med_decode_throughput + + throughput = (input_len + output_len) * batch_size / tot_latency + rank_print( + f"Total. latency: {tot_latency:6.3f} s, throughput: {throughput:9.2f} token/s" + ) + measurement_results["total_latency"] = tot_latency + measurement_results["overall_throughput"] = throughput + return measurement_results + + +def latency_test( + server_args, + port_args, + bench_args, + tp_rank, +): + # Set CPU affinity + if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"): + set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, tp_rank) + + # Configure the logger + configure_logger(server_args, prefix=f" TP{tp_rank}") + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + # Load the model + model_runner, tokenizer = load_model(server_args, port_args, tp_rank) + + # Prepare inputs for warm up + reqs = prepare_synthetic_inputs_for_latency_test( + bench_args.batch_size[0], bench_args.input_len[0] + ) + + # Warm up + rank_print("Warmup ...") + latency_test_run_once( + bench_args.run_name, + model_runner, + rank_print, + reqs, + bench_args.batch_size[0], + bench_args.input_len[0], + 8, # shorter decoding to speed up the warmup + server_args.device, + profile=False, + profile_filename_prefix="", # not used + ) + + rank_print("Benchmark ...") + + # Run the sweep + result_list = [] + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + reqs = prepare_synthetic_inputs_for_latency_test(bs, il) + ret = latency_test_run_once( + bench_args.run_name, + model_runner, + rank_print, + reqs, + bs, + il, + ol, + server_args.device, + bench_args.profile if tp_rank == 0 else None, + bench_args.profile_filename_prefix, + ) + if ret is not None: + result_list.append(ret) + + # Write results in jsonlines format on rank 0. + if tp_rank == 0 and bench_args.result_filename: + with open(bench_args.result_filename, "a") as fout: + for result in result_list: + fout.write(json.dumps(result) + "\n") + + +def main(server_args, bench_args): + _set_envs_and_config(server_args) + + if server_args.model_path: + if bench_args.correctness_test: + work_func = correctness_test + else: + work_func = latency_test + else: + raise ValueError( + "Provide --model-path for running the tests or " + "provide --result-filename for plotting the results" + ) + + port_args = PortArgs.init_new(server_args) + + if server_args.tp_size == 1: + work_func(server_args, port_args, bench_args, 0) + else: + workers = [] + for tp_rank in range(server_args.tp_size): + proc = multiprocessing.Process( + target=work_func, + args=( + server_args, + port_args, + bench_args, + tp_rank, + ), + ) + proc.start() + workers.append(proc) + + for proc in workers: + proc.join() + + proc.terminate() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format="%(message)s", + ) + + try: + main(server_args, bench_args) + finally: + if server_args.tp_size != 1: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/python/sglang/bench_one_batch_server.py b/python/sglang/bench_one_batch_server.py new file mode 100644 index 0000000..1fc4ff5 --- /dev/null +++ b/python/sglang/bench_one_batch_server.py @@ -0,0 +1,192 @@ +""" +Benchmark the latency of running a single batch with a server. + +This script launches a server and uses the HTTP interface. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). + +Usage: +python3 -m sglang.bench_one_batch_server --model meta-llama/Meta-Llama-3.1-8B --batch-size 1 16 64 --input-len 1024 --output-len 8 + +python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 +""" + +import argparse +import dataclasses +import itertools +import json +import multiprocessing +import os +import time +from typing import Tuple + +import numpy as np +import requests + +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree + + +@dataclasses.dataclass +class BenchArgs: + run_name: str = "default" + batch_size: Tuple[int] = (1,) + input_len: Tuple[int] = (1024,) + output_len: Tuple[int] = (16,) + result_filename: str = "result.jsonl" + base_url: str = "" + skip_warmup: bool = False + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--run-name", type=str, default=BenchArgs.run_name) + parser.add_argument( + "--batch-size", type=int, nargs="+", default=BenchArgs.batch_size + ) + parser.add_argument( + "--input-len", type=int, nargs="+", default=BenchArgs.input_len + ) + parser.add_argument( + "--output-len", type=int, nargs="+", default=BenchArgs.output_len + ) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument("--base-url", type=str, default=BenchArgs.base_url) + parser.add_argument("--skip-warmup", action="store_true") + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +def launch_server_internal(server_args): + try: + launch_server(server_args) + except Exception as e: + raise e + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +def launch_server_process(server_args: ServerArgs): + proc = multiprocessing.Process(target=launch_server_internal, args=(server_args,)) + proc.start() + base_url = f"http://{server_args.host}:{server_args.port}" + timeout = 600 + + start_time = time.time() + while time.time() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + } + response = requests.get(f"{base_url}/v1/models", headers=headers) + if response.status_code == 200: + return proc, base_url + except requests.RequestException: + pass + time.sleep(10) + raise TimeoutError("Server failed to start within the timeout period.") + + +def run_one_case( + url: str, + batch_size: int, + input_len: int, + output_len: int, + run_name: str, + result_filename: str, +): + input_ids = [ + [int(x) for x in np.random.randint(0, high=16384, size=(input_len,))] + for _ in range(batch_size) + ] + + tic = time.time() + response = requests.post( + url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + }, + ) + latency = time.time() - tic + + _ = response.json() + output_throughput = batch_size * output_len / latency + overall_throughput = batch_size * (input_len + output_len) / latency + + print(f"batch size: {batch_size}") + print(f"latency: {latency:.2f} s") + print(f"output throughput: {output_throughput:.2f} token/s") + print(f"(input + output) throughput: {overall_throughput:.2f} token/s") + + if result_filename: + with open(result_filename, "a") as fout: + res = { + "run_name": run_name, + "batch_size": batch_size, + "input_len": input_len, + "output_len": output_len, + "latency": round(latency, 4), + "output_throughput": round(output_throughput, 2), + "overall_throughput": round(overall_throughput, 2), + } + fout.write(json.dumps(res) + "\n") + + +def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs): + if bench_args.base_url: + proc, base_url = None, bench_args.base_url + else: + proc, base_url = launch_server_process(server_args) + + # warmup + if not bench_args.skip_warmup: + run_one_case( + base_url, + batch_size=16, + input_len=1024, + output_len=16, + run_name="", + result_filename="", + ) + + # benchmark + try: + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + run_one_case( + base_url, + bs, + il, + ol, + bench_args.run_name, + bench_args.result_filename, + ) + finally: + if proc: + kill_process_tree(proc.pid) + + print(f"\nResults are saved to {bench_args.result_filename}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + run_benchmark(server_args, bench_args) diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py new file mode 100644 index 0000000..95cd6a3 --- /dev/null +++ b/python/sglang/bench_serving.py @@ -0,0 +1,1579 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +""" + +import argparse +import asyncio +import json +import os +import pickle +import random +import resource +import sys +import time +import traceback +import warnings +from argparse import ArgumentParser +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm +from transformers import ( + AutoTokenizer, + PreTrainedTokenizer, + PreTrainedTokenizerBase, + PreTrainedTokenizerFast, +) + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60) +ASSISTANT_SUFFIX = "Assistant:" + +global args + + +@dataclass +class RequestFuncInput: + prompt: str + api_url: str + prompt_len: int + output_len: int + model: str + lora_name: str + extra_request_body: Dict[str, Any] + + +@dataclass +class RequestFuncOutput: + generated_text: str = "" + success: bool = False + latency: float = 0.0 + ttft: float = 0.0 # Time to first token + itl: List[float] = field(default_factory=list) # List of inter-token latencies + prompt_len: int = 0 + error: str = "" + output_len: int = 0 + + +def remove_prefix(text: str, prefix: str) -> str: + return text[len(prefix) :] if text.startswith(prefix) else text + + +def remove_suffix(text: str, suffix: str) -> str: + return text[: -len(suffix)] if text.endswith(suffix) else text + + +def get_auth_headers() -> Dict[str, str]: + api_key = os.environ.get("OPENAI_API_KEY") + if api_key: + return {"Authorization": f"Bearer {api_key}"} + else: + return {} + + +# trt llm does not support ignore_eos +# https://github.com/triton-inference-server/tensorrtllm_backend/issues/505 +async def async_request_trt_llm( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith("generate_stream") + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "accumulate_tokens": True, + "text_input": request_func_input.prompt, + "temperature": 0.000001, + "top_p": 1.0, + "max_tokens": request_func_input.output_len, + "stream": True, + "min_length": request_func_input.output_len, + "end_id": 1048576, + **request_func_input.extra_request_body, + } + if args.disable_ignore_eos: + del payload["min_length"] + del payload["end_id"] + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post(url=api_url, json=payload) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data:") + + data = json.loads(chunk) + output.generated_text += data["text_output"] + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + + output.latency = most_recent_timestamp - st + output.success = True + output.output_len = request_func_input.output_len + + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_truss( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + prompt = request_func_input.prompt + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["delta"]["content"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["delta"]["content"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_sglang_generate( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + prompt = request_func_input.prompt + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "text": prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": request_func_input.output_len, + "ignore_eos": not args.disable_ignore_eos, + }, + "stream": not args.disable_stream, + "lora_path": request_func_input.lora_name, + "return_logprob": args.return_logprob, + "logprob_start_len": -1, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + last_output_len = 0 + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + # print(chunk_bytes) + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["text"]: + timestamp = time.perf_counter() + generated_text = data["text"] + output_len = data["meta_info"]["completion_tokens"] + + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + num_new_tokens = output_len - last_output_len + if num_new_tokens == 0: + continue + adjust_itl = ( + timestamp - most_recent_timestamp + ) / num_new_tokens + output.itl.extend([adjust_itl] * num_new_tokens) + + most_recent_timestamp = timestamp + last_output_len = output_len + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + print(f"{output.error=}") + + if pbar: + pbar.update(1) + return output + + +async def async_request_gserver( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + raise NotImplementedError() + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def get_model(pretrained_model_name_or_path: str) -> str: + if os.getenv("SGLANG_USE_MODELSCOPE", "false").lower() == "true": + import huggingface_hub.constants + from modelscope import snapshot_download + + model_path = snapshot_download( + model_id=pretrained_model_name_or_path, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"], + ) + + return model_path + return pretrained_model_name_or_path + + +def get_tokenizer( + pretrained_model_name_or_path: str, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + if pretrained_model_name_or_path.endswith( + ".json" + ) or pretrained_model_name_or_path.endswith(".model"): + from sglang.srt.hf_transformers_utils import get_tokenizer + + return get_tokenizer(pretrained_model_name_or_path) + + if pretrained_model_name_or_path is not None and not os.path.exists( + pretrained_model_name_or_path + ): + pretrained_model_name_or_path = get_model(pretrained_model_name_or_path) + return AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=True + ) + + +def get_dataset(args, tokenizer): + if args.dataset_name == "sharegpt": + input_requests = sample_sharegpt_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + fixed_output_len=args.sharegpt_output_len, + context_len=args.sharegpt_context_len, + prompt_suffix=args.prompt_suffix, + apply_chat_template=args.apply_chat_template, + ) + elif args.dataset_name == "random": + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path=args.dataset_path, + ) + elif args.dataset_name == "generated-shared-prefix": + input_requests = sample_generated_shared_prefix_requests( + num_groups=args.gsp_num_groups, + prompts_per_group=args.gsp_prompts_per_group, + system_prompt_len=args.gsp_system_prompt_len, + question_len=args.gsp_question_len, + output_len=args.gsp_output_len, + tokenizer=tokenizer, + args=args, + ) + else: + raise ValueError(f"Unknown dataset: {args.dataset_name}") + return input_requests + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_sglang_generate, + "sglang-native": async_request_sglang_generate, + "sglang-oai": async_request_openai_completions, + "vllm": async_request_openai_completions, + "lmdeploy": async_request_openai_completions, + "trt": async_request_trt_llm, + "gserver": async_request_gserver, + "truss": async_request_truss, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p95_itl_ms: float + p99_itl_ms: float + max_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + + +SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + + +def download_and_cache_file(url: str, filename: Optional[str] = None): + """Read and cache a file from a url.""" + if filename is None: + filename = os.path.join("/tmp", url.split("/")[-1]) + + # Check if the cache file already exists + if os.path.exists(filename): + return filename + + print(f"Downloading from {url} to {filename}") + + # Stream the response to show the progress bar + response = requests.get(url, stream=True) + response.raise_for_status() # Check for request errors + + # Total size of the file in bytes + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 # Download in chunks of 1KB + + # Use tqdm to display the progress bar + with open(filename, "wb") as f, tqdm( + desc=filename, + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + bar.update(len(chunk)) + + return filename + + +def sample_sharegpt_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + fixed_output_len: Optional[int] = None, + context_len: Optional[int] = None, + prompt_suffix: Optional[str] = "", + apply_chat_template=False, +) -> List[Tuple[str, int, int]]: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path) and dataset_path == "": + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + + # Filter out the conversations with less than 2 turns. + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + # Only keep the first two turns of each conversation. + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: List[Tuple[str, int, int]] = [] + for i in range(len(dataset)): + if len(filtered_dataset) == num_requests: + break + + # Tokenize the prompts and completions. + prompt = dataset[i][0] + if prompt_suffix: + prompt = ( + remove_suffix(prompt, ASSISTANT_SUFFIX) + + prompt_suffix + + ASSISTANT_SUFFIX + ) + + if apply_chat_template: + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=False, + ) + prompt = prompt.replace(tokenizer.bos_token, "") + + prompt_token_ids = tokenizer.encode(prompt) + completion = dataset[i][1] + completion_token_ids = tokenizer.encode(completion) + prompt_len = len(prompt_token_ids) + output_len = ( + len(completion_token_ids) if fixed_output_len is None else fixed_output_len + ) + + if prompt_len < 2 or output_len < 2: + # Prune too short sequences. + continue + + if context_len and prompt_len + output_len > context_len: + # Prune too long sequences. + continue + + filtered_dataset.append((prompt, prompt_len, output_len)) + + print(f"#Input tokens: {np.sum([x[1] for x in filtered_dataset])}") + print(f"#Output tokens: {np.sum([x[2] for x in filtered_dataset])}") + return filtered_dataset + + +def sample_random_requests( + input_len: int, + output_len: int, + num_prompts: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + dataset_path: str, +) -> List[Tuple[str, int, int]]: + + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * range_ratio), + output_len + 1, + size=num_prompts, + ) + + if True: + # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + # Only keep the first two turns of each conversation. + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + input_requests: List[Tuple[str, int, int]] = [] + for data in dataset: + i = len(input_requests) + if i == num_prompts: + break + + # Tokenize the prompts and completions. + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + # Skip empty prompt + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + prompt = tokenizer.decode(input_ids) + input_requests.append((prompt, int(input_lens[i]), int(output_lens[i]))) + else: + # Sample token ids from random integers. This can cause some NaN issues. + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts) + input_requests = [] + for i in range(num_prompts): + prompt = tokenizer.decode( + [ + (offsets[i] + i + j) % tokenizer.vocab_size + for j in range(input_lens[i]) + ] + ) + input_requests.append((prompt, int(input_lens[i]), int(output_lens[i]))) + + print(f"#Input tokens: {np.sum(input_lens)}") + print(f"#Output tokens: {np.sum(output_lens)}") + return input_requests + + +def gen_prompt(tokenizer, token_num): + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return tokenizer.decode(selected_tokens) + + +def get_gen_prefix_cache_path(args, tokenizer): + """Create cache directory under ~/.cache/sglang/benchmark""" + cache_dir = Path.home() / ".cache" / "sglang" / "benchmark" + + # Create a unique cache filename based on the generation parameters + cache_key = ( + f"gen_shared_prefix_{args.gsp_num_groups}_{args.gsp_prompts_per_group}_" + f"{args.gsp_system_prompt_len}_{args.gsp_question_len}_{args.gsp_output_len}_" + f"{tokenizer.__class__.__name__}.pkl" + ) + return cache_dir / cache_key + + +def sample_generated_shared_prefix_requests( + num_groups: int, + prompts_per_group: int, + system_prompt_len: int, + question_len: int, + output_len: int, + tokenizer: PreTrainedTokenizerBase, + args: argparse.Namespace, +) -> List[Tuple[str, int, int]]: + """Generate benchmark requests with shared system prompts using random tokens and caching.""" + cache_path = get_gen_prefix_cache_path(args, tokenizer) + + # Try to load from cache first + if cache_path.exists(): + print(f"\nLoading cached generated input data from {cache_path}") + with open(cache_path, "rb") as f: + return pickle.load(f) + + print("\nGenerating new input data...") + + # Generate system prompts for each group + system_prompts = [] + for _ in range(num_groups): + system_prompt = gen_prompt(tokenizer, system_prompt_len) + system_prompts.append(system_prompt) + + # Generate questions + questions = [] + for _ in range(num_groups * prompts_per_group): + question = gen_prompt(tokenizer, question_len) + questions.append(question) + + # Combine system prompts with questions + input_requests = [] + total_input_tokens = 0 + total_output_tokens = 0 + + for group_idx in tqdm(range(num_groups), desc="Generating system prompt"): + system_prompt = system_prompts[group_idx] + for prompt_idx in tqdm( + range(prompts_per_group), desc="Generating questions", leave=False + ): + question = questions[group_idx * prompts_per_group + prompt_idx] + full_prompt = f"{system_prompt}\n\n{question}" + prompt_len = len(tokenizer.encode(full_prompt)) + + input_requests.append((full_prompt, prompt_len, output_len)) + total_input_tokens += prompt_len + total_output_tokens += output_len + + # Shuffle questions + random.shuffle(input_requests) + + # Print statistics + print(f"\nGenerated shared prefix dataset statistics:") + print(f"Number of groups: {num_groups}") + print(f"Prompts per group: {prompts_per_group}") + print(f"Total prompts: {len(input_requests)}") + print(f"Total input tokens: {total_input_tokens}") + print(f"Total output tokens: {total_output_tokens}") + print( + f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + ) + print( + f"Average question length: {sum(len(tokenizer.encode(q)) for q in questions) / len(questions):.1f} tokens\n" + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Caching generated input data to {cache_path}") + with open(cache_path, "wb") as f: + pickle.dump(input_requests, f) + + return input_requests + + +async def get_request( + input_requests: List[Tuple[str, int, int]], + request_rate: float, +) -> AsyncGenerator[Tuple[str, int, int], None]: + input_requests = iter(input_requests) + for request in input_requests: + yield request + + if request_rate == float("inf"): + # If the request rate is infinity, then we don't need to wait. + continue + + # Sample the request interval from the exponential distribution. + interval = np.random.exponential(1.0 / request_rate) + # The next request will be sent after the interval. + await asyncio.sleep(interval) + + +def calculate_metrics( + input_requests: List[Tuple[str, int, int]], + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + for i in range(len(outputs)): + if outputs[i].success: + output_len = outputs[i].output_len + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode(outputs[i].generated_text, add_special_tokens=False) + ) + retokenized_output_lens.append(retokenized_output_len) + total_input += input_requests[i][1] + if output_len > 1: + tpots.append((outputs[i].latency - outputs[i].ttft) / (output_len - 1)) + itls += outputs[i].itl + ttfts.append(outputs[i].ttft) + + e2e_latencies.append(outputs[i].latency) + + completed += 1 + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p95_itl_ms=np.percentile(itls or 0, 95) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + max_itl_ms=np.max(itls or 0) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + ) + + return metrics, output_lens + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[Tuple[str, int, int]], + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_names: List[str], + extra_request_body: Dict[str, Any], + profile: bool, + pd_seperated: bool = False, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, pbar): + if semaphore is None: + return await request_func(request_func_input=request_func_input, pbar=pbar) + async with semaphore: + return await request_func(request_func_input=request_func_input, pbar=pbar) + + # Warmup + print("Starting initial single prompt test run...") + test_prompt, test_prompt_len, test_output_len = input_requests[0] + if lora_names != None and len(lora_names) != 0: + lora_name = lora_names[0] + else: + lora_name = None + + test_input = RequestFuncInput( + model=model_id, + prompt=test_prompt, + api_url=api_url, + prompt_len=test_prompt_len, + output_len=min(test_output_len, 32), + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + test_output = await request_func(request_func_input=test_input) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + # Flush cache + if "sglang" in backend: + requests.post(base_url + "/flush_cache", headers=get_auth_headers()) + + time.sleep(1.0) + + # Start profiler + if profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + pbar = None if disable_tqdm else tqdm(total=len(input_requests)) + + # Run all requests + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_request(input_requests, request_rate): + prompt, prompt_len, output_len = request + if lora_names != None and len(lora_names) != 0: + idx = random.randint(0, len(lora_names) - 1) + lora_name = lora_names[idx] + else: + lora_name = None + + request_func_input = RequestFuncInput( + model=model_id, + prompt=prompt, + api_url=api_url, + prompt_len=prompt_len, + output_len=output_len, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + tasks.append( + asyncio.create_task( + limited_request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + # Stop profiler + if profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=base_url + "/stop_profile") + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + if "sglang" in backend: + server_info = requests.get(base_url + "/get_server_info") + if pd_seperated: + accept_length = server_info.json()["decode"][0].get( + "avg_spec_accept_length", None + ) + else: + accept_length = server_info.json().get("avg_spec_accept_length", None) + else: + accept_length = None + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print( + "{:<40} {:<10}".format( + "Max reqeuest concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + if accept_length: + print("{:<40} {:<10.2f}".format("Accept length:", accept_length)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print("{s:{c}^{n}}".format(s="Inter-Token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P95 ITL (ms):", metrics.p95_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": request_rate, + "max_concurrency": max_concurrency, + "sharegpt_output_len": args.sharegpt_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p95_itl_ms": metrics.p95_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "accept_length": accept_length, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "random": + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_sharegpt.jsonl" + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result.update( + { + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + } + ) + return result + + +def check_chat_template(model_path): + try: + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + return "chat_template" in tokenizer.init_kwargs + except Exception as e: + print(f"Fail to load tokenizer config with error={e}") + return False + + +def set_global_args(args_: argparse.Namespace): + """Set the global args.""" + global args + args = args_ + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + print(f"benchmark_args={args}") + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "sglang-native": 30000, + "sglang-oai": 30000, + "lmdeploy": 23333, + "vllm": 8000, + "trt": 8000, + "gserver": 9988, + "truss": 8080, + }.get(args.backend, 30000) + + model_url = ( + f"{args.base_url}/v1/models" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models" + ) + + if args.backend in ["sglang", "sglang-native"]: + api_url = ( + f"{args.base_url}/generate" + if args.base_url + else f"http://{args.host}:{args.port}/generate" + ) + elif args.backend in ["sglang-oai", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/completions" + ) + elif args.backend == "trt": + api_url = ( + f"{args.base_url}/v2/models/ensemble/generate_stream" + if args.base_url + else f"http://{args.host}:{args.port}/v2/models/ensemble/generate_stream" + ) + if args.model is None: + print("Please provide a model using `--model` when using `trt` backend.") + sys.exit(1) + elif args.backend == "gserver": + api_url = args.base_url if args.base_url else f"{args.host}:{args.port}" + args.model = args.model or "default" + elif args.backend == "truss": + api_url = ( + f"{args.base_url}/v1/models/model:predict" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models/model:predict" + ) + base_url = ( + f"http://{args.host}:{args.port}" if args.base_url is None else args.base_url + ) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url, headers=get_auth_headers()) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + if not check_chat_template(args.model): + print( + "\nWARNING It is recommended to use the `Chat` or `Instruct` model for benchmarking.\n" + "Because when the tokenizer counts the output tokens, if there is gibberish, it might count incorrectly.\n" + ) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + tokenizer = get_tokenizer(tokenizer_id) + input_requests = get_dataset(args, tokenizer) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_names=args.lora_name, + extra_request_body=extra_request_body, + profile=args.profile, + pd_seperated=args.pd_seperated, + ) + ) + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") + + +class LoRAPathAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, []) + for lora_name in values: + getattr(namespace, self.dest).append(lora_name) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=["sharegpt", "random", "generated-shared-prefix"], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--lora-name", + type=str, + nargs="*", + default=None, + action=LoRAPathAction, + help="The names of LoRA adapters. You can provide a list of names in the format {name} {name} {name}...", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + parser.add_argument( + "--pd-seperated", + action="store_true", + help="Benchmark PD disaggregation server", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + args = parser.parse_args() + run_benchmark(args) diff --git a/python/sglang/check_env.py b/python/sglang/check_env.py new file mode 100644 index 0000000..d525da9 --- /dev/null +++ b/python/sglang/check_env.py @@ -0,0 +1,305 @@ +"""Check environment configurations and dependency versions.""" + +import importlib.metadata +import os +import resource +import subprocess +import sys +from collections import OrderedDict, defaultdict + +import torch + +from sglang.srt.utils import is_hip + + +def is_cuda_v2(): + return torch.version.cuda is not None + + +# List of packages to check versions +PACKAGE_LIST = [ + "sglang", + "sgl_kernel", + "flashinfer", + "triton", + "transformers", + "torchao", + "numpy", + "aiohttp", + "fastapi", + "hf_transfer", + "huggingface_hub", + "interegular", + "modelscope", + "orjson", + "outlines", + "packaging", + "psutil", + "pydantic", + "multipart", + "zmq", + "torchao", + "uvicorn", + "uvloop", + "vllm", + "xgrammar", + "openai", + "tiktoken", + "anthropic", + "litellm", + "decord", +] + + +def get_package_versions(packages): + """ + Get versions of specified packages. + """ + versions = {} + for package in packages: + package_name = package.split("==")[0].split(">=")[0].split("<=")[0] + try: + version = importlib.metadata.version(package_name) + versions[package_name] = version + except ModuleNotFoundError: + versions[package_name] = "Module Not Found" + return versions + + +def get_cuda_info(): + """ + Get CUDA-related information if available. + """ + if is_cuda_v2(): + cuda_info = {"CUDA available": torch.cuda.is_available()} + + if cuda_info["CUDA available"]: + cuda_info.update(_get_gpu_info()) + cuda_info.update(_get_cuda_version_info()) + + return cuda_info + elif is_hip(): + cuda_info = {"ROCM available": torch.cuda.is_available()} + + if cuda_info["ROCM available"]: + cuda_info.update(_get_gpu_info()) + cuda_info.update(_get_cuda_version_info()) + + return cuda_info + + +def _get_gpu_info(): + """ + Get information about available GPUs. + """ + devices = defaultdict(list) + capabilities = defaultdict(list) + for k in range(torch.cuda.device_count()): + devices[torch.cuda.get_device_name(k)].append(str(k)) + capability = torch.cuda.get_device_capability(k) + capabilities[f"{capability[0]}.{capability[1]}"].append(str(k)) + + gpu_info = {} + for name, device_ids in devices.items(): + gpu_info[f"GPU {','.join(device_ids)}"] = name + + if len(capabilities) == 1: + # All GPUs have the same compute capability + cap, gpu_ids = list(capabilities.items())[0] + gpu_info[f"GPU {','.join(gpu_ids)} Compute Capability"] = cap + else: + # GPUs have different compute capabilities + for cap, gpu_ids in capabilities.items(): + gpu_info[f"GPU {','.join(gpu_ids)} Compute Capability"] = cap + + return gpu_info + + +def _get_cuda_version_info(): + """ + Get CUDA version information. + """ + if is_cuda_v2(): + from torch.utils.cpp_extension import CUDA_HOME + + cuda_info = {"CUDA_HOME": CUDA_HOME} + + if CUDA_HOME and os.path.isdir(CUDA_HOME): + cuda_info.update(_get_nvcc_info()) + cuda_info.update(_get_cuda_driver_version()) + + return cuda_info + elif is_hip(): + from torch.utils.cpp_extension import ROCM_HOME as ROCM_HOME + + cuda_info = {"ROCM_HOME": ROCM_HOME} + + if ROCM_HOME and os.path.isdir(ROCM_HOME): + cuda_info.update(_get_nvcc_info()) + cuda_info.update(_get_cuda_driver_version()) + + return cuda_info + else: + cuda_info = {"CUDA_HOME": ""} + return cuda_info + + +def _get_nvcc_info(): + """ + Get NVCC version information. + """ + if is_cuda_v2(): + from torch.utils.cpp_extension import CUDA_HOME + + try: + nvcc = os.path.join(CUDA_HOME, "bin/nvcc") + nvcc_output = ( + subprocess.check_output(f'"{nvcc}" -V', shell=True) + .decode("utf-8") + .strip() + ) + return { + "NVCC": nvcc_output[ + nvcc_output.rfind("Cuda compilation tools") : nvcc_output.rfind( + "Build" + ) + ].strip() + } + except subprocess.SubprocessError: + return {"NVCC": "Not Available"} + elif is_hip(): + from torch.utils.cpp_extension import ROCM_HOME + + try: + hipcc = os.path.join(ROCM_HOME, "bin/hipcc") + hipcc_output = ( + subprocess.check_output(f'"{hipcc}" --version', shell=True) + .decode("utf-8") + .strip() + ) + return { + "HIPCC": hipcc_output[ + hipcc_output.rfind("HIP version") : hipcc_output.rfind("AMD clang") + ].strip() + } + except subprocess.SubprocessError: + return {"HIPCC": "Not Available"} + else: + return {"NVCC": "Not Available"} + + +def _get_cuda_driver_version(): + """ + Get CUDA driver version. + """ + versions = set() + if is_cuda_v2(): + try: + output = subprocess.check_output( + [ + "nvidia-smi", + "--query-gpu=driver_version", + "--format=csv,noheader,nounits", + ] + ) + versions = set(output.decode().strip().split("\n")) + if len(versions) == 1: + return {"CUDA Driver Version": versions.pop()} + else: + return {"CUDA Driver Versions": ", ".join(sorted(versions))} + except subprocess.SubprocessError: + return {"CUDA Driver Version": "Not Available"} + elif is_hip(): + try: + output = subprocess.check_output( + [ + "rocm-smi", + "--showdriverversion", + "--csv", + ] + ) + versions = set(output.decode().strip().split("\n")) + versions.discard("name, value") + ver = versions.pop() + ver = ver.replace('"Driver version", ', "").replace('"', "") + + return {"ROCM Driver Version": ver} + except subprocess.SubprocessError: + return {"ROCM Driver Version": "Not Available"} + else: + return {"CUDA Driver Version": "Not Available"} + + +def get_gpu_topology(): + """ + Get GPU topology information. + """ + if is_cuda_v2(): + try: + result = subprocess.run( + ["nvidia-smi", "topo", "-m"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return "\n" + result.stdout if result.returncode == 0 else None + except subprocess.SubprocessError: + return None + elif is_hip(): + try: + result = subprocess.run( + ["rocm-smi", "--showtopotype"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return "\n" + result.stdout if result.returncode == 0 else None + except subprocess.SubprocessError: + return None + else: + return None + + +def get_hypervisor_vendor(): + try: + output = subprocess.check_output(["lscpu"], text=True) + for line in output.split("\n"): + if "Hypervisor vendor:" in line: + return line.split(":")[1].strip() + return None + except: + return None + + +def check_env(): + """ + Check and print environment information. + """ + env_info = OrderedDict() + env_info["Python"] = sys.version.replace("\n", "") + env_info.update(get_cuda_info()) + env_info["PyTorch"] = torch.__version__ + env_info.update(get_package_versions(PACKAGE_LIST)) + + gpu_topo = get_gpu_topology() + if gpu_topo: + if is_cuda_v2(): + env_info["NVIDIA Topology"] = gpu_topo + elif is_hip(): + env_info["AMD Topology"] = gpu_topo + + hypervisor_vendor = get_hypervisor_vendor() + if hypervisor_vendor: + env_info["Hypervisor vendor"] = hypervisor_vendor + + ulimit_soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + env_info["ulimit soft"] = ulimit_soft + + for k, v in env_info.items(): + print(f"{k}: {v}") + + +if __name__ == "__main__": + check_env() diff --git a/python/sglang/global_config.py b/python/sglang/global_config.py new file mode 100644 index 0000000..c331e64 --- /dev/null +++ b/python/sglang/global_config.py @@ -0,0 +1,49 @@ +"""Global configurations""" + +import os + + +class GlobalConfig: + """ + Store some global constants. + + See also python/sglang/srt/managers/schedule_batch.py::global_server_args_dict, which stores + many global runtime arguments as well. + """ + + def __init__(self): + # Verbosity level + # 0: do not output anything + # 2: output final text after every run + self.verbosity = 0 + + # Default backend of the language + self.default_backend = None + + # Runtime constants: New generation token ratio estimation + self.default_init_new_token_ratio = float( + os.environ.get("SGLANG_INIT_NEW_TOKEN_RATIO", 0.7) + ) + self.default_min_new_token_ratio_factor = float( + os.environ.get("SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR", 0.14) + ) + self.default_new_token_ratio_decay_steps = float( + os.environ.get("SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS", 600) + ) + + # Runtime constants: others + self.retract_decode_steps = 20 + self.flashinfer_workspace_size = os.environ.get( + "FLASHINFER_WORKSPACE_SIZE", 384 * 1024 * 1024 + ) + + # Output tokenization configs + self.skip_special_tokens_in_output = True + self.spaces_between_special_tokens_in_out = True + + # Language frontend interpreter optimization configs + self.enable_precache_with_tracing = True + self.enable_parallel_encoding = True + + +global_config = GlobalConfig() diff --git a/python/sglang/lang/__init__.py b/python/sglang/lang/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/sglang/lang/backend/__init__.py b/python/sglang/lang/backend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/sglang/lang/backend/anthropic.py b/python/sglang/lang/backend/anthropic.py new file mode 100644 index 0000000..5a36bd9 --- /dev/null +++ b/python/sglang/lang/backend/anthropic.py @@ -0,0 +1,77 @@ +from typing import List, Optional, Union + +import numpy as np + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import anthropic +except ImportError as e: + anthropic = e + + +class Anthropic(BaseBackend): + def __init__(self, model_name, *args, **kwargs): + super().__init__() + + if isinstance(anthropic, Exception): + raise anthropic + + self.model_name = model_name + self.chat_template = get_chat_template("claude") + self.client = anthropic.Anthropic(*args, **kwargs) + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + if messages and messages[0]["role"] == "system": + system = messages.pop(0)["content"] + else: + system = "" + + ret = self.client.messages.create( + model=self.model_name, + system=system, + messages=messages, + **sampling_params.to_anthropic_kwargs(), + ) + comp = ret.content[0].text + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + if messages and messages[0]["role"] == "system": + system = messages.pop(0)["content"] + else: + system = "" + + with self.client.messages.stream( + model=self.model_name, + system=system, + messages=messages, + **sampling_params.to_anthropic_kwargs(), + ) as stream: + for text in stream.text_stream: + yield text, {} diff --git a/python/sglang/lang/backend/base_backend.py b/python/sglang/lang/backend/base_backend.py new file mode 100644 index 0000000..725c0a9 --- /dev/null +++ b/python/sglang/lang/backend/base_backend.py @@ -0,0 +1,82 @@ +from typing import Callable, List, Optional, Union + +from sglang.lang.chat_template import get_chat_template +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + + +class BaseBackend: + def __init__(self) -> None: + self.support_concate_and_append = False + self.chat_template = get_chat_template("default") + + def get_model_name(self): + raise NotImplementedError() + + def get_chat_template(self): + return self.chat_template + + def cache_prefix(self, prefix_str: str): + pass + + def uncache_prefix(self, rid: str): + pass + + def end_request(self, rid: Union[str, List[str]]): + pass + + def begin_program(self, s: StreamExecutor): + pass + + def end_program(self, s: Union[StreamExecutor, List[StreamExecutor]]): + pass + + def commit_lazy_operations(self, s: StreamExecutor): + pass + + def fork_program( + self, + src: StreamExecutor, + dst: List[StreamExecutor], + position_ids_offset: Optional[List[int]] = None, + ): + pass + + def fill_image(self, s: StreamExecutor): + pass + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + raise NotImplementedError() + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + raise NotImplementedError() + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: Optional[ChoicesSamplingMethod] = None, + ) -> ChoicesDecision: + raise NotImplementedError() + + def concatenate_and_append(self, src_rids: List[str], dst_rid: str): + raise NotImplementedError() + + def shutdown(self): + pass + + def flush_cache(self): + pass + + def get_server_info(self): + pass diff --git a/python/sglang/lang/backend/litellm.py b/python/sglang/lang/backend/litellm.py new file mode 100644 index 0000000..5803b54 --- /dev/null +++ b/python/sglang/lang/backend/litellm.py @@ -0,0 +1,90 @@ +from typing import Mapping, Optional + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import litellm +except ImportError as e: + litellm = e + litellm.num_retries = 1 + + +class LiteLLM(BaseBackend): + def __init__( + self, + model_name, + chat_template=None, + api_key=None, + organization: Optional[str] = None, + base_url: Optional[str] = None, + timeout: Optional[float] = 600, + max_retries: Optional[int] = litellm.num_retries, + default_headers: Optional[Mapping[str, str]] = None, + ): + super().__init__() + + if isinstance(litellm, Exception): + raise litellm + + self.model_name = model_name + + self.chat_template = chat_template or get_chat_template_by_model_path( + model_name + ) + + self.client_params = { + "api_key": api_key, + "organization": organization, + "base_url": base_url, + "timeout": timeout, + "max_retries": max_retries, + "default_headers": default_headers, + } + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + ret = litellm.completion( + model=self.model_name, + messages=messages, + **self.client_params, + **sampling_params.to_litellm_kwargs(), + ) + comp = ret.choices[0].message.content + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + ret = litellm.completion( + model=self.model_name, + messages=messages, + stream=True, + **self.client_params, + **sampling_params.to_litellm_kwargs(), + ) + for chunk in ret: + text = chunk.choices[0].delta.content + if text is not None: + yield text, {} diff --git a/python/sglang/lang/backend/openai.py b/python/sglang/lang/backend/openai.py new file mode 100644 index 0000000..c6e31f7 --- /dev/null +++ b/python/sglang/lang/backend/openai.py @@ -0,0 +1,471 @@ +import dataclasses +import logging +import time +import warnings +from typing import Callable, List, Optional, Union + +import numpy as np + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import ChatTemplate, get_chat_template_by_model_path +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import openai + import tiktoken +except ImportError as e: + openai = tiktoken = e + + +logger = logging.getLogger(__name__) + + +def create_logit_bias_int(tokenizer): + """Get logit bias for integer numbers.""" + int_token_ids = [] + + tokens = tokenizer._mergeable_ranks + for token, token_id in tokens.items(): + s = tokenizer.decode([token_id]) + if all([c.isdigit() for c in s]) or s in [" "]: + int_token_ids.append(token_id) + if len(int_token_ids) >= 300: # OpenAI API limit + break + special_tokens = tokenizer._special_tokens + mask = {t: 100 for t in int_token_ids[:299]} + mask[special_tokens["<|endoftext|>"]] = 100 + return mask + + +INSTRUCT_MODEL_NAMES = [ + "gpt-3.5-turbo-instruct", +] + + +@dataclasses.dataclass +class TokenUsage: + prompt_tokens: int + completion_tokens: int + + def reset(self): + self.prompt_tokens = self.completion_tokens = 0 + + +class OpenAI(BaseBackend): + def __init__( + self, + model_name: str, + is_chat_model: Optional[bool] = None, + chat_template: Optional[ChatTemplate] = None, + is_azure: bool = False, + *args, + **kwargs, + ): + super().__init__() + + if isinstance(openai, Exception): + raise openai + + if is_azure: + self.client = openai.AzureOpenAI(*args, **kwargs) + else: + self.client = openai.OpenAI(*args, **kwargs) + + self.model_name = model_name + try: + self.tokenizer = tiktoken.encoding_for_model(model_name) + except KeyError: + self.tokenizer = tiktoken.get_encoding("cl100k_base") + self.logit_bias_int = create_logit_bias_int(self.tokenizer) + + self.chat_template = chat_template or get_chat_template_by_model_path( + model_name + ) + + if is_chat_model is not None: + self.is_chat_model = is_chat_model + else: + if model_name in INSTRUCT_MODEL_NAMES: + self.is_chat_model = False + else: + self.is_chat_model = True + + self.chat_prefix = self.chat_template.role_prefix_and_suffix["assistant"][0] + + # Usage + self.token_usage = TokenUsage(0, 0) + + # API speculative execution + # TODO(ying): This does not support multi-threading (run_batch) + self.spec_kwargs = {} + self.spec_format = [] + self.spec_max_num_tries = 3 + + def get_chat_template(self): + return self.chat_template + + def _prepare_spec_execution( + self, + sampling_params: SglSamplingParams, + num_api_spec_tokens: int, + spec_var_name: str, + ): + if "max_tokens" not in self.spec_kwargs: + self.spec_kwargs["max_tokens"] = num_api_spec_tokens + else: + assert self.spec_kwargs["max_tokens"] == num_api_spec_tokens + + params = sampling_params.to_openai_kwargs() + for key, value in params.items(): + if key in ["stop"]: + continue + if key in ["max_tokens"]: + warnings.warn( + "The parameter max_tokens will be overwritten by speculated number of tokens." + ) + continue + if key not in self.spec_kwargs: + self.spec_kwargs[key] = value + else: + assert ( + value == self.spec_kwargs[key] + ), "sampling parameters should be consistent if turn on api speculative execution." + self.spec_format.append( + {"text": "", "stop": params["stop"], "name": spec_var_name} + ) + return "", {} + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + spec_var_name: str = None, + ): + if sampling_params.dtype is None: + if self.is_chat_model: + if s.num_api_spec_tokens is None: + if not s.text_.endswith(self.chat_prefix): + raise RuntimeError( + "This use case is not supported if api speculative execution is off. " + "For OpenAI chat models, sgl.gen must be right after sgl.assistant. " + "Example of adding api speculative execution: @function(num_api_spec_tokens=128)." + ) + prompt = s.messages_ + else: + return self._prepare_spec_execution( + sampling_params, s.num_api_spec_tokens, spec_var_name + ) + else: + prompt = s.text_ + + kwargs = sampling_params.to_openai_kwargs() + if self.model_name.startswith("o1") or self.model_name.startswith("o3"): + kwargs.pop("max_tokens", None) + else: + kwargs.pop("max_completion_tokens", None) + + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=prompt, + **kwargs, + ) + # Keep the returned list (or string) as is. + elif sampling_params.dtype in [str, "str", "string"]: + assert ( + not self.is_chat_model + ), "constrained type not supported on chat model" + kwargs = sampling_params.to_openai_kwargs() + kwargs.pop("stop") + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.text_ + '"', + stop='"', + **kwargs, + ) + # Wrap each element in quotes if we have a list. + if isinstance(comp, list): + comp = ['"' + x + '"' for x in comp] + else: + comp = '"' + comp + '"' + elif sampling_params.dtype in [int, "int"]: + assert ( + not self.is_chat_model + ), "constrained type not supported on chat model" + kwargs = sampling_params.to_openai_kwargs() + kwargs.pop("stop") + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.text_, + logit_bias=self.logit_bias_int, + stop=[" "], + **kwargs, + ) + # Leave as a list if that's what is returned. + else: + raise ValueError(f"Unknown dtype: {sampling_params.dtype}") + + return comp, {} + + def spec_fill(self, value: str): + assert self.is_chat_model + self.spec_format.append({"text": value, "stop": None, "name": None}) + + def spec_pattern_match(self, comp): + for i, term in enumerate(self.spec_format): + text = term["text"] + if text != "": + if comp.startswith(text): + comp = comp[len(text) :] + else: + return False + else: + pos = comp.find(term["stop"]) + if pos != -1: + term["text"] = comp[:pos] + comp = comp[pos:] + else: + if i == len(self.spec_format) - 1: + term["text"] = comp + else: + return False + return True + + def role_end_generate( + self, + s: StreamExecutor, + ): + if s.num_api_spec_tokens is None or not s.text_.endswith(self.chat_prefix): + return + + comp = "" + if not all(x["name"] is None for x in self.spec_format): + # TODO(ying): throw errors or warnings + for i in range(self.spec_max_num_tries): + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.messages_, + **self.spec_kwargs, + ) + # Use a string for pattern matching. + comp_for_match = comp[0] if isinstance(comp, list) else comp + if self.spec_pattern_match(comp_for_match): + break + + for term in self.spec_format: + s.text_ += term["text"] + name = term["name"] + if name is not None: + s.variables[name] = term["text"] + s.meta_info[name] = {} + s.variable_event[name].set() + + self.spec_kwargs = {} + self.spec_format = [] + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if sampling_params.dtype is None: + if self.is_chat_model: + if not s.text_.endswith(self.chat_prefix): + raise RuntimeError( + "This use case is not supported. " + "For OpenAI chat models, sgl.gen must be right after sgl.assistant" + ) + prompt = s.messages_ + else: + prompt = s.text_ + + kwargs = sampling_params.to_openai_kwargs() + generator = openai_completion_stream( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=prompt, + **kwargs, + ) + return generator + else: + raise ValueError(f"Unknown dtype: {sampling_params.dtype}") + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ) -> ChoicesDecision: + """Note: `choices_method` is not used by the OpenAI backend.""" + if self.is_chat_model: + raise NotImplementedError( + "select/choices is not supported for chat models. " + "Please try to use a non-chat model such as gpt-3.5-turbo-instruct" + ) + + n_choices = len(choices) + token_ids = [self.tokenizer.encode(x) for x in choices] + scores = [0] * n_choices + valid = [len(x) > 0 for x in token_ids] + prompt_tokens = self.tokenizer.encode(s.text_) + + max_len = max([len(x) for x in token_ids]) + for step in range(max_len): + # Build logit bias + logit_bias = {} + for i in range(n_choices): + if valid[i]: + logit_bias[token_ids[i][step]] = 100 + + # Call API + ret = self.client.completions.create( + model=self.model_name, + prompt=prompt_tokens, + logit_bias=logit_bias, + max_tokens=1, + temperature=temperature, + ) + ret_str = ret.choices[0].text + ret_token = self.tokenizer.encode(ret_str)[0] + self.token_usage.prompt_tokens += ret.usage.prompt_tokens + self.token_usage.completion_tokens = ret.usage.completion_tokens + + # TODO: + # 1. return logits as the scores + # 2. compute logits of the full choice + # 3. consider chunk-based decoding + + # Update valid + hit = False + for i in range(n_choices): + if valid[i]: + if step == len(token_ids[i]) - 1: + valid[i] = False + + if ret_token == token_ids[i][step]: + scores[i] += 1 + hit = True + else: + valid[i] = False + assert hit + + if np.sum(valid) <= 1: + break + + prompt_tokens.append(ret_token) + + return ChoicesDecision( + decision=choices[np.argmax(scores)], + meta_info={"scores": scores}, + ) + + +def openai_completion( + client, token_usage, is_chat=None, retries=3, prompt=None, **kwargs +) -> Union[str, List[str]]: + # if "ebnf" is in kwargs, warn and remove + if "ebnf" in kwargs: + warnings.warn("EBNF is not officially supported by OpenAI endpoints. Ignoring.") + del kwargs["ebnf"] + + for attempt in range(retries): + try: + if is_chat: + if "stop" in kwargs and kwargs["stop"] is None: + kwargs.pop("stop") + ret = client.chat.completions.create(messages=prompt, **kwargs) + if len(ret.choices) == 1: + comp = ret.choices[0].message.content + else: + comp = [c.message.content for c in ret.choices] + else: + ret = client.completions.create(prompt=prompt, **kwargs) + if isinstance(prompt, (list, tuple)): + comp = [c.text for c in ret.choices] + else: + comp = ret.choices[0].text + if len(ret.choices) > 1: + comp = [c.text for c in ret.choices] + + token_usage.prompt_tokens += ret.usage.prompt_tokens + token_usage.completion_tokens += ret.usage.completion_tokens + break + except (openai.APIError, openai.APIConnectionError, openai.RateLimitError) as e: + logger.error(f"OpenAI Error: {e}. Waiting 5 seconds...") + time.sleep(5) + if attempt == retries - 1: + raise e + except Exception as e: + logger.error(f"RuntimeError {e}.") + raise e + + return comp + + +def openai_completion_stream( + client, token_usage, is_chat=None, retries=3, prompt=None, **kwargs +): + # if "ebnf" is in kwargs, warn and remove + if "ebnf" in kwargs: + warnings.warn("EBNF is not officially supported by OpenAI endpoints. Ignoring.") + del kwargs["ebnf"] + + for attempt in range(retries): + try: + if is_chat: + if "stop" in kwargs and kwargs["stop"] is None: + kwargs.pop("stop") + generator = client.chat.completions.create( + messages=prompt, + stream=True, + stream_options={"include_usage": True}, + **kwargs, + ) + for ret in generator: + if len(ret.choices) == 0: + continue + try: + content = ret.choices[0].delta.content + except IndexError: + content = None + yield content or "", {} + else: + generator = client.completions.create( + prompt=prompt, + stream=True, + stream_options={"include_usage": True}, + **kwargs, + ) + for ret in generator: + if len(ret.choices) == 0: + continue + content = ret.choices[0].text + yield content or "", {} + + token_usage.prompt_tokens += ret.usage.prompt_tokens + token_usage.completion_tokens += ret.usage.completion_tokens + break + except (openai.APIError, openai.APIConnectionError, openai.RateLimitError) as e: + logger.error(f"OpenAI Error: {e}. Waiting 5 seconds...") + time.sleep(5) + if attempt == retries - 1: + raise e + except Exception as e: + logger.error(f"RuntimeError {e}.") + raise e diff --git a/python/sglang/lang/backend/runtime_endpoint.py b/python/sglang/lang/backend/runtime_endpoint.py new file mode 100644 index 0000000..1cd3d52 --- /dev/null +++ b/python/sglang/lang/backend/runtime_endpoint.py @@ -0,0 +1,500 @@ +import atexit +import json +import multiprocessing +import warnings +from typing import Dict, List, Optional, Union + +import aiohttp +import requests + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import ( + REGEX_BOOL, + REGEX_FLOAT, + REGEX_INT, + REGEX_STR, + SglSamplingParams, +) +from sglang.utils import http_request + + +class RuntimeEndpoint(BaseBackend): + def __init__( + self, + base_url: str, + api_key: Optional[str] = None, + verify: Optional[str] = None, + chat_template_name: Optional[str] = None, + ): + super().__init__() + self.support_concate_and_append = True + + self.base_url = base_url + self.api_key = api_key + self.verify = verify + + res = http_request( + self.base_url + "/get_model_info", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + self.model_info = res.json() + + if chat_template_name: + self.chat_template = get_chat_template(chat_template_name) + else: + self.chat_template = get_chat_template_by_model_path( + self.model_info["model_path"] + ) + + def get_model_name(self): + return self.model_info["model_path"] + + def flush_cache(self): + res = http_request( + self.base_url + "/flush_cache", + api_key=self.api_key, + verify=self.verify, + method="POST", + ) + self._assert_success(res) + + def get_server_info(self): + res = http_request( + self.base_url + "/get_server_info", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + return res.json() + + def get_chat_template(self): + return self.chat_template + + def cache_prefix(self, prefix_str: str): + res = http_request( + self.base_url + "/generate", + json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}}, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def commit_lazy_operations(self, s: StreamExecutor): + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def fill_image(self, s: StreamExecutor): + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def _handle_dtype_to_regex(self, sampling_params: SglSamplingParams): + if sampling_params.dtype is None: + return + + if sampling_params.stop == (): + sampling_params.stop = [] + + dtype_regex = None + if sampling_params.dtype in ["int", int]: + + dtype_regex = REGEX_INT + sampling_params.stop.extend([" ", "\n"]) + elif sampling_params.dtype in ["float", float]: + + dtype_regex = REGEX_FLOAT + sampling_params.stop.extend([" ", "\n"]) + elif sampling_params.dtype in ["str", str]: + + dtype_regex = REGEX_STR + elif sampling_params.dtype in ["bool", bool]: + + dtype_regex = REGEX_BOOL + else: + raise RuntimeError(f"Invalid dtype: {sampling_params.dtype}") + + if dtype_regex is not None and sampling_params.regex is not None: + warnings.warn( + f"Both dtype and regex are set. Only dtype will be used. dtype: {sampling_params.dtype}, regex: {sampling_params.regex}" + ) + + sampling_params.regex = dtype_regex + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + self._handle_dtype_to_regex(sampling_params) + data = { + "text": s.text_, + "sampling_params": { + "skip_special_tokens": global_config.skip_special_tokens_in_output, + "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out, + **sampling_params.to_srt_kwargs(), + }, + } + + for item in [ + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + data[item] = value + + self._add_images(s, data) + + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + obj = res.json() + comp = obj["text"] + return comp, obj["meta_info"] + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + self._handle_dtype_to_regex(sampling_params) + + data = { + "text": s.text_, + "sampling_params": { + "skip_special_tokens": global_config.skip_special_tokens_in_output, + "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out, + **sampling_params.to_srt_kwargs(), + }, + } + + for item in [ + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + data[item] = value + + data["stream"] = True + self._add_images(s, data) + + res = http_request( + self.base_url + "/generate", + json=data, + stream=True, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + pos = 0 + + for chunk in res.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + chunk_text = data["text"][pos:] + meta_info = data["meta_info"] + pos += len(chunk_text) + yield chunk_text, meta_info + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ) -> ChoicesDecision: + assert temperature <= 1e-5 + + # Cache common prefix + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + obj = self._generate_http_request(s, data) + prompt_len = obj["meta_info"]["prompt_tokens"] + logprob_start_len = max(prompt_len - 2, 0) # For token healing + + # Compute logprob + data = { + "text": [s.text_ + c for c in choices], + "sampling_params": { + "max_new_tokens": 0, + "temperature": 0, + }, + "return_logprob": True, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + } + obj = self._generate_http_request(s, data) + + input_token_logprobs = [r["meta_info"]["input_token_logprobs"] for r in obj] + output_token_logprobs = [r["meta_info"]["output_token_logprobs"] for r in obj] + normalized_prompt_logprobs = [ + compute_normalized_prompt_logprobs(r["meta_info"]["input_token_logprobs"]) + for r in obj + ] + + # Remove extra token if no token healing occurred + for i in range(len(input_token_logprobs)): + healed_token_str = input_token_logprobs[i][0][-1] + if s.text_.endswith(healed_token_str): + healed_token_logprob = input_token_logprobs[i][0][0] + normalized_prompt_logprobs[i] = ( + normalized_prompt_logprobs[i] * len(input_token_logprobs[i]) + - healed_token_logprob + ) / (len(input_token_logprobs[i]) - 1) + input_token_logprobs[i] = input_token_logprobs[i][1:] + + # Compute unconditional logprobs if required + if choices_method.requires_unconditional_logprobs: + input_ids = [[el[1] for el in subl] for subl in input_token_logprobs] + data = { + "input_ids": input_ids, + "sampling_params": {"max_new_tokens": 0}, + "return_logprob": True, + } + obj = self._generate_http_request(s, data) + unconditional_token_logprobs = [ + r["meta_info"]["input_token_logprobs"] for r in obj + ] + else: + unconditional_token_logprobs = None + + return choices_method( + choices=choices, + normalized_prompt_logprobs=normalized_prompt_logprobs, + input_token_logprobs=input_token_logprobs, + output_token_logprobs=output_token_logprobs, + unconditional_token_logprobs=unconditional_token_logprobs, + ) + + def concatenate_and_append(self, src_rids: List[str], dst_rid: str): + res = http_request( + self.base_url + "/concate_and_append_request", + json={"src_rids": src_rids, "dst_rid": dst_rid}, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def _generate_http_request(self, s: StreamExecutor, data): + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + return res.json() + + def _add_images(self, s: StreamExecutor, data): + if s.images_: + assert len(s.images_) == 1, "Only support one image." + data["image_data"] = s.images_[0][1] + + def _assert_success(self, res): + if res.status_code != 200: + raise RuntimeError(res.json()) + + +def compute_normalized_prompt_logprobs(input_logprobs): + values = [x[0] for x in input_logprobs if x[0]] + return sum(values) / len(values) + + +class Runtime: + """ + A wrapper for the HTTP server. + This is used for launching the server in a python program without + using the command line interface. + + It is mainly used for the frontend language. + You should use the Engine class if you want to do normal offline processing without the frontend language. + """ + + def __init__( + self, + log_level: str = "error", + *args, + **kwargs, + ): + """See the arguments in server_args.py::ServerArgs""" + # We delay the import of any `sglang.srt` components in `sglang.lang`, so users can run + # client code without installing SRT server and its dependency if they want. + from sglang.srt.entrypoints.http_server import launch_server + from sglang.srt.server_args import ServerArgs + from sglang.srt.utils import is_port_available + + self.server_args = ServerArgs(*args, log_level=log_level, **kwargs) + + # Pre-allocate ports + for port in range(self.server_args.port, 40000): + if is_port_available(port): + break + self.server_args.port = port + + self.url = self.server_args.url() + self.generate_url = self.url + "/generate" + + # NOTE: We store pid instead of proc to fix some issues during __delete__ + self.pid = None + pipe_reader, pipe_writer = multiprocessing.Pipe(duplex=False) + + proc = multiprocessing.Process( + target=launch_server, + args=(self.server_args, pipe_writer), + ) + proc.start() + pipe_writer.close() + self.pid = proc.pid + + # Before python program terminates, call shutdown implicitly. Therefore, users don't have to explicitly call .shutdown() + atexit.register(self.shutdown) + + # TODO: remove this pipe_writer mechanism and use `/health_generate` instead. + try: + init_state = pipe_reader.recv() + except EOFError: + init_state = "" + + if init_state != "ready": + self.shutdown() + raise RuntimeError( + "Initialization failed. Please see the error messages above." + ) + + self.endpoint = RuntimeEndpoint(self.url) + + def shutdown(self): + from sglang.srt.utils import kill_process_tree + + if self.pid is not None: + kill_process_tree(self.pid) + self.pid = None + + def cache_prefix(self, prefix: str): + self.endpoint.cache_prefix(prefix) + + def get_tokenizer(self): + from sglang.srt.hf_transformers_utils import get_tokenizer + + return get_tokenizer( + self.server_args.tokenizer_path, + tokenizer_mode=self.server_args.tokenizer_mode, + trust_remote_code=self.server_args.trust_remote_code, + revision=self.server_args.revision, + ) + + async def async_generate( + self, + prompt: str, + sampling_params: Optional[Dict] = None, + ): + if self.server_args.skip_tokenizer_init: + json_data = { + "input_ids": prompt, + "sampling_params": sampling_params, + "stream": True, + } + else: + json_data = { + "text": prompt, + "sampling_params": sampling_params, + "stream": True, + } + pos = 0 + + timeout = aiohttp.ClientTimeout(total=3 * 3600) + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with session.post(self.generate_url, json=json_data) as response: + async for chunk, _ in response.content.iter_chunks(): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]\n\n": + break + data = json.loads(chunk[5:].strip("\n")) + if "text" in data: + cur = data["text"][pos:] + if cur: + yield cur + pos += len(cur) + else: + yield data + + add_request = async_generate + + def generate( + self, + prompt: Union[str, List[str]], + sampling_params: Optional[Dict] = None, + return_logprob: Optional[Union[List[bool], bool]] = False, + logprob_start_len: Optional[Union[List[int], int]] = None, + top_logprobs_num: Optional[Union[List[int], int]] = None, + lora_path: Optional[List[Optional[str]]] = None, + ): + json_data = { + "text": prompt, + "sampling_params": sampling_params, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + "lora_path": lora_path, + } + assert not isinstance(lora_path, list) or len(lora_path) == len(prompt) + response = requests.post( + self.url + "/generate", + json=json_data, + ) + return json.dumps(response.json()) + + def encode( + self, + prompt: Union[str, List[str], List[Dict], List[List[Dict]]], + ): + json_data = {"text": prompt} + response = requests.post(self.url + "/encode", json=json_data) + return json.dumps(response.json()) + + async def get_server_info(self): + async with aiohttp.ClientSession() as session: + async with session.get(f"{self.url}/get_server_info") as response: + if response.status == 200: + return await response.json() + else: + error_data = await response.json() + raise RuntimeError( + f"Failed to get server info. {error_data['error']['message']}" + ) + + def __del__(self): + self.shutdown() diff --git a/python/sglang/lang/backend/vertexai.py b/python/sglang/lang/backend/vertexai.py new file mode 100644 index 0000000..c27733b --- /dev/null +++ b/python/sglang/lang/backend/vertexai.py @@ -0,0 +1,149 @@ +import os +import warnings +from typing import Optional + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import vertexai + from vertexai.preview.generative_models import ( + GenerationConfig, + GenerativeModel, + Image, + ) +except ImportError as e: + GenerativeModel = e + + +class VertexAI(BaseBackend): + def __init__(self, model_name, safety_settings=None): + super().__init__() + + if isinstance(GenerativeModel, Exception): + raise GenerativeModel + + project_id = os.environ["GCP_PROJECT_ID"] + location = os.environ.get("GCP_LOCATION") + vertexai.init(project=project_id, location=location) + + self.model_name = model_name + self.chat_template = get_chat_template("default") + self.safety_settings = safety_settings + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + prompt = self.messages_to_vertexai_input(s.messages_) + else: + # single-turn + prompt = ( + self.text_to_vertexai_input(s.text_, s.cur_images) + if s.cur_images + else s.text_ + ) + ret = GenerativeModel(self.model_name).generate_content( + prompt, + generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()), + safety_settings=self.safety_settings, + ) + + comp = ret.text + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + prompt = self.messages_to_vertexai_input(s.messages_) + else: + # single-turn + prompt = ( + self.text_to_vertexai_input(s.text_, s.cur_images) + if s.cur_images + else s.text_ + ) + generator = GenerativeModel(self.model_name).generate_content( + prompt, + stream=True, + generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()), + safety_settings=self.safety_settings, + ) + for ret in generator: + yield ret.text, {} + + def text_to_vertexai_input(self, text, images): + input = [] + # split with image token + text_segs = text.split(self.chat_template.image_token) + for image_path, image_base64_data in images: + text_seg = text_segs.pop(0) + if text_seg != "": + input.append(text_seg) + input.append(Image.from_bytes(image_base64_data)) + text_seg = text_segs.pop(0) + if text_seg != "": + input.append(text_seg) + return input + + def messages_to_vertexai_input(self, messages): + vertexai_message = [] + # from openai message format to vertexai message format + for msg in messages: + if isinstance(msg["content"], str): + text = msg["content"] + else: + text = msg["content"][0]["text"] + + if msg["role"] == "system": + warnings.warn("Warning: system prompt is not supported in VertexAI.") + vertexai_message.append( + { + "role": "user", + "parts": [{"text": "System prompt: " + text}], + } + ) + vertexai_message.append( + { + "role": "model", + "parts": [{"text": "Understood."}], + } + ) + continue + if msg["role"] == "user": + vertexai_msg = { + "role": "user", + "parts": [{"text": text}], + } + elif msg["role"] == "assistant": + vertexai_msg = { + "role": "model", + "parts": [{"text": text}], + } + + # images + if isinstance(msg["content"], list) and len(msg["content"]) > 1: + for image in msg["content"][1:]: + assert image["type"] == "image_url" + vertexai_msg["parts"].append( + { + "inline_data": { + "data": image["image_url"]["url"].split(",")[1], + "mime_type": "image/jpeg", + } + } + ) + + vertexai_message.append(vertexai_msg) + return vertexai_message diff --git a/python/sglang/lang/chat_template.py b/python/sglang/lang/chat_template.py new file mode 100644 index 0000000..8554d28 --- /dev/null +++ b/python/sglang/lang/chat_template.py @@ -0,0 +1,555 @@ +from dataclasses import dataclass +from enum import Enum, auto +from typing import Callable, Dict, List, Tuple + + +class ChatTemplateStyle(Enum): + PLAIN = auto() + LLAMA2 = auto() + + +@dataclass +class ChatTemplate: + name: str + default_system_prompt: str + role_prefix_and_suffix: Dict[str, Tuple[str, str]] + stop_str: List[str] = () + image_token: str = "" + audio_token: str = ""), + }, + image_token=" \n", + ) +) + +register_chat_template( + ChatTemplate( + name="llama-2-chat", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("<>\n", "\n<>\n\n"), + "user": ("[INST] ", " [/INST]"), + "assistant": ("", " "), + }, + style=ChatTemplateStyle.LLAMA2, + ) +) + +register_chat_template( + ChatTemplate( + name="llama-3-instruct", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_header_id|>system<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "user": ( + "<|start_header_id|>user<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "assistant": ( + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "<|eot_id|>", + ), + }, + stop_str=("<|eot_id|>",), + image_token="<|image|>", + ) +) + +# https://huggingface.co/openbmb/MiniCPM-V-2_6 +register_chat_template( + ChatTemplate( + name="minicpmv", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", " "), + "user": ("user:", " "), + "assistant": ("assistant:", ""), + }, + stop_str=("<|im_end|>", "<|endoftext|>"), + image_token="(./)", + ) +) + +register_chat_template( + ChatTemplate( + name="janus-pro", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "User": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + image_token="\n", + ) +) + +# https://huggingface.co/openbmb/MiniCPM-o-2_6 +register_chat_template( + ChatTemplate( + name="minicpmo", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", " "), + "user": ("user:", " "), + "assistant": ("assistant:", ""), + }, + stop_str=("<|im_end|>", "<|endoftext|>"), + image_token="(./)", + audio_token="()", + ) +) + +# The difference between "llama-3-instruct-llava" and "llama-3-instruct" is that llava uses a different image_token. +register_chat_template( + ChatTemplate( + name="llama-3-instruct-llava", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_header_id|>system<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "user": ( + "<|start_header_id|>user<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "assistant": ( + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "<|eot_id|>", + ), + }, + stop_str=("<|eot_id|>",), + image_token="\n", + ) +) + +# Reference: https://modelscope.cn/models/01ai/Yi-1.5-34B-Chat/file/view/master?fileName=tokenizer_config.json&status=1 +register_chat_template( + ChatTemplate( + name="yi-1.5", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", ""), + "user": ("<|im_start|>user\n", "<|im_end|>\n<|im_start|>assistant\n"), + "assistant": ("", "<|im_end|>\n"), + }, + style=ChatTemplateStyle.PLAIN, + stop_str=("<|im_end|>",), + ) +) + +# Reference: https://github.com/01-ai/Yi/tree/main/VL#major-difference-with-llava +register_chat_template( + ChatTemplate( + name="yi-vl", + default_system_prompt=( + "This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. Read all the images carefully, and respond to the human's questions with informative, helpful, detailed and polite answers." + "这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。" + ), + role_prefix_and_suffix={ + "system": ("", "\n\n"), + "user": ("### Human:", "\n"), + "assistant": ("### Assistant:", "\n"), + }, + image_token=" \n", + ) +) + +register_chat_template( + ChatTemplate( + name="gemma-it", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", ""), + "user": ("user\n", "\n"), + "assistant": ("model\n", "\n"), + }, + style=ChatTemplateStyle.PLAIN, + ) +) + +register_chat_template( + ChatTemplate( + name="dbrx-instruct", + default_system_prompt="You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER'S QUERY.", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>"), + "user": ("\n<|im_start|>user\n", "<|im_end|>"), + "assistant": ("\n<|im_start|>assistant\n", "<|im_end|>"), + }, + stop_str=("<|im_end|>",), + ) +) + +register_chat_template( + ChatTemplate( + name="c4ai-command-r", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>", + "<|END_OF_TURN_TOKEN|>", + ), + "user": ("<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|END_OF_TURN_TOKEN|>"), + "assistant": ( + "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "<|END_OF_TURN_TOKEN|>", + ), + }, + style=ChatTemplateStyle.PLAIN, + ) +) + +register_chat_template( + ChatTemplate( + name="granite-3-instruct", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_of_role|>system<|end_of_role|>", + "<|end_of_text|>", + ), + "user": ( + "<|start_of_role|>user<|end_of_role|>", + "<|end_of_text|>", + ), + "assistant": ( + "<|start_of_role|>assistant<|end_of_role|>", + "<|end_of_text|>", + ), + }, + stop_str=("<|end_of_text|>",), + ) +) + +register_chat_template( + ChatTemplate( + name="deepseek-v3", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "user": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + ) +) + + +@register_chat_template_matching_function +def match_deepseek(model_path: str): + if ( + "deepseek-v3" in model_path.lower() or "deepseek-r1" in model_path.lower() + ) and "base" not in model_path.lower(): + return get_chat_template("deepseek-v3") + + +@register_chat_template_matching_function +def match_deepseek_janus_pro(model_path: str): + if "janus" in model_path.lower(): + return get_chat_template("janus-pro") + + +@register_chat_template_matching_function +def match_dbrx(model_path: str): + if "dbrx" in model_path.lower() and "instruct" in model_path.lower(): + return get_chat_template("dbrx-instruct") + + +@register_chat_template_matching_function +def match_vicuna(model_path: str): + if "vicuna" in model_path.lower(): + return get_chat_template("vicuna_v1.1") + if "llava-v1.5" in model_path.lower(): + return get_chat_template("vicuna_v1.1") + if "llava-next-video-7b" in model_path.lower(): + return get_chat_template("vicuna_v1.1") + + +@register_chat_template_matching_function +def match_llama2_chat(model_path: str): + model_path = model_path.lower() + if "llama-2" in model_path and "chat" in model_path: + return get_chat_template("llama-2-chat") + if ( + "mistral" in model_path or "mixtral" in model_path + ) and "instruct" in model_path: + return get_chat_template("llama-2-chat") + if "codellama" in model_path and "instruct" in model_path: + return get_chat_template("llama-2-chat") + + +@register_chat_template_matching_function +def match_llama3_instruct(model_path: str): + model_path = model_path.lower() + if "llama-3" in model_path and "instruct" in model_path: + return get_chat_template("llama-3-instruct") + + +@register_chat_template_matching_function +def match_chat_ml(model_path: str): + # import pdb;pdb.set_trace() + model_path = model_path.lower() + if "tinyllama" in model_path: + return get_chat_template("chatml") + # Now the suffix for qwen2 chat model is "instruct" + if "qwen" in model_path and "vl" in model_path: + return get_chat_template("qwen2-vl") + if "qwen" in model_path: + if "vl" in model_path: + return get_chat_template("qwen2-vl") + if ("chat" in model_path or "instruct" in model_path) and ( + "llava" not in model_path + ): + return get_chat_template("qwen") + if ( + "llava-v1.6-34b" in model_path + or "llava-v1.6-yi-34b" in model_path + or "llava-next-video-34b" in model_path + or "llava-onevision-qwen2" in model_path + ): + return get_chat_template("chatml-llava") + + +@register_chat_template_matching_function +def match_chat_yi(model_path: str): + model_path = model_path.lower() + if "yi-vl" in model_path and "llava" not in model_path: + return get_chat_template("yi-vl") + elif "yi-1.5" in model_path and "chat" in model_path: + return get_chat_template("yi-1.5") + + +@register_chat_template_matching_function +def match_gemma_it(model_path: str): + model_path = model_path.lower() + if "gemma" in model_path and "it" in model_path: + return get_chat_template("gemma-it") + + +@register_chat_template_matching_function +def match_openbmb_minicpm(model_path: str): + model_path = model_path.lower() + if "minicpm-v" in model_path: + return get_chat_template("minicpmv") + elif "minicpm-o" in model_path: + return get_chat_template("minicpmo") + + +@register_chat_template_matching_function +def match_c4ai_command_r(model_path: str): + model_path = model_path.lower() + if "c4ai-command-r" in model_path: + return get_chat_template("c4ai-command-r") + + +@register_chat_template_matching_function +def match_granite_instruct(model_path: str): + model_path = model_path.lower() + # When future versions of Granite are released, this code may + # need to be updated. For now, assume that the Granite 3.0 + # template works across the board. + if "granite" in model_path and "instruct" in model_path: + return get_chat_template("granite-3-instruct") + + +@register_chat_template_matching_function +def match_gemma3_instruct(model_path: str): + model_path = model_path.lower() + if "gemma-3" in model_path and "1b" not in model_path: + # gemma-3-1b-it is completion model + return get_chat_template("gemma-it") + + +if __name__ == "__main__": + messages = [ + {"role": "system", "content": None}, # None means default + # {"role": "system", "content": "You are a helpful, respectful and honest assistant."}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi!"}, + {"role": "user", "content": "What can you do?"}, + {"role": "assistant", "content": "I can chat with you."}, + ] + + template = get_chat_template("llama-2-chat") + print(template.get_prompt(messages)) diff --git a/python/sglang/lang/choices.py b/python/sglang/lang/choices.py new file mode 100644 index 0000000..e52c6b3 --- /dev/null +++ b/python/sglang/lang/choices.py @@ -0,0 +1,164 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import numpy as np + + +@dataclass +class ChoicesDecision: + decision: str + meta_info: Optional[Dict[str, Any]] = None + + +class ChoicesSamplingMethod(ABC): + + @property + def requires_unconditional_logprobs(self) -> bool: + return False + + @abstractmethod + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: ... + + +class TokenLengthNormalized(ChoicesSamplingMethod): + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option with the highest token length normalized prompt logprob.""" + best_choice = choices[np.argmax(normalized_prompt_logprobs)] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + +token_length_normalized = TokenLengthNormalized() + + +class GreedyTokenSelection(ChoicesSamplingMethod): + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option based on greedy logprob selection. For overlapping options + where one option is a subset of a longer option, extend the shorter option using + its average logprob for comparison against the longer option.""" + + num_options = len(choices) + max_tokens = max(len(option) for option in input_token_logprobs) + logprob_matrix = self._build_logprob_matrix( + input_token_logprobs, max_tokens, num_options + ) + remaining = self._greedy_selection(logprob_matrix, num_options, max_tokens) + + best_choice = choices[remaining[0]] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + "greedy_logprob_matrix": logprob_matrix.tolist(), + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + def _build_logprob_matrix(self, input_token_logprobs, max_tokens, num_options): + logprob_matrix = np.zeros((num_options, max_tokens)) + for i, option in enumerate(input_token_logprobs): + actual_logprobs = [token[0] for token in option] + avg_logprob = np.mean(actual_logprobs) + logprob_matrix[i, : len(option)] = actual_logprobs + if len(option) < max_tokens: + logprob_matrix[i, len(option) :] = avg_logprob + return logprob_matrix + + def _greedy_selection(self, logprob_matrix, num_options, max_tokens): + remaining = np.arange(num_options) + for j in range(max_tokens): + max_logprob = np.max(logprob_matrix[remaining, j]) + remaining = remaining[logprob_matrix[remaining, j] == max_logprob] + if len(remaining) == 1: + break + return remaining + + +greedy_token_selection = GreedyTokenSelection() + + +class UnconditionalLikelihoodNormalized(ChoicesSamplingMethod): + + @property + def requires_unconditional_logprobs(self) -> bool: + return True + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option with the highest average token logprob once normalized by + the unconditional token logprobs. + + The first unconditional token logprob is assumed to be None. If so, it is + replaced with 0 for the purposes of normalization.""" + + if unconditional_token_logprobs is None: + raise ValueError( + "Unconditional token logprobs are required for this method." + ) + + normalized_unconditional_prompt_logprobs = self._normalize_logprobs( + input_token_logprobs, unconditional_token_logprobs + ) + + best_choice = choices[np.argmax(normalized_unconditional_prompt_logprobs)] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + "unconditional_token_logprobs": unconditional_token_logprobs, + "normalized_unconditional_prompt_logprobs": normalized_unconditional_prompt_logprobs, + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + def _normalize_logprobs(self, input_token_logprobs, unconditional_token_logprobs): + normalized_unconditional_prompt_logprobs = [] + for inputs, unconditionals in zip( + input_token_logprobs, unconditional_token_logprobs + ): + inputs_logprobs = np.array([token[0] for token in inputs]) + unconditionals_logprobs = np.array([token[0] for token in unconditionals]) + unconditionals_logprobs[0] = unconditionals_logprobs[0] or 0 + normalized_unconditional_prompt_logprobs.append( + float(np.mean(inputs_logprobs - unconditionals_logprobs)) + ) + return normalized_unconditional_prompt_logprobs + + +unconditional_likelihood_normalized = UnconditionalLikelihoodNormalized() diff --git a/python/sglang/lang/compiler.py b/python/sglang/lang/compiler.py new file mode 100644 index 0000000..5e1b411 --- /dev/null +++ b/python/sglang/lang/compiler.py @@ -0,0 +1,237 @@ +import multiprocessing +from concurrent.futures import ThreadPoolExecutor +from queue import Queue +from typing import List, Union + +from sglang.global_config import global_config +from sglang.lang.interpreter import ProgramState, StreamExecutor, cache_program +from sglang.lang.ir import ( + SglArgument, + SglConstantText, + SglExpr, + SglSamplingParams, + SglVariable, +) + + +def compile_func(function, backend): + tracer = function.trace(backend=backend) + compiler = CompiledFunction(tracer, function) + return compiler + + +class CompiledFunction: + def __init__(self, tracer, function): + self.function = function + + self.last_node = CompGraphNode(tracer.last_node) + self.expr_to_node = {} + self.build_graph(tracer) + self.topological_sort() + + def build_graph(self, tracer): + self.nodes = [self.last_node] + self.expr_to_node[tracer.last_node] = self.nodes[-1] + + rename_pid = {} + + visited = set([tracer.last_node]) + head = 0 + while head < len(self.nodes): + cur_node = self.nodes[head] + + # add prev node + prev_node = cur_node.expr.prev_node + if prev_node is not None: + if prev_node not in visited: + visited.add(prev_node) + self.nodes.append(CompGraphNode(prev_node)) + self.expr_to_node[prev_node] = self.nodes[-1] + cur_node.prev_node = self.expr_to_node[prev_node] + self.expr_to_node[prev_node].add_next_node(cur_node) + + # add source node + if isinstance(cur_node.expr, SglVariable): + if cur_node.expr.name in tracer.variables: + source = tracer.variables[cur_node.expr.name].source + else: + source = cur_node.expr.source + if source not in visited: + visited.add(source) + self.nodes.append(CompGraphNode(source)) + self.expr_to_node[source] = self.nodes[-1] + cur_node.source_node = self.expr_to_node[source] + self.expr_to_node[source].add_next_node(cur_node) + head += 1 + + # rename pid + if cur_node.expr.pid not in rename_pid: + rename_pid[cur_node.expr.pid] = len(rename_pid) + cur_node.expr.pid = rename_pid[cur_node.expr.pid] + + def topological_sort(self): + prevd = {} + cand = Queue() + for x in self.nodes: + prevd[x] = (x.prev_node is not None) + (x.source_node is not None) + if prevd[x] == 0: + cand.put(x) + new_list = [] + while cand.qsize() > 0: + head = cand.get() + new_list.append(head) + for x in head.next_nodes: + prevd[x] -= 1 + if prevd[x] == 0: + cand.put(x) + self.nodes = new_list + + def print_graph( + self, + ): + for node in self.nodes: + print(node) + + def run_internal( + self, + backend, + kwargs, + default_sampling_para, + ): + stream_executor_ids = set([x.expr.pid for x in self.nodes]) + stream_executors = {} + for x in stream_executor_ids: + arguments = kwargs if x == self.last_node.expr.pid else {} + stream_executors[x] = StreamExecutor( + backend, arguments, default_sampling_para, None, False + ) + for node in self.nodes: + se_id = node.expr.pid + expr = node.expr + if isinstance(expr, SglVariable): + # Make a copy for SglVariable + expr = SglVariable(expr.name, expr.source) + expr.source_stream_executor = stream_executors[ + node.source_node.expr.pid + ] + elif isinstance(expr, SglArgument): + # Substitute SglArgument + expr = kwargs[expr.name] + stream_executors[se_id].submit(expr) + for stream_executor in stream_executors.values(): + stream_executor.end() + return ProgramState(stream_executors[self.last_node.expr.pid]) + + def run( + self, + *, + max_new_tokens: int = 128, + stop: Union[str, List[str]] = (), + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + backend=None, + **kwargs, + ): + backend = backend or global_config.default_backend + + kwargs.update(self.function.bind_arguments) + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + stop=stop, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ) + + return self.run_internal(backend, kwargs, default_sampling_para) + + def run_batch( + self, + batch_kwargs, + *, + max_new_tokens: int = 128, + stop: Union[str, List[str]] = (), + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + backend=None, + num_threads: Union[str, int] = "auto", + ): + assert isinstance(batch_kwargs, (list, tuple)) + if len(batch_kwargs) == 0: + return [] + assert isinstance(batch_kwargs[0], dict) + + backend = backend or global_config.default_backend + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + stop=stop, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ) + + # Extract prefix by tracing and cache it + if len(batch_kwargs) > 1: + cache_program(self.function, backend) + + # Run all programs + if num_threads == "auto": + num_threads = multiprocessing.cpu_count() + num_threads = min(num_threads, len(batch_kwargs)) + + if num_threads == 1: + rets = [] + for arguments in batch_kwargs: + rets.append( + self.run_internal(backend, arguments, default_sampling_para) + ) + else: + with ThreadPoolExecutor(num_threads) as executor: + futures = [] + for arguments in batch_kwargs: + futures.append( + executor.submit( + self.run_internal, backend, arguments, default_sampling_para + ) + ) + rets = [f.result() for f in futures] + rets[-1].sync() + + return rets + + +class CompGraphNode: + def __init__( + self, expr: SglExpr, prev_node=None, next_nodes=None, source_node=None + ): + self.expr = expr + self.next_nodes = next_nodes or [] + self.prev_node = prev_node + self.source_node = source_node + + def add_next_node(self, other): + self.next_nodes.append(other) + + def __repr__(self): + re = f"stream {self.expr.pid:2d}: " + re += f"%{self.expr.node_id} = " + if self.prev_node is not None: + re += f"%{self.prev_node.expr.node_id} + " + re += repr(self.expr) + return re diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py new file mode 100644 index 0000000..1ac9164 --- /dev/null +++ b/python/sglang/lang/interpreter.py @@ -0,0 +1,1021 @@ +"""The interpreter that executes SGL programs""" + +import asyncio +import contextvars +import copy +import multiprocessing +import queue +import threading +import uuid +import warnings +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from typing import Any, Callable, Dict, List, Optional + +import tqdm + +from sglang.global_config import global_config +from sglang.lang.ir import ( + SglCommitLazy, + SglConcateAndAppend, + SglConstantText, + SglExpr, + SglExprList, + SglGen, + SglImage, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglVariable, + SglVarScopeBegin, + SglVarScopeEnd, + SglVideo, +) +from sglang.utils import ( + encode_image_base64, + encode_video_base64, + get_exception_traceback, +) + + +def run_internal(state, program, func_args, func_kwargs, sync): + try: + state.ret_value = program.func(state, *func_args, **func_kwargs) + except Exception as e: + raise e + finally: + state.stream_executor.end() + + if sync: + state.stream_executor.sync() + + if global_config.verbosity >= 2: + print(state.text()) + + +def run_program( + program, + backend, + func_args, + func_kwargs, + default_sampling_para, + stream, + sync=False, + use_thread=True, +): + if hasattr(backend, "endpoint"): + backend = backend.endpoint + assert backend is not None, "Please specify a backend" + func_kwargs.update(program.bind_arguments) + stream_executor = StreamExecutor( + backend, + func_kwargs, + default_sampling_para, + chat_template=None, + stream=stream, + num_api_spec_tokens=program.num_api_spec_tokens, + use_thread=use_thread, + ) + state = ProgramState(stream_executor) + + if stream: + t = threading.Thread( + target=run_internal, args=(state, program, func_args, func_kwargs, sync) + ) + t.start() + return state + else: + run_internal(state, program, func_args, func_kwargs, sync) + return state + + +def run_program_batch( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, + generator_style=False, +): + if hasattr(backend, "endpoint"): + backend = backend.endpoint + + # Pre-cache the common prefix for a batch. The prefix is extracted by tracing the program. + if global_config.enable_precache_with_tracing and len(batch_arguments) > 1: + cache_program(program, backend) + + # Run all programs + if num_threads == "auto": + num_threads = max(96, multiprocessing.cpu_count() * 16) + num_threads = min(num_threads, len(batch_arguments)) + + if generator_style: + return _run_program_batch_generator( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, + ) + + # Original code path when generator_style=False + if num_threads == 1: + rets = [] + if progress_bar: + for arguments in tqdm.tqdm(batch_arguments): + rets.append( + run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + else: + for arguments in batch_arguments: + rets.append( + run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + else: + if progress_bar: + pbar = tqdm.tqdm(total=len(batch_arguments)) + + with ThreadPoolExecutor(num_threads) as executor: + futures = [] + for arguments in batch_arguments: + futures.append( + executor.submit( + run_program, + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + if progress_bar: + futures[-1].add_done_callback(lambda _: pbar.update()) + + rets = [f.result() for f in futures] + rets[-1].sync() + + if progress_bar: + pbar.close() + + return rets + + +def _run_program_batch_generator( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, +): + """Helper function that yields results one by one using chunking to avoid overwhelming ThreadPoolExecutor.""" + if num_threads == 1: + iterator = tqdm.tqdm(batch_arguments) if progress_bar else batch_arguments + for arguments in iterator: + yield run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + else: + pbar = tqdm.tqdm(total=len(batch_arguments)) if progress_bar else None + + # Process in chunks to avoid overwhelming ThreadPoolExecutor + # Otherwise, ThreadPoolExecutor.submit will block after adding certain number of tasks + # so we will never reach "yield" until all tasks are done + chunk_size = 200 + + with ThreadPoolExecutor(num_threads) as executor: + for chunk_start in range(0, len(batch_arguments), chunk_size): + chunk_end = min(chunk_start + chunk_size, len(batch_arguments)) + chunk_futures = [] + + # Submit chunk of tasks + for i in range(chunk_start, chunk_end): + future = executor.submit( + run_program, + program, + backend, + (), + batch_arguments[i], + default_sampling_para, + False, + True, + ) + if pbar: + future.add_done_callback(lambda _: pbar.update()) + chunk_futures.append(future) + + # Yield results from this chunk as they complete + for future in chunk_futures: + yield future.result() + + if pbar: + pbar.close() + + +def cache_program(program, backend): + from sglang.lang.tracer import extract_prefix_by_tracing + + prefix = extract_prefix_by_tracing(program, backend) + if prefix and len(prefix) > 64: + backend.cache_prefix(prefix) + + +class StreamExecutor: + """A stream executor that executes SGL expressions in a background thread.""" + + def __init__( + self, + backend, + arguments, + default_sampling_para, + chat_template, + stream, + num_api_spec_tokens=None, + use_thread=True, + ): + from sglang.lang.backend.base_backend import BaseBackend + + self.sid = uuid.uuid4().hex + self.backend: BaseBackend = backend + self.arguments: Dict[str, Any] = arguments + self.default_sampling_para = default_sampling_para + self.stream = stream + + self.variables = {} # Dict[name: str -> value: str] + self.variable_event = {} # Dict[name: str -> event: threading.Event] + self.meta_info = {} # Dict[name: str -> info: str] + self.is_finished = False + self.error_ = None + + # For completion + self.text_ = "" # The full text + + # For chat + self.messages_ = [] # The messages in the OpenAI API format + self.chat_template = chat_template or self.backend.get_chat_template() + self.cur_role = None + self.cur_role_begin_pos = None + + # For vision + self.images_ = [] + self.cur_images = [] + + # For fork/join + self.fork_start_text_pos = None + + # For speculative execution + self.num_api_spec_tokens = num_api_spec_tokens + self.speculated_text = "" + + # Worker thread + self.use_thread = use_thread + if self.use_thread: + self.queue = queue.Queue() + + def _run_worker_in_context(): + self._thread_worker_func() + + self.worker = threading.Thread( + target=contextvars.copy_context().run, args=(_run_worker_in_context,) + ) + self.worker.start() + + # For streaming + if stream: + self.stream_text_event = threading.Event() + self.stream_var_event = {} + else: + self.stream_text_event = None + self.stream_var_event = None + + def submit(self, expr: SglExpr): + self._init_var_event(expr) + + if self.use_thread: + self.queue.put(expr) + else: + self._execute(expr) + + def sync(self): + if self.use_thread: + self.queue.join() + + def get_var(self, name): + if name in self.variable_event: + self.variable_event[name].wait() + return self.variables[name] + + def set_var(self, name, value): + self.variables[name] = value + + def get_meta_info(self, name, timeout=None): + if name in self.variable_event: + got = self.variable_event[name].wait(timeout) + if not got: + raise TimeoutError(f"Timeout while waiting for event '{name}'") + ret = self.meta_info.get(name, None) + return ret + + def fork( + self, + size: int = 1, + position_ids_offset: Optional[List[int]] = None, + ): + if size > 1 and str(self.text_): + self.submit(SglCommitLazy()) + + self.sync() + size = int(size) + + exes = [ + StreamExecutor( + self.backend, + self.arguments, + self.default_sampling_para, + self.chat_template, + self.stream, + ) + for _ in range(size) + ] + for i in range(size): + exes[i].variables = dict(self.variables) + exes[i].text_ = str(self.text_) + exes[i].messages_ = list(self.messages_) + exes[i].cur_role = self.cur_role + exes[i].cur_role_begin_pos = self.cur_role_begin_pos + exes[i].fork_start_text_pos = len(self.text_) + exes[i].images_ = list(self.images_) + + # TODO(ying): handle API speculative execution + + return exes + + def text(self): + self.sync() + return self.text_ + + def messages(self): + self.sync() + return self.messages_ + + def error(self): + self.sync() + return self.error_ + + def end(self): + if self.use_thread: + if self.worker.is_alive(): + self.queue.put(None) + self.backend.end_program(self) + + def _thread_worker_func(self): + error = None + + while True: + expr = self.queue.get() + if expr is None: + self.queue.task_done() + break + + try: + self._execute(expr) + except Exception as e: + warnings.warn(f"Error in stream_executor: {get_exception_traceback()}") + error = e + break + self.queue.task_done() + if self.stream_text_event: + self.stream_text_event.set() + + # Clean the queue and events + if error is not None: + try: + while True: + self.queue.task_done() + self.queue.get_nowait() + except queue.Empty: + pass + for name in self.variable_event: + self.variable_event[name].set() + if self.stream_var_event: + for name in self.stream_var_event: + self.stream_var_event[name].set() + self.error_ = error + + if self.stream_text_event: + self.stream_text_event.set() + + self.is_finished = True + + def _execute(self, other): + if isinstance(other, str): + other = SglConstantText(other) + + assert isinstance(other, SglExpr), f"{other}" + + if isinstance(other, SglConstantText): + self._execute_fill(other.value) + elif isinstance(other, SglGen): + self._execute_gen(other) + elif isinstance(other, SglSelect): + self._execute_select(other) + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute(x) + elif isinstance(other, SglRoleBegin): + self._execute_role_begin(other) + elif isinstance(other, SglRoleEnd): + self._execute_role_end(other) + elif isinstance(other, SglImage): + self._execute_image(other) + elif isinstance(other, SglVideo): + self._execute_video(other) + elif isinstance(other, SglVariable): + self._execute_variable(other) + elif isinstance(other, SglVarScopeBegin): + self._execute_var_scope_begin(other) + elif isinstance(other, SglVarScopeEnd): + self._execute_var_scope_end(other) + elif isinstance(other, SglCommitLazy): + self._execute_commit_lazy_operations(other) + elif isinstance(other, SglConcateAndAppend): + if ( + global_config.enable_parallel_encoding + and self.backend.support_concate_and_append + ): + self._execute_concatenate_and_append_kv_cache(other) + else: + self._execute_concatenate_and_append_text(other) + else: + raise ValueError(f"Unknown type: {type(other)}") + + def _execute_fill(self, value: str, prefix=False): + value = str(value) + + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + and not prefix + ): + self.backend.spec_fill(value) + return + + if self.speculated_text.startswith(value): + self.speculated_text = self.speculated_text[len(value) :] + else: + self.speculated_text = "" + + self.text_ += value + + def _execute_image(self, expr: SglImage): + path = expr.path + + base64_data = encode_image_base64(path) + + self.images_.append((path, base64_data)) + self.cur_images.append((path, base64_data)) + self.text_ += self.chat_template.image_token + + def _execute_video(self, expr: SglVideo): + path = expr.path + num_frames = expr.num_frames + + base64_data = encode_video_base64(path, num_frames) + + self.images_.append((path, base64_data)) + self.cur_images.append((path, base64_data)) + self.text_ += self.chat_template.image_token + + def _spec_gen(self, sampling_params): + stop = sampling_params.stop + max_new_tokens = sampling_params.max_new_tokens + meta_info = {} + + def regen(): + nonlocal meta_info + + sampling_params.max_new_tokens = max( + sampling_params.max_new_tokens, self.num_api_spec_tokens + ) + sampling_params.stop = None + self.speculated_text, meta_info = self.backend.generate( + self, sampling_params=sampling_params + ) + + def find_stop(): + if isinstance(stop, str): + return self.speculated_text.find(stop) + elif isinstance(stop, (tuple, list)): + pos = -1 + for stop_str in stop: + stop_pos = self.speculated_text.find(stop_str) + if stop_pos != -1 and (pos == -1 or stop_pos < pos): + pos = stop_pos + return pos + else: + raise Exception("Wrong type of stop in sampling parameters.") + + if stop is None: + if len(self.speculated_text) < max_new_tokens: + regen() + comp = self.speculated_text[:max_new_tokens] + self.speculated_text = self.speculated_text[max_new_tokens:] + elif isinstance(stop, (str, list, tuple)): + if self.speculated_text == "": + regen() + stop_pos = find_stop() + if stop_pos == -1: + stop_pos = min( + sampling_params.max_new_tokens, + len(self.speculated_text), + ) + comp = self.speculated_text[:stop_pos] + self.speculated_text = self.speculated_text[stop_pos:] + else: + raise ValueError("Wrong type of stop in sampling parameters.") + + return comp, meta_info + + def _execute_gen(self, expr: SglGen): + sampling_params = self._resolve_sampling_params(expr.sampling_params) + name = expr.name + if not self.stream: + if self.num_api_spec_tokens is None: + comp, meta_info = self.backend.generate( + self, + sampling_params=sampling_params, + ) + + else: + if self.backend.is_chat_model: + # Speculative execution on models with only chat interface. + # Store the calls into a temporary list. + # They will be lazily executed later. + comp, meta_info = self.backend.generate( + self, + sampling_params=sampling_params, + spec_var_name=name, + ) + return + + else: # Speculative execution on models with completion interface + comp, meta_info = self._spec_gen(sampling_params) + if isinstance(comp, list): + self.text_ += comp[0] + else: + assert isinstance(comp, str) + self.text_ += comp + + self.variables[name] = comp + self.meta_info[name] = meta_info + self.variable_event[name].set() + else: + assert ( + self.num_api_spec_tokens is None + ), "stream is not supported with api speculative execution" + generator = self.backend.generate_stream( + self, sampling_params=sampling_params + ) + + self.variables[name] = "" + self.stream_var_event[name].set() + + for comp, meta_info in generator: + self.text_ += comp + self.variables[name] += comp + self.meta_info[name] = meta_info + self.stream_var_event[name].set() + self.stream_text_event.set() + + self.variable_event[name].set() + self.stream_var_event[name].set() + + def _execute_select(self, expr: SglSelect): + choices_decision = self.backend.select( + self, expr.choices, expr.temperature, expr.choices_method + ) + if expr.name is not None: + name = expr.name + self.variables[name] = choices_decision.decision + self.meta_info[name] = choices_decision.meta_info + self.variable_event[name].set() + if self.stream_var_event: + self.stream_var_event[name].set() + self.text_ += choices_decision.decision + + def _execute_variable(self, expr: SglVariable): + src_executor = expr.source_stream_executor + value = src_executor.get_var(expr.name) + self._execute_fill(value) + + def _execute_role_begin(self, expr: SglRoleBegin): + assert self.cur_role is None, "Nested roles are not allowed." + + if len(self.messages_) == 0 and expr.role != "system": + # Insert the default system message + default_system = self.chat_template.default_system_prompt + if default_system: + self._execute_role_begin(SglRoleBegin("system")) + self._execute_fill(default_system) + self._execute_role_end(SglRoleEnd("system")) + + self.cur_role = expr.role + + prefix, _ = self.chat_template.get_prefix_and_suffix(expr.role, self.messages_) + + self._execute_fill(prefix, prefix=True) + self.cur_role_begin_pos = len(self.text_) + + def _execute_role_end(self, expr: SglRoleEnd): + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + ): + # Execute the stored lazy generation calls + self.backend.role_end_generate(self) + self.cur_role = None + + new_text = self.text_[self.cur_role_begin_pos :].lstrip() + + _, suffix = self.chat_template.get_prefix_and_suffix(expr.role, self.messages_) + self._execute_fill(suffix) + + if self.cur_images: + # OpenAI vision API format + last_msg = { + "role": expr.role, + "content": [{"type": "text", "text": new_text}], + } + for image_path, image_base64_data in self.cur_images: + last_msg["content"].append( + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_base64_data}" + }, + } + ) + self.messages_.append(last_msg) + self.cur_images = [] + else: + # OpenAI chat API format + self.messages_.append({"role": expr.role, "content": new_text}) + + def _execute_var_scope_begin(self, expr: SglVarScopeBegin): + self.variables[expr.name] = int(len(self.text_)) + + def _execute_var_scope_end(self, expr: SglVarScopeEnd): + self.variables[expr.name] = self.text_[self.variables[expr.name] :] + self.variable_event[expr.name].set() + + def _execute_commit_lazy_operations(self, expr: SglCommitLazy): + self.backend.commit_lazy_operations(self) + + def _execute_concatenate_and_append_text(self, expr: SglConcateAndAppend): + new_text = "" + for s in expr.states: + exe = s.stream_executor + exe.sync() + new_text += exe.text_[exe.fork_start_text_pos :] + + self._execute_fill(new_text) + + def _execute_concatenate_and_append_kv_cache(self, expr: SglConcateAndAppend): + self_len = len(self.text_) + + for i, s in enumerate(expr.states): + exe = s.stream_executor + exe.submit(SglCommitLazy()) + + for i, s in enumerate(expr.states): + exe = s.stream_executor + exe.sync() + assert exe.fork_start_text_pos == self_len + self.text_ += exe.text_[exe.fork_start_text_pos :] + + src_rids = [state.stream_executor.sid for state in expr.states] + self.backend.concatenate_and_append(src_rids, self.sid) + + def _init_var_event(self, expr): + if isinstance(expr, (SglGen, SglSelect, SglVarScopeBegin)): + self.variable_event[expr.name] = threading.Event() + if self.stream: + self.stream_var_event[expr.name] = threading.Event() + elif isinstance(expr, SglExprList): + for e in expr.expr_list: + self._init_var_event(e) + + def _resolve_sampling_params(self, sampling_params): + """ + Construct sampling param based on default + override values + + The default values of sampling are populated in `default_sampling_para` via sgl.function.run(...sampling_args) + , and `sampling_params` contains the override values from sgl.gen(). + + Here we use default_sampling_para as the base and override the values if they exist in `sampling_params`. + It also extends the stop tokens based on the chat template. + """ + + # deepcopy is required because the dict has lists inside + clone = copy.deepcopy(self.default_sampling_para) + + for item in [ + "max_new_tokens", + "min_new_tokens", + "n", + "stop", + "stop_token_ids", + "temperature", + "top_p", + "top_k", + "min_p", + "frequency_penalty", + "presence_penalty", + "ignore_eos", + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + "dtype", + "regex", + "json_schema", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + setattr(clone, item, value) + + if self.chat_template.stop_str: + if clone.stop == (): + clone.stop = [] + elif isinstance(clone.stop, str): + clone.stop = [clone.stop] + clone.stop += self.chat_template.stop_str + + return clone + + def __del__(self): + self.end() + + +class ProgramState: + """The state of an SGL program.""" + + def __init__(self, stream_executor: StreamExecutor): + self.stream_executor = stream_executor + + def _role_common(self, name: str, expr: Optional[SglExpr] = None): + if expr is not None: + role_expr = SglExprList([SglRoleBegin(name), expr, SglRoleEnd(name)]) + self.stream_executor.submit(role_expr) + return role_expr + else: + + @contextmanager + def role_scope(): + self.stream_executor.submit(SglRoleBegin(name)) + yield + self.stream_executor.submit(SglRoleEnd(name)) + + return role_scope() + + def system(self, expr: Optional[SglExpr] = None): + return self._role_common("system", expr) + + def user(self, expr: Optional[SglExpr] = None): + return self._role_common("user", expr) + + def assistant(self, expr: Optional[SglExpr] = None): + return self._role_common("assistant", expr) + + @contextmanager + def var_scope(self, name: str): + self.stream_executor.submit(SglVarScopeBegin(name)) + yield + self.stream_executor.submit(SglVarScopeEnd(name)) + + def fork( + self, + size: int = 1, + position_ids_offset: Optional[List[int]] = None, + ): + stream_executors = self.stream_executor.fork(size, position_ids_offset) + states = [ProgramState(x) for x in stream_executors] + state_group = ProgramStateGroup(states, self) + return state_group + + @contextmanager + def copy(self, position_ids_offset: Optional[List[int]] = None): + state_group = self.fork(1, position_ids_offset) + try: + yield state_group[0] + finally: + state_group.join() + + def text(self): + return self.stream_executor.text() + + def messages(self): + return self.stream_executor.messages() + + def sync(self): + return self.stream_executor.sync() + + def error(self): + return self.stream_executor.error() + + def text_iter(self, var_name: Optional[str] = None): + if self.stream_executor.stream: + prev = 0 + if var_name is None: + event = self.stream_executor.stream_text_event + while True: + event.wait() + event.clear() + out = str(self.stream_executor.text_[prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.is_finished: + break + else: + event = None + while not event: + if var_name in self.stream_executor.stream_var_event: + event = self.stream_executor.stream_var_event[var_name] + if self.stream_executor.is_finished: + yield "" + return + + while True: + event.wait() + event.clear() + out = str(self.stream_executor.variables[var_name][prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.variable_event[var_name].is_set(): + break + else: + if var_name is None: + yield self.text() + else: + yield self.get_var(var_name) + + async def text_async_iter( + self, var_name: Optional[str] = None, return_meta_data: bool = False + ): + loop = asyncio.get_running_loop() + + if self.stream_executor.stream: + prev = 0 + if var_name is None: + event = self.stream_executor.stream_text_event + while True: + await loop.run_in_executor(None, event.wait) + event.clear() + out = str(self.stream_executor.text_[prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.is_finished: + break + else: + event = None + while not event: + if var_name in self.stream_executor.stream_var_event: + event = self.stream_executor.stream_var_event[var_name] + if self.stream_executor.is_finished: + yield "" + return + + while True: + await loop.run_in_executor(None, event.wait) + event.clear() + out = str(self.stream_executor.variables[var_name][prev:]) + prev += len(out) + if out: + if return_meta_data: + yield out, self.stream_executor.meta_info[var_name] + else: + yield out + if self.stream_executor.variable_event[var_name].is_set(): + break + else: + if var_name is None: + yield self.text() + else: + yield self.get_var(var_name) + + def get_var(self, name): + return self.stream_executor.get_var(name) + + def set_var(self, name, value): + return self.stream_executor.set_var(name, value) + + def get_meta_info(self, name): + return self.stream_executor.get_meta_info(name) + + def __iadd__(self, other): + if other is None: + raise ValueError("Tried to append None to state.") + self.stream_executor.submit(other) + return self + + def __getitem__(self, name): + return self.get_var(name) + + def __setitem__(self, name, value): + self.set_var(name, value) + + def __contains__(self, name): + return name in self.stream_executor.variables + + def __del__(self): + self.stream_executor.end() + + def __repr__(self) -> str: + return f"ProgramState({self.text()})" + + +class ProgramStateGroup: + def __init__( + self, states: List[ProgramState], src_state: Optional[ProgramState] = None + ): + self.states = states + self.src_state = src_state + + def join(self, mode: str = "gather_variable"): + if mode == "gather_variable": + # Copy variables back + src_vars = self.src_state.stream_executor.variables + src_var_set = set(src_vars.keys()) + for child_state in self.states: + child_state.stream_executor.sync() + child_vars = child_state.stream_executor.variables + new_vars = set(child_vars.keys()) - src_var_set + + for k in new_vars: + if k in src_vars: + src_vars[k].append(child_vars[k]) + else: + src_vars[k] = [child_vars[k]] + elif mode == "concate_and_append": + # Concatenate and append KV cache + self.src_state += SglConcateAndAppend(self.states) + # Need a sync here. Otherwise, `states` can be deleted. + self.src_state.stream_executor.sync() + else: + raise ValueError(f"Invalid join mode: {mode}") + + for s in self.states: + s.stream_executor.end() + + def __getitem__(self, i: int): + return self.states[i] + + def __setitem__(self, i: int, value): + assert self.states[i] == value + + def __iadd__(self, other): + if isinstance(other, Callable): + # lambda function + for i in range(len(self.states)): + self.states[i] += other(i) + elif isinstance(other, SglExpr): + for i in range(len(self.states)): + self.states[i] += other + elif isinstance(other, (list, tuple)): + for i in range(len(self.states)): + self.states[i] += other[i] + else: + raise ValueError(f"Invalid value: {other}") + + return self diff --git a/python/sglang/lang/ir.py b/python/sglang/lang/ir.py new file mode 100644 index 0000000..f1775bc --- /dev/null +++ b/python/sglang/lang/ir.py @@ -0,0 +1,608 @@ +"""The intermediate representation.""" + +import dataclasses +import inspect +import warnings +from typing import List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.choices import ChoicesSamplingMethod + +REGEX_INT = r"[-+]?[0-9]+[ \n]*" +REGEX_FLOAT = r"[-+]?[0-9]*\.?[0-9]+[ \n]*" +REGEX_BOOL = r"(True|False)" +REGEX_STR = r"\"[\w\d\s]*\"" # bugs with regex r"\".*\"" in interegular pkg + + +@dataclasses.dataclass +class SglSamplingParams: + max_new_tokens: int = 128 + min_new_tokens: int = 0 + n: int = 1 + stop: Union[str, List[str]] = () + stop_token_ids: Optional[List[int]] = () + temperature: float = 1.0 + top_p: float = 1.0 + top_k: int = -1 # -1 means disable + min_p: float = 0.0 + frequency_penalty: float = 0.0 + presence_penalty: float = 0.0 + ignore_eos: bool = False + return_logprob: Optional[bool] = None + logprob_start_len: Optional[int] = (None,) + top_logprobs_num: Optional[int] = (None,) + return_text_in_logprobs: Optional[bool] = (None,) + json_schema: Optional[str] = None + + # for constrained generation, not included in to_xxx_kwargs + dtype: Optional[str] = None + regex: Optional[str] = None + + def clone(self): + return SglSamplingParams( + self.max_new_tokens, + self.min_new_tokens, + self.n, + self.stop, + self.stop_token_ids, + self.temperature, + self.top_p, + self.top_k, + self.min_p, + self.frequency_penalty, + self.presence_penalty, + self.ignore_eos, + self.return_logprob, + self.logprob_start_len, + self.top_logprobs_num, + self.return_text_in_logprobs, + self.json_schema, + ) + + def to_openai_kwargs(self): + # OpenAI does not support top_k, so we drop it here + if self.regex is not None: + warnings.warn("Regular expression is not supported in the OpenAI backend.") + return { + "max_tokens": self.max_new_tokens, + "max_completion_tokens": self.max_new_tokens, + "n": self.n, + "stop": self.stop or None, + "temperature": self.temperature, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + } + + def to_vertexai_kwargs(self): + if self.regex is not None: + warnings.warn( + "Regular expression is not supported in the VertexAI backend." + ) + return { + "candidate_count": 1, + "max_output_tokens": self.max_new_tokens, + "stop_sequences": self.stop, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k if self.top_k > 0 else None, + } + + def to_anthropic_kwargs(self): + # Anthropic does not support frequency_penalty or presence_penalty, so we drop it here + if self.regex is not None: + warnings.warn( + "Regular expression is not supported in the Anthropic backend." + ) + return { + "max_tokens": self.max_new_tokens, + "stop_sequences": ( + self.stop if isinstance(self.stop, (list, tuple)) else [self.stop] + ), + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + } + + def to_litellm_kwargs(self): + if self.regex is not None: + warnings.warn("Regular expression is not supported in the LiteLLM backend.") + return { + "max_tokens": self.max_new_tokens, + "stop": self.stop or None, + "temperature": self.temperature, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + } + + def to_srt_kwargs(self): + return { + "max_new_tokens": self.max_new_tokens, + "min_new_tokens": self.min_new_tokens, + "n": self.n, + "stop": self.stop, + "stop_token_ids": self.stop_token_ids, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "min_p": self.min_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + "ignore_eos": self.ignore_eos, + "regex": self.regex, + "json_schema": self.json_schema, + } + + +class SglFunction: + def __init__(self, func, num_api_spec_tokens=None, bind_arguments=None): + self.func = func + self.num_api_spec_tokens = num_api_spec_tokens + self.bind_arguments = bind_arguments or {} + self.pin_prefix_rid = None + + # Parse arguments + argspec = inspect.getfullargspec(func) + assert argspec.args[0] == "s", 'The first argument must be "s"' + self.arg_names = argspec.args[1:] + self.arg_defaults = argspec.defaults if argspec.defaults is not None else [] + + def bind(self, **kwargs): + assert all(key in self.arg_names for key in kwargs) + + new_bind_dict = {**self.bind_arguments, **kwargs} + return SglFunction(self.func, bind_arguments=new_bind_dict) + + def run( + self, + *args, + max_new_tokens: int = 128, + n: int = 1, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + ignore_eos: bool = False, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + stream: bool = False, + backend=None, + use_thread: bool = True, + **kwargs, + ): + from sglang.lang.interpreter import run_program + + # avoid using [] as the default arg: https://nikos7am.com/posts/mutable-default-arguments/ + if stop is None: + stop = [] + if stop_token_ids is None: + stop_token_ids = [] + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + ) + backend = backend or global_config.default_backend + return run_program( + self, + backend, + args, + kwargs, + default_sampling_para, + stream, + use_thread=use_thread, + ) + + def run_batch( + self, + batch_kwargs, + *, + max_new_tokens: int = 128, + n: int = 1, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + ignore_eos: bool = False, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + backend=None, + num_threads: Union[str, int] = "auto", + progress_bar: bool = False, + generator_style: bool = False, + ): + from sglang.lang.interpreter import run_program_batch + + if stop is None: + stop = [] + if stop_token_ids is None: + stop_token_ids = [] + + assert isinstance(batch_kwargs, (list, tuple)) + if len(batch_kwargs) == 0: + return [] + if not isinstance(batch_kwargs[0], dict): + num_programs = len(batch_kwargs) + # change the list of argument values to dict of arg_name -> arg_value + batch_kwargs = [ + {self.arg_names[i]: v for i, v in enumerate(arg_values)} + for arg_values in batch_kwargs + if isinstance(arg_values, (list, tuple)) + and len(self.arg_names) - len(self.arg_defaults) + <= len(arg_values) + <= len(self.arg_names) + ] + # Ensure to raise an exception if the number of arguments mismatch + if len(batch_kwargs) != num_programs: + raise Exception("Given arguments mismatch the SGL function signature") + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + ) + backend = backend or global_config.default_backend + return run_program_batch( + self, + backend, + batch_kwargs, + default_sampling_para, + num_threads, + progress_bar, + generator_style=generator_style, + ) + + def trace(self, *, backend=None, **kwargs): + from sglang.lang.tracer import trace_program + + backend = backend or global_config.default_backend + return trace_program(self, kwargs, backend) + + def cache(self, backend=None): + from sglang.lang.interpreter import cache_program + + backend = backend or global_config.default_backend + return cache_program(self, backend) + + def compile(self, *, backend=None): + from sglang.lang.compiler import compile_func + + return compile_func(self, backend) + + def __call__(self, *args, **kwargs): + from sglang.lang.tracer import TracingScope + + tracing_scope = TracingScope.get_current_scope() + if tracing_scope is None: + return self.run(*args, **kwargs) + else: + kwargs["backend"] = tracing_scope.tracer_state.backend + return self.trace(*args, **kwargs) + + +class SglExpr: + node_ct = 0 + + def __init__(self): + self.node_id = SglExpr.node_ct + self.prev_node = None + self.pid = None + SglExpr.node_ct += 1 + + def __add__(self, other): + if isinstance(other, str): + other = SglConstantText(other) + assert isinstance(other, SglExpr) + + return self.concatenate_ir(self, other) + + def __radd__(self, other): + if isinstance(other, str): + other = SglConstantText(other) + assert isinstance(other, SglExpr), f"{other}" + + return self.concatenate_ir(other, self) + + def concatenate_ir(self, a, b): + if isinstance(a, SglExprList): + if isinstance(b, SglExprList): + return SglExprList(a.expr_list + b.expr_list) + else: + return SglExprList(a.expr_list + [b]) + elif isinstance(b, SglExprList): + return SglExprList([a] + b.expr_list) + + return SglExprList([a, b]) + + def print_graph_dfs(self): + ret = [""] + visited = set() + + def dfs_print(x): + if x is None or x in visited: + return + visited.add(x) + + # Print dependency + if x.prev_node is not None: + dfs_print(x.prev_node) + + if isinstance(x, SglExprList): + for y in x.expr_list: + dfs_print(y) + # elif isinstance(x, SglRole): + # dfs_print(x.expr) + elif isinstance(x, SglVariable): + dfs_print(x.source) + + # Print the node itself + if isinstance(x, (SglFork, SglGetForkItem)): + ret[0] += f"%{x.node_id} = {x}\n" + else: + if x.prev_node is not None: + ret[0] += ( + f"%{x.node_id} = %{x.prev_node.node_id} + " + str(x) + "\n" + ) + else: + ret[0] += f"%{x.node_id} = " + str(x) + "\n" + + dfs_print(self) + return ret[0] + + +class SglExprList(SglExpr): + def __init__(self, expr_list: List[SglExpr]): + super().__init__() + self.expr_list = expr_list + + def __repr__(self): + return f"ExprList({self.expr_list})" + + +class SglArgument(SglExpr): + def __init__(self, name: str, value: str): + super().__init__() + self.name = name + self.value = value + + def __repr__(self): + return f"Argument(name={self.name}, value={repr(self.value)})" + + def __len__(self): + return len(self.value) + + def __getitem__(self, i): + return self.value[i] + + def __int__(self): + return self.value + + def __bool__(self): + return self.value + + def __format__(self, *args): + raise TypeError( + "Cannot put argument inside a f-string. " + "This is not compatible with the tracer. " + ) + + +class SglImage(SglExpr): + def __init__(self, path: str): + self.path = path + + def __repr__(self) -> str: + return f"SglImage({self.path})" + + +class SglVideo(SglExpr): + def __init__(self, path: str, num_frames: int): + self.path = path + self.num_frames = num_frames + + def __repr__(self) -> str: + return f"SglVideo({self.path}, {self.num_frames})" + + +class SglGen(SglExpr): + def __init__( + self, + name: Optional[str] = None, + max_new_tokens: Optional[int] = None, + min_new_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + dtype: Optional[type] = None, + regex: Optional[str] = None, + json_schema: Optional[str] = None, + ): + """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + super().__init__() + self.name = name + self.sampling_params = SglSamplingParams( + max_new_tokens=max_new_tokens, + min_new_tokens=min_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + dtype=dtype, + regex=regex, + json_schema=json_schema, + ) + + def __repr__(self): + return f"Gen('{self.name}')" + + +class SglConstantText(SglExpr): + def __init__(self, value: str): + super().__init__() + self.value = value + + def __repr__(self): + return f"Constant({repr(self.value)})" + + +class SglRoleBegin(SglExpr): + def __init__(self, role: str): + super().__init__() + self.role = role + + def __repr__(self): + return f"RoleBegin({self.role})" + + +class SglRoleEnd(SglExpr): + def __init__(self, role: str): + super().__init__() + self.role = role + + def __repr__(self): + return f"RoleEnd({self.role})" + + +class SglSelect(SglExpr): + + def __init__( + self, + name: str, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ): + super().__init__() + self.name = name + self.choices = choices + self.temperature = temperature + self.choices_method = choices_method + + def __repr__(self): + return f"Select({self.name}, choices={self.choices}, choices_method={self.choices_method})" + + +class SglFork(SglExpr): + def __init__(self, number: int, position_ids_offset=None): + super().__init__() + self.number = number + self.position_ids_offset = position_ids_offset + + def __repr__(self): + return ( + f"Fork(%{self.prev_node.node_id}, number={self.number}, " + f"position_ids_offset={self.position_ids_offset})" + ) + + +class SglGetForkItem(SglExpr): + def __init__(self, index: int): + super().__init__() + self.index = index + + def __repr__(self): + return f"GetForkItem(%{self.prev_node.node_id}, index={self.index})" + + +class SglVariable(SglExpr): + def __init__(self, name: str, source): + super().__init__() + self.name = name + self.source = source + + def __repr__(self): + return f"Variable('{self.name}', source=%{self.source.node_id})" + + +class SglVarScopeBegin(SglExpr): + def __init__(self, name: str): + super().__init__() + self.name = name + + def __repr__(self): + return f"VarScopeBegin('{self.name}')" + + +class SglVarScopeEnd(SglExpr): + def __init__(self, name: str): + super().__init__() + self.name = name + + def __repr__(self): + return f"VarScopeEnd('{self.name}')" + + +class SglConcateAndAppend(SglExpr): + def __init__(self, states): + super().__init__() + self.states = states + + def __repr__(self): + return f"ConcatenateAndAppend('{self.states}')" + + +class SglCommitLazy(SglExpr): + def __init__(self): + super().__init__() + + def __repr__(self): + return "CommitLazy()" diff --git a/python/sglang/lang/tracer.py b/python/sglang/lang/tracer.py new file mode 100644 index 0000000..69f035b --- /dev/null +++ b/python/sglang/lang/tracer.py @@ -0,0 +1,283 @@ +"""Tracing a program.""" + +import uuid +from typing import Any, Callable, Dict, List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.interpreter import ProgramState, ProgramStateGroup +from sglang.lang.ir import ( + SglArgument, + SglCommitLazy, + SglConcateAndAppend, + SglConstantText, + SglExpr, + SglExprList, + SglFork, + SglFunction, + SglGen, + SglGetForkItem, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglVariable, + SglVarScopeBegin, + SglVarScopeEnd, +) + + +class StopTracing(Exception): + pass + + +def extract_prefix_by_tracing(program, backend): + # Create dummy arguments + dummy_arguments = {name: SglArgument(name, None) for name in program.arg_names} + arguments = dummy_arguments + arguments.update(program.bind_arguments) + + # Trace + tracer = TracerProgramState(backend, arguments, only_trace_prefix=True) + try: + with TracingScope(tracer): + tracer.ret_value = program.func(tracer, **arguments) + except (StopTracing, TypeError, AttributeError): + # Some exceptions may not be catched + pass + + # Run and cache prefix + prefix = "" + for expr in tracer.flatten_nodes(): + if isinstance(expr, SglConstantText): + prefix += expr.value + else: + break + return prefix + + +def trace_program(program, arguments, backend): + # Create dummy backend + if backend is None: + backend = BaseBackend() + + # Create dummy arguments + dummy_arguments = { + name: SglArgument(name, None) + for name in program.arg_names + if name not in arguments + } + arguments.update(dummy_arguments) + arguments.update(program.bind_arguments) + + # Trace + tracer = TracerProgramState(backend, arguments, only_trace_prefix=False) + with TracingScope(tracer): + tracer.ret_value = program.func(tracer, **arguments) + return tracer + + +class TracerProgramState(ProgramState): + def __init__(self, backend, arguments, only_trace_prefix): + self.pid = uuid.uuid4().hex + self.backend = backend + self.arguments: Dict[str, Any] = arguments + self.only_trace_prefix = only_trace_prefix + + if hasattr(backend, "endpoint"): + self.backend = backend.endpoint + + self.nodes = [] + self.last_node = None + self.variables = {} + self.ret_value = None + + # For completion + + # For chat + self.messages_ = [] + self.cur_role = None + self.chat_template = self.backend.get_chat_template() + + # For multi states + self.child_states = [] + + cur_scope = TracingScope.get_current_scope() + if cur_scope is not None: + cur_scope.add_child_state(self) + + ################################## + ########### Public API ########### + ################################## + + def fork(self, size: int = 1, position_ids_offset: Optional[List[int]] = None): + assert size >= 1 + + if self.only_trace_prefix: + raise StopTracing() + + fork_node = SglFork(size) + fork_node.prev_node = self.last_node + + states = [ + TracerProgramState(self.backend, self.arguments, self.only_trace_prefix) + for _ in range(size) + ] + + for i in range(size): + node = SglGetForkItem(i) + node.prev_node = fork_node + states[i].last_node = node + states[i].variables = dict(self.variables) + states[i].messages_ = list(self.messages_) + states[i].cur_role = self.cur_role + states[i].chat_template = self.chat_template + + state_group = ProgramStateGroup(states, self) + + return state_group + + ################################## + ########## Internal API ########## + ################################## + + def _append_node(self, other: SglExpr): + self.nodes.append(other) + other.prev_node = self.last_node + self.last_node = other + + def _execute(self, other: SglExpr): + if isinstance(other, str): + other = SglConstantText(other) + + other.pid = self.pid + + if isinstance(other, SglConstantText): + self._execute_fill(other) + elif isinstance(other, SglGen): + self._execute_gen(other) + elif isinstance(other, SglSelect): + self._execute_select(other) + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute(x) + elif isinstance(other, SglRoleBegin): + self._execute_role_begin(other) + elif isinstance(other, SglRoleEnd): + self._execute_role_end(other) + elif isinstance(other, SglVarScopeBegin): + self._execute_var_scope_begin(other) + elif isinstance(other, SglVarScopeEnd): + self._execute_var_scope_end(other) + else: + if self.only_trace_prefix: + raise StopTracing() + else: + self._append_node(other) + + return self + + def __iadd__(self, other): + self._execute(other) + return self + + def _execute_fill(self, expr: SglConstantText): + if isinstance(expr, str): + expr = SglConstantText(expr) + self._append_node(expr) + + def _execute_gen(self, expr: SglGen): + name = expr.name if expr.name is not None else "gen_" + str(len(self.variables)) + new_node = SglVariable(name, source=expr) + self.variables[name] = new_node + self._append_node(expr) + + def _execute_select(self, expr: SglSelect): + name = ( + expr.name if expr.name is not None else "select_" + str(len(self.variables)) + ) + new_node = SglVariable(name, source=expr) + self.variables[name] = new_node + self._append_node(expr) + + def _execute_role_begin(self, expr: SglRoleBegin): + assert self.cur_role is None, "Nested roles are not allowed." + + if len(self.messages_) == 0 and expr.role != "system": + # Insert default system message + default_system = self.chat_template.default_system_prompt + if default_system: + self._execute_role_begin(SglRoleBegin("system")) + self._execute_fill(default_system) + self._execute_role_end(SglRoleEnd("system")) + + self.cur_role = expr.role + + prefix, suffix = self.chat_template.get_prefix_and_suffix( + expr.role, self.messages_ + ) + + self._execute_fill(prefix) + + def _execute_role_end(self, expr: SglRoleEnd): + prefix, suffix = self.chat_template.get_prefix_and_suffix( + expr.role, self.messages_ + ) + + self._execute_fill(suffix) + + self.messages_.append({"role": expr.role, "content": ""}) + + self.cur_role = None + + def _execute_var_scope_end(self, expr: SglVarScopeEnd): + new_node = SglVariable(name, source=self.last_node) + self.variables[name] = new_node + + def get_var(self, name): + ret = self.arguments.get(name, None) + if ret is not None: + return ret + + v = self.variables[name] + return SglVariable(v.name, v.source) + + def flatten_nodes(self): + def traverse(cur): + if isinstance(cur, SglExprList): + for child in cur.expr_list: + traverse(child) + else: + ret.append(cur) + + ret = [] + for x in self.nodes: + traverse(x) + return ret + + def __del__(self): + pass + + +class TracingScope: + cur_scope = None + + def __init__(self, tracer_state: TracerProgramState): + self.tracer_state = tracer_state + self.last_scope = TracingScope.cur_scope + + def __enter__(self): + TracingScope.cur_scope = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + TracingScope.cur_scope = self.last_scope + + @staticmethod + def get_current_scope(): + return TracingScope.cur_scope + + def add_child_state(self, state: TracerProgramState): + cur_scope = self + while cur_scope is not None: + cur_scope.tracer_state.child_states.append(state) + cur_scope = cur_scope.last_scope diff --git a/python/sglang/launch_server.py b/python/sglang/launch_server.py new file mode 100644 index 0000000..caae7b0 --- /dev/null +++ b/python/sglang/launch_server.py @@ -0,0 +1,16 @@ +"""Launch the inference server.""" + +import os +import sys + +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import prepare_server_args +from sglang.srt.utils import kill_process_tree + +if __name__ == "__main__": + server_args = prepare_server_args(sys.argv[1:]) + + try: + launch_server(server_args) + finally: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/python/sglang/llama3_eval.py b/python/sglang/llama3_eval.py new file mode 100644 index 0000000..35bd4a7 --- /dev/null +++ b/python/sglang/llama3_eval.py @@ -0,0 +1,316 @@ +# Adapt from https://github.com/fw-ai/llm_eval_meta + +import argparse +import asyncio +import os +import pickle +import re +import shutil +from collections import defaultdict +from dataclasses import dataclass + +import httpx +import numpy as np +import openai +import transformers +from datasets import load_dataset +from openai import AsyncOpenAI +from tqdm import tqdm + +# Mapping providers to their clients and models +provider_to_models = { + "b10": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, + "oai": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, + "sgl": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, +} + + +async def fetch_responses( + client, prompt, semaphore, index, provider, model_size, output_dir, max_tokens +): + output_file = os.path.join(output_dir, f"response_{index}.pkl") + if os.path.exists(output_file): + print(f"File {output_file} already exists, skipping.") + return + + async with semaphore: + response = await client.completions.create( + model=provider_to_models[provider][model_size], + prompt=prompt, + temperature=0.0, + max_tokens=max_tokens, + ) + if isinstance(response, openai.BadRequestError): + with open(output_file, "wb") as f: + pickle.dump("bad_response", f) + assert isinstance(response, openai.types.completion.Completion) + # Save response to a file + with open(output_file, "wb") as f: + pickle.dump(response, f) + + +TASK_TO_MAX_TOKENS = { + "evals__mmlu__details": 1, + "evals__mmlu__0_shot__cot__details": 1024, + # Official meta uses 1024, but a small % (.05) of questions are answered correctly after relaxing + "evals__mmlu_pro__details": 2048, + "evals__gsm8k__details": 1024, +} + +TASK_TO_EVAL_SET = { + "mmlu": "evals__mmlu__details", + "mmlu_cot": "evals__mmlu__0_shot__cot__details", + "mmlu_pro": "evals__mmlu_pro__details", + "gsm8k": "evals__gsm8k__details", +} + + +class CustomAsyncHTTPXClient(httpx.AsyncClient): + async def send(self, request: httpx.Request, *args, **kwargs) -> httpx.Response: + request.url = httpx.URL( + f"https://model-{os.getenv('MODEL_ID')}.api.baseten.co/development/predict" + ) + return await super().send(request, *args, **kwargs) + + +def get_client(provider): + if provider not in "b10": + if os.getenv("OPENAI_API_KEY") == None: + os.environ["OPENAI_API_KEY"] = "EMPTY" + return { + "oai": AsyncOpenAI(base_url="http://127.0.0.1:8000/v1/"), + "b10": AsyncOpenAI( + api_key=f"Api-Key {os.getenv('OPENAI_API_KEY')}", + base_url=f"https://model-{os.getenv('MODEL_ID')}.api.baseten.co/development/predict", + http_client=CustomAsyncHTTPXClient(), + ), + "sgl": AsyncOpenAI(base_url="http://127.0.0.1:30000/v1/"), + }[provider] + + +# Define the benchmark function +async def benchmark(args): + ds = load_dataset( + "meta-llama/Llama-3.1-405B-Instruct-evals", + f"Llama-3.1-405B-Instruct-{TASK_TO_EVAL_SET[args.task]}", + ) + semaphore = asyncio.Semaphore(args.concurrency) # Limit to 16 concurrent tasks + + if args.num_examples is None: + args.num_examples = len(ds["latest"]["input_final_prompts"]) + prompts = ds["latest"]["input_final_prompts"][: args.num_examples] + + # Create the output directory if it does not exist + os.makedirs(args.output_dir, exist_ok=True) + + tasks = [] + # Create the tasks with tqdm progress bar + max_tokens = TASK_TO_MAX_TOKENS[TASK_TO_EVAL_SET[args.task]] + client = get_client(args.provider) + for idx, prompt in enumerate(tqdm(prompts, desc="Creating tasks")): + tasks.append( + asyncio.create_task( + fetch_responses( + client, + f"<|begin_of_text|>{prompt[0]}", + semaphore, + idx, + args.provider, + args.model_size, + args.output_dir, + max_tokens=max_tokens, + ) + ) + ) + + # Run the tasks with tqdm progress bar + for future in tqdm( + asyncio.as_completed(tasks), total=len(tasks), desc="Processing tasks" + ): + await future + + +def get_mmlu_answer(response): + if response is not None: + return response.choices[0].text.lstrip().rstrip().upper().replace(".", "") + return None + + +def get_mmlu_cot_answer(response): + pattern = r"The best answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "").replace("*", "") + + pattern = r"the best answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + pattern = r"The correct answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + pattern = r"the correct answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + +def get_answer_gsm8k(response): + pattern = r"The final answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + s = match.group(1) + for ok_symbol in ["%", "$"]: + s = s.replace(ok_symbol, "") + return s + + +TASK_TO_ANSWER_EXTRACTOR = { + "evals__mmlu__details": get_mmlu_answer, + "evals__mmlu__0_shot__cot__details": get_mmlu_cot_answer, + "evals__gsm8k__details": get_answer_gsm8k, + "evals__mmlu_pro__details": get_mmlu_cot_answer, +} + + +def get_dataset_from_task(task, response_path, model_size): + ds_405b = load_dataset( + f"meta-llama/Llama-3.1-405B-Instruct-evals", + f"Llama-3.1-405B-Instruct-{task}", + ) + ds_405b_hash_order = [x[0] for x in ds_405b["latest"]["input_final_prompts_hash"]] + + if "70b" in model_size or "8b" in model_size: + if "70" in model_size: + ref_model_ds = load_dataset( + f"meta-llama/Llama-3.1-70B-Instruct-evals", + f"Llama-3.1-70B-Instruct-{task}", + ) + else: + ref_model_ds = load_dataset( + f"meta-llama/Llama-3.1-8B-Instruct-evals", + f"Llama-3.1-8B-Instruct-{task}", + ) + + hash_to_row = {} + for row in ref_model_ds["latest"]: + hash_to_row[row["input_final_prompts_hash"][0]] = row + reordered_rows = [] + for prompt_hash in ds_405b_hash_order: + reordered_rows.append(hash_to_row[prompt_hash]) + ref_model_ds["latest"] = reordered_rows + return ref_model_ds + + return ds_405b + + +def analyze(task, response_path, model_size): + ds = get_dataset_from_task(task, response_path, model_size) + + responses = [] + total = len(ds["latest"]) + + for i in range(0, total): + response = pickle.load( + open(os.path.join(response_path, f"response_{i}.pkl"), "rb") + ) + responses.append(response) + + @dataclass + class Stats: + correct: int = 0 + total: int = 0 + meta_correct: int = 0 + + average: float = None + + subtask_name_to_stats = defaultdict(lambda: Stats()) + + for response, ds_row in zip(responses, ds["latest"]): + model_answer = TASK_TO_ANSWER_EXTRACTOR[task](response) + + subtask = ds_row["subtask_name"] + + is_eval_correct = model_answer in ds_row["input_correct_responses"] + if is_eval_correct: + subtask_name_to_stats[subtask].correct += 1 + + if ds_row["is_correct"]: + subtask_name_to_stats[subtask].meta_correct += 1 + + subtask_name_to_stats[subtask].total += 1 + + micro_stats = Stats() + for subtask, stats in subtask_name_to_stats.items(): + stats.average = stats.correct / stats.total + stats.meta_average = stats.meta_correct / stats.total + + micro_stats.correct += stats.correct + micro_stats.total += stats.total + micro_stats.meta_correct += stats.meta_correct + + micro_stats.average = micro_stats.correct / micro_stats.total + micro_stats.meta_average = micro_stats.meta_correct / micro_stats.total + + print("Macro average", np.mean([x.average for x in subtask_name_to_stats.values()])) + print( + "Meta Macro average", + np.mean([x.meta_average for x in subtask_name_to_stats.values()]), + ) + print("Micro average", micro_stats.average) + print("Meta Micro average", micro_stats.meta_average) + + +# Entry point for the script +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Script to run model with specified parameters." + ) + parser.add_argument( + "--model-size", + type=str, + default="8b", + help="Size of the model (e.g., 8b or 70b)", + ) + parser.add_argument( + "--provider", + type=str, + default="sgl", + help="Provider name (e.g., sgl, oai, b10)", + ) + parser.add_argument( + "--task", + type=str, + required=True, + help="Task (e.g., mmlu, mmlu_cot, mmlu_pro, gsm8k)", + ) + parser.add_argument( + "--num-examples", type=int, default=None, help="Number of examples to process" + ) + parser.add_argument("--concurrency", type=int, default=16) + parser.add_argument( + "--output-dir", + type=str, + default="tmp-output-dir", + help="Directory to save responses", + ) + + args = parser.parse_args() + asyncio.run(benchmark(args)) + analyze(TASK_TO_EVAL_SET[args.task], args.output_dir, args.model_size) + shutil.rmtree("tmp-output-dir", ignore_errors=True) diff --git a/python/sglang/srt/_custom_ops.py b/python/sglang/srt/_custom_ops.py new file mode 100644 index 0000000..d0bc512 --- /dev/null +++ b/python/sglang/srt/_custom_ops.py @@ -0,0 +1,150 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/_custom_ops.py +import logging +import os +from typing import List, Tuple + +import torch +import torch.library + +from sglang.srt.utils import get_bool_env_var, is_hip, is_hpu + +logger = logging.getLogger(__name__) +use_vllm_custom_allreduce = get_bool_env_var( + "USE_VLLM_CUSTOM_ALLREDUCE", default="false" +) + +if not is_hpu(): + # ROCm does not use vllm custom allreduce + if use_vllm_custom_allreduce and not is_hip(): + try: + import vllm._C + except ImportError as e: + logger.warning("Failed to import from vllm._C with %r", e) + else: + try: + import sgl_kernel + except ImportError as e: + logger.warning("Failed to import from custom_ar with %r", e) + + +if use_vllm_custom_allreduce and not is_hip(): + # vLLM custom allreduce + def init_custom_ar( + ipc_tensors: List[torch.Tensor], + rank_data: torch.Tensor, + rank: int, + full_nvlink: bool, + ) -> int: + return torch.ops._C_custom_ar.init_custom_ar( + ipc_tensors, rank_data, rank, full_nvlink + ) + + def all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + reg_buffer: int, + reg_buffer_sz_bytes: int, + ) -> None: + torch.ops._C_custom_ar.all_reduce(fa, inp, out, reg_buffer, reg_buffer_sz_bytes) + + def dispose(fa: int) -> None: + torch.ops._C_custom_ar.dispose(fa) + + def meta_size() -> int: + return torch.ops._C_custom_ar.meta_size() + + def register_buffer(fa: int, ipc_tensors: List[int]) -> None: + return torch.ops._C_custom_ar.register_buffer(fa, ipc_tensors) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]: + return torch.ops._C_custom_ar.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[List[int]], offsets: List[List[int]] + ) -> None: + torch.ops._C_custom_ar.register_graph_buffers(fa, handles, offsets) + +else: + if is_hip(): + # ROCM custom allreduce + + def init_custom_ar( + meta: torch.Tensor, + rank_data: torch.Tensor, + handles: List[str], + offsets: List[int], + rank: int, + full_nvlink: bool, + ) -> int: + return sgl_kernel.allreduce.init_custom_ar( + meta, rank_data, handles, offsets, rank, full_nvlink + ) + + def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + sgl_kernel.allreduce.all_reduce_reg(fa, inp, out) + + def all_reduce_unreg( + fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, out: torch.Tensor + ) -> None: + sgl_kernel.allreduce.all_reduce_unreg(fa, inp, reg_buffer, out) + + def dispose(fa: int) -> None: + sgl_kernel.allreduce.dispose(fa) + + def meta_size() -> int: + return sgl_kernel.allreduce.meta_size() + + def register_buffer( + fa: int, t: torch.Tensor, handles: List[str], offsets: List[int] + ) -> None: + return sgl_kernel.allreduce.register_buffer(fa, t, handles, offsets) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[torch.Tensor, List[int]]: + return sgl_kernel.allreduce.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[str], offsets: List[List[int]] + ) -> None: + sgl_kernel.allreduce.register_graph_buffers(fa, handles, offsets) + + def allocate_meta_buffer(size: int) -> torch.Tensor: + return sgl_kernel.allreduce.allocate_meta_buffer(size) + + def get_meta_buffer_ipc_handle(inp: torch.Tensor) -> torch.Tensor: + return sgl_kernel.allreduce.get_meta_buffer_ipc_handle(inp) + + else: + # TRTLLM custom allreduce + def init_custom_ar( + rank_id: int, + world_size: int, + rank_data_base: torch.Tensor, + buffers: List[int], + tmp_result_buffers: List[int], + barrier_in: List[int], + barrier_out: List[int], + ) -> int: + return sgl_kernel.init_custom_reduce( + rank_id, + world_size, + rank_data_base, + buffers, + tmp_result_buffers, + barrier_in, + barrier_out, + ) + + def all_reduce(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + sgl_kernel.custom_reduce(fa, inp, out) + + def dispose(fa: int) -> None: + sgl_kernel.custom_dispose(fa) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]: + return sgl_kernel.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[List[int]], offsets: List[List[int]] + ) -> None: + sgl_kernel.register_graph_buffers(fa, handles, offsets) diff --git a/python/sglang/srt/aio_rwlock.py b/python/sglang/srt/aio_rwlock.py new file mode 100644 index 0000000..deda1fe --- /dev/null +++ b/python/sglang/srt/aio_rwlock.py @@ -0,0 +1,100 @@ +import asyncio + + +class RWLock: + def __init__(self): + # Protects internal state + self._lock = asyncio.Lock() + + # Condition variable used to wait for state changes + self._cond = asyncio.Condition(self._lock) + + # Number of readers currently holding the lock + self._readers = 0 + + # Whether a writer is currently holding the lock + self._writer_active = False + + # How many writers are queued waiting for a turn + self._waiting_writers = 0 + + @property + def reader_lock(self): + """ + A context manager for acquiring a shared (reader) lock. + + Example: + async with rwlock.reader_lock: + # read-only access + """ + return _ReaderLock(self) + + @property + def writer_lock(self): + """ + A context manager for acquiring an exclusive (writer) lock. + + Example: + async with rwlock.writer_lock: + # exclusive access + """ + return _WriterLock(self) + + async def acquire_reader(self): + async with self._lock: + # Wait until there is no active writer or waiting writer + # to ensure fairness. + while self._writer_active or self._waiting_writers > 0: + await self._cond.wait() + self._readers += 1 + + async def release_reader(self): + async with self._lock: + self._readers -= 1 + # If this was the last reader, wake up anyone waiting + # (potentially a writer or new readers). + if self._readers == 0: + self._cond.notify_all() + + async def acquire_writer(self): + async with self._lock: + # Increment the count of writers waiting + self._waiting_writers += 1 + try: + # Wait while either a writer is active or readers are present + while self._writer_active or self._readers > 0: + await self._cond.wait() + self._writer_active = True + finally: + # Decrement waiting writers only after we've acquired the writer lock + self._waiting_writers -= 1 + + async def release_writer(self): + async with self._lock: + self._writer_active = False + # Wake up anyone waiting (readers or writers) + self._cond.notify_all() + + +class _ReaderLock: + def __init__(self, rwlock: RWLock): + self._rwlock = rwlock + + async def __aenter__(self): + await self._rwlock.acquire_reader() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._rwlock.release_reader() + + +class _WriterLock: + def __init__(self, rwlock: RWLock): + self._rwlock = rwlock + + async def __aenter__(self): + await self._rwlock.acquire_writer() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._rwlock.release_writer() diff --git a/python/sglang/srt/code_completion_parser.py b/python/sglang/srt/code_completion_parser.py new file mode 100644 index 0000000..94a98b0 --- /dev/null +++ b/python/sglang/srt/code_completion_parser.py @@ -0,0 +1,174 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Completion templates.""" + + +import dataclasses +import json +import logging +import os +from enum import auto + +from sglang.srt.openai_api.protocol import ChatCompletionRequest + +logger = logging.getLogger(__name__) +completion_template_name = None + + +class FimPosition: + """Postion of fim middle token.""" + + MIDDLE = auto() + END = auto() + + +@dataclasses.dataclass +class CompletionTemplate: + """A class that manages completion prompt templates. only for code completion currently.""" + + # The name of this template + name: str + + # the fim begin token + fim_begin_token: str + + # The fim middle token + fim_middle_token: str + + # The fim end token + fim_end_token: str + + # The position of the fim middle token + fim_position: FimPosition + + +# A global registry for all completion templates +completion_templates: dict[str, CompletionTemplate] = {} + + +def load_completion_template_for_openai_api(completion_template_arg): + global completion_template_name + + logger.info( + f"Use completion template for the OpenAI-compatible API server: {completion_template_arg}" + ) + + if not completion_template_exists(completion_template_arg): + if not os.path.exists(completion_template_arg): + raise RuntimeError( + f"Completion template {completion_template_arg} is not a built-in template name " + "or a valid completion template file path." + ) + + assert completion_template_arg.endswith( + ".json" + ), "unrecognized format of completion template file" + with open(completion_template_arg, "r") as filep: + template = json.load(filep) + try: + fim_position = FimPosition[template["fim_position"]] + except KeyError: + raise ValueError( + f"Unknown fim position: {template['fim_position']}" + ) from None + register_completion_template( + CompletionTemplate( + name=template["name"], + fim_begin_token=template["fim_begin_token"], + fim_middle_token=template["fim_middle_token"], + fim_end_token=template["fim_end_token"], + fim_position=fim_position, + ), + override=True, + ) + completion_template_name = template["name"] + else: + completion_template_name = completion_template_arg + + +def register_completion_template(template: CompletionTemplate, override: bool = False): + """Register a new completion template.""" + if not override: + assert ( + template.name not in completion_templates + ), f"{template.name} has been registered." + + completion_templates[template.name] = template + + +def completion_template_exists(template_name: str) -> bool: + return template_name in completion_templates + + +def is_completion_template_defined() -> bool: + global completion_template_name + return completion_template_name != None + + +def generate_completion_prompt_from_request(request: ChatCompletionRequest) -> str: + global completion_template_name + if request.suffix == "": + return request.prompt + + return generate_completion_prompt( + request.prompt, request.suffix, completion_template_name + ) + + +def generate_completion_prompt(prompt: str, suffix: str, template_name: str) -> str: + + completion_template = completion_templates[template_name] + fim_begin_token = completion_template.fim_begin_token + fim_middle_token = completion_template.fim_middle_token + fim_end_token = completion_template.fim_end_token + fim_position = completion_template.fim_position + + if fim_position == FimPosition.MIDDLE: + prompt = f"{fim_begin_token}{prompt}{fim_middle_token}{suffix}{fim_end_token}" + elif fim_position == FimPosition.END: + prompt = f"{fim_begin_token}{prompt}{fim_end_token}{suffix}{fim_middle_token}" + + return prompt + + +register_completion_template( + CompletionTemplate( + name="deepseek_coder", + fim_begin_token="<｜fim▁begin｜>", + fim_middle_token="<｜fim▁hole｜>", + fim_end_token="<｜fim▁end｜>", + fim_position=FimPosition.MIDDLE, + ) +) + + +register_completion_template( + CompletionTemplate( + name="star_coder", + fim_begin_token="", + fim_middle_token="", + fim_end_token="", + fim_position=FimPosition.END, + ) +) + +register_completion_template( + CompletionTemplate( + name="qwen_coder", + fim_begin_token="<|fim_prefix|>", + fim_middle_token="<|fim_middle|>", + fim_end_token="<|fim_suffix|>", + fim_position=FimPosition.END, + ) +) diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py new file mode 100644 index 0000000..1e8370b --- /dev/null +++ b/python/sglang/srt/configs/__init__.py @@ -0,0 +1,13 @@ +from sglang.srt.configs.chatglm import ChatGLMConfig +from sglang.srt.configs.dbrx import DbrxConfig +from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config +from sglang.srt.configs.exaone import ExaoneConfig +from sglang.srt.configs.janus_pro import MultiModalityConfig + +__all__ = [ + "ExaoneConfig", + "ChatGLMConfig", + "DbrxConfig", + "DeepseekVL2Config", + "MultiModalityConfig", +] diff --git a/python/sglang/srt/configs/chatglm.py b/python/sglang/srt/configs/chatglm.py new file mode 100644 index 0000000..9370c21 --- /dev/null +++ b/python/sglang/srt/configs/chatglm.py @@ -0,0 +1,78 @@ +# Adapted from +# https://github.com/THUDM/ChatGLM2-6B +# https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/chatglm.py + +# ChatGLM2 and ChatGLM3 share the same config. +# ChatGLM4 is officially supported by Huggingface +# transformers >= 4.46.0 is required +# https://huggingface.co/docs/transformers/en/model_doc/glm +from transformers import PretrainedConfig + + +class ChatGLMConfig(PretrainedConfig): + model_type = "chatglm" + attribute_map = { + "num_hidden_layers": "num_layers", + "n_head_kv": "multi_query_group_num", + } + + def __init__( + self, + num_layers=28, + padded_vocab_size=65024, + hidden_size=4096, + ffn_hidden_size=13696, + kv_channels=128, + num_attention_heads=32, + seq_length=2048, + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-5, + rmsnorm=True, + apply_residual_connection_post_layernorm=False, + post_layer_norm=True, + add_bias_linear=False, + add_qkv_bias=False, + interleaved_qkv=False, + bias_dropout_fusion=True, + multi_query_attention=False, + multi_query_group_num=1, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=True, + fp32_residual_connection=False, + quantization_bit=0, + pre_seq_len=None, + prefix_projection=False, + **kwargs + ): + self.num_layers = num_layers + self.vocab_size = padded_vocab_size + self.padded_vocab_size = padded_vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.kv_channels = kv_channels + self.num_attention_heads = num_attention_heads + self.seq_length = seq_length + # It is to be compatible with long lora. + self.max_position_embeddings = seq_length + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.layernorm_epsilon = layernorm_epsilon + self.rmsnorm = rmsnorm + self.apply_residual_connection_post_layernorm = ( + apply_residual_connection_post_layernorm + ) + self.post_layer_norm = post_layer_norm + self.add_bias_linear = add_bias_linear + self.add_qkv_bias = add_qkv_bias + self.bias_dropout_fusion = bias_dropout_fusion + self.multi_query_attention = multi_query_attention + self.multi_query_group_num = multi_query_group_num + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.fp32_residual_connection = fp32_residual_connection + self.quantization_bit = quantization_bit + self.pre_seq_len = pre_seq_len + self.prefix_projection = prefix_projection + self.interleaved_qkv = interleaved_qkv + super().__init__(**kwargs) diff --git a/python/sglang/srt/configs/dbrx.py b/python/sglang/srt/configs/dbrx.py new file mode 100644 index 0000000..75ccbde --- /dev/null +++ b/python/sglang/srt/configs/dbrx.py @@ -0,0 +1,279 @@ +# Adapted from +# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py +# https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/dbrx.py +"""Dbrx configuration.""" + +from typing import Any, Optional + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore + + +class DbrxAttentionConfig(PretrainedConfig): + """Configuration class for Dbrx Attention. + + [`DbrxAttention`] class. It is used to instantiate attention layers + according to the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + attn_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the attention layers. + clip_qkv (`float`, *optional*, defaults to None): + If not `None`, clip the queries, keys, and values in the attention layer to this value. + kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads. + rope_theta (float): The base frequency for rope. + """ + + def __init__( + self, + attn_pdrop: float = 0, + clip_qkv: Optional[float] = None, + kv_n_heads: int = 1, + rope_theta: float = 10000.0, + **kwargs: Any, + ): + super().__init__(**kwargs) + self.attn_pdrop = attn_pdrop + self.clip_qkv = clip_qkv + self.kv_n_heads = kv_n_heads + self.rope_theta = rope_theta + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["attn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + "You are using a model of type %s to instantiate a model of " + "type %s. This is not supported for all configurations of " + "models and can yield errors.", + config_dict["model_type"], + cls.model_type, + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxFFNConfig(PretrainedConfig): + """Configuration class for Dbrx FFN. + + [`DbrxFFN`] class. It is used to instantiate feedforward layers according to + the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + ffn_act_fn (dict, optional): A dict specifying activation function for the FFN. + The dict should have a key 'name' with the value being the name of + the activation function along with any additional keyword arguments. + ffn_hidden_size (int, optional): The hidden size of the feedforward network. + moe_num_experts (int, optional): The number of experts in the mixture of experts layer. + moe_top_k (int, optional): The number of experts to use in the mixture of experts layer. + moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer. + moe_loss_weight (float, optional): The loss weight for the mixture of experts layer. + moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights. + uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment. + This should only be used for benchmarking purposes. + """ + + def __init__( + self, + ffn_act_fn: Optional[dict] = None, + ffn_hidden_size: int = 3584, + moe_num_experts: int = 4, + moe_top_k: int = 1, + moe_jitter_eps: Optional[float] = None, + moe_loss_weight: float = 0.01, + moe_normalize_expert_weights: Optional[float] = 1, + uniform_expert_assignment: bool = False, + **kwargs: Any, + ): + super().__init__() + if ffn_act_fn is None: + ffn_act_fn = {"name": "silu"} + self.ffn_act_fn = ffn_act_fn + self.ffn_hidden_size = ffn_hidden_size + self.moe_num_experts = moe_num_experts + self.moe_top_k = moe_top_k + self.moe_jitter_eps = moe_jitter_eps + self.moe_loss_weight = moe_loss_weight + self.moe_normalize_expert_weights = moe_normalize_expert_weights + self.uniform_expert_assignment = uniform_expert_assignment + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["ffn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + "You are using a model of type %s to instantiate a model of " + "type %s. This is not supported for all " + "configurations of models and can yield errors.", + config_dict["model_type"], + cls.model_type, + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxConfig(PretrainedConfig): + """Configuration class for Dbrx. + + [`DbrxModel`]. It is used to instantiate a Dbrx model according to the + specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + d_model (`int`, *optional*, defaults to 6144): + Dimensionality of the embeddings and hidden states. + n_heads (`int`, *optional*, defaults to 48): + Number of attention heads for each attention layer in the Transformer encoder. + n_layers (`int`, *optional*, defaults to 40): + Number of hidden layers in the Transformer encoder. + max_seq_len (`int`, *optional*, defaults to 32768): + The maximum sequence length of the model. + vocab_size (`int`, *optional*, defaults to 100352): + Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by + the `inputs_ids` passed when calling [`DbrxModel`]. + resid_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability applied to the attention output before combining with residual. + emb_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the embedding layer. + attn_config (`dict`, *optional*): + A dictionary used to configure the model's attention module. + ffn_config (`dict`, *optional*): + A dictionary used to configure the model's FFN module. + use_cache (`bool`, *optional*, defaults to `False`): + Whether or not the model should return the last key/values attentions (not used by all models). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether or not the router logits should be returned by the model. Enabling this will also + allow the model to output the auxiliary loss. See [here]() for more details + router_aux_loss_coef (`float`, *optional*, defaults to 0.001): + The aux loss factor for the total loss. + + + Example: + ```python + >>> from transformers import DbrxConfig, DbrxModel + + >>> # Initializing a Dbrx configuration + >>> configuration = DbrxConfig() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = DbrxModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "dbrx" + attribute_map = { + "num_attention_heads": "n_heads", + "hidden_size": "d_model", + "num_hidden_layers": "n_layers", + "max_position_embeddings": "max_seq_len", + } + + def __init__( + self, + d_model: int = 2048, + n_heads: int = 16, + n_layers: int = 24, + max_seq_len: int = 2048, + vocab_size: int = 32000, + resid_pdrop: float = 0.0, + emb_pdrop: float = 0.0, + attn_config: Optional[DbrxAttentionConfig] = None, + ffn_config: Optional[DbrxFFNConfig] = None, + use_cache: bool = True, + initializer_range: float = 0.02, + output_router_logits: bool = False, + router_aux_loss_coef: float = 0.05, + **kwargs: Any, + ): + if attn_config is None: + self.attn_config = DbrxAttentionConfig() + elif isinstance(attn_config, dict): + self.attn_config = DbrxAttentionConfig(**attn_config) + else: + self.attn_config = attn_config + + if ffn_config is None: + self.ffn_config = DbrxFFNConfig() + elif isinstance(ffn_config, dict): + self.ffn_config = DbrxFFNConfig(**ffn_config) + else: + self.ffn_config = ffn_config + + self.d_model = d_model + self.n_heads = n_heads + self.n_layers = n_layers + self.max_seq_len = max_seq_len + self.vocab_size = vocab_size + self.resid_pdrop = resid_pdrop + self.emb_pdrop = emb_pdrop + self.use_cache = use_cache + self.initializer_range = initializer_range + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + + tie_word_embeddings = kwargs.pop("tie_word_embeddings", False) + if tie_word_embeddings: + raise ValueError("tie_word_embeddings is not supported for Dbrx models.") + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) diff --git a/python/sglang/srt/configs/deepseekvl2.py b/python/sglang/srt/configs/deepseekvl2.py new file mode 100644 index 0000000..a045492 --- /dev/null +++ b/python/sglang/srt/configs/deepseekvl2.py @@ -0,0 +1,676 @@ +import math +import os +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch +from PIL import Image, ImageOps +from transformers import ( + AutoProcessor, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) + + +def select_best_resolution(image_size, candidate_resolutions): + # used for cropping + original_width, original_height = image_size + best_fit = None + max_effective_resolution = 0 + min_wasted_resolution = float("inf") + + for width, height in candidate_resolutions: + scale = min(width / original_width, height / original_height) + downscaled_width, downscaled_height = int(original_width * scale), int( + original_height * scale + ) + effective_resolution = min( + downscaled_width * downscaled_height, original_width * original_height + ) + wasted_resolution = (width * height) - effective_resolution + + if effective_resolution > max_effective_resolution or ( + effective_resolution == max_effective_resolution + and wasted_resolution < min_wasted_resolution + ): + max_effective_resolution = effective_resolution + min_wasted_resolution = wasted_resolution + best_fit = (width, height) + + return best_fit + + +class DictOutput(object): + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + input_ids: torch.LongTensor + target_ids: torch.LongTensor + images: torch.Tensor + images_seq_mask: torch.BoolTensor + images_spatial_crop: torch.LongTensor + + def __len__(self): + return len(self.input_ids) + + +class ImageTransform(object): + def __init__( + self, + mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + normalize: bool = True, + ): + self.mean = mean + self.std = std + self.normalize = normalize + + # only load torchvision.transforms when needed + try: + import torchvision.transforms as T + + # FIXME: add version check for gguf + except ImportError as err: + raise ImportError( + "Please install torchvision via `pip install torchvision` to use Deepseek-VL2." + ) from err + + transform_pipelines = [T.ToTensor()] + + if normalize: + transform_pipelines.append(T.Normalize(mean, std)) + + self.transform = T.Compose(transform_pipelines) + + def __call__(self, pil_img: Image.Image): + x = self.transform(pil_img) + return x + + +class DeepseekVLV2Processor(ProcessorMixin): + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + attributes = ["tokenizer"] + + def __init__( + self, + tokenizer: LlamaTokenizerFast, + candidate_resolutions: Tuple[Tuple[int, int]], + patch_size: int, + downsample_ratio: int, + image_mean: Tuple[float, float, float] = (0.5, 0.5, 0.5), + image_std: Tuple[float, float, float] = (0.5, 0.5, 0.5), + normalize: bool = True, + image_token: str = "", + pad_token: str = "<｜▁pad▁｜>", + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + + self.candidate_resolutions = candidate_resolutions + self.image_size = candidate_resolutions[0][0] + self.patch_size = patch_size + self.image_mean = image_mean + self.image_std = image_std + self.normalize = normalize + self.downsample_ratio = downsample_ratio + + self.image_transform = ImageTransform( + mean=image_mean, std=image_std, normalize=normalize + ) + self.tokenizer = tokenizer + # must set this，padding side with make a difference in batch inference + self.tokenizer.padding_side = "left" + + # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id' + if tokenizer.pad_token is None: + self.tokenizer.add_special_tokens({"pad_token": pad_token}) + + # add image token + image_token_id = self.tokenizer.vocab.get(image_token) + if image_token_id is None: + special_tokens = [image_token] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + self.image_token_id = self.tokenizer.vocab.get(image_token) + + # add five special tokens for grounding-related tasks + # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|> + special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + # add special tokens for SFT data + special_tokens = ["<|User|>", "<|Assistant|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + self.image_token = image_token + self.pad_token = pad_token + self.add_special_token = add_special_token + self.sft_format = sft_format + self.mask_prompt = mask_prompt + self.ignore_id = ignore_id + + super().__init__( + tokenizer, + **kwargs, + ) + + def format_messages_v2(self, messages, pil_images, max_req_input_len=-1): + """play the role of format_messages_v2 and get_images_info in the last version""" + tokenized_data = [] + masked_tokenized_data = [] # labels + images_list = [] + images_seq_mask = [] + images_spatial_crop = [] + + image_index = 0 + image_token_cnt = messages.count(self.image_token) + tokenized_str, images, seq_mask, spatial_crop = self.tokenize_with_images( + messages, + pil_images[image_index : image_index + image_token_cnt], + bos=False, + eos=True, + cropping=len(pil_images) <= 2, + max_req_input_len=max_req_input_len, + ) + + image_index = image_token_cnt + tokenized_data += tokenized_str + if self.mask_prompt: + masked_tokenized_data += [self.ignore_id] * len(tokenized_str) + else: + masked_tokenized_data += tokenized_str + images_list += images + images_seq_mask += seq_mask + images_spatial_crop += spatial_crop + + assert len(tokenized_data) == len( + images_seq_mask + ), f"format_messages_v2: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + return ( + tokenized_data, + masked_tokenized_data, + images_list, + images_seq_mask, + images_spatial_crop, + ) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def pad_id(self): + return self.tokenizer.pad_token_id + + def encode(self, text: str, bos: bool = True, eos: bool = False): + t = self.tokenizer.encode(text, add_special_tokens=False) + + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + + return t + + def decode(self, t: List[int], **kwargs) -> str: + return self.tokenizer.decode(t, **kwargs) + + def process_one( + self, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + apply_sft_format (bool): if prompt is not None, then apply the SFT format to prompt; + if conversations is not None, then it will always apply the SFT format to conversations; + inference_mode (bool): if True, then remove the last eos token; + system_prompt (str): the system prompt; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + assert ( + prompt is None or conversations is None + ), "prompt and conversations cannot be used at the same time." + + ( + tokenized_str, + masked_tokenized_str, + images_list, + images_seq_mask, + images_spatial_crop, + ) = self.format_messages_v2(conversations, images, max_req_input_len) + + assert ( + len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) + ), ( + f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, " + f"imags_seq_mask's length {len(images_seq_mask)}, are not equal" + ) + + input_ids = torch.LongTensor(tokenized_str) + target_ids = torch.LongTensor(masked_tokenized_str) + images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) + + # set input_ids < 0 | input_ids == self.image_token_id as ignore_id + target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( + self.ignore_id + ) + input_ids[input_ids < 0] = self.pad_id + + if inference_mode: + assert input_ids[-1] == self.eos_id + input_ids = input_ids[:-1] + target_ids = target_ids[:-1] + images_seq_mask = images_seq_mask[:-1] + + if len(images_list) == 0: + images = torch.zeros((1, 3, self.image_size, self.image_size)) + images_spatial_crop = torch.zeros((1, 2), dtype=torch.long) + else: + images = torch.stack(images_list, dim=0) + images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) + + prepare = VLChatProcessorOutput( + input_ids=input_ids, + target_ids=target_ids, + images=images, + images_seq_mask=images_seq_mask, + images_spatial_crop=images_spatial_crop, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + **kwargs, + ): + prepare = self.process_one( + prompt=prompt, + conversations=conversations, + images=images, + apply_sft_format=apply_sft_format, + inference_mode=inference_mode, + system_prompt=system_prompt, + max_req_input_len=max_req_input_len, + ) + + return prepare + + def find_all_indices(self, messages, target_value): + indices = [] + for index, item in enumerate(messages): + if item == target_value: + indices.append(index) + return indices + + def tokenize_with_images( + self, + conversation: str, + images: List[Image.Image], + bos: bool = True, + eos: bool = True, + cropping: bool = True, + max_req_input_len: int = -1, + ): + """Tokenize text with tags.""" + images_list, images_seq_mask, images_spatial_crop = [], [], [] + text_splits = conversation.split(self.image_token) + tokenized_str = [] + for text_sep, image in zip(text_splits, images): + """encode text_sep""" + tokenized_sep = self.encode(text_sep, bos=False, eos=False) + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """select best resolution for anyres""" + if cropping: + best_width, best_height = select_best_resolution( + image.size, self.candidate_resolutions + ) + else: + best_width, best_height = self.image_size, self.image_size + # print(image.size, (best_width, best_height)) # check the select_best_resolutions func + + """process the global view""" + global_view = ImageOps.pad( + image, + (self.image_size, self.image_size), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + images_list.append(self.image_transform(global_view)) + + """process the local views""" + local_view = ImageOps.pad( + image, + (best_width, best_height), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + for i in range(0, best_height, self.image_size): + for j in range(0, best_width, self.image_size): + images_list.append( + self.image_transform( + local_view.crop( + (j, i, j + self.image_size, i + self.image_size) + ) + ) + ) + + """record height / width crop num""" + num_width_tiles, num_height_tiles = ( + best_width // self.image_size, + best_height // self.image_size, + ) + images_spatial_crop.append([num_width_tiles, num_height_tiles]) + + """add image tokens""" + h = w = math.ceil( + (self.image_size // self.patch_size) / self.downsample_ratio + ) + # global views tokens h * (w + 1), 1 is for line seperator + tokenized_image = [self.image_token_id] * h * (w + 1) + # add a seperator between global and local views + tokenized_image += [self.image_token_id] + # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1) + tokenized_image += ( + [self.image_token_id] + * (num_height_tiles * h) + * (num_width_tiles * w + 1) + ) + + tokenized_str += tokenized_image + images_seq_mask += [True] * len(tokenized_image) + # print(width_crop_num, height_crop_num, len(tokenized_image)) # test the correctness of the number of image-related tokens + + """process the last text split""" + tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) + # deal with video, limit with request len + if max_req_input_len > -1: + if max_req_input_len < len(tokenized_sep) + len(tokenized_str) - 1: + rest = max_req_input_len - len(tokenized_sep) - 1 - 1024 + tokenized_str = tokenized_str[:rest] + images_seq_mask = images_seq_mask[:rest] + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """add the bos and eos tokens""" + if bos: + tokenized_str = [self.bos_id] + tokenized_str + images_seq_mask = [False] + images_seq_mask + if eos: + tokenized_str = tokenized_str + [self.eos_id] + images_seq_mask = images_seq_mask + [False] + + assert len(tokenized_str) == len( + images_seq_mask + ), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + return tokenized_str, images_list, images_seq_mask, images_spatial_crop + + +class DeepseekVL2VisionEncoderConfig(PretrainedConfig): + model_type: str = "vision" + + model_name: str = "siglip_large_patch16_384" + image_size: int = 384 + patch_size: int = 16 + width: int = 1024 + layers: int = 24 + heads: int = 16 + mlp_ratio: int = 4 + global_pool: str = "map" + ignore_head: bool = True + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + weight_init: str = "skip" + deterministic: bool = False + num_recomputing_layers: int = 0 + + def __init__( + self, + model_name: str = "siglip_large_patch16_384", + image_size: int = 384, + patch_size: int = 16, + width: int = 1024, + layers: int = 24, + heads: int = 16, + mlp_ratio: int = 4, + global_pool: str = "map", + ignore_head: bool = True, + class_token: bool = False, + num_classes: int = 0, + use_checkpoint: bool = False, + **kwargs, + ): + self.model_name = model_name + self.image_size = image_size + self.patch_size = patch_size + self.width = width + self.layers = layers + self.heads = heads + self.mlp_ratio = mlp_ratio + self.global_pool = global_pool + self.ignore_head = ignore_head + self.class_token = class_token + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + + super().__init__(**kwargs) + + +class DeepseekVL2MlpProjectorConfig(PretrainedConfig): + model_type = "mlp_projector" + projector_type: str = "downsample_mlp_gelu" + input_dim: int = 1152 + n_embed: int = 2048 + depth: int = 2 + mlp_ratio: int = 1 + downsample_ratio: int = 2 + token_pooling: bool = False + + def __init__( + self, + projector_type: str = "downsample_mlp_gelu", + input_dim: int = 1152, + n_embed: int = 2048, + depth: int = 2, + mlp_ratio: int = 1, + downsample_ratio: int = 2, + **kwargs, + ): + self.projector_type = projector_type + self.input_dim = input_dim + self.n_embed = n_embed + self.depth = depth + self.mlp_ratio = mlp_ratio + self.downsample_ratio = downsample_ratio + + super().__init__(**kwargs) + + +class DeepseekV2Config(PretrainedConfig): + + model_type = "deepseek_v2" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=102400, + hidden_size=4096, + intermediate_size=11008, + moe_intermediate_size=1407, + num_hidden_layers=30, + num_attention_heads=32, + num_key_value_heads=32, + n_shared_experts=None, + n_routed_experts=None, + ep_size=1, + routed_scaling_factor=1.0, + kv_lora_rank=512, + q_lora_rank=1536, + qk_rope_head_dim=64, + v_head_dim=128, + qk_nope_head_dim=128, + topk_method="gready", + n_group=None, + topk_group=None, + num_experts_per_tok=None, + moe_layer_freq=1, + first_k_dense_replace=0, + norm_topk_prob=False, + scoring_func="softmax", + aux_loss_alpha=0.001, + seq_aux=True, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=100000, + eos_token_id=100001, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + use_mla=True, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.moe_intermediate_size = moe_intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.n_shared_experts = n_shared_experts + self.n_routed_experts = n_routed_experts + self.ep_size = ep_size + self.routed_scaling_factor = routed_scaling_factor + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.topk_method = topk_method + self.n_group = n_group + self.topk_group = topk_group + self.num_experts_per_tok = num_experts_per_tok + self.moe_layer_freq = moe_layer_freq + self.first_k_dense_replace = first_k_dense_replace + self.norm_topk_prob = norm_topk_prob + self.scoring_func = scoring_func + self.aux_loss_alpha = aux_loss_alpha + self.seq_aux = seq_aux + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = float(rms_norm_eps) + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.use_mla = use_mla + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +class DeepseekVL2Config(PretrainedConfig): + model_type = "deepseek_vl_v2" + vision_config: DeepseekVL2VisionEncoderConfig + projector_config: DeepseekVL2MlpProjectorConfig + language_config: DeepseekV2Config + + tile_tag: str = "2D" + global_view_pos: str = "head" + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384),) + + def __init__( + self, + tile_tag: str = "tile_tag", + global_view_pos: str = "head", + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384),), + **kwargs, + ): + super().__init__(**kwargs) + + vision_config = kwargs.get("vision_config", {}) + self.vision_config = DeepseekVL2VisionEncoderConfig(**vision_config) + + projector_config = kwargs.get("projector_config", {}) + self.projector_config = DeepseekVL2MlpProjectorConfig(**projector_config) + + language_config = kwargs.get("language_config", {}) + if isinstance(language_config, DeepseekV2Config): + self.language_config = language_config + else: + self.language_config = DeepseekV2Config(**language_config) + + self.tile_tag = tile_tag + self.global_view_pos = global_view_pos + self.candidate_resolutions = candidate_resolutions + self.architectures = ["DeepseekVL2ForCausalLM"] + + +AutoProcessor.register(DeepseekVL2Config, DeepseekVLV2Processor) diff --git a/python/sglang/srt/configs/device_config.py b/python/sglang/srt/configs/device_config.py new file mode 100644 index 0000000..d95e848 --- /dev/null +++ b/python/sglang/srt/configs/device_config.py @@ -0,0 +1,17 @@ +import logging +from typing import Optional + +import torch + +logger = logging.getLogger(__name__) + + +class DeviceConfig: + device: Optional[torch.device] + + def __init__(self, device: str = "cuda") -> None: + if device in ["cuda", "xpu", "hpu", "cpu"]: + self.device_type = device + else: + raise RuntimeError(f"Not supported device type: {device}") + self.device = torch.device(self.device_type) diff --git a/python/sglang/srt/configs/exaone.py b/python/sglang/srt/configs/exaone.py new file mode 100644 index 0000000..7b0a2d2 --- /dev/null +++ b/python/sglang/srt/configs/exaone.py @@ -0,0 +1,195 @@ +# coding=utf-8 +# Copyright 2024 The LG AI Research EXAONE Lab. All rights reserved. +# Copyright 2024 The LG CNS AI Engineering Team. +# Copyright 2023-2024 SGLang Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" EXAONE model configuration """ +from typing import Any, Dict + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: Dict[str, Any] = {} + + +# ruff: noqa: E501 +class ExaoneConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a :class:`~transformers.ExaoneModel`. It is used to + instantiate a EXAONE model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the Exaone + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 102400): + Vocabulary size of the EXAONE model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.ExaoneModel`. Vocabulary size of the model. + Defines the different tokens that can be represented by the `inputs_ids` passed to the forward method of + :class:`~transformers.EXAONEModel`. + max_position_embeddings (:obj:`int`, `optional`, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + hidden_size (:obj:`int`, `optional`, defaults to 2048): + Dimensionality of the encoder layers and the pooler layer. + num_layers (:obj:`int`, `optional`, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (:obj:`int`, `optional`): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + intermediate_size (:obj:`int`, `optional`, defaults to `hidden_size * 4`): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"silu"`): + The non-linear activation function (function or string) in the decoder. + rope_theta (:obj:`float`, `optional`, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (:obj:`Dict`, `optional`): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (:obj:`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (:obj:`float`, `optional`): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (:obj:`int`, `optional`): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (:obj:`float`, `optional`): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (:obj:`float`, `optional`): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (:obj:`float`, `optional`): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (:obj:`List[float]`, `optional`): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (:obj:`List[float]`, `optional`): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (:obj:`float`, `optional`): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (:obj:`float`, `optional`): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + embed_dropout (:obj:`float`, `optional`, defaults to 0.0): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (:obj:`float`, `optional`, defaults to 0.0): + The dropout ratio for the attention probabilities. + layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5): + The epsilon used by the layer normalization layers. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if ``configs.is_decoder=True``. + bos_token_id (:obj:`int`, `optional`, defaults to 0): + Beginning of stream token id. + eos_token_id (:obj:`int`, `optional`, defaults to 2): + End of stream token id. + tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to tie weight embeddings + gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`): + If True, use gradient checkpointing to save memory at the expense of slower backward pass. + + Example:: + + >>> from transformers import EXAONEModel, ExaoneConfig + + >>> # Initializing a EXAONE configuration + >>> configuration = ExaoneConfig() + + >>> # Initializing a model from configuration + >>> model = EXAONEModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.configs + """ + + model_type = "exaone" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_hidden_layers": "num_layers"} + + def __init__( + self, + vocab_size=102400, + max_position_embeddings=2048, + hidden_size=2048, + num_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + intermediate_size=None, + activation_function="silu", + rope_theta=10000.0, + rope_scaling=None, + embed_dropout=0.0, + attention_dropout=0.0, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + use_cache=True, + bos_token_id=0, + eos_token_id=2, + tie_word_embeddings=True, + **kwargs + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.num_layers = num_layers + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_layers + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + if intermediate_size: + self.intermediate_size = intermediate_size + else: + self.intermediate_size = hidden_size * 4 + self.activation_function = activation_function + self.embed_dropout = embed_dropout + self.attention_dropout = attention_dropout + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs + ) diff --git a/python/sglang/srt/configs/janus_pro.py b/python/sglang/srt/configs/janus_pro.py new file mode 100644 index 0000000..ad254ed --- /dev/null +++ b/python/sglang/srt/configs/janus_pro.py @@ -0,0 +1,628 @@ +# Adapted from: +# https://github.com/deepseek-ai/Janus/tree/main/janus/models + +from dataclasses import dataclass +from typing import Dict, List, Tuple, Union + +import numpy as np +import PIL +import torch +from PIL.Image import Image +from transformers import ( + BaseImageProcessor, + BatchFeature, + LlamaConfig, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) +from transformers.image_utils import to_numpy_array + +from sglang.srt.configs.utils import register_image_processor, register_processor +from sglang.srt.mm_utils import expand2square + + +class DictToObject(dict): + def __init__(self, dictionary): + super(self).__init__(dictionary) + + for key, value in dictionary.items(): + if isinstance(value, dict): + value = DictToObject(value) + setattr(self, key, value) + + +class VisionConfig(PretrainedConfig): + model_type = "vision" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenAlignerConfig(PretrainedConfig): + model_type = "gen_aligner" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenHeadConfig(PretrainedConfig): + model_type = "gen_head" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class AlignerConfig(PretrainedConfig): + model_type = "aligner" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenVisionConfig(PretrainedConfig): + model_type = "gen_vision" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +@dataclass +class SigLIPVisionCfg: + width: int = 1152 + layers: Union[Tuple[int, int, int, int], int] = 27 + heads: int = 16 + patch_size: int = 14 + image_size: Union[Tuple[int, int], int] = 336 + global_pool: str = "map" + mlp_ratio: float = 3.7362 + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + + +class MultiModalityConfig(PretrainedConfig): + model_type = "multi_modality" + vision_config: VisionConfig + aligner_config: AlignerConfig + + gen_vision_config: GenVisionConfig + gen_aligner_config: GenAlignerConfig + gen_head_config: GenHeadConfig + + language_config: LlamaConfig + + def __init__(self, **kwargs): + super().__init__(**kwargs) + vision_config = kwargs.get("vision_config", {}) + self.vision_config = VisionConfig(**vision_config) + + aligner_config = kwargs.get("aligner_config", {}) + self.aligner_config = AlignerConfig(**aligner_config) + + gen_vision_config = kwargs.get("gen_vision_config", {}) + self.gen_vision_config = GenVisionConfig(**gen_vision_config) + + gen_aligner_config = kwargs.get("gen_aligner_config", {}) + self.gen_aligner_config = GenAlignerConfig(**gen_aligner_config) + + gen_head_config = kwargs.get("gen_head_config", {}) + self.gen_head_config = GenHeadConfig(**gen_head_config) + + language_config = kwargs.get("language_config", {}) + if isinstance(language_config, LlamaConfig): + self.language_config = language_config + else: + self.language_config = LlamaConfig(**language_config) + + +class VLMImageProcessor(BaseImageProcessor): + model_input_names = ["pixel_values"] + + def __init__( + self, + image_size: int, + min_size: int = 14, + image_mean: Union[Tuple[float, float, float], List[float]] = ( + 0.48145466, + 0.4578275, + 0.40821073, + ), + image_std: Union[Tuple[float, float, float], List[float]] = ( + 0.26862954, + 0.26130258, + 0.27577711, + ), + rescale_factor: float = 1.0 / 255.0, + do_normalize: bool = True, + **kwargs, + ): + super().__init__(**kwargs) + + self.image_size = image_size + self.rescale_factor = rescale_factor + self.image_mean = image_mean + self.image_std = image_std + self.min_size = min_size + self.do_normalize = do_normalize + + if image_mean is None: + self.background_color = (127, 127, 127) + else: + self.background_color = tuple([int(x * 255) for x in image_mean]) + + def resize(self, pil_img: Image) -> np.ndarray: + """ + + Args: + pil_img (PIL.Image): [H, W, 3] in PIL.Image in RGB + + Returns: + x (np.ndarray): [3, self.image_size, self.image_size] + """ + + width, height = pil_img.size + max_size = max(width, height) + + size = [ + max(int(height / max_size * self.image_size), self.min_size), + max(int(width / max_size * self.image_size), self.min_size), + ] + + if width <= 0 or height <= 0 or size[0] <= 0 or size[1] <= 0: + # print(f"orig size = {pil_img.size}, new size = {size}") + raise ValueError("Invalid size!") + + def resize( + pil_img, size, interpolation=PIL.Image.Resampling.BICUBIC, antialias=True + ): + if isinstance(size, int): + w, h = pil_img.size + if (w <= h and w == size) or (h <= w and h == size): + return pil_img + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + size = (ow, oh) + else: + size = (size[1], size[0]) + + return pil_img.resize( + size, resample=interpolation, reducing_gap=None if antialias else 3.0 + ) + + pil_img = resize( + pil_img, size, interpolation=PIL.Image.Resampling.BICUBIC, antialias=True + ) + + pil_img = expand2square(pil_img, self.background_color) + x = to_numpy_array(pil_img) + + # [H, W, 3] -> [3, H, W] + x = np.transpose(x, (2, 0, 1)) + + return x + + def preprocess(self, images, return_tensors: str = "pt", **kwargs) -> BatchFeature: + # resize and pad to [self.image_size, self.image_size] + # then convert from [H, W, 3] to [3, H, W] + if not isinstance(images, list): + images = [images] + images: List[np.ndarray] = [self.resize(image) for image in images] + images = [image[:3, ...] for image in images] + + # rescale from [0, 255] -> [0, 1] + images = [ + self.rescale( + image=image, + scale=self.rescale_factor, + input_data_format="channels_first", + ) + for image in images + ] + + # normalize + if self.do_normalize: + images = [ + self.normalize( + image=image, + mean=self.image_mean, + std=self.image_std, + input_data_format="channels_first", + ) + for image in images + ] + data = {"pixel_values": images} + return BatchFeature(data=data, tensor_type=return_tensors) + + @property + def default_shape(self): + return [3, self.image_size, self.image_size] + + +class DictOutput(object): + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + sft_format: str + input_ids: torch.Tensor + pixel_values: torch.Tensor + num_image_tokens: torch.IntTensor + + def __len__(self): + return len(self.input_ids) + + +@dataclass +class BatchedVLChatProcessorOutput(DictOutput): + sft_format: List[str] + input_ids: torch.Tensor + pixel_values: torch.Tensor + attention_mask: torch.Tensor + images_seq_mask: torch.BoolTensor + images_emb_mask: torch.BoolTensor + + +# FIXME: had to place Official Processor here, since image_processor module would not be imported in all threads, +# hence AutoProcessor registration would not be affective in some cases +class VLChatProcessor(ProcessorMixin): + image_processor_class = "AutoImageProcessor" + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + + attributes = ["image_processor", "tokenizer"] + + def __init__( + self, + image_processor: VLMImageProcessor, + tokenizer: LlamaTokenizerFast, + image_tag: str = "", + image_start_tag: str = "", + image_end_tag: str = "", + pad_tag: str = "<｜▁pad▁｜>", + num_image_tokens: int = 576, + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + self.image_processor = image_processor + self.tokenizer = tokenizer + + image_id = self.tokenizer.vocab.get(image_tag) + if image_id is None: + special_tokens = [image_tag] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + # print(f"Add image tag = {image_tag} to the tokenizer") + + self.image_tag = image_tag + self.image_start_tag = image_start_tag + self.image_end_tag = image_end_tag + self.pad_tag = pad_tag + + self.num_image_tokens = num_image_tokens + self.add_special_token = add_special_token + self.sft_format = sft_format + self.ignore_id = ignore_id + + super().__init__( + image_processor, + tokenizer, + **kwargs, + ) + + @property + def image_token(self): + return self.image_tag + + @property + def image_id(self) -> int: + image_id = self.tokenizer.vocab.get(self.image_tag) + return image_id + + @property + def image_start_id(self): + image_start_id = self.tokenizer.vocab.get(self.image_start_tag) + return image_start_id + + @property + def image_end_id(self): + image_end_id = self.tokenizer.vocab.get(self.image_end_tag) + return image_end_id + + @property + def image_start_token(self): + return self.image_start_tag + + @property + def image_end_token(self): + return self.image_end_tag + + @property + def pad_id(self): + pad_id = self.tokenizer.vocab.get(self.pad_tag) + return pad_id + + def add_image_token( + self, + image_indices: List[int], + input_ids: torch.LongTensor, + ): + """ + + Args: + image_indices (List[int]): [index_0, index_1, ..., index_j] + input_ids (torch.LongTensor): [N] + + Returns: + input_ids (torch.LongTensor): [N + image tokens] + num_image_tokens (torch.IntTensor): [n_images] + """ + + input_slices = [] + + start = 0 + for index in image_indices: + if self.add_special_token: + end = index + 1 + else: + end = index + + # original text tokens + input_slices.append(input_ids[start:end]) + + # add boi, image tokens, eoi and set the mask as False + input_slices.append(self.image_start_id * torch.ones((1), dtype=torch.long)) + input_slices.append( + self.image_id * torch.ones((self.num_image_tokens,), dtype=torch.long) + ) + input_slices.append(self.image_end_id * torch.ones((1), dtype=torch.long)) + start = index + 1 + + # the left part + input_slices.append(input_ids[start:]) + + # concat all slices + input_ids = torch.cat(input_slices, dim=0) + num_image_tokens = torch.IntTensor([self.num_image_tokens] * len(image_indices)) + + return input_ids, num_image_tokens + + def process_one( + self, + prompt: str = None, + images: List[Image] = None, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + images (List[ImageType]): the list of images; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + sft_format = prompt + # tokenize + input_ids = self.tokenizer.encode(sft_format) + input_ids = torch.LongTensor(input_ids) + + # add image tokens to the input_ids + image_token_mask: torch.Tensor = (input_ids == self.image_id).to(torch.bool) + image_indices = image_token_mask.nonzero() + input_ids, num_image_tokens = self.add_image_token( + image_indices=image_indices, + input_ids=input_ids, + ) + + # load images + images_outputs = self.image_processor(images, return_tensors="pt") + + prepare = VLChatProcessorOutput( + sft_format=sft_format, + input_ids=input_ids, + pixel_values=images_outputs.pixel_values, + num_image_tokens=num_image_tokens, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image] = None, + force_batchify: bool = True, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + force_batchify (bool): force batchify the inputs; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + prepare = self.process_one( + prompt=prompt, conversations=conversations, images=images + ) + + if force_batchify: + prepare = self.batchify([prepare]) + + return prepare + + def batchify( + self, prepare_list: List[VLChatProcessorOutput] + ) -> BatchedVLChatProcessorOutput: + """ + Preprocesses the inputs for multimodal inference. + + Args: + prepare_list (List[VLChatProcessorOutput]): A list of VLChatProcessorOutput. + + Returns: + BatchedVLChatProcessorOutput: A dictionary of the inputs to use for multimodal inference. + """ + + batch_size = len(prepare_list) + sft_format = [] + n_images = [] + seq_lens = [] + for prepare in prepare_list: + n_images.append(len(prepare.num_image_tokens)) + seq_lens.append(len(prepare)) + + input_token_max_len = max(seq_lens) + max_n_images = max(1, max(n_images)) + + batched_input_ids = torch.full( + (batch_size, input_token_max_len), self.pad_id + ).long() # FIXME + batched_attention_mask = torch.zeros((batch_size, input_token_max_len)).long() + batched_pixel_values = torch.zeros( + (batch_size, max_n_images, *self.image_processor.default_shape) + ).float() + batched_images_seq_mask = torch.zeros((batch_size, input_token_max_len)).bool() + batched_images_emb_mask = torch.zeros( + (batch_size, max_n_images, self.num_image_tokens) + ).bool() + + for i, prepare in enumerate(prepare_list): + input_ids = prepare.input_ids + seq_len = len(prepare) + n_image = len(prepare.num_image_tokens) + # left-padding + batched_attention_mask[i, -seq_len:] = 1 + batched_input_ids[i, -seq_len:] = torch.LongTensor(input_ids) + batched_images_seq_mask[i, -seq_len:] = input_ids == self.image_id + + if n_image > 0: + batched_pixel_values[i, :n_image] = prepare.pixel_values + for j, n_image_tokens in enumerate(prepare.num_image_tokens): + batched_images_emb_mask[i, j, :n_image_tokens] = True + + sft_format.append(prepare.sft_format) + + batched_prepares = BatchedVLChatProcessorOutput( + input_ids=batched_input_ids, + attention_mask=batched_attention_mask, + pixel_values=batched_pixel_values, + images_seq_mask=batched_images_seq_mask, + images_emb_mask=batched_images_emb_mask, + sft_format=sft_format, + ) + + return batched_prepares + + +class VLMImageProcessorConfig(PretrainedConfig): + model_type = "deepseek_vlm" + image_size: int + min_size: int + image_mean: Union[Tuple[float, float, float], List[float]] + image_std: Union[Tuple[float, float, float], List[float]] + rescale_factor: float + do_normalize: bool + + def __init__( + self, + image_size: int, + min_size: int = 14, + image_mean: Union[Tuple[float, float, float], List[float]] = ( + 0.48145466, + 0.4578275, + 0.40821073, + ), + image_std: Union[Tuple[float, float, float], List[float]] = ( + 0.26862954, + 0.26130258, + 0.27577711, + ), + rescale_factor: float = 1.0 / 255.0, + do_normalize: bool = True, + **kwargs, + ): + self.image_size = image_size + self.min_size = min_size + self.image_mean = image_mean + self.image_std = image_std + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + + super().__init__(**kwargs) + + +register_processor(MultiModalityConfig, VLChatProcessor) +register_image_processor(MultiModalityConfig, VLMImageProcessor) diff --git a/python/sglang/srt/configs/load_config.py b/python/sglang/srt/configs/load_config.py new file mode 100644 index 0000000..be9a40b --- /dev/null +++ b/python/sglang/srt/configs/load_config.py @@ -0,0 +1,89 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +import enum +import json +import logging +from dataclasses import dataclass, field +from typing import List, Optional, Union + +from sglang.srt.utils import is_hip + +logger = logging.getLogger(__name__) + + +class LoadFormat(str, enum.Enum): + AUTO = "auto" + PT = "pt" + SAFETENSORS = "safetensors" + NPCACHE = "npcache" + DUMMY = "dummy" + SHARDED_STATE = "sharded_state" + GGUF = "gguf" + BITSANDBYTES = "bitsandbytes" + MISTRAL = "mistral" + LAYERED = "layered" + JAX = "jax" + REMOTE = "remote" + + +@dataclass +class LoadConfig: + """ + download_dir: Directory to download and load the weights, default to the + default cache directory of huggingface. + load_format: The format of the model weights to load: + "auto" will try to load the weights in the safetensors format and + fall back to the pytorch bin format if safetensors format is + not available. + "pt" will load the weights in the pytorch bin format. + "safetensors" will load the weights in the safetensors format. + "npcache" will load the weights in pytorch format and store + a numpy cache to speed up the loading. + "dummy" will initialize the weights with random values, which is + mainly for profiling. + "bitsandbytes" will load nf4 type weights. + ignore_patterns: The list of patterns to ignore when loading the model. + Default to "original/**/*" to avoid repeated loading of llama's + checkpoints. + decryption_key_file: If set, decrypts the output files with a password read + from this file (after PBKDF2). + """ + + load_format: Union[str, LoadFormat] = LoadFormat.AUTO + download_dir: Optional[str] = None + model_loader_extra_config: Optional[Union[str, dict]] = field(default_factory=dict) + ignore_patterns: Optional[Union[List[str], str]] = None + decryption_key_file: Optional[str] = None + + def __post_init__(self): + model_loader_extra_config = self.model_loader_extra_config or {} + if isinstance(model_loader_extra_config, str): + self.model_loader_extra_config = json.loads(model_loader_extra_config) + self._verify_load_format() + + if self.ignore_patterns is not None and len(self.ignore_patterns) > 0: + logger.info( + "Ignoring the following patterns when downloading weights: %s", + self.ignore_patterns, + ) + else: + self.ignore_patterns = ["original/**/*"] + + def _verify_load_format(self) -> None: + if not isinstance(self.load_format, str): + return + + load_format = self.load_format.lower() + self.load_format = LoadFormat(load_format) + + rocm_not_supported_load_format: List[str] = [] + if is_hip() and load_format in rocm_not_supported_load_format: + rocm_supported_load_format = [ + f + for f in LoadFormat.__members__ + if (f not in rocm_not_supported_load_format) + ] + raise ValueError( + f"load format '{load_format}' is not supported in ROCm. " + f"Supported load formats are " + f"{rocm_supported_load_format}" + ) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py new file mode 100644 index 0000000..8bc8d12 --- /dev/null +++ b/python/sglang/srt/configs/model_config.py @@ -0,0 +1,512 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import logging +import math +from enum import IntEnum, auto +from typing import List, Optional, Set, Union + +import torch +from transformers import PretrainedConfig + +from sglang.srt.hf_transformers_utils import get_config, get_context_length +from sglang.srt.layers.quantization import QUANTIZATION_METHODS +from sglang.srt.utils import get_bool_env_var, is_hip + +logger = logging.getLogger(__name__) + + +class AttentionArch(IntEnum): + MLA = auto() + MHA = auto() + + +class ModelConfig: + def __init__( + self, + model_path: str, + trust_remote_code: bool = True, + revision: Optional[str] = None, + context_length: Optional[int] = None, + model_override_args: Optional[str] = None, + is_embedding: Optional[bool] = None, + dtype: str = "auto", + quantization: Optional[str] = None, + override_config_file: Optional[str] = None, + ) -> None: + self.model_path = model_path + self.revision = revision + self.quantization = quantization + + # Parse args + self.maybe_pull_model_tokenizer_from_remote() + self.model_override_args = json.loads(model_override_args) + kwargs = {} + if override_config_file and override_config_file.strip(): + kwargs["_configuration_file"] = override_config_file.strip() + + self.hf_config = get_config( + self.model_path, + trust_remote_code=trust_remote_code, + revision=revision, + model_override_args=self.model_override_args, + **kwargs, + ) + self.hf_text_config = get_hf_text_config(self.hf_config) + + # Check model type + self.is_generation = is_generation_model( + self.hf_config.architectures, is_embedding + ) + self.is_multimodal = is_multimodal_model(self.hf_config.architectures) + self.is_multimodal_gen = is_multimodal_gen_model(self.hf_config.architectures) + self.is_image_gen = is_image_gen_model(self.hf_config.architectures) + self.is_audio_model = is_audio_model(self.hf_config.architectures) + self.is_encoder_decoder = is_encoder_decoder_model(self.hf_config.architectures) + self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype) + + # Derive context length + derived_context_len = get_context_length(self.hf_text_config) + if context_length is not None: + if context_length > derived_context_len: + if get_bool_env_var( + "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN", default="True" + ): + logger.warning( + f"Warning: User-specified context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). " + f"This may lead to incorrect model outputs or CUDA errors." + ) + self.context_len = context_length + else: + raise ValueError( + f"User-specified context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). " + f"This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config. " + f"To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1" + ) + else: + self.context_len = context_length + else: + self.context_len = derived_context_len + + # Unify the config keys for hf_text_config + self.head_dim = getattr( + self.hf_text_config, + "head_dim", + self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads, + ) + + # FIXME: temporary special judge for MLA architecture + if ( + "DeepseekV2ForCausalLM" in self.hf_config.architectures + or "DeepseekV3ForCausalLM" in self.hf_config.architectures + or "DeepseekV3ForCausalLMNextN" in self.hf_config.architectures + ): + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + self.v_head_dim = self.hf_config.v_head_dim + + # Handle rope scaling with yarn + self.scaling = 1 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim) + if self.hf_config.rope_scaling: + mscale_all_dim = self.hf_config.rope_scaling.get( + "mscale_all_dim", False + ) + scaling_factor = self.hf_config.rope_scaling["factor"] + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) + self.scaling = self.scaling * mscale * mscale + + elif "MiniCPM3ForCausalLM" in self.hf_config.architectures: + self.head_dim = 128 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + elif "DeepseekVL2ForCausalLM" in self.hf_config.architectures: + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_text_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim + else: + self.attention_arch = AttentionArch.MHA + + self.num_attention_heads = self.hf_text_config.num_attention_heads + self.num_key_value_heads = getattr( + self.hf_text_config, "num_key_value_heads", None + ) + + # for Dbrx and MPT models + if self.hf_config.model_type in ["dbrx", "mpt"]: + self.num_key_value_heads = getattr( + self.hf_config.attn_config, "kv_n_heads", None + ) + + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + self.hidden_size = self.hf_text_config.hidden_size + self.num_hidden_layers = self.hf_text_config.num_hidden_layers + self.vocab_size = self.hf_text_config.vocab_size + + # Verify quantization + self._verify_quantization() + + # Cache attributes + self.hf_eos_token_id = self.get_hf_eos_token_id() + self.image_token_id = getattr(self.hf_config, "image_token_id", None) + + # adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py#L289 + def get_total_num_kv_heads(self) -> int: + """Returns the total number of KV heads.""" + # For GPTBigCode & Falcon: + # NOTE: for falcon, when new_decoder_architecture is True, the + # multi_query flag is ignored and we use n_head_kv for the number of + # KV heads. + falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"] + new_decoder_arch_falcon = ( + self.hf_config.model_type in falcon_model_types + and getattr(self.hf_config, "new_decoder_architecture", False) + ) + if not new_decoder_arch_falcon and getattr( + self.hf_text_config, "multi_query", False + ): + # Multi-query attention, only one KV head. + # Currently, tensor parallelism is not supported in this case. + return 1 + + # For DBRX and MPT + if self.hf_config.model_type in ["mpt"]: + if "kv_n_heads" in self.hf_config.attn_config: + return self.hf_config.attn_config["kv_n_heads"] + return self.hf_config.num_attention_heads + if self.hf_config.model_type in ["dbrx"]: + return getattr( + self.hf_config.attn_config, + "kv_n_heads", + self.hf_config.num_attention_heads, + ) + + attributes = [ + # For Falcon: + "n_head_kv", + "num_kv_heads", + # For LLaMA-2: + "num_key_value_heads", + # For ChatGLM: + "multi_query_group_num", + ] + for attr in attributes: + num_kv_heads = getattr(self.hf_text_config, attr, None) + if num_kv_heads is not None: + return num_kv_heads + + # For non-grouped-query attention models, the number of KV heads is + # equal to the number of attention heads. + return self.hf_text_config.num_attention_heads + + def get_num_kv_heads(self, tensor_parallel_size) -> int: + """Returns the number of KV heads per GPU.""" + total_num_kv_heads = self.get_total_num_kv_heads() + # If tensor parallelism is used, we divide the number of KV heads by + # the tensor parallel size. We will replicate the KV heads in the + # case where the number of KV heads is smaller than the tensor + # parallel size so each GPU has at least one KV head. + return max(1, total_num_kv_heads // tensor_parallel_size) + + # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py + def _parse_quant_hf_config(self): + quant_cfg = getattr(self.hf_config, "quantization_config", None) + if quant_cfg is None: + # compressed-tensors uses a "compression_config" key + quant_cfg = getattr(self.hf_config, "compression_config", None) + return quant_cfg + + # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py + def _verify_quantization(self) -> None: + supported_quantization = [*QUANTIZATION_METHODS] + rocm_supported_quantization = [ + "awq", + "gptq", + "fp8", + "compressed_tensors", + "compressed-tensors", + "fbgemm_fp8", + "w8a8_fp8", + ] + optimized_quantization_methods = [ + "fp8", + "marlin", + "modelopt", + "gptq_marlin_24", + "gptq_marlin", + "awq_marlin", + "fbgemm_fp8", + "compressed_tensors", + "compressed-tensors", + "experts_int8", + "w8a8_int8", + "w8a8_fp8", + ] + compatible_quantization_methods = { + "w8a8_int8": ["compressed-tensors", "compressed_tensors"], + "w8a8_fp8": ["compressed-tensors", "compressed_tensors"], + } + if self.quantization is not None: + self.quantization = self.quantization.lower() + + # Parse quantization method from the HF model config, if available. + quant_cfg = self._parse_quant_hf_config() + + if quant_cfg is not None: + quant_method = quant_cfg.get("quant_method", "").lower() + + # Detect which checkpoint is it + for _, method in QUANTIZATION_METHODS.items(): + quantization_override = method.override_quantization_method( + quant_cfg, self.quantization + ) + if quantization_override: + quant_method = quantization_override + self.quantization = quantization_override + break + + # Verify quantization configurations. + if self.quantization is None: + self.quantization = quant_method + elif self.quantization != quant_method: + if ( + self.quantization not in compatible_quantization_methods + or quant_method + not in compatible_quantization_methods[self.quantization] + ): + raise ValueError( + "Quantization method specified in the model config " + f"({quant_method}) does not match the quantization " + f"method specified in the `quantization` argument " + f"({self.quantization})." + ) + + if self.quantization is not None: + if self.quantization not in supported_quantization: + raise ValueError( + f"Unknown quantization method: {self.quantization}. Must " + f"be one of {supported_quantization}." + ) + if is_hip() and self.quantization not in rocm_supported_quantization: + raise ValueError( + f"{self.quantization} quantization is currently not " + f"supported in ROCm." + ) + if self.quantization not in optimized_quantization_methods: + logger.warning( + "%s quantization is not fully " + "optimized yet. The speed can be slower than " + "non-quantized models.", + self.quantization, + ) + + def get_hf_eos_token_id(self) -> Optional[Set[int]]: + eos_ids = getattr(self.hf_config, "eos_token_id", None) + if eos_ids: + # it can be either int or list of int + eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids) + return eos_ids + + def maybe_pull_model_tokenizer_from_remote(self) -> None: + """ + Pull the model config files to a temporary + directory in case of remote. + + Args: + model: The model name or path. + + """ + from sglang.srt.connector import create_remote_connector + from sglang.srt.utils import is_remote_url + + if is_remote_url(self.model_path): + logger.info("Pulling model configs from remote...") + # BaseConnector implements __del__() to clean up the local dir. + # Since config files need to exist all the time, so we DO NOT use + # with statement to avoid closing the client. + client = create_remote_connector(self.model_path) + if is_remote_url(self.model_path): + client.pull_files(allow_pattern=["*config.json"]) + self.model_weights = self.model_path + self.model_path = client.get_local_dir() + + +def get_hf_text_config(config: PretrainedConfig): + """Get the "sub" config relevant to llm for multi modal models. + No op for pure text models. + """ + class_name = config.architectures[0] + if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"): + # We support non-hf version of llava models, so we do not want to + # read the wrong values from the unused default text_config. + # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as + # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`. + setattr(config, "torch_dtype", torch.float16) + return config + + if hasattr(config, "text_config"): + # The code operates under the assumption that text_config should have + # `num_attention_heads` (among others). Assert here to fail early + # if transformers config doesn't align with this assumption. + assert hasattr(config.text_config, "num_attention_heads") + return config.text_config + if hasattr(config, "language_config"): + return config.language_config + else: + return config + + +# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +_STR_DTYPE_TO_TORCH_DTYPE = { + "half": torch.float16, + "float16": torch.float16, + "float": torch.float32, + "float32": torch.float32, + "bfloat16": torch.bfloat16, +} + + +# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +def _get_and_verify_dtype( + config: PretrainedConfig, + dtype: Union[str, torch.dtype], +) -> torch.dtype: + # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct + # because config.torch_dtype can be None. + config_dtype = getattr(config, "torch_dtype", None) + if config_dtype is None: + config_dtype = torch.float32 + + if isinstance(dtype, str): + dtype = dtype.lower() + if dtype == "auto": + if config_dtype == torch.float32: + if config.model_type.startswith("gemma"): + if config.model_type == "gemma": + gemma_version = "" + else: + gemma_version = config.model_type[5] + logger.info( + f"For Gemma {gemma_version}, we downcast float32 to bfloat16 instead " + "of float16 by default. Please specify `dtype` if you " + "want to use float16." + ) + torch_dtype = torch.bfloat16 + else: + # Following the common practice, we use float16 for float32 + # models. + torch_dtype = torch.float16 + else: + torch_dtype = config_dtype + else: + if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: + raise ValueError(f"Unknown dtype: {dtype}") + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype] + elif isinstance(dtype, torch.dtype): + torch_dtype = dtype + else: + raise ValueError(f"Unknown dtype: {dtype}") + + # Verify the dtype. + if torch_dtype != config_dtype: + if torch_dtype == torch.float32: + # Upcasting to float32 is allowed. + logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) + pass + elif config_dtype == torch.float32: + # Downcasting from float32 to float16 or bfloat16 is allowed. + logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) + pass + else: + # Casting between float16 and bfloat16 is allowed with a warning. + logger.warning("Casting %s to %s.", config_dtype, torch_dtype) + + return torch_dtype + + +def is_generation_model(model_architectures: List[str], is_embedding: bool = False): + # We have two ways to determine whether a model is a generative model. + # 1. Check the model architecture + # 2. check the `is_embedding` server args + + if ( + "LlamaEmbeddingModel" in model_architectures + or "MistralModel" in model_architectures + or "LlamaForSequenceClassification" in model_architectures + or "LlamaForSequenceClassificationWithNormal_Weights" in model_architectures + or "InternLM2ForRewardModel" in model_architectures + or "Qwen2ForRewardModel" in model_architectures + or "Qwen2ForSequenceClassification" in model_architectures + or "CLIPModel" in model_architectures + ): + return False + else: + return not is_embedding + + +multimodal_model_archs = [ + "DeepseekVL2ForCausalLM", + "Gemma3ForConditionalGeneration", + "Grok1VForCausalLM", + "Grok1AForCausalLM", + "LlavaLlamaForCausalLM", + "LlavaMistralForCausalLM", + "LlavaQwenForCausalLM", + "LlavaVidForCausalLM", + "MiniCPMO", + "MiniCPMV", + "MultiModalityCausalLM", + "MllamaForConditionalGeneration", + "Qwen2VLForConditionalGeneration", + "Qwen2_5_VLForConditionalGeneration", + "CLIPModel", +] + + +def is_multimodal_model(model_architectures: List[str]): + if any( + multi_model_arch in model_architectures + for multi_model_arch in multimodal_model_archs + ): + return True + else: + return False + + +def is_multimodal_gen_model(model_architectures: List[str]): + return False + + +def is_image_gen_model(model_architectures: List[str]): + return False + + +def is_audio_model(model_architectures: List[str]): + return False + + +def is_encoder_decoder_model(model_architectures: List[str]): + return "MllamaForConditionalGeneration" in model_architectures + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 diff --git a/python/sglang/srt/configs/utils.py b/python/sglang/srt/configs/utils.py new file mode 100644 index 0000000..8403d95 --- /dev/null +++ b/python/sglang/srt/configs/utils.py @@ -0,0 +1,25 @@ +from typing import Type + +from transformers import ( + AutoImageProcessor, + AutoProcessor, + BaseImageProcessor, + PretrainedConfig, + ProcessorMixin, +) + + +def register_image_processor( + config: Type[PretrainedConfig], image_processor: Type[BaseImageProcessor] +): + """ + register customized hf image processor while removing hf impl + """ + AutoImageProcessor.register(config, None, image_processor, None, exist_ok=True) + + +def register_processor(config: Type[PretrainedConfig], processor: Type[ProcessorMixin]): + """ + register customized hf processor while removing hf impl + """ + AutoProcessor.register(config, processor, exist_ok=True) diff --git a/python/sglang/srt/connector/__init__.py b/python/sglang/srt/connector/__init__.py new file mode 100644 index 0000000..829644c --- /dev/null +++ b/python/sglang/srt/connector/__init__.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 + +import enum +import logging + +from sglang.srt.connector.base_connector import ( + BaseConnector, + BaseFileConnector, + BaseKVConnector, +) +from sglang.srt.connector.redis import RedisConnector +from sglang.srt.connector.s3 import S3Connector +from sglang.srt.utils import parse_connector_type + +logger = logging.getLogger(__name__) + + +class ConnectorType(str, enum.Enum): + FS = "filesystem" + KV = "KV" + + +def create_remote_connector(url, device="cpu") -> BaseConnector: + connector_type = parse_connector_type(url) + if connector_type == "redis": + return RedisConnector(url) + elif connector_type == "s3": + return S3Connector(url) + else: + raise ValueError(f"Invalid connector type: {url}") + + +def get_connector_type(client: BaseConnector) -> ConnectorType: + if isinstance(client, BaseKVConnector): + return ConnectorType.KV + if isinstance(client, BaseFileConnector): + return ConnectorType.FS + + raise ValueError(f"Invalid connector type: {client}") + + +__all__ = [ + "BaseConnector", + "BaseFileConnector", + "BaseKVConnector", + "RedisConnector", + "S3Connector", + "ConnectorType", + "create_remote_connector", + "get_connector_type", +] diff --git a/python/sglang/srt/connector/base_connector.py b/python/sglang/srt/connector/base_connector.py new file mode 100644 index 0000000..a9c00d0 --- /dev/null +++ b/python/sglang/srt/connector/base_connector.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import signal +import tempfile +from abc import ABC, abstractmethod +from typing import Generator, List, Optional, Tuple + +import torch + + +class BaseConnector(ABC): + """ + For fs connector such as s3: + :/// + + For kv connector such as redis: + ://://keys/ + ://files/ + """ + + def __init__(self, url: str, device: torch.device = "cpu"): + self.url = url + self.device = device + self.closed = False + self.local_dir = tempfile.mkdtemp() + for sig in (signal.SIGINT, signal.SIGTERM): + existing_handler = signal.getsignal(sig) + signal.signal(sig, self._close_by_signal(existing_handler)) + + def get_local_dir(self): + return self.local_dir + + @abstractmethod + def weight_iterator( + self, rank: int = 0 + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + raise NotImplementedError() + + @abstractmethod + def pull_files( + self, + allow_pattern: Optional[List[str]] = None, + ignore_pattern: Optional[List[str]] = None, + ) -> None: + raise NotImplementedError() + + def close(self): + if self.closed: + return + + self.closed = True + if os.path.exists(self.local_dir): + shutil.rmtree(self.local_dir) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __del__(self): + self.close() + + def _close_by_signal(self, existing_handler=None): + + def new_handler(signum, frame): + self.close() + if existing_handler: + existing_handler(signum, frame) + + return new_handler + + +class BaseKVConnector(BaseConnector): + + @abstractmethod + def get(self, key: str) -> Optional[torch.Tensor]: + raise NotImplementedError() + + @abstractmethod + def getstr(self, key: str) -> Optional[str]: + raise NotImplementedError() + + @abstractmethod + def set(self, key: str, obj: torch.Tensor) -> None: + raise NotImplementedError() + + @abstractmethod + def setstr(self, key: str, obj: str) -> None: + raise NotImplementedError() + + @abstractmethod + def list(self, prefix: str) -> List[str]: + raise NotImplementedError() + + +class BaseFileConnector(BaseConnector): + """ + List full file names from remote fs path and filter by allow pattern. + + Args: + allow_pattern: A list of patterns of which files to pull. + + Returns: + list[str]: List of full paths allowed by the pattern + """ + + @abstractmethod + def glob(self, allow_pattern: str) -> List[str]: + raise NotImplementedError() diff --git a/python/sglang/srt/connector/redis.py b/python/sglang/srt/connector/redis.py new file mode 100644 index 0000000..761594f --- /dev/null +++ b/python/sglang/srt/connector/redis.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 + +import logging +from typing import Generator, List, Optional, Tuple +from urllib.parse import urlparse + +import torch + +from sglang.srt.connector import BaseKVConnector +from sglang.srt.connector.serde import create_serde +from sglang.srt.connector.utils import pull_files_from_db + +logger = logging.getLogger(__name__) + + +class RedisConnector(BaseKVConnector): + + def __init__(self, url: str, device: torch.device = "cpu"): + import redis + + super().__init__(url, device) + parsed_url = urlparse(url) + self.connection = redis.Redis(host=parsed_url.hostname, port=parsed_url.port) + self.model_name = parsed_url.path.lstrip("/") + # TODO: more serde options + self.s, self.d = create_serde("safe") + + def get(self, key: str) -> Optional[torch.Tensor]: + val = self.connection.get(key) + + if val is None: + logger.error("Key %s not found", key) + return None + + return self.d.from_bytes(val) + + def getstr(self, key: str) -> Optional[str]: + val = self.connection.get(key) + if val is None: + logger.error("Key %s not found", key) + return None + + return val.decode("utf-8") + + def set(self, key: str, tensor: torch.Tensor) -> None: + assert tensor is not None + self.connection.set(key, self.s.to_bytes(tensor)) + + def setstr(self, key: str, obj: str) -> None: + self.connection.set(key, obj) + + def list(self, prefix: str) -> List[str]: + cursor = 0 + all_keys: List[bytes] = [] + + while True: + ret: Tuple[int, List[bytes]] = self.connection.scan( + cursor=cursor, match=f"{prefix}*" + ) # type: ignore + cursor, keys = ret + all_keys.extend(keys) + if cursor == 0: + break + + return [key.decode("utf-8") for key in all_keys] + + def weight_iterator( + self, rank: int = 0 + ) -> Generator[Tuple[str, bytes], None, None]: + keys = self.list(f"{self.model_name}/keys/rank_{rank}/") + for key in keys: + val = self.get(key) + key = key.removeprefix(f"{self.model_name}/keys/rank_{rank}/") + yield key, val + + def pull_files( + self, + allow_pattern: Optional[List[str]] = None, + ignore_pattern: Optional[List[str]] = None, + ) -> None: + pull_files_from_db(self, self.model_name, allow_pattern, ignore_pattern) + + def close(self): + self.connection.close() + super().close() diff --git a/python/sglang/srt/connector/s3.py b/python/sglang/srt/connector/s3.py new file mode 100644 index 0000000..7bef8f5 --- /dev/null +++ b/python/sglang/srt/connector/s3.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 + +import fnmatch +import os +from pathlib import Path +from typing import Generator, Optional, Tuple + +import torch + +from sglang.srt.connector import BaseFileConnector + + +def _filter_allow(paths: list[str], patterns: list[str]) -> list[str]: + return [ + path + for path in paths + if any(fnmatch.fnmatch(path, pattern) for pattern in patterns) + ] + + +def _filter_ignore(paths: list[str], patterns: list[str]) -> list[str]: + return [ + path + for path in paths + if not any(fnmatch.fnmatch(path, pattern) for pattern in patterns) + ] + + +def list_files( + s3, + path: str, + allow_pattern: Optional[list[str]] = None, + ignore_pattern: Optional[list[str]] = None, +) -> tuple[str, str, list[str]]: + """ + List files from S3 path and filter by pattern. + + Args: + s3: S3 client to use. + path: The S3 path to list from. + allow_pattern: A list of patterns of which files to pull. + ignore_pattern: A list of patterns of which files not to pull. + + Returns: + tuple[str, str, list[str]]: A tuple where: + - The first element is the bucket name + - The second element is string represent the bucket + and the prefix as a dir like string + - The third element is a list of files allowed or + disallowed by pattern + """ + parts = path.removeprefix("s3://").split("/") + prefix = "/".join(parts[1:]) + bucket_name = parts[0] + + objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=prefix) + paths = [obj["Key"] for obj in objects.get("Contents", [])] + + paths = _filter_ignore(paths, ["*/"]) + if allow_pattern is not None: + paths = _filter_allow(paths, allow_pattern) + + if ignore_pattern is not None: + paths = _filter_ignore(paths, ignore_pattern) + + return bucket_name, prefix, paths + + +class S3Connector(BaseFileConnector): + + def __init__(self, url: str) -> None: + import boto3 + + super().__init__(url) + self.client = boto3.client("s3") + + def glob(self, allow_pattern: Optional[list[str]] = None) -> list[str]: + bucket_name, _, paths = list_files( + self.client, path=self.url, allow_pattern=allow_pattern + ) + return [f"s3://{bucket_name}/{path}" for path in paths] + + def pull_files( + self, + allow_pattern: Optional[list[str]] = None, + ignore_pattern: Optional[list[str]] = None, + ) -> None: + """ + Pull files from S3 storage into the temporary directory. + + Args: + s3_model_path: The S3 path of the model. + allow_pattern: A list of patterns of which files to pull. + ignore_pattern: A list of patterns of which files not to pull. + + """ + bucket_name, base_dir, files = list_files( + self.client, self.url, allow_pattern, ignore_pattern + ) + if len(files) == 0: + return + + for file in files: + destination_file = os.path.join(self.local_dir, file.removeprefix(base_dir)) + local_dir = Path(destination_file).parent + os.makedirs(local_dir, exist_ok=True) + self.client.download_file(bucket_name, file, destination_file) + + def weight_iterator( + self, rank: int = 0 + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + from sglang.srt.model_loader.weight_utils import ( + runai_safetensors_weights_iterator, + ) + + # only support safetensor files now + hf_weights_files = self.glob(allow_pattern=["*.safetensors"]) + return runai_safetensors_weights_iterator(hf_weights_files) + + def close(self): + self.client.close() + super().close() diff --git a/python/sglang/srt/connector/serde/__init__.py b/python/sglang/srt/connector/serde/__init__.py new file mode 100644 index 0000000..394dba0 --- /dev/null +++ b/python/sglang/srt/connector/serde/__init__.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 + +# inspired by LMCache +from typing import Optional, Tuple + +import torch + +from sglang.srt.connector.serde.safe_serde import SafeDeserializer, SafeSerializer +from sglang.srt.connector.serde.serde import Deserializer, Serializer + + +def create_serde(serde_type: str) -> Tuple[Serializer, Deserializer]: + s: Optional[Serializer] = None + d: Optional[Deserializer] = None + + if serde_type == "safe": + s = SafeSerializer() + d = SafeDeserializer(torch.uint8) + else: + raise ValueError(f"Unknown serde type: {serde_type}") + + return s, d + + +__all__ = [ + "Serializer", + "Deserializer", + "SafeSerializer", + "SafeDeserializer", + "create_serde", +] diff --git a/python/sglang/srt/connector/serde/safe_serde.py b/python/sglang/srt/connector/serde/safe_serde.py new file mode 100644 index 0000000..0163af9 --- /dev/null +++ b/python/sglang/srt/connector/serde/safe_serde.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 + +from typing import Union + +import torch +from safetensors.torch import load, save + +from sglang.srt.connector.serde.serde import Deserializer, Serializer + + +class SafeSerializer(Serializer): + + def __init__(self): + super().__init__() + + def to_bytes(self, t: torch.Tensor) -> bytes: + return save({"tensor_bytes": t.cpu().contiguous()}) + + +class SafeDeserializer(Deserializer): + + def __init__(self, dtype): + super().__init__(dtype) + + def from_bytes_normal(self, b: Union[bytearray, bytes]) -> torch.Tensor: + return load(bytes(b))["tensor_bytes"].to(dtype=self.dtype) + + def from_bytes(self, b: Union[bytearray, bytes]) -> torch.Tensor: + return self.from_bytes_normal(b) diff --git a/python/sglang/srt/connector/serde/serde.py b/python/sglang/srt/connector/serde/serde.py new file mode 100644 index 0000000..3d6f804 --- /dev/null +++ b/python/sglang/srt/connector/serde/serde.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 + +import abc +from abc import ABC, abstractmethod + +import torch + + +class Serializer(ABC): + + @abstractmethod + def to_bytes(self, t: torch.Tensor) -> bytes: + """ + Serialize a pytorch tensor to bytes. The serialized bytes should contain + both the data and the metadata (shape, dtype, etc.) of the tensor. + + Input: + t: the input pytorch tensor, can be on any device, in any shape, + with any dtype + + Returns: + bytes: the serialized bytes + """ + raise NotImplementedError + + +class Deserializer(metaclass=abc.ABCMeta): + + def __init__(self, dtype): + self.dtype = dtype + + @abstractmethod + def from_bytes(self, bs: bytes) -> torch.Tensor: + """ + Deserialize a pytorch tensor from bytes. + + Input: + bytes: a stream of bytes + + Output: + torch.Tensor: the deserialized pytorch tensor + """ + raise NotImplementedError diff --git a/python/sglang/srt/connector/utils.py b/python/sglang/srt/connector/utils.py new file mode 100644 index 0000000..6cd12da --- /dev/null +++ b/python/sglang/srt/connector/utils.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +from typing import Optional +from urllib.parse import urlparse + +from sglang.srt.connector import BaseConnector + + +def parse_model_name(url: str) -> str: + """ + Parse the model name from the url. + Only used for db connector + """ + parsed_url = urlparse(url) + return parsed_url.path.lstrip("/") + + +def pull_files_from_db( + connector: BaseConnector, + model_name: str, + allow_pattern: Optional[list[str]] = None, + ignore_pattern: Optional[list[str]] = None, +) -> None: + prefix = f"{model_name}/files/" + local_dir = connector.get_local_dir() + files = connector.list(prefix) + + for file in files: + destination_file = os.path.join(local_dir, file.removeprefix(prefix)) + local_dir = Path(destination_file).parent + os.makedirs(local_dir, exist_ok=True) + with open(destination_file, "wb") as f: + f.write(connector.getstr(file).encode("utf-8")) diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py new file mode 100644 index 0000000..5d1b0da --- /dev/null +++ b/python/sglang/srt/constrained/base_grammar_backend.py @@ -0,0 +1,194 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The baseclass of a backend for grammar-guided constrained decoding.""" + +import logging +from abc import ABC, abstractmethod +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from threading import Event, Lock +from typing import Dict, List, Optional, Tuple + +import torch + +from sglang.srt.server_args import ServerArgs + +logger = logging.getLogger(__name__) + + +class BaseGrammarObject(ABC): + @abstractmethod + def try_jump_forward(self, tokenizer) -> Optional[Tuple[List[int], str]]: + """ + Try to jump forward in the grammar. + + Returns: + A jump forward helper which may be used in `jump_forward_str_state`. + None if the jump forward is not possible. + """ + raise NotImplementedError + + @abstractmethod + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + """ + Jump forward for the grammar. + + Returns: + A tuple of the jump forward string and the next state of the grammar + (which can be used in `jump_and_retokenize` if needed). + """ + raise NotImplementedError + + @abstractmethod + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ) -> None: + """ + Jump forward occurs, and update the grammar state if needed. + """ + raise NotImplementedError + + @abstractmethod + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + raise NotImplementedError + + @abstractmethod + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + raise NotImplementedError + + @staticmethod + @abstractmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + raise NotImplementedError + + @staticmethod + @abstractmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor) -> None: + raise NotImplementedError + + @abstractmethod + def copy(self) -> "BaseGrammarObject": + raise NotImplementedError + + +@dataclass +class CacheEntry: + value: Optional[BaseGrammarObject] + event: Event + + +class BaseGrammarBackend(ABC): + def __init__(self): + self.executor = ThreadPoolExecutor() + self.cache: Dict[Tuple[str, str], CacheEntry] = {} + self.cache_lock = Lock() + + def _not_supported(self, key_type: str, key_string: str) -> None: + logger.warning(f"Skip unsupported {key_type}: {key_type}={key_string}") + + def dispatch_fallback( + self, key_type: str, key_string: str + ) -> Optional[BaseGrammarObject]: + """ + This function should not be reached in any case. + """ + raise ValueError(f"Invalid key_type: {key_type}={key_string}") + + @abstractmethod + def dispatch_json(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("json", key_string) + + @abstractmethod + def dispatch_regex(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("regex", key_string) + + @abstractmethod + def dispatch_ebnf(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("ebnf", key_string) + + @abstractmethod + def dispatch_structural_tag(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("structural_tag", key_string) + + def _init_value_dispatch(self, key: Tuple[str, str]) -> Optional[BaseGrammarObject]: + key_type, key_string = key + if key_type == "json": + return self.dispatch_json(key_string) + elif key_type == "regex": + return self.dispatch_regex(key_string) + elif key_type == "ebnf": + return self.dispatch_ebnf(key_string) + elif key_type == "structural_tag": + return self.dispatch_structural_tag(key_string) + else: + return self.dispatch_fallback(key_type, key_string) + + def _init_value(self, key: Tuple[str, str]) -> Optional[BaseGrammarObject]: + with self.cache_lock: + if key in self.cache: + cache_hit = True + entry = self.cache[key] + else: + cache_hit = False + entry = CacheEntry(None, Event()) + self.cache[key] = entry + + if cache_hit: + entry.event.wait() + else: + entry.value = self._init_value_dispatch(key) + entry.event.set() + return entry.value.copy() if entry.value else None + + def get_cached_value(self, key: Tuple[str, str]) -> Optional[BaseGrammarObject]: + with self.cache_lock: + entry = self.cache.get(key) + if not entry or not entry.event.is_set(): + return None + val = self.cache[key].value + return val.copy() if val else None + + def get_future_value(self, key: Tuple[str, str]) -> Future: + return self.executor.submit(self._init_value, key) + + def reset(self): + with self.cache_lock: + self.cache.clear() + + +def create_grammar_backend(server_args: ServerArgs, tokenizer, vocab_size): + if server_args.grammar_backend == "outlines": + from sglang.srt.constrained.outlines_backend import OutlinesGrammarBackend + + grammar_backend = OutlinesGrammarBackend( + tokenizer, + whitespace_pattern=server_args.constrained_json_whitespace_pattern, + ) + elif server_args.grammar_backend == "xgrammar": + from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend + + grammar_backend = XGrammarGrammarBackend(tokenizer, vocab_size=vocab_size) + elif server_args.grammar_backend == "llguidance": + from sglang.srt.constrained.llguidance_backend import GuidanceBackend + + grammar_backend = GuidanceBackend( + tokenizer=tokenizer, + whitespace_pattern=server_args.constrained_json_whitespace_pattern, + ) + else: + raise ValueError(f"Invalid grammar backend: {server_args.grammar_backend}") + + return grammar_backend diff --git a/python/sglang/srt/constrained/llguidance_backend.py b/python/sglang/srt/constrained/llguidance_backend.py new file mode 100644 index 0000000..24893a4 --- /dev/null +++ b/python/sglang/srt/constrained/llguidance_backend.py @@ -0,0 +1,151 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constrained decoding with llguidance backend.""" + +import json +import os +from typing import List, Optional, Tuple + +import llguidance +import llguidance.hf +import llguidance.torch +import torch +from llguidance.gbnf_to_lark import any_to_lark + +from sglang.srt.constrained.base_grammar_backend import ( + BaseGrammarBackend, + BaseGrammarObject, +) + + +class GuidanceGrammar(BaseGrammarObject): + def __init__( + self, llguidance_tokenizer: llguidance.LLTokenizer, serialized_grammar: str + ): + self.llguidance_tokenizer = llguidance_tokenizer + self.serialized_grammar = serialized_grammar + + # TODO: add support for fast-forward tokens in the future + self.ll_interpreter = llguidance.LLInterpreter( + self.llguidance_tokenizer, + self.serialized_grammar, + enable_backtrack=False, + enable_ff_tokens=False, + log_level=int(os.environ.get("LLGUIDANCE_LOG_LEVEL", "1")), + ) + self.pending_ff_tokens: list[int] = [] + self.finished = False + self.bitmask = None + + def try_jump_forward(self, tokenizer) -> Optional[Tuple[List[int], str]]: + if len(self.pending_ff_tokens) > 0: + s = self.llguidance_tokenizer.decode_str(self.pending_ff_tokens) + ff_tokens = self.pending_ff_tokens + self.pending_ff_tokens = [] + return (ff_tokens, s) + + return None + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + return "", -1 + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + pass + + def accept_token(self, token: int): + backtrack, ff_tokens = self.ll_interpreter.commit_token(token) + if len(ff_tokens) > 0 and backtrack == 0: + # first token is last generated token + ff_tokens = ff_tokens[1:] + self.pending_ff_tokens.extend(ff_tokens) + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + if len(self.pending_ff_tokens) > 0: + # if we have pending fast-forward tokens, + # just return them immediately + ff_token = self.pending_ff_tokens.pop(0) + vocab_mask[idx, :] = 0 + vocab_mask[idx, ff_token // 32] = 1 << (ff_token % 32) + return + + if self.ll_interpreter.has_pending_stop(): + self.finished = True + + llguidance.torch.fill_next_token_bitmask(self.ll_interpreter, vocab_mask, idx) + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + if self.bitmask is None or self.bitmask.shape[0] < batch_size: + # only create bitmask when batch gets larger + self.bitmask = llguidance.torch.allocate_token_bitmask( + batch_size, self.llguidance_tokenizer.vocab_size + ) + bitmask = self.bitmask + else: + bitmask = self.bitmask[:batch_size] + + return bitmask + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + return vocab_mask.to(device, non_blocking=True) + + @staticmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor) -> None: + llguidance.torch.apply_token_bitmask_inplace(logits, vocab_mask) + + def copy(self): + return GuidanceGrammar( + llguidance_tokenizer=self.llguidance_tokenizer, + serialized_grammar=self.serialized_grammar, + ) + + +class GuidanceBackend(BaseGrammarBackend): + def __init__(self, tokenizer, whitespace_pattern: Optional[str] = None): + super().__init__() + + self.tokenizer = tokenizer + self.whitespace_flexible = ( + True if whitespace_pattern == "whitespace_flexible" else False + ) + self.llguidance_tokenizer = llguidance.hf.from_tokenizer(self.tokenizer, None) + + def _from_serialized(self, serialized_grammar) -> GuidanceGrammar: + return GuidanceGrammar( + llguidance_tokenizer=self.llguidance_tokenizer, + serialized_grammar=serialized_grammar, + ) + + def dispatch_json(self, key_string: str) -> GuidanceGrammar: + json_schema = key_string + compiler = llguidance.JsonCompiler(whitespace_flexible=self.whitespace_flexible) + serialized_grammar = compiler.compile(json_schema) + return self._from_serialized(serialized_grammar) + + def dispatch_regex(self, key_string: str) -> GuidanceGrammar: + compiler = llguidance.RegexCompiler() + serialized_grammar = compiler.compile(regex=key_string) + return self._from_serialized(serialized_grammar) + + def dispatch_ebnf(self, key_string: str) -> GuidanceGrammar: + compiler = llguidance.LarkCompiler() + serialized_grammar = compiler.compile(any_to_lark(key_string)) + return self._from_serialized(serialized_grammar) + + def dispatch_structural_tag(self, key_string: str): + return super().dispatch_structural_tag(key_string) diff --git a/python/sglang/srt/constrained/outlines_backend.py b/python/sglang/srt/constrained/outlines_backend.py new file mode 100644 index 0000000..7a05bb2 --- /dev/null +++ b/python/sglang/srt/constrained/outlines_backend.py @@ -0,0 +1,188 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constrained decoding with outlines backend.""" + +import json +import logging +from typing import Dict, List, Optional, Tuple, Union + +import interegular +import torch +from outlines.fsm.guide import RegexGuide +from outlines.models.transformers import TransformerTokenizer +from pydantic import BaseModel + +from sglang.srt.constrained.base_grammar_backend import ( + BaseGrammarBackend, + BaseGrammarObject, +) +from sglang.srt.constrained.outlines_jump_forward import OutlinesJumpForwardMap + +try: + from outlines.fsm.json_schema import build_regex_from_schema +except ImportError: + from outlines_core.fsm.json_schema import build_regex_from_schema + + +logger = logging.getLogger(__name__) + + +class OutlinesGrammar(BaseGrammarObject): + def __init__( + self, + guide: RegexGuide, + jump_forward_map: Union[OutlinesJumpForwardMap, None], + ) -> None: + self.guide = guide + self.jump_forward_map = jump_forward_map + self.state = 0 + self.finished = False + + def accept_token(self, token: int): + self.state = self.guide.get_next_state(self.state, token) + + def try_jump_forward(self, tokenizer) -> Optional[Tuple]: + if not self.jump_forward_map: + return None + + jump_forward_bytes = self.jump_forward_map.jump_forward_byte(self.state) + if jump_forward_bytes is None or len(jump_forward_bytes) <= 1: + return None + + # preprocess the jump forward string + suffix_bytes = [] + continuation_range = range(0x80, 0xC0) + cur_state = self.state + while ( + len(jump_forward_bytes) and jump_forward_bytes[0][0] in continuation_range + ): + # continuation bytes + byte_edge = jump_forward_bytes.pop(0) + suffix_bytes.append(byte_edge[0]) + cur_state = byte_edge[1] + + suffix_tokens = [f"<0x{hex(b)[2:].upper()}>" for b in suffix_bytes] + suffix_ids = tokenizer.convert_tokens_to_ids(suffix_tokens) + return suffix_ids, cur_state + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + _, cur_state = helper + return self.jump_forward_map.jump_forward_symbol(cur_state) + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + self.state = next_state + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + return torch.zeros(batch_size, vocab_size, dtype=torch.bool, device=device) + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + return vocab_mask + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + tokens = torch.tensor( + self.guide.get_next_instruction(self.state).tokens, dtype=torch.int64 + ).to(vocab_mask.device, non_blocking=True) + vocab_mask = vocab_mask[idx] + vocab_mask.fill_(1) + vocab_mask.scatter_(0, tokens, torch.zeros_like(tokens, dtype=torch.bool)) + + @staticmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor): + logits.masked_fill_(vocab_mask, float("-inf")) + + def copy(self): + return OutlinesGrammar(self.guide, self.jump_forward_map) + + +class OutlinesGrammarBackend(BaseGrammarBackend): + def __init__( + self, + tokenizer, + whitespace_pattern: bool, + ): + super().__init__() + + try: + self.outlines_tokenizer = TransformerTokenizer(tokenizer) + except AttributeError: + # FIXME: tmp fix for chatglm2 & chatglm3 (pad_token_id=0) + origin_pad_token_id = tokenizer.pad_token_id + + def fset(self, value): + self._value = value + + type(tokenizer).pad_token_id = property( + fget=type(tokenizer).pad_token_id.fget, fset=fset + ) + self.outlines_tokenizer = TransformerTokenizer(tokenizer) + self.outlines_tokenizer.tokenizer.pad_token_id = origin_pad_token_id + self.outlines_tokenizer.pad_token_id = origin_pad_token_id + self.outlines_tokenizer.pad_token = ( + self.outlines_tokenizer.tokenizer.pad_token + ) + self.outlines_tokenizer.vocabulary = ( + self.outlines_tokenizer.tokenizer.get_vocab() + ) + self.whitespace_pattern = whitespace_pattern + + def _compile_regex(self, regex: str) -> Optional[OutlinesGrammar]: + try: + if hasattr(RegexGuide, "from_regex"): + # outlines >= 0.1.1 + guide = RegexGuide.from_regex(regex, self.outlines_tokenizer) + else: + # outlines <= 0.0.46 + guide = RegexGuide(regex, self.outlines_tokenizer) + except interegular.patterns.InvalidSyntax as e: + logger.warning(f"skip invalid regex schema: {regex=}, {e=}") + return None + + jump_forward_map = None + return OutlinesGrammar(guide, jump_forward_map) + + def dispatch_ebnf(self, key_string: str): + return super().dispatch_ebnf(key_string) + + def dispatch_structural_tag(self, key_string: str): + return super().dispatch_structural_tag(key_string) + + def dispatch_json(self, key_string: str): + try: + regex = build_regex_from_object( + key_string, + whitespace_pattern=self.whitespace_pattern, + ) + except (NotImplementedError, json.decoder.JSONDecodeError) as e: + logger.warning(f"Skip invalid json_schema: json_schema={key_string}, {e=}") + return self._compile_regex(regex) + + def dispatch_regex(self, key_string: str): + return self._compile_regex(key_string) + + +def build_regex_from_object( + object: Union[str, BaseModel, Dict], whitespace_pattern: Optional[str] = None +): + if isinstance(object, type(BaseModel)): + schema = json.dumps(object.model_json_schema()) + elif isinstance(object, Dict): + schema = json.dumps(object) + else: + schema = object + return build_regex_from_schema(schema, whitespace_pattern) diff --git a/python/sglang/srt/constrained/outlines_jump_forward.py b/python/sglang/srt/constrained/outlines_jump_forward.py new file mode 100644 index 0000000..5e32c40 --- /dev/null +++ b/python/sglang/srt/constrained/outlines_jump_forward.py @@ -0,0 +1,187 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Faster constrained decoding with jump forward decoding / compressed finite state machine. +Reference: https://lmsys.org/blog/2024-02-05-compressed-fsm/ +""" + +import dataclasses +import logging +from collections import defaultdict + +import interegular +from interegular import InvalidSyntax +from outlines.caching import cache as disk_cache + +try: + # outlines >= 0.1.0 + from outlines_core.fsm.outlines_core_rs import FSMInfo + from outlines_core.fsm.regex import make_byte_level_fsm, make_deterministic_fsm +except ImportError: + # outlines <= 0.0.46 + from outlines.fsm.regex import FSMInfo, make_byte_level_fsm, make_deterministic_fsm + +IP_REGEX = r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class JumpEdge: + symbol: str = None + symbol_next_state: int = None + byte: int = None + byte_next_state: int = None + + +@disk_cache() +def init_state_to_jump_forward(regex_string): + try: + regex_pattern = interegular.parse_pattern(regex_string) + except InvalidSyntax as e: + logger.warning(f"skip invalid regex: {regex_string}, {e=}") + return + + byte_fsm = make_byte_level_fsm(regex_pattern.to_fsm().reduce(), keep_utf8=True) + regex_fsm, _ = make_deterministic_fsm(byte_fsm) + + fsm_info: FSMInfo = regex_fsm.fsm_info + + symbol_to_id = fsm_info.alphabet_symbol_mapping + id_to_symbol = {} + for symbol, id_ in symbol_to_id.items(): + id_to_symbol.setdefault(id_, []).append(symbol) + + transitions = fsm_info.transitions + + outgoings_ct = defaultdict(int) + # NOTE(lsyin): Final states can lead to terminate, so they have one outgoing edge naturally + for s in fsm_info.finals: + outgoings_ct[s] = 1 + + state_to_jump_forward = {} + for (state, id_), next_state in transitions.items(): + if id_ == fsm_info.alphabet_anything_value: + # Arbitrarily symbol cannot be recognized as jump forward + continue + + symbols = id_to_symbol[id_] + for c in symbols: + if len(c) > 1: + # Skip byte level transitions like c = "5E" + continue + + outgoings_ct[state] += 1 + if outgoings_ct[state] > 1: + if state in state_to_jump_forward: + del state_to_jump_forward[state] + break + + state_to_jump_forward[state] = JumpEdge( + symbol=c, + symbol_next_state=next_state, + ) + + # Process the byte level jump forward + outgoings_ct = defaultdict(int) + for s in fsm_info.finals: + outgoings_ct[s] = 1 + + for (state, id_), next_state in transitions.items(): + if id_ == fsm_info.alphabet_anything_value: + continue + symbols = id_to_symbol[id_] + for c in symbols: + byte_ = None + if len(c) == 1 and ord(c) < 0x80: + # ASCII character + byte_ = ord(c) + elif len(c) > 1: + # FIXME: This logic is due to the leading \x00 + # https://github.com/outlines-dev/outlines/pull/930 + byte_ = int(symbols[0][1:], 16) + + if byte_ is not None: + outgoings_ct[state] += 1 + if outgoings_ct[state] > 1: + if state in state_to_jump_forward: + del state_to_jump_forward[state] + break + e = state_to_jump_forward.get(state, JumpEdge()) + e.byte = byte_ + e.byte_next_state = next_state + state_to_jump_forward[state] = e + + return state_to_jump_forward + + +class OutlinesJumpForwardMap: + def __init__(self, regex_string): + self.state_to_jump_forward = init_state_to_jump_forward(regex_string) + + def jump_forward_symbol(self, state): + jump_forward_str = "" + next_state = state + while state in self.state_to_jump_forward: + e = self.state_to_jump_forward[state] + if e.symbol is None: + break + jump_forward_str += e.symbol + next_state = e.symbol_next_state + state = next_state + + return jump_forward_str, next_state + + def jump_forward_byte(self, state): + if state not in self.state_to_jump_forward: + return None + + jump_forward_bytes = [] + next_state = None + while state in self.state_to_jump_forward: + e = self.state_to_jump_forward[state] + assert e.byte is not None and e.byte_next_state is not None + jump_forward_bytes.append((e.byte, e.byte_next_state)) + next_state = e.byte_next_state + state = next_state + + return jump_forward_bytes + + def is_jump_forward_symbol_state(self, state): + return ( + state in self.state_to_jump_forward + and self.state_to_jump_forward[state].symbol is not None + ) + + +def test_main(regex_string): + jump_forward_map = OutlinesJumpForwardMap(regex_string) + for state, e in jump_forward_map.state_to_jump_forward.items(): + if e.symbol is not None: + jump_forward_str, next_state = jump_forward_map.jump_forward_symbol(state) + print(f"{state} -> {next_state}", jump_forward_str) + bytes_ = jump_forward_map.jump_forward_byte(state) + print(f"{state} -> {bytes_[-1][1]}", [hex(b) for b, _ in bytes_]) + + +if __name__ == "__main__": + import outlines + + outlines.caching.clear_cache() + test_main(r"The google's DNS sever address is " + IP_REGEX) + test_main(r"霍格沃茨特快列车|霍比特人比尔博") + # 霍格: \xe9\x9c\x8d \xe6\xa0\xbc ... + # 霍比: \xe9\x9c\x8d \xe6\xaf\x94 ... + + test_main(r"[-+]?[0-9]+[ ]*") diff --git a/python/sglang/srt/constrained/xgrammar_backend.py b/python/sglang/srt/constrained/xgrammar_backend.py new file mode 100644 index 0000000..4df3ae2 --- /dev/null +++ b/python/sglang/srt/constrained/xgrammar_backend.py @@ -0,0 +1,183 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constrained decoding with xgrammar backend.""" + +import json +import logging +from typing import List, Optional, Tuple, Union + +import torch +from xgrammar import ( + CompiledGrammar, + GrammarCompiler, + GrammarMatcher, + StructuralTagItem, + TokenizerInfo, + allocate_token_bitmask, + apply_token_bitmask_inplace, +) + +from sglang.srt.constrained.base_grammar_backend import ( + BaseGrammarBackend, + BaseGrammarObject, +) + +logger = logging.getLogger(__name__) + + +MAX_ROLLBACK_TOKENS = 200 + + +class XGrammarGrammar(BaseGrammarObject): + + def __init__( + self, + matcher: GrammarMatcher, + vocab_size: int, + ctx: CompiledGrammar, + override_stop_tokens: Optional[Union[List[int], int]], + ) -> None: + self.matcher = matcher + self.vocab_size = vocab_size + self.ctx = ctx + self.override_stop_tokens = override_stop_tokens + self.finished = False + + def accept_token(self, token: int): + assert self.matcher.accept_token(token) + + def try_jump_forward(self, tokenizer) -> Optional[Tuple[List[int], str]]: + s = self.matcher.find_jump_forward_string() + if s: + return [], s + return None + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + _, data = helper + return data, -1 + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + k = 0 + for i, old_id in enumerate(old_output_ids): + if old_id == new_output_ids[i]: + k = i + 1 + else: + break + + # rollback to the last token that is the same + if k < len(old_output_ids): + self.matcher.rollback(len(old_output_ids) - k) + + for i in range(k, len(new_output_ids)): + assert self.matcher.accept_token(new_output_ids[i]) + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + return allocate_token_bitmask(batch_size, vocab_size) + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + self.matcher.fill_next_token_bitmask(vocab_mask, idx) + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + return vocab_mask.to(device, non_blocking=True) + + @staticmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor) -> None: + apply_token_bitmask_inplace(logits, vocab_mask) + + def copy(self): + matcher = GrammarMatcher( + self.ctx, + max_rollback_tokens=MAX_ROLLBACK_TOKENS, + override_stop_tokens=self.override_stop_tokens, + ) + return XGrammarGrammar( + matcher, self.vocab_size, self.ctx, self.override_stop_tokens + ) + + +class XGrammarGrammarBackend(BaseGrammarBackend): + def __init__( + self, + tokenizer, + vocab_size: int, + ): + super().__init__() + + tokenizer_info = TokenizerInfo.from_huggingface( + tokenizer, vocab_size=vocab_size + ) + override_stop_tokens = None + + self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info) + self.vocab_size = vocab_size + self.override_stop_tokens = override_stop_tokens + + def _from_context(self, ctx: CompiledGrammar) -> XGrammarGrammar: + matcher = GrammarMatcher(ctx, max_rollback_tokens=MAX_ROLLBACK_TOKENS) + return XGrammarGrammar(matcher, self.vocab_size, ctx, self.override_stop_tokens) + + def dispatch_json(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + if key_string == "$$ANY$$": + ctx = self.grammar_compiler.compile_builtin_json_grammar() + else: + ctx = self.grammar_compiler.compile_json_schema(schema=key_string) + except RuntimeError as e: + logging.warning(f"Skip invalid json_schema: json_schema={key_string}, {e=}") + return None + return self._from_context(ctx) + + def dispatch_ebnf(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + ctx = self.grammar_compiler.compile_grammar(key_string) + except RuntimeError as e: + logging.warning(f"Skip invalid ebnf: ebnf={key_string}, {e=}") + return None + return self._from_context(ctx) + + def dispatch_regex(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + ctx = self.grammar_compiler.compile_regex(key_string) + except RuntimeError as e: + logging.warning(f"Skip invalid regex: regex={key_string}, {e=}") + return None + return self._from_context(ctx) + + def dispatch_structural_tag(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + structural_tag = json.loads(key_string) + tags = [ + StructuralTagItem( + begin=structure["begin"], + schema=json.dumps(structure["schema"]), + end=structure["end"], + ) + for structure in structural_tag["structures"] + ] + ctx = self.grammar_compiler.compile_structural_tag( + tags, structural_tag["triggers"] + ) + except RuntimeError as e: + logging.warning(f"Skip invalid regex: regex={key_string}, {e=}") + return None + return self._from_context(ctx) + + def reset(self): + if self.grammar_compiler: + self.grammar_compiler.clear_cache() diff --git a/python/sglang/srt/conversation.py b/python/sglang/srt/conversation.py new file mode 100644 index 0000000..70152a6 --- /dev/null +++ b/python/sglang/srt/conversation.py @@ -0,0 +1,736 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Conversation chat templates.""" + +# Adapted from +# https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py +import dataclasses +from enum import IntEnum, auto +from typing import Dict, List, Optional, Tuple, Union + +from sglang.srt.openai_api.protocol import ChatCompletionRequest + + +class SeparatorStyle(IntEnum): + """Separator styles.""" + + ADD_COLON_SINGLE = auto() + ADD_COLON_TWO = auto() + ADD_COLON_SPACE_SINGLE = auto() + NO_COLON_SINGLE = auto() + NO_COLON_TWO = auto() + ADD_NEW_LINE_SINGLE = auto() + LLAMA2 = auto() + LLAMA3 = auto() + CHATGLM = auto() + CHATML = auto() + CHATINTERN = auto() + DOLLY = auto() + RWKV = auto() + PHOENIX = auto() + ROBIN = auto() + FALCON_CHAT = auto() + CHATGLM3 = auto() + DEEPSEEK_CHAT = auto() + METAMATH = auto() + DeepSeekVL2 = auto() + QWEN2_VL_EMBED = auto() + GEMMA3 = auto() + + +@dataclasses.dataclass +class Conversation: + """A class that manages prompt templates and keeps all conversation history.""" + + # The name of this template + name: str + # The template of the system prompt + system_template: str = "{system_message}" + # The system message + system_message: str = "" + # The names of two roles + roles: Tuple[str] = ("USER", "ASSISTANT") + # All messages. Each item is (role, message). + messages: List[List[str]] = () + # The number of few shot examples + offset: int = 0 + # The separator style and configurations + sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE + sep: str = "\n" + sep2: str = None + # Stop criteria (the default one is EOS token) + stop_str: Union[str, List[str]] = None + # The string that represents an image token in the prompt + image_token: str = "" + audio_token: str = "

Prompt conversation

+{% for message in prompt_messages %} +{{ message_to_html(message) | safe }} +{% endfor %} +

Sampled message

+{{ message_to_html(next_message) | safe }} +

Results

+

Correct Answer: {{ correct_answer }}

+

Extracted Answer: {{ extracted_answer }}

+

Score: {{ score }}

+""" + + +def format_multichoice_question(row): + return QUERY_TEMPLATE_MULTICHOICE.format(**row) + + +def check_equality(sampler: SamplerBase, expr1: str, expr2: str): + prompt = EQUALITY_TEMPLATE % {"expression1": expr1, "expression2": expr2} + response = sampler([dict(content=prompt, role="user")]) + return response.lower().strip() == "yes" + + +def _compute_stat(values: list, stat: str): + if stat == "mean": + return np.mean(values) + elif stat == "std": + return np.std(values) + elif stat == "min": + return np.min(values) + elif stat == "max": + return np.max(values) + else: + raise ValueError(f"Unknown {stat =}") + + +def aggregate_results( + single_eval_results: List[SingleEvalResult], + default_stats: Tuple[str] = ("mean", "std"), + name2stats: Optional[Dict[str, Tuple[str]]] = None, +) -> EvalResult: + """ + Aggregate results from multiple evaluations into a single EvalResult. + """ + name2stats = name2stats or {} + name2values = defaultdict(list) + htmls = [] + convos = [] + for single_eval_result in single_eval_results: + for name, value in single_eval_result.metrics.items(): + name2values[name].append(value) + if single_eval_result.score is not None: + name2values["score"].append(single_eval_result.score) + htmls.append(single_eval_result.html) + convos.append(single_eval_result.convo) + final_metrics = {} + for name, values in name2values.items(): + stats = name2stats.get(name, default_stats) + for stat in stats: + key = name if stat == "mean" else f"{name}:{stat}" + final_metrics[key] = _compute_stat(values, stat) + return EvalResult( + score=final_metrics.pop("score", None), + metrics=final_metrics, + htmls=htmls, + convos=convos, + ) + + +def map_with_progress(f: callable, xs: List[Any], num_threads: int): + """ + Apply f to each element of xs, using a ThreadPool, and show progress. + """ + if os.getenv("debug"): + return list(map(f, tqdm(xs, total=len(xs)))) + else: + with ThreadPool(min(num_threads, len(xs))) as pool: + return list(tqdm(pool.imap(f, xs), total=len(xs))) + + +jinja_env = jinja2.Environment( + loader=jinja2.BaseLoader(), + undefined=jinja2.StrictUndefined, + autoescape=jinja2.select_autoescape(["html", "xml"]), +) +_message_template = """ +
+
+ {{ role }} + {% if variant %}({{ variant }}){% endif %} +
+
+ 
{{ content }}
+
+
+""" + + +def message_to_html(message: Message) -> str: + """ + Generate HTML snippet (inside a
) for a message. + """ + return jinja_env.from_string(_message_template).render( + role=message["role"], + content=message["content"], + variant=message.get("variant", None), + ) + + +jinja_env.globals["message_to_html"] = message_to_html + + +_report_template = """ + + + + + + {% if metrics %} +

Metrics

+ + + + + + + + + + {% for name, value in metrics.items() %} + + + + + {% endfor %} +
MetricValue
Score{{ score | float | round(3) }}
{{ name }}{{ value }}
+ {% endif %} +

Examples

+ {% for html in htmls %} + {{ html | safe }} +
+ {% endfor %} + + +""" + + +def make_report(eval_result: EvalResult) -> str: + """ + Create a standalone HTML report from an EvalResult. + """ + return jinja_env.from_string(_report_template).render( + score=eval_result.score, + metrics=eval_result.metrics, + htmls=eval_result.htmls, + ) + + +def make_report_from_example_htmls(htmls: List[str]): + """ + Create a standalone HTML report from a list of example htmls + """ + return jinja_env.from_string(_report_template).render( + score=None, metrics={}, htmls=htmls + ) + + +def download_dataset(path, url): + print(f"Downloading dataset {path} from {url}") + try: + response = requests.get(url, stream=True) + response.raise_for_status() + + total_size = int(response.headers.get("content-length", 0)) + block_size = 8192 + + with open(path, "wb") as f, tqdm( + desc="Downloading", + total=total_size, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as progress_bar: + for data in response.iter_content(block_size): + size = f.write(data) + progress_bar.update(size) + + print(f"Dataset downloaded and saved to {path}") + except requests.RequestException as e: + raise Exception(f"Failed to download dataset: {e}") + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") diff --git a/python/sglang/test/simple_eval_gpqa.py b/python/sglang/test/simple_eval_gpqa.py new file mode 100644 index 0000000..ec2abb4 --- /dev/null +++ b/python/sglang/test/simple_eval_gpqa.py @@ -0,0 +1,93 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +GPQA: A Graduate-Level Google-Proof Q&A Benchmark +David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, Samuel R. Bowman +https://arxiv.org/abs/2311.12022 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + MessageList, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) + + +class GPQAEval(Eval): + def __init__( + self, + filename: str, + num_examples: Optional[int], + num_threads: int, + n_repeats: int = 1, + ): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + rng = random.Random(0) + if num_examples: + assert n_repeats == 1, "n_repeats only supported for num_examples" + examples = rng.sample(examples, num_examples) + examples = examples * n_repeats + examples = [ + example | {"permutation": rng.sample(range(4), 4)} for example in examples + ] + self.examples = examples + self.n_repeats = n_repeats + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + choices = [ + row["Correct Answer"], + row["Incorrect Answer 1"], + row["Incorrect Answer 2"], + row["Incorrect Answer 3"], + ] + choices = [choices[i] for i in row["permutation"]] + correct_index = choices.index(row["Correct Answer"]) + correct_answer = "ABCD"[correct_index] + choices_dict = dict( + A=choices[0], + B=choices[1], + C=choices[2], + D=choices[3], + Question=row["Question"], + ) + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(choices_dict), role="user" + ) + ] + response_text = sampler(prompt_messages) + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == correct_answer else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={"chars": len(response_text)}, + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/simple_eval_humaneval.py b/python/sglang/test/simple_eval_humaneval.py new file mode 100644 index 0000000..fe6f32f --- /dev/null +++ b/python/sglang/test/simple_eval_humaneval.py @@ -0,0 +1,133 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +HumanEval: Evaluating Large Language Models Trained on Code +Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba +https://arxiv.org/abs/2107.03374 https://github.com/openai/human-eval/ +""" + +import random +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, List, Optional + +import tqdm + +try: + from human_eval.data import read_problems + from human_eval.evaluation import estimate_pass_at_k + from human_eval.execution import check_correctness # , unsafe_execute +except (ImportError, ModuleNotFoundError): + print("\nPlease install human-eval at https://github.com/openai/human-eval.\n") + raise + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + + +def evaluate_functional_correctness( + sample: Dict[str, str], + completions: List[str], + n_workers: int = 4, + timeout: float = 3.0, +): + """ + Evaluates the functional correctness of generated samples, and writes + results to f"{sample_file}_results.jsonl.gz" + """ + import copy + + # Check the generated samples against test suites. + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = [] + for i, completion in enumerate(completions): + args = (sample, completion, timeout, i) + future = executor.submit(check_correctness, *args) + futures.append(future) + results = [] + for future in as_completed(futures): + result = future.result() + results.append(result) + passed = [int(r["passed"]) for r in results] + return passed + + +class HumanEval(Eval): + def __init__( + self, + num_examples: Optional[int], + num_threads: int, + num_samples_per_task: int = 5, + ks_passes: List[int] = [1, 2, 5], + timeout: int = 120, + ): + self.seed = 0 + self.examples = read_problems() + self.examples = list(self.examples.values()) + + self._num_examples = num_examples + if self._num_examples: + self.examples = random.Random(self.seed).sample(self.examples, num_examples) + self._num_samples_per_task = num_samples_per_task + self._ks_passes = ks_passes + self._timeout = timeout + self._num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + instruction = "Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n" + + def find_code(completion): + pattern = re.compile(r"```python\n(.*?)```", re.DOTALL) + matches = pattern.findall(completion) + extracted_answer = matches[0] if len(matches) >= 1 else completion + extracted_answer = extracted_answer[ + extracted_answer.find(":\n ") + 2 : + ] # remove signature + return extracted_answer + + def fn(sample: Dict[str, str]): + prompt_messages = [ + sampler._pack_message( + role="user", content=instruction + sample["prompt"] + ) + ] + completions = [ + find_code(sampler(prompt_messages)) + for _ in range(self._num_samples_per_task) + ] + results = evaluate_functional_correctness(sample, completions) + total = len(results) + correct = sum(results) + score = sum(results) / len(results) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=completions[0], role="assistant"), + score=score, + correct_answer=[1] * len(results), + extracted_answer=results, + ) + convo = prompt_messages + [ + dict(content=completion, role="assistant") for completion in completions + ] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={ + f"pass@{k}": estimate_pass_at_k([total], [correct], k) + # this will be aggrated so no need of .mean() + for k in self._ks_passes + if total >= k + }, + ) + + results = common.map_with_progress( + fn, self.examples, num_threads=self._num_threads + ) + return common.aggregate_results(results) diff --git a/python/sglang/test/simple_eval_math.py b/python/sglang/test/simple_eval_math.py new file mode 100644 index 0000000..74c49ab --- /dev/null +++ b/python/sglang/test/simple_eval_math.py @@ -0,0 +1,73 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Mathematical Problem Solving With the MATH Dataset +Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2103.03874 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + check_equality, +) + +QUERY_TEMPLATE = """ +Solve the following math problem step by step. The last line of your response should be of the form Answer: $ANSWER (without quotes) where $ANSWER is the answer to the problem. + +{Question} + +Remember to put your answer on its own line after "Answer:", and you do not need to use a \\boxed command. +""".strip() + + +class MathEval(Eval): + def __init__( + self, + filename: str, + equality_checker: SamplerBase, + num_examples: Optional[int], + num_threads: int, + ): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.equality_checker = equality_checker + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + prompt_messages = [ + sampler._pack_message(content=QUERY_TEMPLATE.format(**row), role="user") + ] + response_text = sampler(prompt_messages) + match = re.search(ANSWER_PATTERN, response_text) + extracted_answer = match.group(1) if match else None + score = float( + check_equality(self.equality_checker, row["Answer"], extracted_answer) + ) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult(html=html, score=score, convo=convo) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/simple_eval_mgsm.py b/python/sglang/test/simple_eval_mgsm.py new file mode 100644 index 0000000..0b0b72a --- /dev/null +++ b/python/sglang/test/simple_eval_mgsm.py @@ -0,0 +1,203 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +MGSM: Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems. +Language Models are Multilingual Chain-of-Thought Reasoners +Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei +https://arxiv.org/abs/2210.03057 reference: https://github.com/google-research/url-nlp +""" + +import re +import urllib +from typing import Optional + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + +ALL_LANGUAGES = ["bn", "de", "en", "es", "fr", "ja", "ru", "sw", "te", "th", "zh"] +LATIN_LANGUAGES = ["de", "en", "es", "fr", "sw"] +NON_LATIN_LANGUAGES = ["bn", "ja", "ru", "te", "th", "zh"] + +LANG_TO_FPATH = { + "bn": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_bn.tsv", + "de": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_de.tsv", + "en": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_en.tsv", + "es": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_es.tsv", + "fr": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_fr.tsv", + "ja": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ja.tsv", + "ru": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ru.tsv", + "sw": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_sw.tsv", + "te": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_te.tsv", + "th": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_th.tsv", + "zh": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_zh.tsv", +} +LANG_TO_INSTRUCTIONS = { + "en": """Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:". + +{input}""", + "bn": """এই গণিতের সমস্যাটি সমাধান করুন। চূড়ান্ত উত্তর দেওয়ার আগে যুক্তিসম্পন্ন পদক্ষেপ প্রদান করুন। চূড়ান্ত উত্তরটি একক সংখ্যা হিসাবে "উত্তর:" এর পরে শেষ লাইনে দিন। "উত্তর:" এর পরে অন্য কিছু যুক্ত করবেন না।. + +{input}""", + "de": """Löse dieses Mathematikproblem. Gib die Schritte zur Begründung an, bevor du die endgültige Antwort in der letzten Zeile alleine im Format "Antwort:" gibst. Füge nichts anderes als die ganzzahlige Antwort nach "Antwort:" hinzu. + +{input}""", + "es": """Resuelve este problema matemático. Proporciona los pasos de razonamiento antes de dar la respuesta final en la última línea por sí misma en el formato de "Respuesta:". No añadas nada más que la respuesta entera después de "Respuesta:". + +{input}""", + "fr": """Résolvez ce problème de mathématiques. Donnez les étapes de raisonnement avant de fournir la réponse finale sur la dernière ligne elle-même dans le format de "Réponse:". N'ajoutez rien d'autre que la réponse entière après "Réponse:". + +{input}""", + "ja": """の数学の問題を解いてください。最終的な答えを出す前に、解答の推論過程を記述してください。そして最後の行には "答え:" の形式で答えを記述し、その後には整数の答え以外何も追加しないでください。 + +{input}""", + "ru": """Решите эту математическую задачу. Объясните шаги рассуждения перед тем, как дать окончательный ответ в последней строке сам по себе в формате "Ответ:". Не добавляйте ничего, кроме целочисленного ответа после "Ответ:". + +{input}""", + "sw": """Suluhisha tatizo hili la hesabu. Toa hatua za mantiki kabla ya kutoa jibu la mwisho kwenye mstari wa mwisho peke yake katika muundo wa "Jibu:". Usiongeze chochote kingine isipokuwa jibu la integer baada ya "Jibu:". + +{input}""", + "te": """ఈ గణిత సమస్యను పరిష్కరించండి. చివరి సమాధానాన్ని ఇవ్వదానికి ముందు తర్కాత్మక అదుగులను ఇవ్వండి. చివరి పంక్తిలో మాత్రమే 'సమాధానం:' అనే ఆకారంలో చివరి సమాధానాద్ని ఇవ్వండి సమాధానం: తర్వాత పూర్ణాంక సమాధానానికి తప్పించి ఎదేనా చేర్చవద్దు. + +{input}""", + "th": """แก้ปัญหาคณิตศาสตร์นี้ ให้ให้ขั้นตอนการใช้เหตุผลก่อนที่จะให้คำตอบสุดท้ายในบรรทัดสุดท้ายโดยอยู่ในรูปแบบ "คำตอบ:" ไม่ควรเพิ่มอะไรนอกจากคำตอบที่เป็นจำนวนเต็มหลังจาก "คำตอบ:" + +{input}""", + "zh": """解决这个数学问题。在最后一行给出答案前，请提供推理步骤。最后一行应该以 "答案: " 的形式独立给出答案。在 "答案：" 后不要添加除整数答案之外的任何内容。 + +{input}""", +} + +LANG_TO_ANSWER_PREFIX = { + "en": "Answer", + "bn": "উত্তর", + "de": "Antwort", + "es": "Respuesta", + "fr": "Réponse", + "ja": "答え", + "ru": "Ответ", + "sw": "Jibu", + "te": "సమాధానం", + "th": "คำตอบ", + "zh": "答案", +} + + +def parse_answer(answer: str, answer_prefix: str) -> str: + if answer_prefix not in answer: + return "" + + answer_text = answer.split(answer_prefix)[-1].strip() + + # find all the numbers (including decimals) in the string + numbers = re.findall(r"\d+\.?\d*", answer_text.replace(",", "")) + + # return the first number (removing trailing decimal point if present), + # or an empty string if there were no numbers + return numbers[-1].rstrip(".") if numbers else "" + + +def score_mgsm(target: str, prediction: str) -> bool: + if "." in prediction: + prediction = prediction.rstrip("0").rstrip(".") + + target = target.replace(",", "") + prediction = prediction.replace(",", "") + + return target == prediction + + +def get_lang_examples(lang: str) -> list[dict[str, str]]: + fpath = LANG_TO_FPATH[lang] + examples = [] + with urllib.request.urlopen(fpath) as f: + for line in f.read().decode("utf-8").splitlines(): + inputs, targets = line.strip().split("\t") + if "." in targets: + raise ValueError(f"targets {targets} contains a decimal point.") + # targets = int(targets.replace(",", "")) + examples.append({"inputs": inputs, "targets": targets, "lang": lang}) + return examples + + +def get_all_examples() -> list[dict[str, str]]: + examples = [] + for lang in ALL_LANGUAGES: + if lang != "en": + continue + examples += get_lang_examples(lang) + return examples + + +class MGSMEval(Eval): + def __init__( + self, + num_examples_per_lang: int = 250, # restrict to a subset of the data for debugging + num_threads: int = 64, + languages: Optional[list[str]] = ALL_LANGUAGES, + ): + if languages is None: + languages = ALL_LANGUAGES + else: + for language in languages: + if language not in ALL_LANGUAGES: + raise ValueError( + f"language {language} is not a valid language. " + f"It should be one in {ALL_LANGUAGES}" + ) + self._languages = languages + self._num_examples_per_lang = num_examples_per_lang + self._num_threads = num_threads + + examples = [] + for lang in self._languages: + lang_examples = get_lang_examples(lang) + examples.extend(lang_examples[: self._num_examples_per_lang]) + self.examples = examples + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(example: dict[str, str]): + language = example["lang"] + latin_language = ( + "group_latin" if language in LATIN_LANGUAGES else "group_non_latin" + ) + correct_answer = example["targets"] + instructoin = LANG_TO_INSTRUCTIONS[language] + prompt_messages = [ + sampler._pack_message( + content=instructoin.format(input=example["inputs"]), role="user" + ) + ] + try: + response_text = sampler(prompt_messages) + except Exception as e: + response_text = "" + + answer_prefix = LANG_TO_ANSWER_PREFIX[language] + extracted_answer = parse_answer(response_text, answer_prefix) + + score = score_mgsm(correct_answer, extracted_answer) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={language: score, latin_language: score}, + ) + + results = common.map_with_progress( + fn, self.examples, num_threads=self._num_threads + ) + return common.aggregate_results(results, default_stats=("mean", "std")) diff --git a/python/sglang/test/simple_eval_mmlu.py b/python/sglang/test/simple_eval_mmlu.py new file mode 100644 index 0000000..36a5c7f --- /dev/null +++ b/python/sglang/test/simple_eval_mmlu.py @@ -0,0 +1,121 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Massive Multitask Language Understanding +Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2009.03300 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) + +subject2category = { + "abstract_algebra": "stem", + "anatomy": "other", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + + +class MMLUEval(Eval): + def __init__(self, filename: str, num_examples: Optional[int], num_threads: int): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(row), role="user" + ) + ] + response_text = sampler(prompt_messages) + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == row["Answer"] else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + category = subject2category.get(row["Subject"], "other") + return SingleEvalResult( + html=html, score=score, metrics={category: score}, convo=convo + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/test_activation.py b/python/sglang/test/test_activation.py new file mode 100644 index 0000000..38366e9 --- /dev/null +++ b/python/sglang/test/test_activation.py @@ -0,0 +1,56 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import GeluAndMul +from sglang.test.test_utils import CustomTestCase + + +class TestGeluAndMul(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gelu_and_mul_test(self, num_tokens, d, dtype, seed): + torch.manual_seed(seed) + + layer = GeluAndMul().to(dtype=dtype) + x = torch.randn(num_tokens, 2 * d, dtype=dtype) + + with torch.inference_mode(): + ref_out = layer.forward_native(x) + out = layer.forward_cuda(x) + + if dtype == torch.bfloat16: + atol = rtol = 1e-2 + else: + atol = rtol = 1e-3 + + self.assertTrue(torch.allclose(out, ref_out, atol=atol, rtol=rtol)) + + def test_gelu_and_mul(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + seed=params[3], + ): + self._run_gelu_and_mul_test(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_block_fp8.py b/python/sglang/test/test_block_fp8.py new file mode 100644 index 0000000..a7c068a --- /dev/null +++ b/python/sglang/test/test_block_fp8.py @@ -0,0 +1,432 @@ +import itertools +import os +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_fp8, + static_quant_fp8, + w8a8_block_fp8_matmul, +) +from sglang.test.test_utils import CustomTestCase + +_is_cuda = torch.cuda.is_available() and torch.version.cuda + + +# For test +def native_per_token_group_quant_fp8( + x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn +): + """Function to perform per-token-group quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Note that only `torch.float8_e4m3fn` is supported for now. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / fp8_max + x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +class TestPerTokenGroupQuantFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + GROUP_SIZE = [64, 128, 256, 512] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_token_group_quant_fp8(self, num_tokens, d, dtype, group_size, seed): + torch.manual_seed(seed) + + x = torch.rand(num_tokens, d, dtype=dtype) + + with torch.inference_mode(): + ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size) + out, scale = per_token_group_quant_fp8(x, group_size) + + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.20) + ) + self.assertTrue(torch.allclose(scale, ref_scale)) + + def test_per_token_group_quant_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.GROUP_SIZE, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + group_size=params[3], + seed=params[4], + ): + self._per_token_group_quant_fp8(*params) + + +# For test +def native_static_quant_fp8(x, x_s, dtype=torch.float8_e4m3fn): + """Function to perform static quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + """ + assert x.is_contiguous(), "`x` is not contiguous" + assert x_s.numel() == 1, "only supports per-tensor scale" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // x.shape[-1], x.shape[-1]) + x_s_inv = 1.0 / x_s + x_q = (x_ * x_s_inv).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + + return x_q, x_s + + +class TestStaticQuantFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _static_quant_fp8(self, num_tokens, d, dtype, seed): + torch.manual_seed(seed) + + x = torch.rand(num_tokens, d, dtype=dtype) + fp8_max = torch.finfo(torch.float8_e4m3fn).max + x_s = x.max() / fp8_max + + with torch.inference_mode(): + ref_out, _ = native_static_quant_fp8(x, x_s) + out, _ = static_quant_fp8(x, x_s, repeat_scale=True) + + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.50) + ) + + def test_static_quant_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + seed=params[3], + ): + self._static_quant_fp8(*params) + + +# For test +def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +class TestW8A8BlockFP8Matmul(CustomTestCase): + + if not _is_cuda: + OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 7, 83, 512, 2048] + NKs = [ + (N, K) + for N in [128, 512, 1024, 4096, 7748, 13824] + for K in [256, 4096, 5120, 3884, 13824] + ] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + else: + # use practical shape in DeepSeek V3 for test + OUT_DTYPES = [torch.bfloat16] + M = [64, 128, 512, 1024, 4096] + NKs = [ + (1536, 7168), + (3072, 1536), + (24576, 7168), + (4096, 512), + (7168, 2048), + (4608, 7168), + (512, 7168), + (7168, 2304), + (7168, 512), + ] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_matmul(self, M, NK, block_size, out_dtype, seed): + N, K = NK + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + + As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale + Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale + + with torch.inference_mode(): + ref_out = native_w8a8_block_fp8_matmul( + A_fp8, B_fp8, As, Bs, block_size, out_dtype + ) + out = w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.001 + ) + + def test_w8a8_block_fp8_matmul(self): + for params in itertools.product( + self.M, + self.NKs, + self.BLOCK_SIZE, + self.OUT_DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + NKs=params[1], + block_size=params[2], + out_dtype=params[3], + seed=params[4], + ): + self._w8a8_block_fp8_matmul(*params) + + +# For test +def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): + """This function performs fused moe with block-wise quantization using native torch.""" + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + _, block_k = block_shape[0], block_shape[1] + a_q, a_s = native_per_token_group_quant_fp8(a, block_k) + # NOTE(HandH1998): Since "index_cuda" not implemented for 'Float8_e4m3fn', we need to cast `float8`` to `float32``. + a_q = a_q.to(torch.float32) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + inter_out = native_w8a8_block_fp8_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype + ) + act_out = SiluAndMul().forward_native(inter_out) + act_out_q, act_out_s = native_per_token_group_quant_fp8(act_out, block_k) + act_out = act_out.to(torch.float32) + out[mask] = native_w8a8_block_fp8_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8BlockFP8FusedMoE(CustomTestCase): + DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 33, 64, 222, 1024 * 128] + N = [128, 1024, 2048] + K = [256, 4096, 5120] + E = [8, 24] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + # BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 * fp8_max + w1 = w1_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 * fp8_max + w2 = w2_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + out = fused_moe( + a, + w1, + w2, + score, + topk, + renormalize=False, + use_fp8_w8a8=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=block_size, + ) + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, score, topk, block_size + ) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.02 + ) + + def test_w8a8_block_fp8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_block_fp8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_block_fp8_ep.py b/python/sglang/test/test_block_fp8_ep.py new file mode 100644 index 0000000..ad8a169 --- /dev/null +++ b/python/sglang/test/test_block_fp8_ep.py @@ -0,0 +1,362 @@ +import itertools +import random +import unittest +from typing import Any, Callable, Dict, List, Optional, Tuple + +import torch + +from sglang.srt.layers.moe.ep_moe.kernels import ( + grouped_gemm_triton, + post_reorder_triton_kernel, + pre_reorder_triton_kernel, + run_moe_ep_preproess, + silu_and_mul_triton_kernel, +) +from sglang.srt.layers.moe.topk import select_experts +from sglang.test.test_utils import CustomTestCase + + +# For test +def ep_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + # ep config + num_experts: int = 256, + fp8_dtype: torch.types = torch.float8_e4m3fn, + num_experts_per_partition: int = 128, + start_expert_id: int = 0, + end_expert_id: int = 127, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + use_fp8_w8a8: bool = False, + w1_scale_inv: Optional[torch.Tensor] = None, + w2_scale_inv: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, +): + use_blockwise_fp8 = block_shape is not None + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + top_k=top_k, + use_grouped_topk=use_grouped_topk, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + # correction_bias=correction_bias, #skip this in test + custom_routing_function=custom_routing_function, + ) + + reorder_topk_ids, src2dst, seg_indptr = run_moe_ep_preproess(topk_ids, num_experts) + + gateup_input = torch.empty( + (int(hidden_states.shape[0] * top_k), hidden_states.shape[1]), + device=hidden_states.device, + dtype=( + fp8_dtype + if (use_fp8_w8a8 and not use_blockwise_fp8) + else hidden_states.dtype + ), + ) + + if use_fp8_w8a8 and not use_blockwise_fp8: + max_value = ( + torch.max(hidden_states).repeat(num_experts_per_partition).to(torch.float32) + ) + w1_input_scale = max_value / torch.finfo(fp8_dtype).max + else: + w1_input_scale = None + + # PreReorder + pre_reorder_triton_kernel[(hidden_states.shape[0],)]( + hidden_states, + gateup_input, + src2dst, + topk_ids, + w1_input_scale, + start_expert_id, + end_expert_id, + top_k, + hidden_states.shape[1], + BLOCK_SIZE=512, + ) + + seg_indptr_cur_rank = seg_indptr[start_expert_id : end_expert_id + 2] + weight_indices_cur_rank = torch.arange( + 0, + num_experts_per_partition, + device=hidden_states.device, + dtype=torch.int64, + ) + + # GroupGemm-0 + gateup_output = torch.empty( + gateup_input.shape[0], + w1.shape[1], + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + gateup_output = grouped_gemm_triton( + a=gateup_input, + b=w1, + c=gateup_output, + batch_size=num_experts_per_partition, + weight_column_major=True, + seg_indptr=seg_indptr_cur_rank, + weight_indices=weight_indices_cur_rank, + use_fp8_w8a8=use_fp8_w8a8, + scale_a=w1_input_scale, + scale_b=w1_scale_inv, + block_shape=block_shape, + ) + + # Act + down_input = torch.empty( + gateup_output.shape[0], + gateup_output.shape[1] // 2, + device=gateup_output.device, + dtype=( + fp8_dtype + if (use_fp8_w8a8 and not use_blockwise_fp8) + else hidden_states.dtype + ), + ) + if use_fp8_w8a8 and not use_blockwise_fp8: + w2_input_scale = torch.ones( + num_experts_per_partition, + dtype=torch.float32, + device=hidden_states.device, + ) + else: + w2_input_scale = None + + silu_and_mul_triton_kernel[(gateup_output.shape[0],)]( + gateup_output, + down_input, + gateup_output.shape[1], + reorder_topk_ids, + w2_input_scale, + start_expert_id, + end_expert_id, + BLOCK_SIZE=512, + ) + + # GroupGemm-1 + down_output = torch.empty( + down_input.shape[0], + w2.shape[1], + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + down_output = grouped_gemm_triton( + a=down_input, + b=w2, + c=down_output, + batch_size=num_experts_per_partition, + weight_column_major=True, + seg_indptr=seg_indptr_cur_rank, + weight_indices=weight_indices_cur_rank, + use_fp8_w8a8=use_fp8_w8a8, + scale_a=w2_input_scale, + scale_b=w2_scale_inv, + block_shape=block_shape, + ) + + # PostReorder + output = torch.empty_like(hidden_states) + post_reorder_triton_kernel[(hidden_states.size(0),)]( + down_output, + output, + src2dst, + topk_ids, + topk_weights, + start_expert_id, + end_expert_id, + top_k, + hidden_states.size(1), + BLOCK_SIZE=512, + ) + return output + + +# test util +def block_dequant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], +) -> Tuple[torch.Tensor, torch.Tensor]: + """This function converts block-wise quantization to tensor-wise quantization. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The outputs are tensor-wise quantization tensor and tensor-wise quantization scale. + Note only float8 is supported for now. + """ + + # process 3D tensor + if x_q_block.dim() == 3: + batch_size = x_q_block.size(0) + return torch.stack( + [block_dequant(x_q_block[b], x_s[b], block_size) for b in range(batch_size)] + ) + + block_n, block_k = block_size[0], block_size[1] + n, k = x_q_block.shape + n_tiles = (n + block_n - 1) // block_n + k_tiles = (k + block_k - 1) // block_k + assert n_tiles == x_s.shape[0] + assert k_tiles == x_s.shape[1] + + x_dq_block = x_q_block.to(torch.float32) + + x_dq_block_tiles = [ + [ + x_dq_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + + for i in range(k_tiles): + for j in range(n_tiles): + x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i] + + return x_dq_block + + +class TestW8A8BlockFP8EPMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 222, 1024, 2048] + N = [128, 1024, 2048] + K = [256, 4096, 5120] + E = [8, 16] + ep_size = [2, 4] + TOP_KS = [2, 4] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_ep_moe( + self, M, N, K, E, ep_size, topk, block_size, dtype, seed + ): + torch.manual_seed(seed) + random.seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=dtype) - 0.5) * 2 * fp8_max + w1 = w1_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=dtype) - 0.5) * 2 * fp8_max + w2 = w2_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + w1_ref = block_dequant(w1, w1_s, block_size).to(dtype) + w2_ref = block_dequant(w2, w2_s, block_size).to(dtype) + + score = torch.randn((M, E), dtype=dtype) + num_experts_per_partition = E // ep_size + cur_rank = random.randint(0, ep_size - 1) + start_id = cur_rank * num_experts_per_partition + end_id = start_id + num_experts_per_partition - 1 + + with torch.inference_mode(): + out = ep_moe( + hidden_states=a, + w1=w1, + w2=w2, + router_logits=score, + top_k=topk, + renormalize=False, + use_fp8_w8a8=True, + w1_scale_inv=w1_s, + w2_scale_inv=w2_s, + block_shape=block_size, + num_experts=E, + num_experts_per_partition=num_experts_per_partition, + start_expert_id=start_id, + end_expert_id=end_id, + ) + ref_out = ep_moe( + hidden_states=a, + w1=w1_ref, + w2=w2_ref, + router_logits=score, + top_k=topk, + renormalize=False, + use_fp8_w8a8=False, + w1_scale_inv=None, + w2_scale_inv=None, + block_shape=None, + num_experts=E, + num_experts_per_partition=num_experts_per_partition, + start_expert_id=start_id, + end_expert_id=end_id, + ) + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / (torch.mean(torch.abs(ref_out.to(torch.float32))) + 1e-6) + < 0.06 + ) + + def test_w8a8_block_fp8_ep_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.ep_size, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + ep_size=params[4], + topk=params[5], + block_size=params[6], + dtype=params[7], + seed=params[8], + ): + self._w8a8_block_fp8_ep_moe(*params) + torch.cuda.empty_cache() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_custom_ops.py b/python/sglang/test/test_custom_ops.py new file mode 100644 index 0000000..6fa343d --- /dev/null +++ b/python/sglang/test/test_custom_ops.py @@ -0,0 +1,88 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/8ca7a71df787ad711ad3ac70a5bd2eb2bb398938/tests/quantization/test_fp8.py + +import pytest +import torch + +from sglang.srt.custom_op import scaled_fp8_quant +from sglang.srt.utils import is_cuda + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_scaled_fp8_quant_per_tensor(dtype) -> None: + + def quantize_ref_per_tensor(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(torch.float8_e4m3fn) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(torch.float8_e4m3fn) + return qweight + + def dequantize_per_tensor(tensor, inv_scale, dtype): + fake_qweight = tensor.to(dtype) + dq_weight = fake_qweight * inv_scale + return dq_weight + + # Note that we use a shape % 8 != 0 to cover edge cases, + # because scaled_fp8_quant is vectorized by 8. + x = (torch.randn(size=(11, 11), device="cuda") * 13).to(dtype) + + # Test Per Tensor Dynamic quantization + # scale = max(abs(x)) / FP8_E4M3_MAX + y, scale = scaled_fp8_quant(x, None) + ref_y = quantize_ref_per_tensor(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_tensor(y, scale, dtype), + dequantize_per_tensor(ref_y, scale, dtype), + ) + + # Test Per Tensor Static quantization + y, _ = scaled_fp8_quant(x, scale) + ref_y = quantize_ref_per_tensor(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_tensor(y, scale, dtype), + dequantize_per_tensor(ref_y, scale, dtype), + ) + + +if is_cuda: + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) + def test_scaled_fp8_quant_per_token_dynamic(dtype) -> None: + def quantize_ref_per_token(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(torch.float8_e4m3fn) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp( + min=finfo.min, max=finfo.max + ) + qweight = qweight.to(torch.float8_e4m3fn) + return qweight + + def dequantize_per_token(tensor, inv_scale, dtype): + fake_qweight = tensor.to(dtype) + dq_weight = fake_qweight * inv_scale + return dq_weight + + # Note that we use a shape % 8 = 0, + # because per_token_quant_fp8 is vectorized by 8 elements. + x = (torch.randn(size=(11, 16), device="cuda") * 13).to(dtype) + + # Test Per Tensor Dynamic quantization + # scale = max(abs(x)) / FP8_E4M3_MAX + y, scale = scaled_fp8_quant(x, None, use_per_token_if_dynamic=True) + ref_y = quantize_ref_per_token(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_token(y, scale, dtype), + dequantize_per_token(ref_y, scale, dtype), + ) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/python/sglang/test/test_dynamic_grad_mode.py b/python/sglang/test/test_dynamic_grad_mode.py new file mode 100644 index 0000000..078c313 --- /dev/null +++ b/python/sglang/test/test_dynamic_grad_mode.py @@ -0,0 +1,58 @@ +import unittest + +import torch + +from sglang.srt.utils import DynamicGradMode +from sglang.test.test_utils import CustomTestCase + + +class TestDynamicGradMode(CustomTestCase): + def test_inference(self): + # Test inference_mode + DynamicGradMode.set_inference_mode(True) + + @DynamicGradMode() + def create_tensor_x(): + return torch.empty(0) + + X = create_tensor_x() + self.assertTrue(not X.requires_grad and X.is_inference()) + + def test_no_grad(self): + # Test no_grad + DynamicGradMode.set_inference_mode(False) + + @DynamicGradMode() + def create_tensor_y(): + return torch.empty(0) + + Y = create_tensor_y() + self.assertTrue(not Y.requires_grad and not Y.is_inference()) + + def test_nested_inference(self): + # Test no_grad nested inference_mode, inference_mode should has higher priority + DynamicGradMode.set_inference_mode(False) + + @DynamicGradMode() + def create_tensor_z(): + with torch.inference_mode(): + return torch.empty(0) + + Z = create_tensor_z() + self.assertTrue(not Z.requires_grad and Z.is_inference()) + + def test_nested_no_grad(self): + # Test inference_mode nested no_grad, inference_mode should has higher priority + DynamicGradMode.set_inference_mode(True) + + @DynamicGradMode() + def create_tensor_w(): + with torch.no_grad(): + return torch.empty(0) + + W = create_tensor_w() + self.assertTrue(not W.requires_grad and W.is_inference()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_layernorm.py b/python/sglang/test/test_layernorm.py new file mode 100644 index 0000000..05b6593 --- /dev/null +++ b/python/sglang/test/test_layernorm.py @@ -0,0 +1,113 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm +from sglang.test.test_utils import CustomTestCase + + +class TestRMSNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 4096] + HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199] + ADD_RESIDUAL = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_rms_norm_test(self, num_tokens, hidden_size, add_residual, dtype, seed): + torch.manual_seed(seed) + + layer = RMSNorm(hidden_size).to(dtype=dtype) + layer.weight.data.normal_(mean=1.0, std=0.1) + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + residual = torch.randn_like(x) * scale if add_residual else None + + with torch.inference_mode(): + ref_out = layer.forward_native(x, residual) + out = layer(x, residual) + + if add_residual: + self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)) + self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)) + else: + self.assertTrue(torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)) + + def test_rms_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.ADD_RESIDUAL, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + add_residual=params[2], + dtype=params[3], + seed=params[4], + ): + self._run_rms_norm_test(*params) + + +class TestGemmaRMSNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 4096] + HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199] + ADD_RESIDUAL = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gemma_rms_norm_test( + self, num_tokens, hidden_size, add_residual, dtype, seed + ): + torch.manual_seed(seed) + + layer = GemmaRMSNorm(hidden_size).to(dtype=dtype) + layer.weight.data.normal_(mean=1.0, std=0.1) + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + residual = torch.randn_like(x) * scale if add_residual else None + + with torch.inference_mode(): + ref_out = layer.forward_native(x, residual) + out = layer(x, residual) + + if add_residual: + self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-3, rtol=1e-3)) + self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-3, rtol=1e-3)) + else: + self.assertTrue(torch.allclose(out, ref_out, atol=1e-3, rtol=1e-3)) + + def test_gemma_rms_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.ADD_RESIDUAL, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + add_residual=params[2], + dtype=params[3], + seed=params[4], + ): + self._run_gemma_rms_norm_test(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_programs.py b/python/sglang/test/test_programs.py new file mode 100644 index 0000000..ca2cdb1 --- /dev/null +++ b/python/sglang/test/test_programs.py @@ -0,0 +1,575 @@ +"""This file contains the SGL programs used for unit testing.""" + +import json +import re +import time + +import numpy as np + +import sglang as sgl +from sglang.utils import download_and_cache_file, read_jsonl + + +def test_few_shot_qa(): + @sgl.function + def few_shot_qa(s, question): + s += "The following are questions with answers.\n\n" + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: What is the capital of Germany?\n" + s += "A: Berlin\n" + s += "Q: What is the capital of Italy?\n" + s += "A: Rome\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + ret = few_shot_qa.run(question="What is the capital of the United States?") + assert "washington" in ret["answer"].strip().lower(), f"answer: {ret['answer']}" + + rets = few_shot_qa.run_batch( + [ + {"question": "What is the capital of Japan?"}, + {"question": "What is the capital of the United Kingdom?"}, + {"question": "What is the capital city of China?"}, + ], + temperature=0.1, + ) + answers = [x["answer"].strip().lower() for x in rets] + assert answers == ["tokyo", "london", "beijing"], f"answers: {answers}" + + +def test_mt_bench(): + @sgl.function + def answer_mt_bench(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + with s.user(): + s += question_2 + with s.assistant(): + s += sgl.gen("answer_2") + + question_1 = "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions." + question_2 = ( + "Rewrite your previous response. Start every sentence with the letter A." + ) + ret = answer_mt_bench.run( + question_1=question_1, question_2=question_2, temperature=0.7, max_new_tokens=64 + ) + assert len(ret.messages()) in [4, 5] + + +def test_select(check_answer): + @sgl.function + def true_or_false(s, statement): + s += "Determine whether the statement below is True, False, or Unknown.\n" + s += "Statement: The capital of France is Pairs.\n" + s += "Answer: True\n" + s += "Statement: " + statement + "\n" + s += "Answer:" + sgl.select("answer", ["True", "False", "Unknown"]) + + ret = true_or_false.run( + statement="The capital of Germany is Berlin.", + ) + if check_answer: + assert ret["answer"] == "True", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + ret = true_or_false.run( + statement="The capital of Canada is Tokyo.", + ) + if check_answer: + assert ret["answer"] == "False", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + ret = true_or_false.run( + statement="Purple is a better color than green.", + ) + if check_answer: + assert ret["answer"] == "Unknown", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + +def test_decode_int(): + @sgl.function + def decode_int(s): + s += "The number of hours in a day is " + sgl.gen_int("hours") + "\n" + s += "The number of days in a year is " + sgl.gen_int("days") + "\n" + + ret = decode_int.run(temperature=0.1) + assert int(ret["hours"]) == 24, ret.text() + assert int(ret["days"]) == 365, ret.text() + + +def test_decode_json_regex(): + @sgl.function + def decode_json(s): + from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR + + s += "Generate a JSON object to describe the basic city information of Paris.\n" + s += "Here are the JSON object:\n" + + # NOTE: we recommend using dtype gen or whole regex string to control the output + + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen(regex=REGEX_STR) + ",\n" + s += ' "population": ' + sgl.gen(regex=REGEX_INT, stop=[" ", "\n"]) + ",\n" + s += ' "area": ' + sgl.gen(regex=REGEX_INT, stop=[" ", "\n"]) + ",\n" + s += ' "latitude": ' + sgl.gen(regex=REGEX_FLOAT, stop=[" ", "\n"]) + "\n" + s += "}" + + ret = decode_json.run(temperature=0.0) + try: + js_obj = json.loads(ret["json_output"]) + except json.decoder.JSONDecodeError: + print("JSONDecodeError", ret["json_output"]) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + +def test_decode_json(): + @sgl.function + def decode_json(s): + s += "Generate a JSON object to describe the basic city information of Paris.\n" + + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen_string() + ",\n" + s += ' "population": ' + sgl.gen_int() + ",\n" + s += ' "area": ' + sgl.gen(dtype=int) + ",\n" + s += ' "country": ' + sgl.gen_string() + ",\n" + s += ' "timezone": ' + sgl.gen(dtype=str) + "\n" + s += "}" + + ret = decode_json.run(max_new_tokens=64) + try: + js_obj = json.loads(ret["json_output"]) + except json.decoder.JSONDecodeError: + print("JSONDecodeError", ret["json_output"]) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + +def test_expert_answer(check_answer=True): + @sgl.function + def expert_answer(s, question): + s += "Question: " + question + "\n" + s += ( + "A good person to answer this question is" + + sgl.gen("expert", stop=[".", "\n"]) + + ".\n" + ) + s += ( + "For example," + + s["expert"] + + " would answer that " + + sgl.gen("answer", stop=".") + + "." + ) + + ret = expert_answer.run(question="What is the capital of France?", temperature=0.1) + + if check_answer: + assert "paris" in ret.text().lower(), f"Answer: {ret.text()}" + + +def test_tool_use(): + def calculate(expression): + return f"{eval(expression)}" + + @sgl.function + def tool_use(s, lhs, rhs): + s += "Please perform computations using a calculator. You can use calculate(expression) to get the results.\n" + s += "For example,\ncalculate(1+2)=3\ncalculate(3*4)=12\n" + s += "Question: What is the product of " + str(lhs) + " and " + str(rhs) + "?\n" + s += ( + "Answer: The answer is calculate(" + + sgl.gen("expression", stop=")") + + ") = " + ) + with s.var_scope("answer"): + s += calculate(s["expression"]) + + lhs, rhs = 257, 983 + ret = tool_use(lhs=lhs, rhs=rhs, temperature=0) + assert int(ret["answer"]) == lhs * rhs + + +def test_react(): + @sgl.function + def react(s, question): + s += """ +Question: Which country does the founder of Microsoft live in? +Thought 1: I need to search for the founder of Microsoft. +Action 1: Search [Founder of Microsoft]. +Observation 1: The founder of Microsoft is Bill Gates. +Thought 2: I need to search for the country where Bill Gates lives in. +Action 2: Search [Where does Bill Gates live]. +Observation 2: Bill Gates lives in the United States. +Thought 3: The answer is the United States. +Action 3: Finish [United States].\n +""" + + s += "Question: " + question + "\n" + + for i in range(1, 5): + s += f"Thought {i}:" + sgl.gen(stop=[".", "\n"]) + ".\n" + s += f"Action {i}: " + sgl.select(f"action_{i}", ["Search", "Finish"]) + + if s[f"action_{i}"] == "Search": + s += " [" + sgl.gen(stop="]") + "].\n" + s += f"Observation {i}:" + sgl.gen(stop=[".", "\n"]) + ".\n" + else: + s += " [" + sgl.gen("answer", stop="]") + "].\n" + break + + ret = react.run( + question="What country does the creator of Linux live in?", + temperature=0.1, + ) + answer = ret["answer"].lower() + assert "finland" in answer or "states" in answer + + +def test_parallel_decoding(): + max_tokens = 64 + fork_size = 5 + + @sgl.function + def parallel_decoding(s, topic): + s += "Act as a helpful assistant.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(fork_size) + + " concise tips, each under 8 words:\n" + ) + + # Generate skeleton + for i in range(1, 1 + fork_size): + s += f"{i}." + sgl.gen(max_tokens=16, stop=[".", "\n"]) + ".\n" + + # Generate detailed tips + forks = s.fork(fork_size) + for i in range(fork_size): + forks[ + i + ] += f"Now, I expand tip {i+1} into a detailed paragraph:\nTip {i+1}:" + forks[i] += sgl.gen("detailed_tip", max_tokens, stop=["\n\n"]) + forks.join() + + # Concatenate tips and summarize + s += "Here are these tips with detailed explanation:\n" + for i in range(fork_size): + s += f"Tip {i+1}:" + forks[i]["detailed_tip"] + "\n" + + s += "\nIn summary," + sgl.gen("summary", max_tokens=512) + + ret = parallel_decoding.run(topic="writing a good blog post", temperature=0.3) + assert isinstance(ret["summary"], str) + + +def test_parallel_encoding(check_answer=True): + max_tokens = 64 + + @sgl.function + def parallel_encoding(s, question, context_0, context_1, context_2): + s += "USER: I will ask a question based on some statements.\n" + s += "ASSISTANT: Sure. I will give the answer.\n" + s += "USER: Please memorize these statements.\n" + + contexts = [context_0, context_1, context_2] + + forks = s.fork(len(contexts)) + forks += lambda i: f"Statement {i}: " + contexts[i] + "\n" + forks.join(mode="concate_and_append") + + s += "Now, please answer the following question. " "Do not list options." + s += "\nQuestion: " + question + "\n" + s += "ASSISTANT:" + sgl.gen("answer", max_tokens=max_tokens) + + ret = parallel_encoding.run( + question="Who is the father of Julian?", + context_0="Ethan is the father of Liam.", + context_1="Noah is the father of Julian.", + context_2="Oliver is the father of Carlos.", + temperature=0, + ) + answer = ret["answer"] + + if check_answer: + assert "Noah" in answer + + +def test_image_qa(): + @sgl.function + def image_qa(s, question): + s += sgl.user(sgl.image("example_image.png") + question) + s += sgl.assistant(sgl.gen("answer")) + + state = image_qa.run( + question="Please describe this image in simple words.", + temperature=0, + max_new_tokens=64, + ) + + assert ( + "taxi" in state.messages()[-1]["content"] + or "car" in state.messages()[-1]["content"] + ), f"{state.messages()[-1]['content']}" + + +def test_stream(): + @sgl.function + def qa(s, question): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question) + s += sgl.assistant(sgl.gen("answer")) + + ret = qa( + question="Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", + stream=True, + ) + out = "" + for chunk in ret.text_iter(): + out += chunk + + ret = qa( + question="Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", + stream=True, + ) + out = "" + for chunk in ret.text_iter("answer"): + out += chunk + + +def test_regex(): + regex = r"((25[0-5]|2[0-4]\d|[01]?\d\d?).){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + + @sgl.function + def regex_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + temperature=0, + regex=regex, + ) + + state = regex_gen.run() + answer = state["answer"] + assert re.match(regex, answer) + + +def test_dtype_gen(): + @sgl.function + def dtype_gen(s): + s += "Q: What is the full name of DNS?\n" + s += "A: The full nams is " + sgl.gen("str_res", dtype=str, stop="\n") + "\n" + s += "Q: Which year was DNS invented?\n" + s += "A: " + sgl.gen("int_res", dtype=int) + "\n" + s += "Q: What is the value of pi?\n" + s += "A: " + sgl.gen("float_res", dtype=float) + "\n" + s += "Q: Is the sky blue?\n" + s += "A: " + sgl.gen("bool_res", dtype=bool) + "\n" + + state = dtype_gen.run() + + try: + state["int_res"] = int(state["int_res"]) + state["float_res"] = float(state["float_res"]) + state["bool_res"] = bool(state["bool_res"]) + # assert state["str_res"].startswith('"') and state["str_res"].endswith('"') + except ValueError: + print(state) + raise + + +def test_completion_speculative(): + @sgl.function(num_api_spec_tokens=64) + def gen_character_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += ( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + ) + s += "\nJob:" + sgl.gen("job", stop="\n") + "\n" + + @sgl.function + def gen_character_no_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += ( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + ) + s += "\nJob:" + sgl.gen("job", stop="\n") + "\n" + + token_usage = sgl.global_config.default_backend.token_usage + + token_usage.reset() + gen_character_spec().sync() + usage_with_spec = token_usage.prompt_tokens + + token_usage.reset() + gen_character_no_spec().sync() + usage_with_no_spec = token_usage.prompt_tokens + + assert ( + usage_with_spec < usage_with_no_spec + ), f"{usage_with_spec} vs {usage_with_no_spec}" + + +def test_chat_completion_speculative(): + @sgl.function(num_api_spec_tokens=256) + def gen_character_spec(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Construct a character within the following format:") + s += sgl.assistant( + "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + ) + s += sgl.user("Please generate new Name, Birthday and Job.\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + gen_character_spec().sync() + + +def test_hellaswag_select(): + """Benchmark the accuracy of sgl.select on the HellaSwag dataset.""" + + def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + # Read data + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = 200 + num_shots = 20 + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_hellaswag(s, question, choices): + s += few_shot_examples + question + s += sgl.select("answer", choices=choices) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.time() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=64, + progress_bar=True, + generator_style=False, + ) + preds = [] + for i, ret in enumerate(rets): + preds.append(choices[i].index(ret["answer"])) + latency = time.time() - tic + + # Compute accuracy + accuracy = np.mean(np.array(preds) == np.array(labels)) + + # Test generator style of run_batch + tic = time.time() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=64, + progress_bar=True, + generator_style=True, + ) + preds_gen = [] + for i, ret in enumerate(rets): + preds_gen.append(choices[i].index(ret["answer"])) + latency_gen = time.time() - tic + + # Compute accuracy + accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels)) + print(f"{accuracy=}, {accuracy_gen=}") + assert np.abs(accuracy_gen - accuracy) < 0.1 + assert np.abs(latency_gen - latency) < 1 + + return accuracy, latency + + +def test_gen_min_new_tokens(): + """ + Validate sgl.gen(min_tokens) functionality. + + The test asks a question where, without a min_tokens constraint, the generated answer is expected to be short. + By enforcing the min_tokens parameter, we ensure the generated answer has at least the specified number of tokens. + We verify that the number of tokens in the answer is >= the min_tokens threshold. + """ + import sglang as sgl + from sglang.srt.hf_transformers_utils import get_tokenizer + + model_path = sgl.global_config.default_backend.endpoint.get_model_name() + MIN_TOKENS, MAX_TOKENS = 64, 128 + + @sgl.function + def convo_1(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant( + sgl.gen("answer", min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS) + ) + + def assert_min_tokens(tokenizer, text): + token_ids = tokenizer.encode(text) + assert ( + len(token_ids) >= MIN_TOKENS + ), f"Generated {len(token_ids)} tokens, min required: {MIN_TOKENS}. Text: {text}" + + tokenizer = get_tokenizer(model_path) + + state = convo_1.run() + assert_min_tokens(tokenizer, state["answer"]) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py new file mode 100644 index 0000000..3955252 --- /dev/null +++ b/python/sglang/test/test_utils.py @@ -0,0 +1,1035 @@ +"""Common utilities for testing and benchmarking""" + +import argparse +import copy +import logging +import os +import random +import subprocess +import threading +import time +import traceback +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from types import SimpleNamespace +from typing import Callable, List, Optional, Tuple + +import numpy as np +import requests +import torch +import torch.nn.functional as F + +from sglang.bench_serving import run_benchmark +from sglang.global_config import global_config +from sglang.lang.backend.openai import OpenAI +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint +from sglang.srt.utils import get_bool_env_var, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.utils import get_exception_traceback + +DEFAULT_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8" +DEFAULT_FP8_MODEL_NAME_FOR_ACCURACY_TEST = "neuralmagic/Meta-Llama-3-8B-Instruct-FP8" +DEFAULT_FP8_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST = ( + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic" +) +DEFAULT_FP8_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST = ( + "nvidia/Llama-3.1-8B-Instruct-FP8" +) +# TODO(yundai424): right now specifying to an older revision since the latest one +# carries kv cache quantization which doesn't work yet +DEFAULT_FP8_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_REVISION = ( + "13858565416dbdc0b4e7a4a677fadfbd5b9e5bb9" +) + +DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct" +DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1" +DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST = "Qwen/Qwen1.5-MoE-A2.7B" +DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct" +DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" +DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8" +DEFAULT_REASONING_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" +DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST = ( + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" +) +DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 1000 +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = "meta-llama/Llama-3.1-8B-Instruct,mistralai/Mistral-7B-Instruct-v0.3,deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct,google/gemma-2-27b-it" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1 = "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4,hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4,hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN = "Qwen/Qwen2.5-1.5B-Instruct" +DEFAULT_SMALL_VLM_MODEL_NAME = "Qwen/Qwen2-VL-2B" + +DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST = "meta-llama/Llama-2-7b-chat-hf" +DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST = "lmsys/sglang-EAGLE-llama2-chat-7B" + +DEFAULT_IMAGE_URL = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" +DEFAULT_VIDEO_URL = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + + +def is_in_ci(): + """Return whether it is in CI runner.""" + return get_bool_env_var("SGLANG_IS_IN_CI") + + +if is_in_ci(): + DEFAULT_PORT_FOR_SRT_TEST_RUNNER = 5157 + DEFAULT_URL_FOR_TEST = "http://127.0.0.1:6157" +else: + DEFAULT_PORT_FOR_SRT_TEST_RUNNER = 1157 + DEFAULT_URL_FOR_TEST = "http://127.0.0.1:2157" + + +def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None): + assert url is not None + + data = { + "inputs": prompt, + "parameters": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "stop_sequences": stop, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + pred = res.json()["generated_text"][0] + return pred + + +def call_generate_vllm(prompt, temperature, max_tokens, stop=None, n=1, url=None): + assert url is not None + + data = { + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + "n": n, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + if n == 1: + pred = res.json()["text"][0][len(prompt) :] + else: + pred = [x[len(prompt) :] for x in res.json()["text"]] + return pred + + +def call_generate_outlines( + prompt, temperature, max_tokens, stop=None, regex=None, n=1, url=None +): + assert url is not None + + data = { + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + "regex": regex, + "n": n, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + if n == 1: + pred = res.json()["text"][0][len(prompt) :] + else: + pred = [x[len(prompt) :] for x in res.json()["text"]] + return pred + + +def call_generate_srt_raw(prompt, temperature, max_tokens, stop=None, url=None): + assert url is not None + + data = { + "text": prompt, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "stop": stop, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + obj = res.json() + pred = obj["text"] + return pred + + +def call_generate_guidance( + prompt, temperature, max_tokens, stop=None, n=1, regex=None, model=None +): + assert model is not None + from guidance import gen + + rets = [] + for _ in range(n): + out = ( + model + + prompt + + gen( + name="answer", + max_tokens=max_tokens, + temperature=temperature, + stop=stop, + regex=regex, + ) + ) + rets.append(out["answer"]) + return rets if n > 1 else rets[0] + + +def call_select_lightllm(context, choices, url=None): + assert url is not None + + scores = [] + for i in range(len(choices)): + data = { + "inputs": context + choices[i], + "parameters": { + "max_new_tokens": 1, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + scores.append(0) + return np.argmax(scores) + + +def call_select_vllm(context, choices, url=None): + assert url is not None + + scores = [] + for i in range(len(choices)): + data = { + "prompt": context + choices[i], + "max_tokens": 1, + "prompt_logprobs": 1, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + scores.append(res.json().get("prompt_score", 0)) + return np.argmax(scores) + + """ + Modify vllm/entrypoints/api_server.py + + if final_output.prompt_logprobs is not None: + score = np.mean([prob[t_id] for t_id, prob in zip(final_output.prompt_token_ids[1:], final_output.prompt_logprobs[1:])]) + ret["prompt_score"] = score + """ + + +def call_select_guidance(context, choices, model=None): + assert model is not None + from guidance import select + + out = model + context + select(choices, name="answer") + return choices.index(out["answer"]) + + +def add_common_other_args_and_parse(parser: argparse.ArgumentParser): + parser.add_argument("--parallel", type=int, default=64) + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=None) + parser.add_argument( + "--backend", + type=str, + required=True, + choices=[ + "vllm", + "outlines", + "lightllm", + "gserver", + "guidance", + "srt-raw", + "llama.cpp", + ], + ) + parser.add_argument("--n-ctx", type=int, default=4096) + parser.add_argument( + "--model-path", type=str, default="meta-llama/Llama-2-7b-chat-hf" + ) + parser.add_argument("--result-file", type=str, default="result.jsonl") + args = parser.parse_args() + + if args.port is None: + default_port = { + "vllm": 21000, + "outlines": 21000, + "lightllm": 22000, + "srt-raw": 30000, + "gserver": 9988, + } + args.port = default_port.get(args.backend, None) + return args + + +def add_common_sglang_args_and_parse(parser: argparse.ArgumentParser): + parser.add_argument("--parallel", type=int, default=64) + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument("--backend", type=str, default="srt") + parser.add_argument("--result-file", type=str, default="result.jsonl") + args = parser.parse_args() + return args + + +def select_sglang_backend(args: argparse.Namespace): + if args.backend.startswith("srt"): + if args.backend == "srt-no-parallel": + global_config.enable_parallel_encoding = False + backend = RuntimeEndpoint(f"{args.host}:{args.port}") + elif args.backend.startswith("gpt-"): + backend = OpenAI(args.backend) + else: + raise ValueError(f"Invalid backend: {args.backend}") + return backend + + +def _get_call_generate(args: argparse.Namespace): + if args.backend == "lightllm": + return partial(call_generate_lightllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "vllm": + return partial(call_generate_vllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "srt-raw": + return partial(call_generate_srt_raw, url=f"{args.host}:{args.port}/generate") + elif args.backend == "gserver": + return partial(call_generate_gserver, url=f"{args.host}:{args.port}") + elif args.backend == "outlines": + return partial(call_generate_outlines, url=f"{args.host}:{args.port}/generate") + elif args.backend == "guidance": + from guidance import models + + model = models.LlamaCpp(args.model_path, n_gpu_layers=-1, n_ctx=args.n_ctx) + call_generate = partial(call_generate_guidance, model=model) + call_generate("Hello,", 1.0, 8, ".") + return call_generate + else: + raise ValueError(f"Invalid backend: {args.backend}") + + +def _get_call_select(args: argparse.Namespace): + if args.backend == "lightllm": + return partial(call_select_lightllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "vllm": + return partial(call_select_vllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "guidance": + from guidance import models + + model = models.LlamaCpp(args.model_path, n_gpu_layers=-1, n_ctx=args.n_ctx) + call_select = partial(call_select_guidance, model=model) + + call_select("Hello,", ["world", "earth"]) + return call_select + else: + raise ValueError(f"Invalid backend: {args.backend}") + + +def get_call_generate(args: argparse.Namespace): + call_generate = _get_call_generate(args) + + def func(*args, **kwargs): + try: + return call_generate(*args, **kwargs) + except Exception: + print("Exception in call_generate:\n" + get_exception_traceback()) + raise + + return func + + +def get_call_select(args: argparse.Namespace): + call_select = _get_call_select(args) + + def func(*args, **kwargs): + try: + return call_select(*args, **kwargs) + except Exception: + print("Exception in call_select:\n" + get_exception_traceback()) + raise + + return func + + +def popen_launch_server( + model: str, + base_url: str, + timeout: float, + api_key: Optional[str] = None, + other_args: list[str] = (), + env: Optional[dict] = None, + return_stdout_stderr: Optional[tuple] = None, + pd_seperated: bool = False, +): + _, host, port = base_url.split(":") + host = host[2:] + + if pd_seperated: + command = "sglang.launch_pd_server" + else: + command = "sglang.launch_server" + + command = [ + "python3", + "-m", + command, + "--model-path", + model, + *[str(x) for x in other_args], + ] + + if pd_seperated: + command.extend( + [ + "--lb-host", + host, + "--lb-port", + port, + ] + ) + else: + command.extend( + [ + "--host", + host, + "--port", + port, + ] + ) + + if api_key: + command += ["--api-key", api_key] + + print(f"command={' '.join(command)}") + + if return_stdout_stderr: + process = subprocess.Popen( + command, + stdout=return_stdout_stderr[0], + stderr=return_stdout_stderr[1], + env=env, + text=True, + ) + else: + process = subprocess.Popen(command, stdout=None, stderr=None, env=env) + + start_time = time.time() + with requests.Session() as session: + while time.time() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"Bearer {api_key}", + } + response = session.get( + f"{base_url}/health_generate", + headers=headers, + ) + if response.status_code == 200: + return process + except requests.RequestException: + pass + + return_code = process.poll() + if return_code is not None: + raise Exception(f"Server unexpectedly exits ({return_code=}).") + + time.sleep(10) + + kill_process_tree(process.pid) + raise TimeoutError("Server failed to start within the timeout period.") + + +def run_with_timeout( + func: Callable, + args: tuple = (), + kwargs: Optional[dict] = None, + timeout: float = None, +): + """Run a function with timeout.""" + ret_value = [] + + def _target_func(): + ret_value.append(func(*args, **(kwargs or {}))) + + t = threading.Thread(target=_target_func) + t.start() + t.join(timeout=timeout) + if t.is_alive(): + raise TimeoutError() + + if not ret_value: + raise RuntimeError() + + return ret_value[0] + + +@dataclass +class TestFile: + name: str + estimated_time: float = 60 + + +def run_unittest_files(files: List[TestFile], timeout_per_file: float): + tic = time.time() + success = True + + for file in files: + filename, estimated_time = file.name, file.estimated_time + process = None + + def run_one_file(filename): + nonlocal process + + filename = os.path.join(os.getcwd(), filename) + print(f".\n.\nBegin:\npython3 {filename}\n.\n.\n", flush=True) + tic = time.time() + + process = subprocess.Popen( + ["python3", filename], stdout=None, stderr=None, env=os.environ + ) + process.wait() + elapsed = time.time() - tic + + print( + f".\n.\nEnd:\n{filename=}, {elapsed=:.0f}, {estimated_time=}\n.\n.\n", + flush=True, + ) + return process.returncode + + try: + ret_code = run_with_timeout( + run_one_file, args=(filename,), timeout=timeout_per_file + ) + assert ( + ret_code == 0 + ), f"expected return code 0, but {filename} returned {ret_code}" + except TimeoutError: + kill_process_tree(process.pid) + time.sleep(5) + print( + f"\nTimeout after {timeout_per_file} seconds when running {filename}\n", + flush=True, + ) + success = False + break + + if success: + print(f"Success. Time elapsed: {time.time() - tic:.2f}s", flush=True) + else: + print(f"Fail. Time elapsed: {time.time() - tic:.2f}s", flush=True) + + return 0 if success else -1 + + +def get_similarities(vec1, vec2): + return F.cosine_similarity(torch.tensor(vec1), torch.tensor(vec2), dim=0) + + +def get_benchmark_args( + base_url="", + dataset_name="", + dataset_path="", + tokenizer="", + num_prompts=500, + sharegpt_output_len=None, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + request_rate=float("inf"), + disable_stream=False, + disable_ignore_eos=False, + seed: int = 0, + pd_seperated: bool = False, +): + return SimpleNamespace( + backend="sglang", + base_url=base_url, + host=None, + port=None, + dataset_name=dataset_name, + dataset_path=dataset_path, + model=None, + tokenizer=tokenizer, + num_prompts=num_prompts, + sharegpt_output_len=sharegpt_output_len, + sharegpt_context_len=sharegpt_context_len, + random_input_len=random_input_len, + random_output_len=random_output_len, + random_range_ratio=0.0, + request_rate=request_rate, + multi=None, + output_file=None, + disable_tqdm=False, + disable_stream=disable_stream, + return_logprob=False, + seed=seed, + disable_ignore_eos=disable_ignore_eos, + extra_request_body=None, + apply_chat_template=False, + profile=None, + lora_name=None, + prompt_suffix="", + pd_seperated=pd_seperated, + ) + + +def run_bench_serving( + model, + num_prompts, + request_rate, + other_server_args, + dataset_name="random", + dataset_path="", + tokenizer=None, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + disable_stream=False, + disable_ignore_eos=False, + need_warmup=False, + seed: int = 0, +): + # Launch the server + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + ) + + # Run benchmark + args = get_benchmark_args( + base_url=base_url, + dataset_name=dataset_name, + dataset_path=dataset_path, + tokenizer=tokenizer, + num_prompts=num_prompts, + random_input_len=random_input_len, + random_output_len=random_output_len, + sharegpt_context_len=sharegpt_context_len, + request_rate=request_rate, + disable_stream=disable_stream, + disable_ignore_eos=disable_ignore_eos, + seed=seed, + ) + + try: + if need_warmup: + warmup_args = copy.deepcopy(args) + warmup_args.num_prompts = 16 + run_benchmark(warmup_args) + res = run_benchmark(args) + finally: + kill_process_tree(process.pid) + + assert res["completed"] == num_prompts + return res + + +def run_bench_serving_multi( + model, + base_url, + other_server_args, + benchmark_args, + need_warmup=False, + pd_seperated=False, +): + # Launch the server + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + pd_seperated=pd_seperated, + ) + + # run benchmark for all + res_l = [] + try: + for args in benchmark_args: + if need_warmup: + warmup_args = copy.deepcopy(args) + warmup_args.num_prompts = 16 + run_benchmark(warmup_args) + + res = run_benchmark(args) + res_l.append((args, res)) + finally: + kill_process_tree(process.pid) + + return res_l + + +def run_bench_one_batch(model, other_args): + command = [ + "python3", + "-m", + "sglang.bench_one_batch", + "--model-path", + model, + "--batch-size", + "1", + "--input", + "128", + "--output", + "8", + *[str(x) for x in other_args], + ] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + try: + stdout, stderr = process.communicate() + output = stdout.decode() + error = stderr.decode() + print(f"Output: {output}", flush=True) + print(f"Error: {error}", flush=True) + + lastline = output.split("\n")[-3] + output_throughput = float(lastline.split(" ")[-2]) + finally: + kill_process_tree(process.pid) + + return output_throughput + + +def lcs(X, Y): + m = len(X) + n = len(Y) + L = [[0] * (n + 1) for _ in range(m + 1)] + + for i in range(m + 1): + for j in range(n + 1): + if i == 0 or j == 0: + L[i][j] = 0 + elif X[i - 1] == Y[j - 1]: + L[i][j] = L[i - 1][j - 1] + 1 + else: + L[i][j] = max(L[i - 1][j], L[i][j - 1]) + + return L[m][n] + + +def calculate_rouge_l(output_strs_list1, output_strs_list2): + """calculate the ROUGE-L score""" + rouge_l_scores = [] + + for s1, s2 in zip(output_strs_list1, output_strs_list2): + lcs_len = lcs(s1, s2) + precision = lcs_len / len(s1) if len(s1) > 0 else 0 + recall = lcs_len / len(s2) if len(s2) > 0 else 0 + if precision + recall > 0: + fmeasure = (2 * precision * recall) / (precision + recall) + else: + fmeasure = 0.0 + rouge_l_scores.append(fmeasure) + + return rouge_l_scores + + +STDERR_FILENAME = "stderr.txt" +STDOUT_FILENAME = "stdout.txt" + + +def read_output(output_lines: List[str], filename: str = STDERR_FILENAME): + """Print the output in real time with another thread.""" + while not os.path.exists(filename): + time.sleep(1) + + pt = 0 + while pt >= 0: + if pt > 0 and not os.path.exists(filename): + break + lines = open(filename).readlines() + for line in lines[pt:]: + print(line, end="", flush=True) + output_lines.append(line) + pt += 1 + time.sleep(0.1) + + +def run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort, +): + other_args = [ + "--chunked-prefill-size", + str(chunked_prefill_size), + "--log-level", + "debug", + ] + if disable_radix_cache: + other_args += ["--disable-radix-cache"] + if enable_mixed_chunk: + other_args += ["--enable-mixed-chunk"] + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + model = DEFAULT_MODEL_NAME_FOR_TEST + port = random.randint(4000, 5000) + base_url = f"http://127.0.0.1:{port}" + + # Create files and launch the server + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + return_stdout_stderr=(stdout, stderr), + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines,)) + t.start() + + # Run the workload + workload_func(base_url, model) + + # Clean up everything + kill_process_tree(process.pid) + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + # Assert success + has_new_server = False + has_leak = False + has_abort = False + for line in output_lines: + if "Uvicorn running" in line: + has_new_server = True + if "leak" in line: + has_leak = True + if "Abort" in line: + has_abort = True + + assert has_new_server + assert not has_leak + if assert_has_abort: + assert has_abort + + +def run_command_and_capture_output(command, env: Optional[dict] = None): + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = subprocess.Popen( + command, stdout=stdout, stderr=stdout, env=env, text=True + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines, STDOUT_FILENAME)) + t.start() + + # Join the process + process.wait() + + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + return output_lines + + +def run_mmlu_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + # Run the eval + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=128, + num_threads=128, + ) + + try: + metrics = run_eval(args) + assert metrics["score"] >= 0.65, f"{metrics=}" + finally: + pass + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def run_mulit_request_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + enable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + def run_one(_): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 8, + }, + }, + ) + ret = response.json() + + with ThreadPoolExecutor(2) as executor: + list(executor.map(run_one, list(range(4)))) + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + enable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + logging.warning("GITHUB_STEP_SUMMARY environment variable not set") + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def run_logprob_check(self: unittest.TestCase, arg: Tuple): + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) = arg + input_ids = list(range(input_len)) + + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + }, + ) + response_json = response.json() + + res = response_json + self.assertEqual(res["meta_info"]["prompt_tokens"], input_len) + self.assertEqual(res["meta_info"]["completion_tokens"], output_len) + + # Test the number of tokens are correct + if return_logprob: + self.assertEqual( + len(res["meta_info"]["input_token_logprobs"]) + logprob_start_len, + res["meta_info"]["prompt_tokens"], + ) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), output_len) + + if top_logprobs_num: + self.assertEqual( + len(res["meta_info"]["input_top_logprobs"]) + logprob_start_len, + res["meta_info"]["prompt_tokens"], + ) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), output_len) + + for i in range(output_len): + self.assertEqual( + len(res["meta_info"]["output_top_logprobs"][i]), + top_logprobs_num, + ) + + # Test the top-1 tokens are the same as output tokens if temperature == 0 + if temperature == 0: + rank = 0 + while rank < len(res["meta_info"]["output_top_logprobs"][i]): + try: + self.assertListEqual( + res["meta_info"]["output_token_logprobs"][i], + res["meta_info"]["output_top_logprobs"][i][rank], + ) + break + except AssertionError: + # There's a tie. Allow the second item in this case. + if ( + res["meta_info"]["output_top_logprobs"][i][rank][0] + == res["meta_info"]["output_top_logprobs"][i][rank + 1][ + 0 + ] + ): + rank += 1 + else: + raise + + +class CustomTestCase(unittest.TestCase): + def _callTestMethod(self, method): + _retry_execution( + lambda: super(CustomTestCase, self)._callTestMethod(method), + max_retry=_get_max_retry(), + ) + + +def _get_max_retry(): + return int(os.environ.get("SGLANG_TEST_MAX_RETRY", "2" if is_in_ci() else "0")) + + +def _retry_execution(fn, max_retry: int): + if max_retry == 0: + fn() + return + + try: + fn() + except Exception as e: + print( + f"retry_execution failed once and will retry. This may be an error or a flaky test. Error: {e}" + ) + traceback.print_exc() + _retry_execution(fn, max_retry=max_retry - 1) diff --git a/python/sglang/utils.py b/python/sglang/utils.py new file mode 100644 index 0000000..bd5acd4 --- /dev/null +++ b/python/sglang/utils.py @@ -0,0 +1,514 @@ +"""Common utilities""" + +import base64 +import importlib +import json +import logging +import os +import random +import signal +import socket +import subprocess +import sys +import time +import traceback +import urllib.request +import weakref +from concurrent.futures import ThreadPoolExecutor +from io import BytesIO +from json import dumps +from typing import Any, Callable, List, Optional, Tuple, Type, Union + +import numpy as np +import requests +from IPython.display import HTML, display +from pydantic import BaseModel +from tqdm import tqdm + +from sglang.srt.utils import kill_process_tree + +logger = logging.getLogger(__name__) + + +def convert_json_schema_to_str(json_schema: Union[dict, str, Type[BaseModel]]) -> str: + """Convert a JSON schema to a string. + Parameters + ---------- + json_schema + The JSON schema. + Returns + ------- + str + The JSON schema converted to a string. + Raises + ------ + ValueError + If the schema is not a dictionary, a string or a Pydantic class. + """ + if isinstance(json_schema, dict): + schema_str = json.dumps(json_schema) + elif isinstance(json_schema, str): + schema_str = json_schema + elif issubclass(json_schema, BaseModel): + schema_str = json.dumps(json_schema.model_json_schema()) + else: + raise ValueError( + f"Cannot parse schema {json_schema}. The schema must be either " + + "a Pydantic class, a dictionary or a string that contains the JSON " + + "schema specification" + ) + return schema_str + + +def get_exception_traceback(): + etype, value, tb = sys.exc_info() + err_str = "".join(traceback.format_exception(etype, value, tb)) + return err_str + + +def is_same_type(values: list): + """Return whether the elements in values are of the same type.""" + if len(values) <= 1: + return True + else: + t = type(values[0]) + return all(isinstance(v, t) for v in values[1:]) + + +def read_jsonl(filename: str): + """Read a JSONL file.""" + with open(filename) as fin: + for line in fin: + if line.startswith("#"): + continue + yield json.loads(line) + + +def dump_state_text(filename: str, states: list, mode: str = "w"): + """Dump program state in a text file.""" + from sglang.lang.interpreter import ProgramState + + with open(filename, mode) as fout: + for i, s in enumerate(states): + if isinstance(s, str): + pass + elif isinstance(s, ProgramState): + s = s.text() + else: + s = str(s) + + fout.write( + "=" * 40 + f" {i} " + "=" * 40 + "\n" + s + "\n" + "=" * 80 + "\n\n" + ) + + +class HttpResponse: + def __init__(self, resp): + self.resp = resp + + def json(self): + return json.loads(self.resp.read()) + + @property + def status_code(self): + return self.resp.status + + +def http_request( + url, + json=None, + stream=False, + api_key=None, + verify=None, + method: Optional[str] = None, +): + """A faster version of requests.post with low-level urllib API.""" + headers = {"Content-Type": "application/json; charset=utf-8"} + + # add the Authorization header if an api key is provided + if api_key is not None: + headers["Authorization"] = f"Bearer {api_key}" + + if stream: + return requests.post(url, json=json, stream=True, headers=headers) + else: + req = urllib.request.Request(url, headers=headers, method=method) + if json is None: + data = None + else: + data = bytes(dumps(json), encoding="utf-8") + + try: + resp = urllib.request.urlopen(req, data=data, cafile=verify) + return HttpResponse(resp) + except urllib.error.HTTPError as e: + return HttpResponse(e) + + +def encode_image_base64(image_path: Union[str, bytes]): + """Encode an image in base64.""" + if isinstance(image_path, str): + with open(image_path, "rb") as image_file: + data = image_file.read() + return base64.b64encode(data).decode("utf-8") + elif isinstance(image_path, bytes): + return base64.b64encode(image_path).decode("utf-8") + else: + # image_path is PIL.WebPImagePlugin.WebPImageFile + image = image_path + buffered = BytesIO() + image.save(buffered, format="PNG") + return base64.b64encode(buffered.getvalue()).decode("utf-8") + + +def encode_frame(frame): + import cv2 # pip install opencv-python-headless + from PIL import Image + + # Convert the frame to RGB (OpenCV uses BGR by default) + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # Convert the frame to PIL Image to easily convert to bytes + im_pil = Image.fromarray(frame) + + # Convert to bytes + buffered = BytesIO() + + # frame_format = str(os.getenv('FRAME_FORMAT', "JPEG")) + + im_pil.save(buffered, format="PNG") + + frame_bytes = buffered.getvalue() + + # Return the bytes of the frame + return frame_bytes + + +def encode_video_base64(video_path: str, num_frames: int = 16): + import cv2 # pip install opencv-python-headless + + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + raise IOError(f"Could not open video file:{video_path}") + + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + print(f"target_frames: {num_frames}") + + frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int) + + frames = [] + for _ in range(total_frames): + ret, frame = cap.read() + if ret: + frames.append(frame) + else: + # Handle the case where the frame could not be read + # print(f"Warning: Could not read frame at index {i}.") + pass + + cap.release() + + # Safely select frames based on frame_indices, avoiding IndexError + frames = [frames[i] for i in frame_indices if i < len(frames)] + + # If there are not enough frames, duplicate the last frame until we reach the target + while len(frames) < num_frames: + frames.append(frames[-1]) + + # Use ThreadPoolExecutor to process and encode frames in parallel + with ThreadPoolExecutor() as executor: + encoded_frames = list(executor.map(encode_frame, frames)) + + # encoded_frames = list(map(encode_frame, frames)) + + # Concatenate all frames bytes + video_bytes = b"".join(encoded_frames) + + # Encode the concatenated bytes to base64 + video_base64 = "video:" + base64.b64encode(video_bytes).decode("utf-8") + + return video_base64 + + +def _is_chinese_char(cp: int): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + +def find_printable_text(text: str): + """Returns the longest printable substring of text that contains only entire words.""" + # Borrowed from https://github.com/huggingface/transformers/blob/061580c82c2db1de9139528243e105953793f7a2/src/transformers/generation/streamers.py#L99 + + # After the symbol for a new line, we flush the cache. + if text.endswith("\n"): + return text + # If the last token is a CJK character, we print the characters. + elif len(text) > 0 and _is_chinese_char(ord(text[-1])): + return text + # Otherwise if the penultimate token is a CJK character, we print the characters except for the last one. + elif len(text) > 1 and _is_chinese_char(ord(text[-2])): + return text[:-1] + # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, + # which may change with the subsequent token -- there are probably smarter ways to do this!) + else: + return text[: text.rfind(" ") + 1] + + +def graceful_registry(sub_module_name: str): + def graceful_shutdown(signum, frame): + logger.info( + f"{sub_module_name} Received signal to shutdown. Performing graceful shutdown..." + ) + if signum == signal.SIGTERM: + logger.info(f"{sub_module_name} recive sigterm") + + signal.signal(signal.SIGTERM, graceful_shutdown) + + +class LazyImport: + """Lazy import to make `import sglang` run faster.""" + + def __init__(self, module_name: str, class_name: str): + self.module_name = module_name + self.class_name = class_name + self._module = None + + def _load(self): + if self._module is None: + module = importlib.import_module(self.module_name) + self._module = getattr(module, self.class_name) + return self._module + + def __getattr__(self, name: str): + module = self._load() + return getattr(module, name) + + def __call__(self, *args, **kwargs): + module = self._load() + return module(*args, **kwargs) + + +def download_and_cache_file(url: str, filename: Optional[str] = None): + """Read and cache a file from a url.""" + if filename is None: + filename = os.path.join("/tmp", url.split("/")[-1]) + + # Check if the cache file already exists + if os.path.exists(filename): + return filename + + print(f"Downloading from {url} to {filename}") + + # Stream the response to show the progress bar + response = requests.get(url, stream=True) + response.raise_for_status() # Check for request errors + + # Total size of the file in bytes + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 # Download in chunks of 1KB + + # Use tqdm to display the progress bar + with open(filename, "wb") as f, tqdm( + desc=filename, + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + bar.update(len(chunk)) + + return filename + + +def is_in_ci(): + from sglang.test.test_utils import is_in_ci + + return is_in_ci() + + +def print_highlight(html_content: str): + if is_in_ci(): + html_content = str(html_content).replace("\n", "
") + display(HTML(f"{html_content}")) + else: + print(html_content) + + +process_socket_map = weakref.WeakKeyDictionary() + + +def reserve_port(host, start=30000, end=40000): + """ + Reserve an available port by trying to bind a socket. + Returns a tuple (port, lock_socket) where `lock_socket` is kept open to hold the lock. + """ + candidates = list(range(start, end)) + random.shuffle(candidates) + + for port in candidates: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + # Attempt to bind to the port on localhost + sock.bind((host, port)) + return port, sock + except socket.error: + sock.close() # Failed to bind, try next port + continue + raise RuntimeError("No free port available.") + + +def release_port(lock_socket): + """ + Release the reserved port by closing the lock socket. + """ + try: + lock_socket.close() + except Exception as e: + print(f"Error closing socket: {e}") + + +def execute_shell_command(command: str) -> subprocess.Popen: + """ + Execute a shell command and return its process handle. + """ + command = command.replace("\\\n", " ").replace("\\", " ") + parts = command.split() + return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT) + + +def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): + """ + Launch the server using the given command. + If no port is specified, a free port is reserved. + """ + if port is None: + port, lock_socket = reserve_port(host) + else: + lock_socket = None + + full_command = f"{command} --port {port}" + process = execute_shell_command(full_command) + + if lock_socket is not None: + process_socket_map[process] = lock_socket + + return process, port + + +def terminate_process(process): + """ + Terminate the process and automatically release the reserved port. + """ + kill_process_tree(process.pid) + + lock_socket = process_socket_map.pop(process, None) + if lock_socket is not None: + release_port(lock_socket) + + +def wait_for_server(base_url: str, timeout: int = None) -> None: + """Wait for the server to be ready by polling the /v1/models endpoint. + + Args: + base_url: The base URL of the server + timeout: Maximum time to wait in seconds. None means wait forever. + """ + start_time = time.time() + while True: + try: + response = requests.get( + f"{base_url}/v1/models", + headers={"Authorization": "Bearer None"}, + ) + if response.status_code == 200: + time.sleep(5) + print_highlight( + """\n + NOTE: Typically, the server runs in a separate terminal. + In this notebook, we run the server and notebook code together, so their outputs are combined. + To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue. + We are running those notebooks in a CI parallel environment, so the throughput is not representative of the actual performance. + """ + ) + break + + if timeout and time.time() - start_time > timeout: + raise TimeoutError("Server did not become ready within timeout period") + except requests.exceptions.RequestException: + time.sleep(1) + + +class TypeBasedDispatcher: + def __init__(self, mapping: List[Tuple[Type, Callable]]): + self._mapping = mapping + + def __call__(self, obj: Any): + for ty, fn in self._mapping: + if isinstance(obj, ty): + return fn(obj) + raise ValueError(f"Invalid object: {obj}") + + +def trim_overlap(existing_text, new_chunk): + """ + Finds the largest suffix of 'existing_text' that is a prefix of 'new_chunk' + and removes that overlap from the start of 'new_chunk'. + """ + max_overlap = 0 + max_possible = min(len(existing_text), len(new_chunk)) + for i in range(max_possible, 0, -1): + if existing_text.endswith(new_chunk[:i]): + max_overlap = i + break + return new_chunk[max_overlap:] + + +def stream_and_merge(llm, prompt, sampling_params): + """ + 1) Streams the text, + 2) Removes chunk overlaps, + 3) Returns the merged text. + """ + final_text = "" + for chunk in llm.generate(prompt, sampling_params, stream=True): + chunk_text = chunk["text"] + cleaned_chunk = trim_overlap(final_text, chunk_text) + final_text += cleaned_chunk + return final_text + + +async def async_stream_and_merge(llm, prompt, sampling_params): + """ + Streams tokens asynchronously, removes chunk overlaps, + and yields the cleaned chunk in real time for printing. + """ + final_text = "" + generator = await llm.async_generate(prompt, sampling_params, stream=True) + async for chunk in generator: + chunk_text = chunk["text"] + cleaned_chunk = trim_overlap(final_text, chunk_text) + final_text += cleaned_chunk + yield cleaned_chunk # yield the non-overlapping portion diff --git a/python/sglang/version.py b/python/sglang/version.py new file mode 100644 index 0000000..2cc054f --- /dev/null +++ b/python/sglang/version.py @@ -0,0 +1 @@ +__version__ = "0.4.4.post3" diff --git a/scripts/ci_install_dependency.sh b/scripts/ci_install_dependency.sh new file mode 100644 index 0000000..3f397ae --- /dev/null +++ b/scripts/ci_install_dependency.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Install the dependency in CI. +set -euxo pipefail + +# Use repo from environment variables, passed from GitHub Actions +FLASHINFER_REPO="${FLASHINFER_REPO:-https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python}" + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +bash "${SCRIPT_DIR}/killall_sglang.sh" + +pip install --upgrade pip +pip uninstall flashinfer -y +pip install -e "python[all]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python + +rm -rf /root/.cache/flashinfer +# Force reinstall flashinfer and torch_memory_saver +pip install flashinfer_python==0.2.3 --find-links ${FLASHINFER_REPO} --force-reinstall --no-deps +pip install sgl-kernel==0.0.5.post4 --force-reinstall + +pip install torch_memory_saver +pip install transformers==4.50.0 sentence_transformers accelerate==1.4.0 peft pandas datasets timm + +# For compling xgrammar kernels +pip install cuda-python nvidia-cuda-nvrtc-cu12 + +pip uninstall vllm -y || true diff --git a/scripts/ci_install_rust.sh b/scripts/ci_install_rust.sh new file mode 100644 index 0000000..519155d --- /dev/null +++ b/scripts/ci_install_rust.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euxo pipefail + +# Check if sudo is available +if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y libssl-dev pkg-config +else + apt-get update + apt-get install -y libssl-dev pkg-config +fi + +# Install rustup (Rust installer and version manager) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + + +# Follow the installation prompts, then reload your shell +. "$HOME/.cargo/env" +source $HOME/.cargo/env + +# Verify installation +rustc --version +cargo --version diff --git a/scripts/deprecated/convert_yi_vl.py b/scripts/deprecated/convert_yi_vl.py new file mode 100644 index 0000000..bdf37ff --- /dev/null +++ b/scripts/deprecated/convert_yi_vl.py @@ -0,0 +1,38 @@ +""" +Convert Yi-VL config into a format useable with SGLang + +Usage: python3 scripts/convert_yi_vl.py --model-path +""" + +import argparse +import json +import os + +from transformers import AutoConfig, AutoTokenizer + + +def add_image_token(model_path: str): + tokenizer = AutoTokenizer.from_pretrained(model_path) + tokenizer.add_tokens([""], special_tokens=True) + + print(tokenizer) + tokenizer.save_pretrained(model_path) + + +def edit_model_config(model_path): + config = AutoConfig.from_pretrained(model_path) + + setattr(config, "architectures", ["YiVLForCausalLM"]) + setattr(config, "image_token_index", 64002) + + print(config) + config.save_pretrained(model_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model-path", type=str) + args = parser.parse_args() + + add_image_token(args.model_path) + edit_model_config(args.model_path) diff --git a/scripts/deprecated/convert_yi_vl.sh b/scripts/deprecated/convert_yi_vl.sh new file mode 100644 index 0000000..07a7aeb --- /dev/null +++ b/scripts/deprecated/convert_yi_vl.sh @@ -0,0 +1,13 @@ +# For 34B Model +mkdir ~/model_weights +cd ~/model_weights +git clone https://huggingface.co/01-ai/Yi-VL-34B +cp ~/model_weights/Yi-VL-34B/vit/clip-vit-H-14-laion2B-s32B-b79K-yi-vl-34B-448/preprocessor_config.json ~/model_weights/Yi-VL-34B +python3 convert_yi_vl.py --model-path ~/model_weights/Yi-VL-34B + +# For 6B Model +mkdir ~/model_weights +cd ~/model_weights +git clone https://huggingface.co/01-ai/Yi-VL-6B +cp ~/model_weights/Yi-VL-6B/vit/clip-vit-H-14-laion2B-s32B-b79K-yi-vl-6B-448/preprocessor_config.json ~/model_weights/Yi-VL-6B +python3 convert_yi_vl.py --model-path ~/model_weights/Yi-VL-6B diff --git a/scripts/deprecated/test_curl.sh b/scripts/deprecated/test_curl.sh new file mode 100644 index 0000000..1c83208 --- /dev/null +++ b/scripts/deprecated/test_curl.sh @@ -0,0 +1,9 @@ +curl http://localhost:30000/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Once upon a time,", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' diff --git a/scripts/deprecated/test_flashinfer.py b/scripts/deprecated/test_flashinfer.py new file mode 100644 index 0000000..d017d11 --- /dev/null +++ b/scripts/deprecated/test_flashinfer.py @@ -0,0 +1,217 @@ +import pytest +import torch +from flashinfer import ( + BatchDecodeWithPagedKVCacheWrapper, + BatchPrefillWithPagedKVCacheWrapper, +) + +from sglang.srt.layers.attention.triton_ops.decode_attention import decode_attention_fwd +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + redundant_attention, +) +from sglang.srt.utils import should_use_tensor_core + +flashinfer_prefill_wrapper = None +flashinfer_decode_wrapper = None + + +@pytest.mark.parametrize("batch_size", [12, 37, 67]) +@pytest.mark.parametrize("kv_len", [54, 97]) +@pytest.mark.parametrize("qo_len", [37, 17]) +@pytest.mark.parametrize("num_kv_heads", [4]) +@pytest.mark.parametrize("num_qo_heads", [32, 4]) +@pytest.mark.parametrize("head_dim", [128]) +def test_batch_prefill_with_paged_kv_cache( + batch_size, + kv_len, + qo_len, + num_kv_heads, + num_qo_heads, + head_dim, +): + init_flashinfer(num_qo_heads, num_kv_heads) + + q = torch.randn(batch_size * qo_len, num_qo_heads, head_dim).to(0).half() + qo_indptr = torch.arange(0, batch_size + 1).to(0).int() * qo_len + total_tokens = kv_len * batch_size + kv_data = torch.randn(total_tokens, 2, num_kv_heads, head_dim).to(0).half() + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full((batch_size,), 1, dtype=torch.int32).to(0) + + # init args for triton kernel + k_extend = ( + kv_data.view(batch_size, kv_len, 2, -1)[:, -qo_len:, 0] + .contiguous() + .view(-1, num_kv_heads, head_dim) + ) + v_extend = ( + kv_data.view(batch_size, kv_len, 2, -1)[:, -qo_len:, 1] + .contiguous() + .view(-1, num_kv_heads, head_dim) + ) + o_triton = torch.empty_like(q) + k_buffer = kv_data[:, 0].view(-1, num_kv_heads, head_dim).contiguous() + v_buffer = kv_data[:, 1].view(-1, num_kv_heads, head_dim).contiguous() + req_to_token = torch.arange(0, total_tokens).to(0).int().view(batch_size, kv_len) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32).to(0) + b_start_loc_extend = torch.arange(0, batch_size).to(0).int() * qo_len + b_seq_len_extend = torch.full((batch_size,), qo_len, dtype=torch.int32).to(0) + max_len_in_batch = kv_len + max_len_extend = qo_len + + extend_attention_fwd( + q, + k_extend, + v_extend, + o_triton, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + None, # b_start_loc = None + b_seq_len, + None, # b_seq_len_prefix = None + b_start_loc_extend, + b_seq_len_extend, + max_len_in_batch, + max_len_extend, + ) + + o_redundant = torch.empty_like(q) + b_start_loc = torch.zeros((batch_size,), dtype=torch.int32).to(0) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], dim=0) + b_seq_len_prefix = b_seq_len - b_seq_len_extend + + redundant_attention( + q, + k_extend, + v_extend, + o_redundant, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + print("Mean: ", torch.mean(torch.abs(o_redundant - o_triton))) + print("Max: ", torch.max(torch.abs(o_redundant - o_triton))) + assert torch.allclose(o_redundant, o_triton, rtol=1e-2, atol=1e-3) + + flashinfer_prefill_wrapper.end_forward() + + flashinfer_prefill_wrapper.begin_forward( + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + 1, + ) + o = flashinfer_prefill_wrapper.forward( + q.contiguous().view(-1, num_qo_heads, head_dim), kv_data + ) + + print("Mean: ", torch.mean(torch.abs(o - o_triton))) + print("Max: ", torch.max(torch.abs(o - o_triton))) + assert torch.allclose(o, o_triton, rtol=1e-2, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [12, 17, 37]) +@pytest.mark.parametrize("kv_len", [54, 127, 537]) +@pytest.mark.parametrize("num_kv_heads", [32]) +@pytest.mark.parametrize("num_qo_heads", [32]) +@pytest.mark.parametrize("head_dim", [128]) +def test_batch_decode_with_paged_kv_cache( + batch_size, + kv_len, + num_kv_heads, + num_qo_heads, + head_dim, +): + # note(lsyin): when pytest, the number of heads cannot change, because triton kernel has a cache + # to test different shape of decode, change the parameters in the __main__, and run decode only once + init_flashinfer(num_qo_heads, num_kv_heads) + + q = torch.randn(batch_size, num_qo_heads, head_dim).to(0).half() + total_tokens = kv_len * batch_size + kv_data = torch.randn(total_tokens, 2, num_kv_heads, head_dim).to(0).half() + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full((batch_size,), 1, dtype=torch.int32).to(0) + + # init args for triton kernel + k_buffer = kv_data[:, 0].view(-1, num_kv_heads, head_dim).contiguous() + v_buffer = kv_data[:, 1].view(-1, num_kv_heads, head_dim).contiguous() + o_triton = torch.empty_like(q) + req_to_token = ( + torch.arange(0, kv_len * batch_size).to(0).int().view(batch_size, kv_len) + ) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_start_loc = torch.arange(0, batch_size).to(0).int() * kv_len + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32).to(0) + max_len_in_batch = kv_len + other_kv_index = 0 + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o_triton, + req_to_token, + b_req_idx, + b_start_loc, + b_seq_len, + max_len_in_batch, + other_kv_index, + total_tokens, + ) + + flashinfer_decode_wrapper.end_forward() + flashinfer_decode_wrapper.begin_forward( + kv_indptr, + kv_indices, + kv_last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + 1, + pos_encoding_mode="NONE", + data_type="float16", + ) + o = flashinfer_decode_wrapper.forward( + q.contiguous().view(-1, num_qo_heads, head_dim), kv_data + ) + + print("Mean: ", torch.mean(torch.abs(o - o_triton))) + print("Max: ", torch.max(torch.abs(o - o_triton))) + assert torch.allclose(o, o_triton, rtol=1e-2, atol=2e-3) + + +def init_flashinfer(num_attention_heads, num_kv_heads): + use_tensor_cores = should_use_tensor_core( + torch.half, num_attention_heads, num_kv_heads + ) + + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device="cuda") + + global flashinfer_prefill_wrapper, flashinfer_decode_wrapper + + flashinfer_prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper( + workspace_buffer, "NHD" + ) + flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( + workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores + ) + + +if __name__ == "__main__": + test_batch_prefill_with_paged_kv_cache(12, 54, 37, 8, 8, 128) + test_batch_prefill_with_paged_kv_cache(37, 1111, 456, 32, 32, 128) + test_batch_decode_with_paged_kv_cache(12, 54, 4, 32, 128) diff --git a/scripts/deprecated/test_httpserver_concurrent.py b/scripts/deprecated/test_httpserver_concurrent.py new file mode 100644 index 0000000..855e51f --- /dev/null +++ b/scripts/deprecated/test_httpserver_concurrent.py @@ -0,0 +1,56 @@ +""" +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C. + +The capital of the United Kindom is London.\nThe capital of the United Kingdom is London.\nThe capital of +""" + +import argparse +import asyncio +import json +import time + +import aiohttp +import requests + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def main(args): + url = f"{args.host}:{args.port}" + task1 = send_request( + url + "/generate", + { + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 128}, + }, + delay=1, + ) + + task2 = send_request( + url + "/generate", + { + "text": "The capital of the United Kindom is", + "sampling_params": {"temperature": 0, "max_new_tokens": 128}, + }, + ) + + rets = await asyncio.gather(task1, task2) + print(rets) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + asyncio.run(main(args)) diff --git a/scripts/deprecated/test_httpserver_decode.py b/scripts/deprecated/test_httpserver_decode.py new file mode 100644 index 0000000..57517a1 --- /dev/null +++ b/scripts/deprecated/test_httpserver_decode.py @@ -0,0 +1,55 @@ +""" +Usage: +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 +python3 test_httpserver_decode.py + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C.\nThe capital of Canada is Ottawa.\nThe capital of Japan is Tokyo +""" + +import argparse +import json + +import requests + + +def test_decode(url, return_logprob=False, top_logprobs_num=0, return_text=False, n=1): + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 32, + "n": n, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + print(json.dumps(response.json())) + print("=" * 100) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + url = f"{args.host}:{args.port}" + + test_decode(url) + test_decode(url, n=3) + + for top_logprobs_num in [0, 3]: + for return_text in [True, False]: + test_decode( + url, + return_logprob=True, + top_logprobs_num=top_logprobs_num, + return_text=return_text, + ) diff --git a/scripts/deprecated/test_httpserver_decode_stream.py b/scripts/deprecated/test_httpserver_decode_stream.py new file mode 100644 index 0000000..616eaf6 --- /dev/null +++ b/scripts/deprecated/test_httpserver_decode_stream.py @@ -0,0 +1,68 @@ +""" +Usage: +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 +python3 test_httpserver_decode_stream.py + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C.\nThe capital of Canada is Ottawa.\nThe capital of Japan is Tokyo +""" + +import argparse +import json + +import requests + + +def test_decode_stream(url, return_logprob, top_logprobs_num): + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + }, + "stream": True, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": True, + "logprob_start_len": 0, + }, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + + if return_logprob: + assert data["meta_info"]["input_token_logprobs"] is not None + assert data["meta_info"]["output_token_logprobs"] is not None + for logprob, token_id, token_text in data["meta_info"][ + "output_token_logprobs" + ][prev:]: + print(f"{token_text:12s}\t{logprob}\t{token_id}", flush=True) + prev = len(data["meta_info"]["output_token_logprobs"]) + else: + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + + print("=" * 100) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + url = f"{args.host}:{args.port}" + + test_decode_stream(url, False, 0) + test_decode_stream(url, True, 0) + test_decode_stream(url, True, 3) diff --git a/scripts/deprecated/test_httpserver_llava.py b/scripts/deprecated/test_httpserver_llava.py new file mode 100644 index 0000000..791fc6d --- /dev/null +++ b/scripts/deprecated/test_httpserver_llava.py @@ -0,0 +1,88 @@ +""" +Usage: +python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000 +python3 test_httpserver_llava.py + +Output: +The image features a man standing on the back of a yellow taxi cab, holding +""" + +import argparse +import asyncio +import json + +import aiohttp +import requests + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + response = [] + for i in range(8): + response.append( + send_request( + url + "/generate", + { + "text": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. USER: \nDescribe this picture ASSISTANT:", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + + response = requests.post( + url + "/generate", + json={ + "text": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. USER: \nDescribe this picture ASSISTANT:", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + }, + "stream": True, + }, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + asyncio.run(test_concurrent(args)) + + test_streaming(args) diff --git a/scripts/deprecated/test_httpserver_reuse.py b/scripts/deprecated/test_httpserver_reuse.py new file mode 100644 index 0000000..ef866af --- /dev/null +++ b/scripts/deprecated/test_httpserver_reuse.py @@ -0,0 +1,42 @@ +""" +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C.\nThe capital of Canada is Ottawa.\nThe capital of Japan is Tokyo +""" + +import argparse + +import requests + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + url = f"{args.host}:{args.port}" + + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + print(response.json()) + + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is Paris.\nThe capital of the United States is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + print(response.json()) diff --git a/scripts/deprecated/test_jump_forward.py b/scripts/deprecated/test_jump_forward.py new file mode 100644 index 0000000..315a50b --- /dev/null +++ b/scripts/deprecated/test_jump_forward.py @@ -0,0 +1,138 @@ +import argparse +from enum import Enum + +from pydantic import BaseModel, constr + +import sglang as sgl +from sglang.srt.constrained.outlines_backend import build_regex_from_object +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +IP_REGEX = r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + +ip_jump_forward = ( + r"The google's DNS sever address is " + + IP_REGEX + + r" and " + + IP_REGEX + + r". " + + r"The google's website domain name is " + + r"www\.(\w)+\.(\w)+" + + r"." +) + + +# fmt: off +@sgl.function +def regex_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + max_tokens=128, + temperature=0, + regex=ip_jump_forward, + ) +# fmt: on + +json_jump_forward = ( + r"""The information about Hogwarts is in the following JSON format\.\n""" + + r"""\n\{\n""" + + r""" "name": "[\w\d\s]*",\n""" + + r""" "country": "[\w\d\s]*",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]+,\n""" + + r""" "population": [-+]?[0-9]+,\n""" + + r""" "top 3 landmarks": \["[\w\d\s]*", "[\w\d\s]*", "[\w\d\s]*"\],\n""" + + r"""\}\n""" +) + +# fmt: off +@sgl.function +def json_gen(s): + s += sgl.gen( + "json", + max_tokens=128, + temperature=0, + regex=json_jump_forward, + ) +# fmt: on + + +class Weapon(str, Enum): + sword = "sword" + axe = "axe" + mace = "mace" + spear = "spear" + bow = "bow" + crossbow = "crossbow" + + +class Armor(str, Enum): + leather = "leather" + chainmail = "chainmail" + plate = "plate" + + +class Character(BaseModel): + name: constr(max_length=10) + age: int + armor: Armor + weapon: Weapon + strength: int + + +@sgl.function +def character_gen(s): + s += "Give me a character description who is a wizard.\n" + s += sgl.gen( + "character", + max_tokens=128, + temperature=0, + regex=build_regex_from_object(Character), + ) + + +def main(args): + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + state = regex_gen.run(temperature=0) + + print("=" * 20, "IP TEST", "=" * 20) + print(state.text()) + + state = json_gen.run(temperature=0) + + print("=" * 20, "JSON TEST", "=" * 20) + print(state.text()) + + state = character_gen.run(temperature=0) + + print("=" * 20, "CHARACTER TEST", "=" * 20) + print(state.text()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args = add_common_sglang_args_and_parse(parser) + main(args) + +# ==================== IP TEST ==================== +# Q: What is the IP address of the Google DNS servers? +# A: The google's DNS sever address is 8.8.8.8 and 8.8.4.4. The google's website domain name is www.google.com. +# ==================== JSON TEST ==================== +# The information about Hogwarts is in the following JSON format. + +# { +# "name": "Hogwarts School of Witchcraft and Wizardry", +# "country": "Scotland", +# "latitude": 55.566667, +# "population": 1000, +# "top 3 landmarks": ["Hogwarts Castle", "The Great Hall", "The Forbidden Forest"], +# } + +# ==================== CHARACTER TEST ==================== +# Give me a character description who is a wizard. +# { "name" : "Merlin", "age" : 500, "armor" : "chainmail" , "weapon" : "sword" , "strength" : 10 } diff --git a/scripts/deprecated/test_robust.py b/scripts/deprecated/test_robust.py new file mode 100644 index 0000000..633e2e6 --- /dev/null +++ b/scripts/deprecated/test_robust.py @@ -0,0 +1,132 @@ +import argparse +import random +import string + +from vllm.transformers_utils.tokenizer import get_tokenizer + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +TOKENIZER = None +RANDOM_PREFILL_LEN = None +RANDOM_DECODE_LEN = None + + +def gen_prompt(token_num): + if RANDOM_PREFILL_LEN: + token_num = random.randint(1, token_num) + + cha_set = string.ascii_letters + string.digits + ret = "".join(random.choices(cha_set, k=token_num)) + while len(TOKENIZER(ret).input_ids) < token_num: + ret += random.choice(cha_set) + + return ret + + +def robust_test_dfs(s, d, args, leaf_states): + if d == 0: + s += "END" + leaf_states.append(s) + return + + s += gen_prompt(args.len_prefill) + forks = s.fork(args.num_fork) + for fork_s in forks: + fork_s += gen_prompt(args.len_prefill) + new_tokens = ( + args.len_decode + if not RANDOM_DECODE_LEN + else random.randint(1, args.len_decode) + ) + fork_s += sgl.gen( + max_tokens=new_tokens, + ignore_eos=True, + ) + + for fork_s in forks: + robust_test_dfs(fork_s, d - 1, args, leaf_states) + + +def robust_test_bfs(s, args, leaf_states): + old_forks = [s] + new_forks = [] + for _ in range(args.depth): + for old_fork in old_forks: + old_fork += gen_prompt(args.len_prefill) + forks = old_fork.fork(args.num_fork) + for fork_s in forks: + fork_s += gen_prompt(args.len_prefill) + new_tokens = ( + args.len_decode + if not RANDOM_DECODE_LEN + else random.randint(1, args.len_decode) + ) + fork_s += sgl.gen( + max_tokens=new_tokens, + ignore_eos=True, + ) + new_forks.extend(forks) + + old_forks = new_forks + new_forks = [] + + for old_fork in old_forks: + old_fork += "END" + leaf_states.append(old_fork) + + +@sgl.function +def robust_test(s, args): + leaf_states = [] + if args.mode == "bfs": + robust_test_bfs(s, args, leaf_states) + else: + robust_test_dfs(s, args.depth, args, leaf_states) + return leaf_states + + +def main(args): + backend = select_sglang_backend(args) + + arguments = [{"args": args} for _ in range(args.num_req)] + + states = robust_test.run_batch( + arguments, temperature=0, backend=backend, num_threads=args.parallel + ) + + with open(f"tmp_robust_{args.mode}.txt", "w") as f: + for state in states: + leaf_states = state.ret_value + for leaf_state in leaf_states: + assert leaf_state.text()[-3:] == "END" + f.write(leaf_state.text()[:-3] + "\n") + + +if __name__ == "__main__": + # fmt: off + parser = argparse.ArgumentParser() + parser.add_argument("--num-req", type=int, default=2) + parser.add_argument("--depth", type=int, default=3) + parser.add_argument("--num-fork", type=int, default=2) + parser.add_argument("--len-prefill", type=int, default=128) + parser.add_argument("--len-decode", type=int, default=128) + parser.add_argument("--random-prefill-len", action="store_true") + parser.add_argument("--random-decode-len", action="store_true") + parser.add_argument("--mode", type=str, default="bfs", choices=["dfs", "bfs"]) + parser.add_argument("--tokenizer", type=str, default = "meta-llama/Llama-2-7b-chat-hf") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--seed", type=int, default=42) + args = add_common_sglang_args_and_parse(parser) + # fmt: on + + RANDOM_PREFILL_LEN = args.random_prefill_len + RANDOM_DECODE_LEN = args.random_decode_len + TOKENIZER = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + random.seed(args.seed) + + main(args) diff --git a/scripts/export_deepseek_nextn.py b/scripts/export_deepseek_nextn.py new file mode 100644 index 0000000..35a06b6 --- /dev/null +++ b/scripts/export_deepseek_nextn.py @@ -0,0 +1,115 @@ +""" +Export NextN layer for DeepSeek-V3/R1 model. The exported model can be used for speculative decoding. + +Usage: +python3 export_deepseek_nextn.py --input-dir /path/to/DeepSeek-V3 --output-dir /path/to/DeepSeek-V3-NextN +""" + +import argparse +import json +import os +import shutil + +from safetensors import safe_open +from safetensors.torch import save_file +from transformers import AutoConfig + + +def get_nextn_layer_id(config): + if not hasattr(config, "num_hidden_layers"): + raise ValueError("'num_hidden_layers' not found in model config.") + return config.num_hidden_layers + + +def update_and_save_config(config, output_dir): + new_config = config.to_dict() + new_config.update( + { + "num_hidden_layers": 1, + "architectures": ["DeepseekV3ForCausalLMNextN"], + } + ) + with open(os.path.join(output_dir, "config.json"), "w") as f: + json.dump(new_config, f, indent=2, ensure_ascii=False, sort_keys=True) + + +def copy_non_safetensors_files(input_dir, output_dir): + for filename in os.listdir(input_dir): + src_file_path = os.path.join(input_dir, filename) + if os.path.isfile(src_file_path) and not filename.endswith(".safetensors"): + dst_file_path = os.path.join(output_dir, filename) + shutil.copy2(src_file_path, dst_file_path) + print(f"All non-safetensors files have been copied to {output_dir}") + + +def export_nextn_layer_parameters(input_dir, output_dir, nextn_layer_id): + prefix = f"model.layers.{nextn_layer_id}" + output_path = os.path.join(output_dir, "nextn_layer_parameters.safetensors") + params = {} + for filename in os.listdir(input_dir): + if not filename.endswith(".safetensors"): + continue + + file_path = os.path.join(input_dir, filename) + print(f"Processing: {filename}") + + try: + with safe_open(file_path, framework="pt") as f: + matching_keys = [k for k in f.keys() if k.startswith(prefix)] + + if not matching_keys: + print(f" No parameters starting with '{prefix}' found") + continue + + for key in matching_keys: + if "embed_tokens" in key or "shared_head.head" in key: + continue + new_key = key.replace(prefix, "model.layers.0") + params[new_key] = f.get_tensor(key) + + except Exception as e: + print(f" Error processing {filename}: {str(e)}") + + if params: + print(f"Saving {len(params)} parameters to {output_path}") + save_file(params, output_path) + else: + print("No matching parameters found.") + + # Update safetensors index + index_path = os.path.join(output_dir, "model.safetensors.index.json") + print(f"Updating safetensors index to {index_path}") + index_data = {"weight_map": {}} + for key in params: + index_data["weight_map"][key] = "nextn_layer_parameters.safetensors" + with open(index_path, "w") as f: + json.dump(index_data, f, indent=4) + + print("All done.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Export NextN layer paramerters for DeepSeek-V3/R1" + ) + parser.add_argument( + "--input-dir", + type=str, + required=True, + help="Input HF model directory.", + ) + parser.add_argument( + "--output-dir", + type=str, + required=True, + help="Output nextn model directory.", + ) + args = parser.parse_args() + + config = AutoConfig.from_pretrained(args.input_dir, trust_remote_code=True) + assert config.num_nextn_predict_layers == 1, "Only 1 nextn layer is supported." + nextn_layer_id = get_nextn_layer_id(config) + os.makedirs(args.output_dir, exist_ok=True) + copy_non_safetensors_files(args.input_dir, args.output_dir) + update_and_save_config(config, args.output_dir) + export_nextn_layer_parameters(args.input_dir, args.output_dir, nextn_layer_id) diff --git a/scripts/fix_corrupted_json.py b/scripts/fix_corrupted_json.py new file mode 100644 index 0000000..67c2980 --- /dev/null +++ b/scripts/fix_corrupted_json.py @@ -0,0 +1,40 @@ +import json +import re +import sys + + +def clean_json_file(input_file, output_file): + try: + # Open the input file with 'replace' option for handling bad characters + with open(input_file, "r", encoding="utf-8", errors="replace") as f: + data = f.read() + + # Replace bad characters (represented by '�' after decoding) with a space + cleaned_data = data.replace("�", " ") + + # Remove control characters (e.g., ASCII control characters like \x00 to \x1F) + # These can cause issues in JSON parsing. + cleaned_data = re.sub(r"[\x00-\x1F]+", " ", cleaned_data) + + # Parse cleaned data as JSON + json_data = json.loads(cleaned_data) + + # Write the cleaned JSON to a new output file + with open(output_file, "w", encoding="utf-8") as f: + json.dump(json_data, f, ensure_ascii=False, indent=4) + + print(f"Cleaned JSON file has been saved to {output_file}") + + except Exception as e: + print(f"Error: {e}") + + +if __name__ == "__main__": + assert len(sys.argv) > 1, "please give the input file path" + if len(sys.argv) == 3: + input_file = sys.argv[1] + output_file = sys.argv[2] + else: + input_file = output_file = sys.argv[1] + + clean_json_file(input_file, output_file) diff --git a/scripts/killall_sglang.sh b/scripts/killall_sglang.sh new file mode 100644 index 0000000..fdafbbb --- /dev/null +++ b/scripts/killall_sglang.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Show current GPU status +nvidia-smi + +# Clean SGLang processes +kill -9 $(ps aux | grep 'sglang::' | grep -v 'grep' | awk '{print $2}') 2>/dev/null +kill -9 $(ps aux | grep 'sglang.launch_server' | grep -v 'grep' | awk '{print $2}') 2>/dev/null +kill -9 $(ps aux | grep 'sglang.bench' | grep -v 'grep' | awk '{print $2}') 2>/dev/null +kill -9 $(ps aux | grep 'sglang.data_parallel' | grep -v 'grep' | awk '{print $2}') 2>/dev/null + +# Clean all GPU processes if any argument is provided +if [ $# -gt 0 ]; then + # Check if sudo is available + if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y lsof + else + apt-get update + apt-get install -y lsof + fi + kill -9 $(nvidia-smi | sed -n '/Processes:/,$p' | grep " [0-9]" | awk '{print $5}') 2>/dev/null + lsof /dev/nvidia* | awk '{print $2}' | xargs kill -9 2>/dev/null +fi + + +# Show GPU status after clean up +nvidia-smi diff --git a/scripts/playground/bench_speculative.py b/scripts/playground/bench_speculative.py new file mode 100644 index 0000000..72faf70 --- /dev/null +++ b/scripts/playground/bench_speculative.py @@ -0,0 +1,280 @@ +""" +Usage: +# single GPU +python3 bench_speculative.py --model-path meta-llama/Llama-2-7b-chat-hf --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B + +# multiple GPU +python3 bench_speculative.py --model-path deepseek-ai/DeepSeek-V3 --speculative-draft-model-path lmsys/DeepSeek-V3-NextN --tp-size 8 --trust-remote-code --batch-size 1 4 8 16 32 --steps 0 1 2 --topk 0 1 2 4 --num_draft_tokens 0 2 4 8 +""" + +import argparse +import asyncio +import json +import os +import time +from types import SimpleNamespace + +import numpy as np +import requests + +from sglang.bench_serving import benchmark, set_global_args +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + kill_process_tree, + popen_launch_server, +) + + +def node0_print(msg): + if server_args.node_rank == 0: + print(msg) + + +prompts = [ + "Human: Give me a fully functional FastAPI server. Show the full, long python code without stop.\n\nAssistant:", + "Human: Imagine you are an experienced Ethereum developer tasked with creating a smart contract for a blockchain messenger. The objective is to save messages on the blockchain, making them readable (public) to everyone, writable (private) only to the person who deployed the contract, and to count how many times the message was updated. Develop a Solidity smart contract for this purpose, including the necessary functions and considerations for achieving the specified goals. Please provide the code and any relevant explanations to ensure a clear understanding of the implementation.\n\nAssistant:", + "Human: Write a travel blog post to Hawaii.\n\nAssistant:", + "Human: I want you to act as an English translator, spelling corrector and improver. I will speak to you in any language and you will detect the language, translate it and answer in the corrected and improved version of my text, in English. I want you to replace my simplified A0-level words and sentences with more beautiful and elegant, upper level English words and sentences. Keep the meaning same, but make them more literary. My first sentence is 'istanbulu cok seviyom burada olmak cok guzel'. Answer in more than 5000 words.\n\nAssistant:", + "Human: I want you to act as a storyteller. You will come up with entertaining stories that are engaging, imaginative and captivating for the audience. It can be fairy tales, educational stories or any other type of stories which has the potential to capture people's attention and imagination. Depending on the target audience, you may choose specific themes or topics for your storytelling session e.g., if it’s children then you can talk about animals; If it’s adults then history-based tales might engage them better etc. Answer in more than 5000 words. My first request is 'I need an interesting story on perseverance.'\n\nAssistant:", + "Human: Solve x^2 = -1. Think step-by-step. Give me a long detailed explanation. \n\nAssistant:", + "Human: Tell me about the president of the USA in wikipedia style.\n\nAssistant:", + "Human: Hello? Who are you? Write code, math, and poem to explanin yourself.\n\nAssistant:", +] + + +class FakeTokenizer: + def encode(self, text: str, add_special_tokens: bool = False): + return [] + + +def send_one_batch(base_url, num_prompts, batch_size): + padded_prompts = (prompts * ((num_prompts + len(prompts) - 1) // len(prompts)))[ + :num_prompts + ] + + # format: (prompt, input_len, output len). We set input_len as a dummy value 0. + input_requests = [(p, 0, 512) for p in padded_prompts] + + # We need to set some dummy values in order to call `benchmark` below. + args = SimpleNamespace( + disable_ignore_eos=False, + disable_stream=False, + return_logprob=False, + backend="sglang", + dataset_name="custom", + num_prompts=None, + sharegpt_output_len=None, + random_input_len=None, + random_output_len=None, + random_range_ratio=None, + output_file=None, + ) + set_global_args(args) + tokenizer = FakeTokenizer() + + # Run benchmark + results = asyncio.run( + benchmark( + backend="sglang", + api_url=f"{base_url}/generate", + base_url=base_url, + model_id="default", + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=float("inf"), + max_concurrency=batch_size, + disable_tqdm=False, + lora_name=None, + extra_request_body={}, + profile=None, + ) + ) + + assert results["completed"] == len(input_requests) + acc_length = results["accept_length"] or 1.0 + avg_output_token = results["total_output_tokens"] / results["completed"] + + server_info = requests.get(base_url + "/get_server_info").json() + # We use 20% percentile instead of median on purpose + step_time = np.percentile(server_info["step_time_dict"][str(batch_size)], 20) + speed = 1 / step_time * acc_length + + return ( + round(acc_length, 3), + round(step_time, 5), + round(speed, 3), + avg_output_token, + ) + + +def main(args, server_args): + base_url = "http://127.0.0.1:20000" + + configs = [] + for batch_size in args.batch_size: + for steps in args.steps: + for topk in args.topk: + for num_draft_tokens in args.num_draft_tokens: + if steps * topk + 1 < num_draft_tokens: + continue + + if (steps == 0 or topk == 0 or num_draft_tokens == 0) and ( + steps + topk + num_draft_tokens != 0 + ): + # steps == 0 and topk == 0 and num_draft_tokens == 0 is a special case for non-speculative decoding. + continue + + configs.append((batch_size, steps, topk, num_draft_tokens)) + + for i in range(args.start, args.end or len(configs)): + batch_size, steps, topk, num_draft_tokens = configs[i] + + node0_print( + f"Start {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}" + ) + + # Create an LLM. + if steps == 0: + other_args = [] + else: + other_args = [ + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + steps, + "--speculative-eagle-topk", + topk, + "--speculative-num-draft-tokens", + num_draft_tokens, + ] + if server_args.speculative_draft_model_path is not None: + other_args.extend( + [ + "--speculative-draft-model-path", + server_args.speculative_draft_model_path, + ] + ) + + other_args.extend( + [ + "--cuda-graph-max-bs", + batch_size, + "--mem-fraction-static", + server_args.mem_fraction_static, + "--tp-size", + server_args.tp_size, + "--max-running-requests", + batch_size, + ] + ) + + if server_args.trust_remote_code: + other_args.extend( + [ + "--trust-remote-code", + ] + ) + + if server_args.enable_flashinfer_mla: + other_args.extend( + [ + "--enable-flashinfer-mla", + ] + ) + if server_args.enable_flashmla: + other_args.extend( + [ + "--enable-flashmla", + ] + ) + + if server_args.quantization: + other_args.extend( + [ + "--quantization", + server_args.quantization, + ] + ) + + process = popen_launch_server( + args.model_path, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={ + "SGLANG_RECORD_STEP_TIME": "1", + **os.environ, + }, + ) + + try: + # Warmup + send_one_batch(base_url, batch_size, batch_size) + + # Benchmark + acc_length, step_time, speed, completion_tokens = send_one_batch( + base_url, max(args.num_prompts, batch_size), batch_size + ) + finally: + kill_process_tree(process.pid) + + node0_print( + f"Finish {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}, {speed=:.2f} token/s, step_time={step_time * 1000:.2f} ms" + ) + + record = { + "batch_size": batch_size, + "steps": steps, + "topk": topk, + "num_draft_tokens": num_draft_tokens, + "acc_length": acc_length, + "step_time": step_time, + "speed": speed, + "completion_tokens": completion_tokens, + } + + with open(args.output, "a") as fout: + fout.write(json.dumps(record) + "\n") + + # Wait for the server to shutdown + time.sleep(5) + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + parser.add_argument( + "--batch-size", + type=int, + nargs="+", + default=(1, 2, 4, 8, 16), + ) + parser.add_argument( + "--steps", + type=int, + nargs="+", + default=(0, 1, 3, 5, 7), # use (0, 1, 2, 3, 4) for large batch size + ) + parser.add_argument( + "--topk", + type=int, + nargs="+", + default=(0, 1, 2, 4, 8), + ) + parser.add_argument( + "--num_draft_tokens", + type=int, + nargs="+", + default=(0, 2, 4, 8, 16, 32), # use (0, 2, 4, 8) for large batch size + ) + parser.add_argument("--num-prompts", type=int, default=16) + parser.add_argument("--start", type=int, default=0) + parser.add_argument("--end", type=int) + parser.add_argument("--output", type=str, default="output.jsonl") + args = parser.parse_args() + server_args: ServerArgs = ServerArgs.from_cli_args(args) + + main(args, server_args) diff --git a/scripts/playground/launch_tgi.sh b/scripts/playground/launch_tgi.sh new file mode 100644 index 0000000..a32405c --- /dev/null +++ b/scripts/playground/launch_tgi.sh @@ -0,0 +1,7 @@ +# Assuming the model is downdloaded at /home/ubuntu/model_weights/Llama-2-7b-chat-hf +docker run --name tgi --rm -ti --gpus all --network host \ + -v /home/ubuntu/model_weights/Llama-2-7b-chat-hf:/Llama-2-7b-chat-hf \ + ghcr.io/huggingface/text-generation-inference:1.1.0 \ + --model-id /Llama-2-7b-chat-hf --num-shard 1 --trust-remote-code \ + --max-input-length 2048 --max-total-tokens 4096 \ + --port 24000 diff --git a/scripts/playground/load_tokenizer.py b/scripts/playground/load_tokenizer.py new file mode 100644 index 0000000..94cf34b --- /dev/null +++ b/scripts/playground/load_tokenizer.py @@ -0,0 +1,14 @@ +import argparse +import code + +from sglang.srt.hf_transformers_utils import get_tokenizer + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--name", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct" + ) + args = parser.parse_args() + + t = get_tokenizer(args.name) + code.interact(local=locals()) diff --git a/scripts/playground/lora/analyzer.py b/scripts/playground/lora/analyzer.py new file mode 100644 index 0000000..15568fc --- /dev/null +++ b/scripts/playground/lora/analyzer.py @@ -0,0 +1,77 @@ +import glob +import json +import os +import re +import sys + +from tqdm import tqdm + +sys.path.append("../../") +from fix_corrupted_json import clean_json_file + +dirpath = "/Users/ying" +output_file_prefix = "analyzed_log" + +time = {} +tot_time = {} +size = {} + +os.system(f"rm {output_file_prefix}*") + +for dirname in glob.glob(os.path.join(dirpath, "trace*")): + print(dirname) + trace_name = dirname.split("/")[-1] + time[trace_name] = {} + size[trace_name] = {} + total_time = 0 + for filename in tqdm(glob.glob(os.path.join(dirname, "*.json"))): + step_name = filename.split("/")[-1].split(".")[0] + step_name = "_".join(step_name.split("_")[1:]) + if "prefill" not in filename and "decode" not in filename: + continue + + match = re.search(r"(prefill|decode)_step_(\d+)\.json", filename) + if match: + phase = match.group(1) + step = match.group(2) + else: + raise Exception(f"Cannot parse {filename}") + + try: + with open(filename, "r") as f: + trace = json.load(f) + except: + clean_json_file(filename, filename) + with open(filename, "r") as f: + trace = json.load(f) + + for event in trace["traceEvents"]: + name = event["name"] + if name in ["profile_prefill_step", "profile_decode_step"]: + dur = event["dur"] / 1e3 + time[trace_name][step_name] = dur + break + total_time += dur + + step = int(step_name.split("_")[-1]) + with open(os.path.join(dirname, f"size_{step}.json"), "r") as f: + size_info = json.load(f) + size[trace_name][step_name] = size_info["size"] + + tot_time[trace_name] = total_time + time[trace_name] = dict( + sorted(time[trace_name].items(), key=lambda x: int(x[0].split("_")[-1])) + ) + size[trace_name] = dict( + sorted(size[trace_name].items(), key=lambda x: int(x[0].split("_")[-1])) + ) + + with open(f"{output_file_prefix}_{trace_name}", "a") as f: + for k, v in time[trace_name].items(): + size_v = size[trace_name][k] + print(f"{k:>15}{v:10.2f}\t{size_v}") + f.write(f"{k:>15}{v:10.2f}\t{size_v}\n") + +with open(f"{output_file_prefix}_total_time", "w") as f: + print(tot_time) + json.dump(tot_time, f) diff --git a/scripts/playground/lora/lora_hf_play.py b/scripts/playground/lora/lora_hf_play.py new file mode 100644 index 0000000..0abddd2 --- /dev/null +++ b/scripts/playground/lora/lora_hf_play.py @@ -0,0 +1,62 @@ +import torch +from peft import PeftModel +from transformers import LlamaForCausalLM, LlamaTokenizer + +MODEL = "mistralai/Mistral-7B-Instruct-v0.3" +# ADAPTER = "winddude/wizardLM-LlaMA-LoRA-7B" +ADAPTER = "/home/ying/test_lora" +HF_TOKEN = "..." + + +prompt = """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""" + + +tokenizer = LlamaTokenizer.from_pretrained(MODEL) + +base_model = LlamaForCausalLM.from_pretrained( + MODEL, + device_map="auto", + # load_in_8bit=True, + torch_dtype=torch.float16, + # use_auth_token=HF_TOKEN, +).cuda() + + +# base model generate +with torch.no_grad(): + output_tensors = base_model.generate( + input_ids=tokenizer(prompt, return_tensors="pt").input_ids.cuda(), + max_new_tokens=32, + do_sample=False, + )[0] + +output = tokenizer.decode(output_tensors, skip_special_tokens=True) +print("======= base output ========") +print(output) + + +# peft model generate +model = PeftModel.from_pretrained( + base_model, + ADAPTER, + torch_dtype=torch.float16, + is_trainable=False, +) + +with torch.no_grad(): + output_tensors = model.generate( + input_ids=tokenizer(prompt, return_tensors="pt").input_ids.cuda(), + max_new_tokens=32, + do_sample=False, + )[0] + +output = tokenizer.decode(output_tensors, skip_special_tokens=True) +print("======= peft output ========") +print(output) diff --git a/scripts/playground/lora/lora_vllm_play.py b/scripts/playground/lora/lora_vllm_play.py new file mode 100644 index 0000000..4f77d8b --- /dev/null +++ b/scripts/playground/lora/lora_vllm_play.py @@ -0,0 +1,30 @@ +from vllm import LLM, SamplingParams +from vllm.lora.request import LoRARequest + +MODEL = "mistralai/Mistral-7B-Instruct-v0.3" +ADAPTER = "/home/ying/test_lora" +prompt = """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""" + + +llm = LLM(model=MODEL, enable_lora=True) + +sampling_params = SamplingParams( + temperature=0, + max_tokens=32, +) + +prompts = [prompt] + +outputs = llm.generate( + prompts, sampling_params, lora_request=LoRARequest("test_lora", 1, ADAPTER) +) + +print(outputs[0].prompt) +print(outputs[0].outputs[0].text) diff --git a/scripts/playground/reference_hf.py b/scripts/playground/reference_hf.py new file mode 100644 index 0000000..14d23fb --- /dev/null +++ b/scripts/playground/reference_hf.py @@ -0,0 +1,197 @@ +""" +Usage: python3 scripts/playground/reference_hf.py --model-path MODEL_PATH --model-type {text,vlm} [--max-new-tokens NUM] [--dtype DTYPE] + --model-path MODEL_PATH: Path to model (default: TinyLlama/TinyLlama-1.1B-Chat-v0.4) + --model-type {text,vlm}: Model type, text or vlm (default: text) + --max-new-tokens NUM: Max new tokens to generate (default: 16) + --dtype DTYPE: Data type for computation (default: float16) +Note: '--model' is deprecated; use '--model-path'. Runs normal_text() for text, vlm_text_with_image() for vlm. + +Reference output: +========== Prompt 0 ========== +prefill logits (final) tensor([-8.3125, -7.1172, 3.3398, ..., -4.9531, -4.1328, -3.4141], + device='cuda:0') + The capital of France is Paris. +The capital of the United States is Washington, D.C. + +========== Prompt 1 ========== +prefill logits (final) tensor([-8.9062, -9.0156, 4.1484, ..., -4.9922, -4.4961, -4.0742], + device='cuda:0') + The capital of the United Kindom is London. +The capital of the United Kingdom is London. +The capital of + +========== Prompt 2 ========== +prefill logits (final) tensor([-9.6328, -9.0547, 4.0234, ..., -5.3047, -4.7148, -4.4609], + device='cuda:0') + Today is a sunny day and I like to go for a walk in the park. +I'm going to the +""" + +import argparse + +import requests +import torch +from PIL import Image +from transformers import ( + AutoModelForCausalLM, + AutoModelForImageTextToText, + AutoProcessor, +) + +from sglang.srt.hf_transformers_utils import get_tokenizer + + +@torch.no_grad() +def vlm_text_with_image(args): + # Load the processor and model for ImageTextToText tasks + processor = AutoProcessor.from_pretrained(args.model_path, trust_remote_code=True) + model = AutoModelForImageTextToText.from_pretrained( + args.model_path, + torch_dtype=args.dtype, + low_cpu_mem_usage=True, + device_map="auto", + trust_remote_code=True, + ) + + torch.cuda.set_device(0) + + # List of image URLs to process + image_urls = [ + "https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true" + ] + + # Conversation template for the processor + conversation = [ + { + "role": "user", + "content": [ + { + "type": "image", + }, + {"type": "text", "text": "Describe this image."}, + ], + } + ] + + max_new_tokens = args.max_new_tokens + + for i, url in enumerate(image_urls): + # Load the image from the URL + image = Image.open(requests.get(url, stream=True).raw) + + # Apply the chat template to the text prompt + # Notice that not all processors support chat templates. + # LLaVA and QWen are two processors that support chat templates. + if not hasattr(processor, "apply_chat_template"): + raise ValueError("The processor does not support chat templates.") + text_prompt = processor.apply_chat_template( + conversation, add_generation_prompt=True + ) + + # Prepare inputs for the model + inputs = processor(text=[text_prompt], images=[image], return_tensors="pt").to( + "cuda:0" + ) + + # Generate output from the model + output_ids = model.generate( + **inputs, do_sample=False, max_new_tokens=max_new_tokens + ) + output_str = processor.decode(output_ids[0]) + + # Get the logits from the model's forward pass + outputs = model.forward(**inputs) + logits = outputs.logits[0, -1, :] + + print(f"\n========== Image {i} ==========") + print("prefill logits (final)", logits) + # TODO(gaocegege): The output contains numerous <|image_pad|> tokens, + # making it cluttered and difficult to read. + # These tokens should be removed or cleaned up for better readability. + print(output_str) + + +@torch.no_grad() +def normal_text(args): + t = get_tokenizer(args.model_path, trust_remote_code=True) + m = AutoModelForCausalLM.from_pretrained( + args.model_path, + torch_dtype=args.dtype, + low_cpu_mem_usage=True, + device_map="auto", + trust_remote_code=True, + ) + + prompts = [ + "The capital of France is", + "The capital of the United Kindom is", + "Today is a sunny day and I like", + ] + max_new_tokens = args.max_new_tokens + + torch.cuda.set_device(0) + + for i, p in enumerate(prompts): + if isinstance(p, str): + input_ids = t.encode(p, return_tensors="pt").to("cuda:0") + else: + input_ids = torch.tensor([p], device="cuda:0") + + output_ids = m.generate( + input_ids, do_sample=False, max_new_tokens=max_new_tokens + ) + output_str = t.decode(output_ids[0]) + + prefill_logits = m.forward(input_ids).logits[0][-1] + + print(f"\n========== Prompt {i} ==========") + print("prefill logits (final)", prefill_logits) + print(output_str) + + +@torch.no_grad() +def synthetic_tokens(args): + m = AutoModelForCausalLM.from_pretrained( + args.model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + m.cuda() + print(m) + + input_len = 256 + output_len = 8 + prompts = [list(range(5, 5 + input_len))] + + for p in prompts: + input_ids = p + for i in range(output_len + 1): + prefill_logits = m.forward(torch.tensor([input_ids], device="cuda")).logits[ + 0 + ][-1] + + if i == 0: + print("prefill logits", prefill_logits) + else: + print("decode", i - 1, prefill_logits) + + input_ids.append(torch.argmax(prefill_logits).item()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-path", + type=str, + default="TinyLlama/TinyLlama-1.1B-Chat-v0.4", + ) + parser.add_argument("--max-new-tokens", type=int, default=16) + + parser.add_argument("--dtype", type=str, default="float16") + + parser.add_argument("--model-type", type=str, default="text") + + args = parser.parse_args() + + if args.model_type == "vlm": + vlm_text_with_image(args) + else: + normal_text(args) diff --git a/scripts/playground/router/test_tree.py b/scripts/playground/router/test_tree.py new file mode 100644 index 0000000..af41c73 --- /dev/null +++ b/scripts/playground/router/test_tree.py @@ -0,0 +1,207 @@ +import random +import string +import time +import unittest +from typing import Dict, List, Tuple + +from tree import MultiTenantRadixTree + + +class TestMultiTenantRadixTree(unittest.TestCase): + def setUp(self): + self.tree = MultiTenantRadixTree() + + def test_insert_exact_match(self): + """Test 1: Basic insert and exact match operations""" + # Insert a single string for one tenant + self.tree.insert("hello", "tenant1") + matched, tenant = self.tree.prefix_match("hello") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + # Insert same string for different tenant + self.tree.insert("hello", "tenant2") + matched, tenant = self.tree.prefix_match("hello") + self.assertIn(tenant, ["tenant1", "tenant2"]) + + # Insert different string for same tenant + self.tree.insert("world", "tenant1") + matched, tenant = self.tree.prefix_match("world") + self.assertEqual(matched, "world") + self.assertEqual(tenant, "tenant1") + + print(self.tree.pretty_print()) + + def test_insert_partial_match(self): + """Test 2: Insert with partial matching scenarios""" + # Test partial matches with common prefixes + self.tree.insert("hello", "tenant1") + print(self.tree.pretty_print()) + self.tree.insert("help", "tenant2") + print(self.tree.pretty_print()) + + # Match exact strings + matched, tenant = self.tree.prefix_match("hello") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + matched, tenant = self.tree.prefix_match("help") + self.assertEqual(matched, "help") + self.assertEqual(tenant, "tenant2") + + # Match partial string + matched, tenant = self.tree.prefix_match("hel") + self.assertEqual(matched, "hel") + self.assertIn(tenant, ["tenant1", "tenant2"]) + + # Match longer string + matched, tenant = self.tree.prefix_match("hello_world") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + def test_insert_edge_cases(self): + """Test 3: Edge cases for insert and match operations""" + # Empty string + self.tree.insert("", "tenant1") + matched, tenant = self.tree.prefix_match("") + self.assertEqual(matched, "") + self.assertEqual(tenant, "tenant1") + + # Single character + self.tree.insert("a", "tenant1") + matched, tenant = self.tree.prefix_match("a") + self.assertEqual(matched, "a") + self.assertEqual(tenant, "tenant1") + + # Very long string + long_str = "a" * 1000 + self.tree.insert(long_str, "tenant1") + matched, tenant = self.tree.prefix_match(long_str) + self.assertEqual(matched, long_str) + self.assertEqual(tenant, "tenant1") + + # Unicode characters + self.tree.insert("你好", "tenant1") + matched, tenant = self.tree.prefix_match("你好") + self.assertEqual(matched, "你好") + self.assertEqual(tenant, "tenant1") + + def test_simple_eviction(self): + """Test 4: Simple eviction scenarios + Tenant1: limit 10 chars + Tenant2: limit 5 chars + + Should demonstrate: + 1. Basic eviction when size limit exceeded + 2. Proper eviction based on last access time + 3. Verification that shared nodes remain intact for other tenants + """ + # Set up size limits + max_size = {"tenant1": 10, "tenant2": 5} + + # Insert strings for both tenants + self.tree.insert("hello", "tenant1") # size 5 + self.tree.insert("hello", "tenant2") # size 5 + self.tree.insert("world", "tenant2") # size 5, total for tenant2 = 10 + + # Verify initial sizes + sizes_before = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_before["tenant1"], 5) # "hello" = 5 + self.assertEqual(sizes_before["tenant2"], 10) # "hello" + "world" = 10 + + # Evict - should remove "hello" from tenant2 as it's the oldest + self.tree.evict_tenant_data(max_size) + + # Verify sizes after eviction + sizes_after = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_after["tenant1"], 5) # Should be unchanged + self.assertEqual(sizes_after["tenant2"], 5) # Only "world" remains + + # Verify "world" remains for tenant2 (was accessed more recently) + matched, tenant = self.tree.prefix_match("world") + self.assertEqual(matched, "world") + self.assertEqual(tenant, "tenant2") + + def test_medium_eviction(self): + """Test 5: Medium complexity eviction scenarios with shared prefixes + Tenant1: limit 10 chars + Tenant2: limit 7 chars (forces one string to be evicted) + + Tree structure after inserts: + └── 'h' [t1, t2] + ├── 'i' [t1, t2] # Oldest for t2 + └── 'e' [t1, t2] + ├── 'llo' [t1, t2] + └── 'y' [t2] # Newest for t2 + + Size calculations: + tenant1: "h"(1) + "i"(1) + "e"(1) + "llo"(3) = 6 chars + tenant2: "h"(1) + "i"(1) + "e"(1) + "llo"(3) + "y"(1) = 7 chars + + After eviction (tenant2 exceeds limit by 1 char): + "hi" should be removed from tenant2 as it's the oldest access + """ + max_size = { + "tenant1": 10, + "tenant2": 6, + } # tenant2 will need to evict one string + + # Create a tree with overlapping prefixes + self.tree.insert("hi", "tenant1") + self.tree.insert("hi", "tenant2") # OLDEST for t2 + + self.tree.insert("hello", "tenant1") + self.tree.insert("hello", "tenant2") + + self.tree.insert("hey", "tenant2") # NEWEST for t2 + + # Verify initial sizes + sizes_before = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_before["tenant1"], 6) # h(1) + i(1) + e(1) + llo(3) = 6 + self.assertEqual( + sizes_before["tenant2"], 7 + ) # h(1) + i(1) + e(1) + llo(3) + y(1) = 7 + + print("\nTree before eviction:") + print(self.tree.pretty_print()) + + # Evict - should remove "hi" from tenant2 as it's the oldest + self.tree.evict_tenant_data(max_size) + + print("\nTree after eviction:") + print(self.tree.pretty_print()) + + # Verify sizes after eviction + sizes_after = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_after["tenant1"], 6) # Should be unchanged + self.assertEqual(sizes_after["tenant2"], 6) # h(1) + e(1) + llo(3) + y(1) = 6 + + def test_advanced_eviction(self): + ... + # Create 4 tenants + # Each tenants keeps adding strings with shared prefixes to thousands usage + # Set a strict limit for each tenant to only 100 + # At the end, check whether all of the tenant is under 100 after eviction + + max_size = {"tenant1": 100, "tenant2": 100, "tenant3": 100, "tenant4": 100} + + prefixes = ["aqwefcisdf", "iajsdfkmade", "kjnzxcvewqe", "iejksduqasd"] + for i in range(100): + for j, prefix in enumerate(prefixes): + random_suffix = "".join(random.choices(string.ascii_letters, k=10)) + self.tree.insert(prefix + random_suffix, f"tenant{j+1}") + + sizes_before = self.tree.get_used_size_per_tenant() + print(sizes_before) + + self.tree.evict_tenant_data(max_size) + + sizes_after = self.tree.get_used_size_per_tenant() + print(sizes_after) + # ensure size_after is below max_size + for tenant, size in sizes_after.items(): + self.assertLessEqual(size, max_size[tenant]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/playground/router/tree.py b/scripts/playground/router/tree.py new file mode 100644 index 0000000..9cbfa7c --- /dev/null +++ b/scripts/playground/router/tree.py @@ -0,0 +1,292 @@ +import time +from collections import defaultdict +from typing import Dict, List + + +class Node: + def __init__(self): + self.children: Dict[str, Node] = dict() + # We choose to use text because most of the use cases are text-to-text, + # so we can save the tokenizing overhead. + self.text: str = "" + # Maps tenant_id to their last access timestamp + self.tenant_last_access_time: Dict[str, float] = dict() + self.parent = None + + +def shared_prefix_length(s1, s2): + min_length = min(len(s1), len(s2)) + for i in range(min_length): + if s1[i] != s2[i]: + return i + return min_length + + +class MultiTenantRadixTree: + """ + Python Reference of Rust implementation of MultiTenantRadixTree + + MultiTenantRadixTree is the overlap of multiple radix trees by different tenant + Each node in the tree can be owned by multiple tenants, allowing for efficient storage of common prefixes + while maintaining tenant isolation. + + Key concepts: + - Tenant: An entity that owns a subset of the stored strings + - Each node tracks which tenants have access to it via tenant_last_access_time + - The tree structure is shared, but queries can be filtered by tenant_id + """ + + def __init__(self): + self.root = Node() + + def insert(self, s: str, tenant_id: str) -> None: + """ + Insert string 's' and associate it with the given tenant_id. + + Args: + s: The string to insert + tenant_id: The identifier of the tenant who owns this string + """ + curr = self.root + curr_idx = 0 + curr.tenant_last_access_time[tenant_id] = time.time() + + while curr_idx < len(s): + matched_node = None + if s[curr_idx] in curr.children: + matched_node = curr.children[s[curr_idx]] + + if matched_node is None: + # No match => create a new node + new_node = Node() + new_node.text = s[curr_idx:] + new_node.parent = curr + + curr.children[s[curr_idx]] = new_node + curr_idx = len(s) + curr = new_node + curr.tenant_last_access_time[tenant_id] = time.time() + else: + shared_len = shared_prefix_length(s[curr_idx:], matched_node.text) + + # 1. If the matched text is shorter than the node text => split the node + if shared_len < len(matched_node.text): + # Split structure: [matched_node] => [new_node] -> [contracted_matched_node] + + matched_text = matched_node.text[:shared_len] + unmatched_text = matched_node.text[shared_len:] + + new_node = Node() + new_node.text = matched_text + new_node.children = {unmatched_text[0]: matched_node} + new_node.parent = curr + new_node.parent.children[matched_text[0]] = new_node + new_node.tenant_last_access_time = ( + matched_node.tenant_last_access_time.copy() + ) + + # Contract matched node + matched_node.text = unmatched_text + matched_node.parent = new_node + + curr_idx += shared_len + curr = new_node + curr.tenant_last_access_time[tenant_id] = time.time() + # 2. If the matched text is longer or equal to the node text => walk down the node + else: + curr_idx += shared_len + curr = matched_node + curr.tenant_last_access_time[tenant_id] = time.time() + + def prefix_match(self, s: str) -> tuple[str, int]: + """ + Match string 's' with multiple tenants' trees in one operation. + + Args: + s: The string to match + + Returns: + Tuple(str, int): The longest prefix of 's' that matches the tree and the first tenant_id that own the matched prefix + """ + curr = self.root + curr_idx = 0 + + ret_text = "" + ret_tenant = None + + while curr_idx < len(s): + matched_node = None + if s[curr_idx] in curr.children: + matched_node = curr.children[s[curr_idx]] + + if matched_node is None: + break + + shared_len = shared_prefix_length(s[curr_idx:], matched_node.text) + if shared_len == len(matched_node.text): + curr_idx += shared_len + curr = matched_node + else: + curr_idx += shared_len + curr = matched_node + break + + selected_tenant = list(curr.tenant_last_access_time.keys())[0] + + # traverse back to the root to update last access time for the selected tenant + while curr != self.root: + curr.tenant_last_access_time[selected_tenant] = time.time() + curr = curr.parent + + return s[:curr_idx], selected_tenant + + def evict_tenant_data(self, max_size_per_tenant: Dict[str, int]) -> None: + """ + Evict data for tenants that have exceeded their storage limits. + + Args: + max_size_per_tenant: Dictionary mapping tenant_id to their maximum allowed storage size + """ + + def leaf_of(node): + """ + If the node is a leaf for a tenant, add tenant_id to the return list + This will return list of tenant ids + If not a leaf for all tenants, return [] + """ + candidates = dict([(k, True) for k in node.tenant_last_access_time.keys()]) + + for n in node.children.values(): + for c in n.tenant_last_access_time.keys(): + candidates[c] = False + + return [k for k, v in candidates.items() if v] + + # maintain a heap with (time, tenant, node) as the value + import heapq + + # 1. traverse the tree to + # a. add all the leaves into a heap (a node with N tenants will be added N times into the heap) + # b. calculate the used size for each tenant + # do a dfs with stack + stack = [self.root] + pq = [] + used_size_per_tenant = defaultdict(int) + + while stack: + curr = stack.pop() + for t in curr.tenant_last_access_time.keys(): + used_size_per_tenant[t] += len(curr.text) + + for c in curr.children.values(): + stack.append(c) + + # if the node is a leaf for a tenant, add the tenant to the heap + tenants = leaf_of(curr) + for t in tenants: + heapq.heappush(pq, (curr.tenant_last_access_time[t], t, curr)) + + # 2. pop the heap + # a. if the tenant's used size is less than the limit, continue + # b. if the tenant's used size is greater than the limit, remove the leaf and update the used size, and add its parent to the heap + while len(pq) > 0: + time, tenant, node = heapq.heappop(pq) + if used_size_per_tenant[tenant] <= max_size_per_tenant[tenant]: + continue + + # remove the leaf + used_size_per_tenant[tenant] -= len(node.text) + del node.tenant_last_access_time[tenant] + # if no children and no tenants, remove the node + if len(node.children) == 0 and len(node.tenant_last_access_time) == 0: + del node.parent.children[node.text[0]] + + # add its parent to the heap + if tenant in leaf_of(node.parent): + heapq.heappush( + pq, + (node.parent.tenant_last_access_time[tenant], tenant, node.parent), + ) + + def get_used_size_per_tenant(self) -> Dict[str, int]: + """ + Calculate the used storage size for each tenant. + + Returns: + Dict[str, int]: A dictionary mapping tenant_id to their used storage size + """ + used_size_per_tenant = defaultdict(int) + + stack = [self.root] + while stack: + curr = stack.pop() + for t in curr.tenant_last_access_time.keys(): + used_size_per_tenant[t] += len(curr.text) + + for c in curr.children.values(): + stack.append(c) + + return used_size_per_tenant + + def remove_tenant(self, tenant_id: str) -> None: + """ + Remove all data associated with a specific tenant from the tree. + This operation maintains the integrity of the shared tree structure while + removing only the specified tenant's access information. + + Args: + tenant_id: The identifier of the tenant whose data should be removed + """ + # TODO: Implementation needed + pass + + def pretty_print(self) -> str: + """ + Returns a string representation of the tree showing the structure, tenant ownership, + and leaf status for each node. + + Returns: + str: A formatted string showing the tree hierarchy with tenant information + """ + + def _node_to_str(node: Node, prefix: str = "", is_last: bool = True) -> str: + # Current node representation + node_str = prefix + node_str += "└── " if is_last else "├── " + + # Add node text + node_str += f"'{node.text}' [" + + # Add tenant information including both timestamp and leaf status + tenant_info = [] + for tid, ts in node.tenant_last_access_time.items(): + time_str = ( + time.strftime("%H:%M:%S.", time.localtime(ts)) + + f"{(ts % 1):0.3f}"[2:] + ) + tenant_info.append(f"{tid} | {time_str}") + + node_str += ", ".join(tenant_info) + node_str += "]\n" + + # Handle children + children = list(node.children.items()) + for i, (char, child) in enumerate(children): + is_last_child = i == len(children) - 1 + # Adjust prefix for children based on whether this is the last child + new_prefix = prefix + (" " if is_last else "│ ") + node_str += _node_to_str(child, new_prefix, is_last_child) + + return node_str + + if not self.root.children: + return "Empty tree" + + # Start with root's children since root itself is just an empty node + result = "" + children = list(self.root.children.items()) + for i, (char, child) in enumerate(children): + is_last = i == len(children) - 1 + result += _node_to_str(child, "", is_last) + + return result diff --git a/scripts/update_kernel_whl_index.py b/scripts/update_kernel_whl_index.py new file mode 100644 index 0000000..8e0a2b5 --- /dev/null +++ b/scripts/update_kernel_whl_index.py @@ -0,0 +1,31 @@ +# Reference: https://github.com/flashinfer-ai/flashinfer/blob/v0.2.0/scripts/update_whl_index.py + +import argparse +import hashlib +import pathlib +import re + + +def update_wheel_index(cuda_version="118"): + index_dir = pathlib.Path(f"sgl-whl/cu{cuda_version}/sgl-kernel") + index_dir.mkdir(exist_ok=True) + base_url = "https://github.com/sgl-project/whl/releases/download" + + for path in sorted(pathlib.Path("sgl-kernel/dist").glob("*.whl")): + with open(path, "rb") as f: + sha256 = hashlib.sha256(f.read()).hexdigest() + ver = re.findall(r"sgl_kernel-([0-9.]+(?:\.post[0-9]+)?)-", path.name)[0] + full_url = f"{base_url}/v{ver}/{path.name}#sha256={sha256}" + with (index_dir / "index.html").open("a") as f: + f.write(f'{path.name}
\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cuda", type=str, default="118") + args = parser.parse_args() + update_wheel_index(args.cuda) + + +if __name__ == "__main__": + main() diff --git a/scripts/version_branch_to_tag.sh b/scripts/version_branch_to_tag.sh new file mode 100644 index 0000000..9f587fb --- /dev/null +++ b/scripts/version_branch_to_tag.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euxo pipefail + +# This script is used for release. +# It tags all remote branches starting with 'v' with the same name as the branch, +# deletes the corresponding branches from the remote, and pushes the tags to the remote repository. + +git fetch origin --prune + +# List all branches starting with 'v' +branches=$(git branch -r | grep 'origin/v' | sed 's/origin\///') + +# Loop through each branch +for branch in $branches; do + echo "Processing branch: $branch" + + # Get the commit hash for the branch + commit_hash=$(git rev-parse origin/$branch) + + # Create a tag with the same name as the branch using the commit hash + git tag $branch $commit_hash + + # Delete the branch from the remote + git push origin --delete $branch +done + +# Push all tags to the remote repository +git push --tags + +echo "All branches starting with 'v' have been tagged, deleted from remote, and pushed to the remote repository." diff --git a/sgl-kernel/.clang-format b/sgl-kernel/.clang-format new file mode 100644 index 0000000..afbd654 --- /dev/null +++ b/sgl-kernel/.clang-format @@ -0,0 +1,15 @@ +BasedOnStyle: Google +IndentWidth: 2 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: Empty +DerivePointerAlignment: false +PointerAlignment: Left +NamespaceIndentation: None +SortIncludes: true +AllowShortLoopsOnASingleLine: false +BinPackParameters: false # Prevents packing parameters in declarations +BinPackArguments: false # Prevents packing arguments in function calls +AlignAfterOpenBracket: AlwaysBreak # Forces a break after the opening parenthesis +AlignOperands: Align # Aligns arguments vertically +PenaltyBreakBeforeFirstCallParameter: 1 # Encourages breaking before the first argument +PenaltyReturnTypeOnItsOwnLine: 100 # Keeps return type with function name diff --git a/sgl-kernel/CMakeLists.txt b/sgl-kernel/CMakeLists.txt new file mode 100644 index 0000000..72cac8e --- /dev/null +++ b/sgl-kernel/CMakeLists.txt @@ -0,0 +1,187 @@ +cmake_minimum_required(VERSION 3.26 FATAL_ERROR) +project(sgl-kernel LANGUAGES CXX CUDA) + +# we only want to download 3rd, but not build them. +# FetchContent_MakeAvailable will build it. +cmake_policy(SET CMP0169 OLD) + +find_package(Python COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT} REQUIRED) + +enable_language(CUDA) +find_package(CUDAToolkit REQUIRED) + +message(STATUS "Detected CUDA_VERSION=${CUDA_VERSION}") +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.8") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.4") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.4") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.1") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.1") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "11.8") + message("CUDA_VERSION ${CUDA_VERSION} >= 11.8") +endif() + +find_package(Torch REQUIRED) + +include(FetchContent) + +FetchContent_Declare( + repo-cutlass + GIT_REPOSITORY https://github.com/NVIDIA/cutlass + GIT_TAG 62750a2b75c802660e4894434dc55e839f322277 + GIT_SHALLOW ON +) +FetchContent_Populate(repo-cutlass) +FetchContent_Declare( + repo-deepgemm + GIT_REPOSITORY https://github.com/deepseek-ai/DeepGEMM + GIT_TAG c57699ac933a93651c34d365797c2d8b41a4765b + GIT_SHALLOW ON +) +FetchContent_Populate(repo-deepgemm) +FetchContent_Declare( + repo-flashinfer + GIT_REPOSITORY https://github.com/sgl-project/flashinfer + GIT_TAG sgl-kernel + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flashinfer) + +include_directories( + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/csrc + ${repo-cutlass_SOURCE_DIR}/include + ${repo-cutlass_SOURCE_DIR}/tools/util/include + ${repo-flashinfer_SOURCE_DIR}/include + ${repo-flashinfer_SOURCE_DIR}/csrc +) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") + +set(SGL_KERNEL_CUDA_FLAGS + "-DNDEBUG" + "-DOPERATOR_NAMESPACE=sgl-kernel" + "-O3" + "-Xcompiler" + "-fPIC" + "-gencode=arch=compute_75,code=sm_75" + "-gencode=arch=compute_80,code=sm_80" + "-gencode=arch=compute_89,code=sm_89" + "-gencode=arch=compute_90,code=sm_90" + "-std=c++17" + "-DFLASHINFER_ENABLE_F16" + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1" + "-DCUTLASS_VERSIONS_GENERATED" + "-DCUTE_USE_PACKED_TUPLE=1" + "-DCUTLASS_TEST_LEVEL=0" + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1" + "-DCUTLASS_DEBUG_TRACE_LEVEL=0" + "--expt-relaxed-constexpr" + "-Xcompiler=-Wconversion" + "-Xcompiler=-fno-strict-aliasing" +) + +option(SGL_KERNEL_ENABLE_SM100A "Enable SM100A" OFF) +option(SGL_KERNEL_ENABLE_SM90A "Enable SM90A" OFF) +option(SGL_KERNEL_ENABLE_BF16 "Enable BF16" ON) +option(SGL_KERNEL_ENABLE_FP8 "Enable FP8" ON) +option(SGL_KERNEL_ENABLE_FP4 "Enable FP4" OFF) + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_SM100A) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_100,code=sm_100" + "-gencode=arch=compute_100a,code=sm_100a" + ) +else() + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-use_fast_math" + ) +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.4" OR SGL_KERNEL_ENABLE_SM90A) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_90a,code=sm_90a" + ) +endif() + +if (SGL_KERNEL_ENABLE_BF16) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DFLASHINFER_ENABLE_BF16" + ) +endif() + +if (SGL_KERNEL_ENABLE_FP8) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DFLASHINFER_ENABLE_FP8" + "-DFLASHINFER_ENABLE_FP8_E4M3" + "-DFLASHINFER_ENABLE_FP8_E5M2" + ) +endif() + +if (SGL_KERNEL_ENABLE_FP4) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DENABLE_NVFP4=1" + ) +endif() + +string(REPLACE "-D__CUDA_NO_HALF_OPERATORS__" "" CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") +string(REPLACE "-D__CUDA_NO_HALF_CONVERSIONS__" "" CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") +string(REPLACE "-D__CUDA_NO_BFLOAT16_CONVERSIONS__" "" CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") +string(REPLACE "-D__CUDA_NO_HALF2_OPERATORS__" "" CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}") + +set(SOURCES + "csrc/allreduce/trt_reduce_internal.cu" + "csrc/allreduce/trt_reduce_kernel.cu" + "csrc/attention/lightning_attention_decode_kernel.cu" + "csrc/elementwise/activation.cu" + "csrc/elementwise/fused_add_rms_norm_kernel.cu" + "csrc/elementwise/rope.cu" + "csrc/gemm/awq_kernel.cu" + "csrc/gemm/bmm_fp8.cu" + "csrc/gemm/cublas_grouped_gemm.cu" + "csrc/gemm/fp8_blockwise_gemm_kernel.cu" + "csrc/gemm/fp8_gemm_kernel.cu" + "csrc/gemm/int8_gemm_kernel.cu" + "csrc/gemm/nvfp4_quant_entry.cu" + "csrc/gemm/nvfp4_quant_kernels.cu" + "csrc/gemm/nvfp4_scaled_mm_entry.cu" + "csrc/gemm/nvfp4_scaled_mm_kernels.cu" + "csrc/gemm/per_tensor_quant_fp8.cu" + "csrc/gemm/per_token_group_quant_8bit.cu" + "csrc/gemm/per_token_quant_fp8.cu" + "csrc/moe/moe_align_kernel.cu" + "csrc/moe/moe_topk_softmax_kernels.cu" + "csrc/speculative/eagle_utils.cu" + "csrc/speculative/speculative_sampling.cu" + "csrc/speculative/packbit.cu" + "csrc/torch_extension.cc" + "${repo-flashinfer_SOURCE_DIR}/csrc/norm.cu" + "${repo-flashinfer_SOURCE_DIR}/csrc/renorm.cu" + "${repo-flashinfer_SOURCE_DIR}/csrc/sampling.cu" +) + +# Support abi3 for build +Python_add_library(common_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SOURCES}) + +target_compile_options(common_ops PRIVATE $<$:${SGL_KERNEL_CUDA_FLAGS}>) + +target_include_directories(common_ops PRIVATE ${TORCH_INCLUDE_DIRS}) + +target_link_libraries(common_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda cublas cublasLt) + +install(TARGETS common_ops LIBRARY DESTINATION "sgl_kernel") + +# JIT Logic +# DeepGEMM + +install(DIRECTORY "${repo-deepgemm_SOURCE_DIR}/deep_gemm/" + DESTINATION "deep_gemm" + PATTERN ".git*" EXCLUDE + PATTERN "__pycache__" EXCLUDE) + +install(DIRECTORY "${repo-cutlass_SOURCE_DIR}/include/cute/" + DESTINATION "deep_gemm/include/cute") + +install(DIRECTORY "${repo-cutlass_SOURCE_DIR}/include/cutlass/" + DESTINATION "deep_gemm/include/cutlass") diff --git a/sgl-kernel/LICENSE b/sgl-kernel/LICENSE new file mode 100644 index 0000000..9c42268 --- /dev/null +++ b/sgl-kernel/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2024 SGLang Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sgl-kernel/Makefile b/sgl-kernel/Makefile new file mode 100644 index 0000000..e8e0910 --- /dev/null +++ b/sgl-kernel/Makefile @@ -0,0 +1,69 @@ +.PHONY: help check-deps install-deps tree ln submodule install build clean rebuild test format update + +# Show help for each target +help: ## Show this help message + @echo "Available targets:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +check-deps: ## Check and install required Python formatting dependencies + @command -v isort >/dev/null 2>&1 || (echo "Installing isort..." && pip install isort) + @command -v black >/dev/null 2>&1 || (echo "Installing black..." && pip install black) + +install-deps: ## Install Python formatting tools (isort and black) + pip install isort black + +tree: ## Show project directory structure + @tree --prune -I "__pycache__|*.egg-info|*.so|build|3rdparty|dist" + +submodule: ## Initialize and update git submodules + @git submodule update --init --recursive + +ln: submodule ## Create compilation database + @rm -rf build && mkdir build && cd build && cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=YES + + +install: submodule ## Install package in development mode + @pip install -e . + +build: submodule ## Build and install wheel package + @rm -rf dist/* || true && export MAX_JOBS=$(nproc) && uv build --wheel -Cbuild-dir=build . --verbose --color=always && pip3 install dist/*whl --force-reinstall --no-deps + +clean: ## Remove build artifacts + @rm -rf build dist *.egg-info + +rebuild: clean submodule build ## Clean and rebuild the project + @echo "Succeed to rebuild" + +test: ## Run all tests + @find tests -name "test_*.py" | xargs -n 1 python3 + +format: check-deps ## Format all source files + @echo "Formatting source files..." + @find csrc tests -name '*.cc' -o -name '*.cu' -o -name '*.cuh' -o -name '*.h' -o -name '*.hpp' | xargs clang-format -i + @find python tests -name '*.py' | xargs isort + @find python tests -name '*.py' | xargs black + @pre-commit run --all-files + +FILES_TO_UPDATE = python/sgl_kernel/version.py \ + pyproject.toml \ + pyproject_rocm.toml + +update: ## Update version numbers across project files. Usage: make update + @if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \ + echo "Version required. Usage: make update "; \ + exit 1; \ + fi + @OLD_VERSION=$$(grep "version" python/sgl_kernel/version.py | cut -d '"' -f2); \ + NEW_VERSION=$(filter-out $@,$(MAKECMDGOALS)); \ + echo "Updating version from $$OLD_VERSION to $$NEW_VERSION"; \ + for file in $(FILES_TO_UPDATE); do \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + else \ + sed -i -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + fi \ + done; \ + echo "Version update complete" + +%: + @: diff --git a/sgl-kernel/README.md b/sgl-kernel/README.md new file mode 100644 index 0000000..e86c262 --- /dev/null +++ b/sgl-kernel/README.md @@ -0,0 +1,73 @@ +# SGL Kernel + +[Kernel Library](https://github.com/sgl-project/sglang/tree/main/sgl-kernel) for SGLang + +[![PyPI](https://img.shields.io/pypi/v/sgl-kernel)](https://pypi.org/project/sgl-kernel) + +## Installation + +For CUDA 11.8: + +```bash +pip3 install sgl-kernel -i https://docs.sglang.ai/whl/cu118 +``` + +For CUDA 12.1 or CUDA 12.4: + +```bash +pip3 install sgl-kernel +``` + +# Developer Guide + +## Development Environment Setup + +Use Docker to set up the development environment. See [Docker setup guide](https://github.com/sgl-project/sglang/blob/main/docs/developer/development_guide_using_docker.md#setup-docker-container). + +Create and enter development container: +```bash +docker run -itd --shm-size 32g --gpus all -v $HOME/.cache:/root/.cache --ipc=host --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` + +## Project Structure + +### Dependencies + +Third-party libraries: + +- [CCCL](https://github.com/NVIDIA/cccl) +- [CUTLASS](https://github.com/NVIDIA/cutlass) +- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) + +### Kernel Development + +Steps to add a new kernel: + +1. Implement the kernel in [csrc](https://github.com/sgl-project/sglang/tree/main/sgl-kernel/csrc) +2. Expose the interface in [include/sgl_kernel_ops.h](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/include/sgl_kernel_ops.h) +3. Create torch extension in [csrc/torch_extension.cc](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/csrc/torch_extension.cc) +4. Update [setup.py](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/setup.py) to include new CUDA source +5. Expose Python interface in [python](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/python/sgl_kernel) + +### Build & Install + +Development build: + +```bash +make build +``` + +Note: + +The `sgl-kernel` is rapidly evolving. If you experience a compilation failure, try using `make rebuild`. + +### Testing & Benchmarking + +1. Add pytest tests in [tests/](https://github.com/sgl-project/sglang/tree/main/sgl-kernel/tests) +2. Add benchmarks using [triton benchmark](https://triton-lang.org/main/python-api/generated/triton.testing.Benchmark.html) in [benchmark/](https://github.com/sgl-project/sglang/tree/main/sgl-kernel/benchmark) +3. Run test suite + +### Release new version + +Update version in [pyproject.toml](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/pyproject.toml) and [version.py](https://github.com/sgl-project/sglang/blob/main/sgl-kernel/python/sgl_kernel/version.py) diff --git a/sgl-kernel/THIRDPARTYNOTICES.txt b/sgl-kernel/THIRDPARTYNOTICES.txt new file mode 100644 index 0000000..e737388 --- /dev/null +++ b/sgl-kernel/THIRDPARTYNOTICES.txt @@ -0,0 +1,455 @@ +Notice for flashinfer-ai/flashinfer +------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------------------- +Some of the code in this project are adapted from other open-source projects with different +licenses. This product also bundles some third-party components under other open source licenses. +This section summarizes those components and their licenses. +See licenses/ for text of these licenses. + +BSD 3-Clause License +-------------------- + +include/flashinfer/attention/hopper/epilogue.cuh +include/flashinfer/attention/hopper/mainloop.cuh +include/flashinfer/attention/hopper/kernel_traits.cuh +include/flashinfer/attention/hopper/named_barrier.cuh +include/flashinfer/attention/hopper/tile_scheduler.cuh +include/flashinfer/attention/hopper/utils.cuh + +BSD 3-Clause "New" License +-------------------------- + +3rdparty/cutlass +include/flashinfer/attention/hopper/block_sparse_gather.cuh + +Notice for NVIDIA/TensorRT-LLM +------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Notice for deepseek-ai/DeepGEMM +------------------------------- + +MIT License + +Copyright (c) 2025 DeepSeek + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/sgl-kernel/benchmark/bench_awq_dequant.py b/sgl-kernel/benchmark/bench_awq_dequant.py new file mode 100644 index 0000000..22280c2 --- /dev/null +++ b/sgl-kernel/benchmark/bench_awq_dequant.py @@ -0,0 +1,118 @@ +import itertools +from typing import List, Tuple + +import torch +import triton +import triton.testing +from sgl_kernel import awq_dequantize +from vllm import _custom_ops as ops + + +def vllm_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + return ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0) + + +def sglang_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + + return awq_dequantize(qweight, scales, qzeros) + + +def calculate_diff(qweight_row: int, qweight_col: int): + """Calculate difference between VLLM and SGLang implementations.""" + device = torch.device("cuda") + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + vllm_out = vllm_awq_dequantize(qweight, scales, qzeros) + sglang_out = sglang_awq_dequantize(qweight, scales, qzeros) + + output_diff = torch.abs(vllm_out.float() - sglang_out.float()).mean().item() + + if torch.allclose( + vllm_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +qweight_row_range = [3584, 18944, 128, 256, 512, 1024] +qweight_cols_range = [448, 576, 4736, 16, 32, 64, 128] + +configs = list(itertools.product(qweight_row_range, qweight_cols_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["qweight_row", "qweight_col"], + x_vals=configs, + line_arg="provider", + line_vals=["vllm", "sglang"], + line_names=["VLLM", "SGL Kernel"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="awq-dequantize-performance", + args={}, + ) +) +def benchmark(qweight_row, qweight_col, provider): + dtype = torch.float16 + device = torch.device("cuda") + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "vllm": + fn = lambda: vllm_awq_dequantize( + qweight.clone(), scales.clone(), qzeros.clone() + ) + elif provider == "sglang": + fn = lambda: sglang_awq_dequantize( + qweight.clone(), scales.clone(), qzeros.clone() + ) + + ms, min_ms, max_ms = triton.testing.do_bench(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + calculate_diff(qweight_row=3584, qweight_col=448) + benchmark.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_cublas_grouped_gemm.py b/sgl-kernel/benchmark/bench_cublas_grouped_gemm.py new file mode 100644 index 0000000..15f4d7b --- /dev/null +++ b/sgl-kernel/benchmark/bench_cublas_grouped_gemm.py @@ -0,0 +1,262 @@ +import argparse + +import torch +import triton +import triton.language as tl +from sgl_kernel import cublas_grouped_gemm + +WEIGHT_CONFIGS = { + "DeepSeek-V2-Lite": { + "num_routed_experts": 64, + "ffn_shapes": [ + [2048, 2816], + [1408, 2048], + ], + }, + "DeepSeek-V2": { + "num_routed_experts": 160, + "ffn_shapes": [ + [5120, 3072], + [1536, 5120], + ], + }, +} + + +# This Triton Grouped Gemm Kernel is adapted from +# https://triton-lang.org/main/getting-started/tutorials/08-grouped-gemm.html +@triton.jit +def grouped_matmul_kernel( + # device tensor of matrices pointers + group_a_ptrs, + group_b_ptrs, + group_c_ptrs, + # device tensor of gemm sizes. its shape is [group_size, 3] + # dim 0 is group_size, dim 1 is the values of of each gemm + group_gemm_sizes, + # device tensor of leading dimension sizes. its shape is [group_size, 3] + # dim 0 is group_size, dim 1 is the values of of each gemm + g_lds, + # Factors for multiplication. + alphas, + betas, + # number of gemms + group_size, + # number of virtual SM + NUM_SM: tl.constexpr, + # tile sizes + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, +): + tile_idx = tl.program_id(0) + last_problem_end = 0 + for g in range(group_size): + # get the gemm size of the current problem + gm = tl.load(group_gemm_sizes + g * 3) + gn = tl.load(group_gemm_sizes + g * 3 + 1) + gk = tl.load(group_gemm_sizes + g * 3 + 2) + num_m_tiles = tl.cdiv(gm, BLOCK_SIZE_M) + num_n_tiles = tl.cdiv(gn, BLOCK_SIZE_N) + num_tiles = num_m_tiles * num_n_tiles + # load multiplication factors + alpha = tl.load(alphas + g) + beta = tl.load(betas + g) + # iterate through the tiles in the current gemm problem + while tile_idx >= last_problem_end and tile_idx < last_problem_end + num_tiles: + # pick up a tile from the current gemm problem + k = gk + lda = tl.load(g_lds + g * 3) + ldb = tl.load(g_lds + g * 3 + 1) + ldc = tl.load(g_lds + g * 3 + 2) + a_ptr = tl.load(group_a_ptrs + g).to(tl.pointer_type(tl.float16)) + b_ptr = tl.load(group_b_ptrs + g).to(tl.pointer_type(tl.float16)) + c_ptr = tl.load(group_c_ptrs + g).to(tl.pointer_type(tl.float16)) + # figure out tile coordinates + tile_idx_in_gemm = tile_idx - last_problem_end + tile_m_idx = tile_idx_in_gemm // num_n_tiles + tile_n_idx = tile_idx_in_gemm % num_n_tiles + + # do regular gemm here + offs_am = tile_m_idx * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_bn = tile_n_idx * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + offs_am[:, None] * lda + offs_k[None, :] + b_ptrs = b_ptr + offs_k[:, None] * ldb + offs_bn[None, :] + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for kk in range(0, tl.cdiv(k, BLOCK_SIZE_K)): + a = tl.load( + a_ptrs, + mask=(offs_am[:, None] < gm) + and (offs_k[None, :] < gk - kk * BLOCK_SIZE_K), + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=(offs_k[:, None] < gk - kk * BLOCK_SIZE_K) + and (offs_bn[None, :] < gn), + other=0.0, + ) + accumulator += tl.dot(a, b) + a_ptrs += BLOCK_SIZE_K + b_ptrs += BLOCK_SIZE_K * ldb + accumulator *= alpha + c = accumulator.to(tl.float16) + + offs_cm = tile_m_idx * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = tile_n_idx * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + ldc * offs_cm[:, None] + offs_cn[None, :] + output_mask = (offs_am[:, None] < gm) and (offs_bn[None, :] < gn) + c += beta * tl.load(c_ptrs, mask=output_mask) + tl.store(c_ptrs, c, mask=output_mask) + + # go to the next tile by advancing NUM_SM + tile_idx += NUM_SM + + # get ready to go to the next gemm problem + last_problem_end = last_problem_end + num_tiles + + +def triton_perf_fn(group_A, group_B, group_C, dtype): + # We put the process of matrix lengths and pointers here out of fairness, + # since cublas_grouped_gemm kernel also does these work. + group_size = len(group_A) + A_addrs = [] + B_addrs = [] + C_addrs = [] + g_sizes = [] + g_lds = [] + alphas = [1.0] * group_size + betas = [0.0] * group_size + for i in range(group_size): + M, N, K = group_A[i].shape[0], group_B[i].shape[1], group_A[i].shape[1] + g_sizes += [M, N, K] + g_lds += [K, N, N] + A_addrs.append(group_A[i].data_ptr()) + B_addrs.append(group_B[i].data_ptr()) + C_addrs.append(group_C[i].data_ptr()) + + d_a_ptrs = torch.tensor(A_addrs, device="cuda") + d_b_ptrs = torch.tensor(B_addrs, device="cuda") + d_c_ptrs = torch.tensor(C_addrs, device="cuda") + d_g_sizes = torch.tensor(g_sizes, dtype=torch.int32, device="cuda") + d_g_lds = torch.tensor(g_lds, dtype=torch.int32, device="cuda") + d_alphas = torch.tensor(alphas, dtype=torch.float32, device="cuda") + d_betas = torch.tensor(betas, dtype=torch.float32, device="cuda") + + NUM_SM = 128 + grid = (NUM_SM,) + grouped_matmul_kernel[grid]( + d_a_ptrs, + d_b_ptrs, + d_c_ptrs, + d_g_sizes, + d_g_lds, + d_alphas, + d_betas, + group_size, + NUM_SM=NUM_SM, + BLOCK_SIZE_M=128, + BLOCK_SIZE_N=128, + BLOCK_SIZE_K=32, + ) + + +def cublas_perf_fn(group_A, group_B, group_C, dtype): + cublas_grouped_gemm(group_A, group_B, group_C, dtype) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["M"], + x_vals=[1, 16, 32, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=[ + "triton", + "cublas", + ], + line_names=[ + "triton", + "cublas", + ], + styles=[("green", "-"), ("blue", "-")], + ylabel="gbps", + plot_name="grouped gemm", + args={}, + ) +) +def benchmark(M, provider, N, K): + group_size = 20 # Number of used experts per gpu is usually around 20 + group_A = [] + group_B_row_major = [] + group_B_col_major = [] + group_C = [] + dtype = torch.float16 + for i in range(group_size): + A = torch.rand((M, K), device="cuda", dtype=dtype) + B_row_major = torch.rand((K, N), device="cuda", dtype=dtype) + B_col_major = torch.rand((N, K), device="cuda", dtype=dtype) + C = torch.empty((M, N), device="cuda", dtype=dtype) + group_A.append(A) + group_B_row_major.append(B_row_major) + group_B_col_major.append(B_col_major) + group_C.append(C) + + quantiles = [0.5, 0.2, 0.8] + if "triton" in provider: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: triton_perf_fn(group_A, group_B_row_major, group_C, dtype), + quantiles=quantiles, + ) + elif "cublas" in provider: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: cublas_perf_fn(group_A, group_B_col_major, group_C, dtype), + quantiles=quantiles, + ) + + gbps = ( + lambda ms: group_size + * (2 * M * N * K + 2 * M * N) + * group_A[0].element_size() + * 1e-9 + / (ms * 1e-3) + ) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["DeepSeek-V2"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-size", + type=int, + default=8, + help="Tensor parallel size", + ) + args = parser.parse_args() + for model in args.models: + assert model in WEIGHT_CONFIGS + num_experts_per_device = ( + WEIGHT_CONFIGS[model]["num_routed_experts"] // args.tp_size + ) + for K, N in WEIGHT_CONFIGS[model]["ffn_shapes"]: + print( + f"{model} N={N} K={K} tp_size={args.tp_size} " + f"group_size=num_experts_per_device={num_experts_per_device}: " + ) + benchmark.run( + print_data=True, + show_plots=True, + save_path="bench_grouped_gemm_res", + N=N, + K=K, + ) + + print("Benchmark finished!") diff --git a/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py b/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py new file mode 100644 index 0000000..a23db75 --- /dev/null +++ b/sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py @@ -0,0 +1,148 @@ +import argparse +import copy +import itertools + +import torch +import triton +from sgl_kernel import fp8_blockwise_scaled_mm +from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm + + +def get_weight_shapes(args): + models_tps = list(itertools.product(args.models, args.tp_sizes)) + # NOTE(HandH1998): The weight shapes only works for DeepSeek-V3. Modify them, if you tune for another different model. + # cannot TP + total = [ + # (512 + 64, 7168), # this weight is not supported by current kernel + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + # only support Deepseek-V3 + SUPPORT_MODEL = ["deepseek-ai/DeepSeek-V3"] + + weight_shapes = [] + for model, tp_size in models_tps: + assert model in SUPPORT_MODEL + for t in total: + new_t = [t[0], t[1], model] + weight_shapes.append(new_t) + for n_t in n_tp: + new_t = [n_t[0] // tp_size, n_t[1], model] + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = [k_t[0], k_t[1] // tp_size, model] + weight_shapes.append(new_t) + return weight_shapes + + +def cdiv(a: int, b: int) -> int: + """Ceiling division.""" + return -(a // -b) + + +def scale_shape(shape, group_shape): + assert len(shape) == len(group_shape) + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 32, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=["vllm", "sgl-kernel"], + line_names=["vllm fp8 blockwise gemm", "sgl-kernel fp8 blockwise gemm"], + styles=[("blue", "-"), ("orange", "-")], + ylabel="GB/s", + plot_name="fp8 blockwise scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + M = batch_size + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a_fp32 = (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + b_fp32 = (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn).t() + + scale_a_group_shape = (1, 128) + scale_b_group_shape = (128, 128) + scale_a_shape = scale_shape(a_fp8.shape, scale_a_group_shape) + scale_b_shape = scale_shape(b_fp8.shape, scale_b_group_shape) + + scale_a = torch.randn(scale_a_shape, device="cuda", dtype=torch.float32) + scale_b = torch.randn(scale_b_shape, device="cuda", dtype=torch.float32) + scale_a = scale_a.t().contiguous().t() + scale_b = scale_b.t().contiguous().t() + + quantiles = [0.5, 0.2, 0.8] + if provider == "sgl-kernel": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_blockwise_scaled_mm( + a_fp8, b_fp8, scale_a, scale_b, torch.float16 + ), + quantiles=quantiles, + ) + if provider == "vllm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: vllm_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, torch.float16), + quantiles=quantiles, + ) + gbps = ( + lambda ms: ( + (2 * M * N * K - M * N) * a_fp8.element_size() + + (3 * M * N) * scale_a.element_size() + ) + * 1e-9 + / (ms * 1e-3) + ) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["deepseek-ai/DeepSeek-V3"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + NK_model_names = get_weight_shapes(args) + for N, K, model_name in NK_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run( + print_data=True, + show_plots=True, + save_path="bench_fp8_blockwise_res", + N=N, + K=K, + ) + + print("Benchmark finished!") diff --git a/sgl-kernel/benchmark/bench_fp8_gemm.py b/sgl-kernel/benchmark/bench_fp8_gemm.py new file mode 100644 index 0000000..c3f8047 --- /dev/null +++ b/sgl-kernel/benchmark/bench_fp8_gemm.py @@ -0,0 +1,164 @@ +import argparse +import copy +import itertools + +import torch +import triton +from sgl_kernel import fp8_scaled_mm as sgl_scaled_mm +from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm +from vllm._custom_ops import scaled_fp8_quant as vllm_scaled_fp8_quant + +# Weight Shapes are in the format +# ([K, N], TP_SPLIT_DIM) +# Example: +# A shape of ([14336, 4096], 0) indicates the following GEMM shape, +# - TP1 : K = 14336, N = 4096 +# - TP2 : K = 7168, N = 4096 +# A shape of ([4096, 6144], 1) indicates the following GEMM shape, +# - TP1 : K = 4096, N = 6144 +# - TP4 : K = 4096, N = 1536 + +# TP1 shapes +WEIGHT_SHAPES = { + "meta-llama/Llama-3.1-8B-Instruct": [ + ([4096, 6144], 1), + ([4096, 4096], 0), + ([4096, 28672], 1), + ([14336, 4096], 0), + ], + "meta-llama/Llama-3.3-70B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 57344], 1), + ([28672, 8192], 0), + ], + "mistralai/Mistral-Large-Instruct-2407": [ + ([12288, 14336], 1), + ([12288, 12288], 0), + ([12288, 57344], 1), + ([28672, 12288], 0), + ], + "Qwen/Qwen2.5-7B-Instruct": [ + ([3584, 4608], 1), + ([3584, 3584], 0), + ([3584, 37888], 1), + ([18944, 3584], 0), + ], + "Qwen/Qwen2.5-32B-Instruct": [ + ([5120, 7168], 1), + ([5120, 5120], 0), + ([5120, 55296], 1), + ([27648, 5120], 0), + ], + "Qwen/Qwen2.5-72B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 59136], 1), + ([29568, 8192], 0), + ], + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": [ + ([2048, 3072], 1), + ([2048, 4096], 1), + ([2048, 2048], 0), + ([2048, 576], 0), + ([2048, 21888], 1), + ([10944, 2048], 0), + ([2048, 2816], 1), + ([1408, 2048], 0), + ], +} + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=[ + "vllm-fp8-fp16", + "vllm-fp8-bf16", + "sglang-fp8-fp16", + "sglang-fp8-bf16", + ], + line_names=[ + "vllm-fp8-fp16", + "vllm-fp8-bf16", + "sglang-fp8-fp16", + "sglang-fp8-bf16", + ], + styles=[("green", "-"), ("green", "--"), ("blue", "-"), ("blue", "--")], + ylabel="GB/s", + plot_name="fp8 scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + # M, N, K = batch_size, 4096, 8192 + M = batch_size + a = torch.ones((M, K), device="cuda") * 5.0 + b = torch.ones((N, K), device="cuda") * 5.0 + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + a_fp8, scale_a_fp8 = vllm_scaled_fp8_quant(a, scale_a) + b_fp8, scale_b_fp8 = vllm_scaled_fp8_quant(b, scale_b) + b_fp8 = b_fp8.t() + quantiles = [0.5, 0.2, 0.8] + + dtype = torch.float16 if "fp16" in provider else torch.bfloat16 + + if "vllm-fp8" in provider: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: vllm_scaled_mm(a_fp8, b_fp8, scale_a_fp8, scale_b_fp8, dtype), + quantiles=quantiles, + ) + elif "sglang-fp8" in provider: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: sgl_scaled_mm( + a_fp8, b_fp8, scale_a_fp8, scale_b_fp8, dtype, bias=None + ), + quantiles=quantiles, + ) + + gbps = lambda ms: (2 * M * N * K + M * N) * a.element_size() * 1e-9 / (ms * 1e-3) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +def prepare_shapes(args): + KN_model_names = [] + models_tps = list(itertools.product(args.models, args.tp_sizes)) + for model, tp_size in models_tps: + assert model in WEIGHT_SHAPES + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model]): + KN[tp_split_dim] = KN[tp_split_dim] // tp_size + KN.append(model) + KN_model_names.append(KN) + return KN_model_names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["meta-llama/Llama-3.1-8B-Instruct"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + KN_model_names = prepare_shapes(args) + for K, N, model_name in KN_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run( + print_data=True, show_plots=True, save_path="bench_fp8_res", N=N, K=K + ) + + print("Benchmark finished!") diff --git a/sgl-kernel/benchmark/bench_int8_gemm.py b/sgl-kernel/benchmark/bench_int8_gemm.py new file mode 100644 index 0000000..c5a7093 --- /dev/null +++ b/sgl-kernel/benchmark/bench_int8_gemm.py @@ -0,0 +1,146 @@ +import argparse +import copy +import itertools + +import torch +import triton +from sgl_kernel import int8_scaled_mm +from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm + + +def to_int8(tensor: torch.Tensor) -> torch.Tensor: + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) + + +WEIGHT_SHAPES = { + "meta-llama/Llama-3.1-8B-Instruct": [ + ([4096, 6144], 1), + ([4096, 4096], 0), + ([4096, 28672], 1), + ([14336, 4096], 0), + ], + "meta-llama/Llama-3.3-70B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 57344], 1), + ([28672, 8192], 0), + ], + "mistralai/Mistral-Large-Instruct-2407": [ + ([12288, 14336], 1), + ([12288, 12288], 0), + ([12288, 57344], 1), + ([28672, 12288], 0), + ], + "Qwen/Qwen2.5-7B-Instruct": [ + ([3584, 4608], 1), + ([3584, 3584], 0), + ([3584, 37888], 1), + ([18944, 3584], 0), + ], + "Qwen/Qwen2.5-32B-Instruct": [ + ([5120, 7168], 1), + ([5120, 5120], 0), + ([5120, 55296], 1), + ([27648, 5120], 0), + ], + "Qwen/Qwen2.5-72B-Instruct": [ + ([8192, 10240], 1), + ([8192, 8192], 0), + ([8192, 59136], 1), + ([29568, 8192], 0), + ], + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": [ + ([2048, 3072], 1), + ([2048, 4096], 1), + ([2048, 2048], 0), + ([2048, 576], 0), + ([2048, 21888], 1), + ([10944, 2048], 0), + ([2048, 2816], 1), + ([1408, 2048], 0), + ], +} + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 32, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=["vllm", "sgl-kernel"], + line_names=["vllm int8 gemm", "sgl-kernel int8 gemm"], + styles=[("blue", "-"), ("orange", "-")], + ylabel="GB/s", + plot_name="int8 scaled matmul", + args={}, + ) +) +def benchmark(batch_size, provider, N, K): + M = batch_size + a = to_int8(torch.randn((M, K), device="cuda") * 5) + b = to_int8(torch.randn((N, K), device="cuda").t() * 5) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + bias = torch.randn((N,), device="cuda", dtype=torch.float16) + + quantiles = [0.5, 0.2, 0.8] + if provider == "sgl-kernel": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: int8_scaled_mm(a, b, scale_a, scale_b, torch.float16, bias), + quantiles=quantiles, + ) + if provider == "vllm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: vllm_scaled_mm(a, b, scale_a, scale_b, torch.float16, bias), + quantiles=quantiles, + ) + gbps = ( + lambda ms: ( + (2 * M * N * K - M * N) * a.element_size() + + (3 * M * N) * scale_a.element_size() + ) + * 1e-9 + / (ms * 1e-3) + ) + return gbps(ms), gbps(max_ms), gbps(min_ms) + + +def prepare_shapes(args): + KN_model_names = [] + models_tps = list(itertools.product(args.models, args.tp_sizes)) + for model, tp_size in models_tps: + assert model in WEIGHT_SHAPES + for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model]): + KN[tp_split_dim] = KN[tp_split_dim] // tp_size + KN.append(model) + KN_model_names.append(KN) + return KN_model_names + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--models", + nargs="+", + type=str, + default=["meta-llama/Llama-3.1-8B-Instruct"], + help="List of models to benchmark", + ) + parser.add_argument( + "--tp-sizes", + nargs="+", + type=int, + default=[1], + help="List of tensor parallel sizes", + ) + args = parser.parse_args() + + KN_model_names = prepare_shapes(args) + for K, N, model_name in KN_model_names: + print(f"{model_name} N={N} K={K}: ") + benchmark.run( + print_data=True, show_plots=True, save_path="bench_int8_res", N=N, K=K + ) + + print("Benchmark finished!") diff --git a/sgl-kernel/benchmark/bench_lightning_attention_decode.py b/sgl-kernel/benchmark/bench_lightning_attention_decode.py new file mode 100644 index 0000000..36bdcca --- /dev/null +++ b/sgl-kernel/benchmark/bench_lightning_attention_decode.py @@ -0,0 +1,299 @@ +import itertools +import math + +import torch +import triton +import triton.language as tl +from sgl_kernel import lightning_attention_decode + + +def next_power_of_2(n): + return 2 ** (int(math.ceil(math.log(n, 2)))) + + +@triton.jit +def _decode_kernel( + Q, + K, + V, + KV, + Out, + S, + b: tl.constexpr, + h: tl.constexpr, + n: tl.constexpr, + d: tl.constexpr, + d_original: tl.constexpr, + e: tl.constexpr, + e_original: tl.constexpr, +): + off_bh = tl.program_id(0) + off_h = off_bh % h + + qk_offset = off_bh * n * d + v_offset = off_bh * n * e + o_offset = off_bh * n * e + kv_offset = off_bh * d * e + + s = tl.load(S + off_h) + ratio = tl.exp(-s) + + d_idx = tl.arange(0, d) + e_idx = tl.arange(0, e) + + # Create masks for original dimensions + d_mask = d_idx < d_original + e_mask = e_idx < e_original + + # Load with masking + q = tl.load(Q + qk_offset + d_idx, mask=d_mask, other=0.0) + k = tl.load(K + qk_offset + d_idx, mask=d_mask, other=0.0) + v = tl.load(V + v_offset + e_idx, mask=e_mask, other=0.0) + + # Load KV with 2D masking + kv = tl.load( + KV + kv_offset + d_idx[:, None] * e + e_idx[None, :], + mask=(d_mask[:, None] & e_mask[None, :]), + other=0.0, + ) + + # Compute outer product using element-wise operations + k_v_prod = k[:, None] * v[None, :] + kv = ratio * kv + k_v_prod + + # Store KV with 2D masking + tl.store( + KV + kv_offset + d_idx[:, None] * e + e_idx[None, :], + kv.to(KV.dtype.element_ty), + mask=(d_mask[:, None] & e_mask[None, :]), + ) + + # Compute matrix-vector multiplication using element-wise operations and reduction + o = tl.sum(q[:, None] * kv, axis=0) + + # Store output with masking + tl.store(Out + o_offset + e_idx, o.to(Out.dtype.element_ty), mask=e_mask) + + +def triton_lightning_attn_decode(q, k, v, kv, s): + """Triton implementation of Lightning Attention decode operation""" + b, h, n, d = q.shape + e = v.shape[-1] + assert n == 1, "Sequence length must be 1 in decode mode" + + # Get padded dimensions (power of 2) + d_padded = next_power_of_2(d) + e_padded = next_power_of_2(e) + + # Create output tensor (padded) + o_padded = torch.empty(b, h, n, e_padded, dtype=v.dtype, device=v.device) + + # Create padded tensors without actually padding the data + q_padded = torch.empty(b, h, n, d_padded, dtype=q.dtype, device=q.device) + k_padded = torch.empty(b, h, n, d_padded, dtype=k.dtype, device=k.device) + v_padded = torch.empty(b, h, n, e_padded, dtype=v.dtype, device=v.device) + kv_padded = torch.empty( + b, h, d_padded, e_padded, dtype=torch.float32, device=kv.device + ) + + # Copy data to padded tensors + q_padded[..., :d] = q + k_padded[..., :d] = k + v_padded[..., :e] = v + kv_padded[..., :d, :e] = kv + + # Launch kernel + grid = (b * h, 1) + _decode_kernel[grid]( + q_padded, + k_padded, + v_padded, + kv_padded, + o_padded, + s, + b=b, + h=h, + n=n, + d=d_padded, + d_original=d, + e=e_padded, + e_original=e, + ) + + # Get unpadded outputs + o = o_padded[..., :e] + kv_out = kv_padded[..., :d, :e] + + return o, kv_out + + +def lightning_attention_decode_naive(q, k, v, past_kv, slope): + """Naive implementation of lightning attention decode""" + original_dtype = q.dtype + ratio = torch.exp(-slope) # [h, 1, 1] + + kv = past_kv + b, h, n, d = q.shape + + output = [] + for i in range(n): + kv = ratio * kv.to(torch.float32) + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + qkv = torch.einsum( + "... n e, ... e d -> ... n d", + q[:, :, i : i + 1].to(torch.float32), + kv.to(torch.float32), + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + + return output.to(original_dtype), kv + + +def lightning_attention_decode_kernel(q, k, v, past_kv, slope, output, new_kv): + return lightning_attention_decode(q, k, v, past_kv, slope, output, new_kv) + + +def calculate_diff(batch_size): + dtype = torch.bfloat16 + device = torch.device("cuda") + num_heads = 64 + head_dim = 96 + seq_len = 1 + + q = torch.randn( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + k = torch.randn( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + v = torch.randn( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + past_kv = torch.randn(batch_size, num_heads, head_dim, head_dim, device=device) + slope = torch.randn(num_heads, 1, 1, device=device) + + output_naive, new_kv_naive = lightning_attention_decode_naive( + q.clone(), k.clone(), v.clone(), past_kv.clone(), slope.clone() + ) + + output_kernel = torch.empty_like(output_naive) + new_kv_kernel = torch.empty_like(new_kv_naive) + lightning_attention_decode_kernel( + q.clone(), + k.clone(), + v.clone(), + past_kv.clone(), + slope.clone(), + output_kernel, + new_kv_kernel, + ) + + output_triton, new_kv_triton = triton_lightning_attn_decode( + q.clone(), k.clone(), v.clone(), past_kv.clone(), slope.clone() + ) + + if ( + torch.allclose(output_naive, output_kernel, atol=1e-2, rtol=1e-2) + and torch.allclose(output_naive, output_triton, atol=1e-2, rtol=1e-2) + and torch.allclose(new_kv_naive, new_kv_kernel, atol=1e-2, rtol=1e-2) + and torch.allclose(new_kv_naive, new_kv_triton, atol=1e-2, rtol=1e-2) + ): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +batch_size_range = [i for i in range(1, 65)] # 1 to 128 +configs = [(bs,) for bs in batch_size_range] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["naive", "kernel", "triton"], + line_names=["PyTorch Naive", "SGL Kernel", "Triton"], + styles=[("blue", "-"), ("red", "-"), ("green", "-")], + ylabel="us", + plot_name="lightning-attention-decode-performance", + args={}, + ) +) +def benchmark(batch_size, provider): + dtype = torch.bfloat16 + device = torch.device("cuda") + num_heads = 64 + head_dim = 96 + seq_len = 1 + + q = torch.randn( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + k = torch.randn( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + v = torch.randn( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + past_kv = torch.randn(batch_size, num_heads, head_dim, head_dim, device=device) + slope = torch.randn(num_heads, 1, 1, device=device) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "naive": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: lightning_attention_decode_naive( + q.clone(), k.clone(), v.clone(), past_kv.clone(), slope.clone() + ), + quantiles=quantiles, + ) + elif provider == "kernel": + output = torch.empty( + batch_size, num_heads, seq_len, head_dim, device=device, dtype=dtype + ) + new_kv = torch.empty(batch_size, num_heads, head_dim, head_dim, device=device) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: lightning_attention_decode_kernel( + q.clone(), + k.clone(), + v.clone(), + past_kv.clone(), + slope.clone(), + output, + new_kv, + ), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: triton_lightning_attn_decode( + q.clone(), k.clone(), v.clone(), past_kv.clone(), slope.clone() + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/lightning_attention_decode_sgl/", + help="Path to save lightning attention decode benchmark results", + ) + args = parser.parse_args() + + # Run correctness test + calculate_diff(batch_size=4) + + # Run performance benchmark + benchmark.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_moe_align_block_size.py b/sgl-kernel/benchmark/bench_moe_align_block_size.py new file mode 100644 index 0000000..4266acf --- /dev/null +++ b/sgl-kernel/benchmark/bench_moe_align_block_size.py @@ -0,0 +1,385 @@ +import argparse +import itertools + +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size +from vllm import _custom_ops as ops + +USE_RANDOM_PERM = False + + +def ceil_div(a, b): + return (a + b - 1) // b + + +@triton.jit +def moe_align_block_size_stage1( + topk_ids_ptr, + tokens_cnts_ptr, + num_experts: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = pid * tokens_per_thread + off_c = (pid + 1) * num_experts + + for i in range(tokens_per_thread): + if start_idx + i < numel: + idx = tl.load(topk_ids_ptr + start_idx + i) + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) + + +@triton.jit +def moe_align_block_size_stage2( + tokens_cnts_ptr, + num_experts: tl.constexpr, +): + pid = tl.program_id(0) + last_cnt = 0 + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) + last_cnt = last_cnt + token_cnt + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) + + +@triton.jit +def moe_align_block_size_stage3( + total_tokens_post_pad_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, +): + last_cumsum = 0 + off_cnt = num_experts * num_experts + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size + tl.store(cumsum_ptr + i, last_cumsum) + tl.store(total_tokens_post_pad_ptr, last_cumsum) + + +@triton.jit +def moe_align_block_size_stage4( + topk_ids_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = tl.load(cumsum_ptr + pid) + end_idx = tl.load(cumsum_ptr + pid + 1) + + for i in range(start_idx, end_idx, block_size): + tl.store(expert_ids_ptr + i // block_size, pid) + + start_idx = pid * tokens_per_thread + off_t = pid * num_experts + + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): + expert_id = tl.load(topk_ids_ptr + i) + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) + tl.store(sorted_token_ids_ptr + rank_post_pad, i) + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) + + +def moe_align_block_size_triton( + topk_ids: torch.Tensor, + num_experts: int, + block_size: int, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor, +) -> None: + numel = topk_ids.numel() + grid = (num_experts,) + tokens_cnts = torch.zeros( + (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device + ) + cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) + tokens_per_thread = ceil_div(numel, num_experts) + + moe_align_block_size_stage1[grid]( + topk_ids, + tokens_cnts, + num_experts, + numel, + tokens_per_thread, + ) + moe_align_block_size_stage2[grid]( + tokens_cnts, + num_experts, + ) + moe_align_block_size_stage3[(1,)]( + num_tokens_post_pad, + tokens_cnts, + cumsum, + num_experts, + block_size, + ) + moe_align_block_size_stage4[grid]( + topk_ids, + sorted_token_ids, + expert_ids, + tokens_cnts, + cumsum, + num_experts, + block_size, + numel, + tokens_per_thread, + ) + + +def calculate_diff(num_tokens, num_experts=256, block_size=128, topk=8): + topk_ids = torch.stack( + [ + torch.randperm(num_experts, dtype=torch.int32, device="cuda")[:topk] + for _ in range(num_tokens) + ] + ) + + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) + sorted_ids_cuda = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + sorted_ids_cuda.fill_(topk_ids.numel()) + max_num_m_blocks = max_num_tokens_padded // block_size + expert_ids_cuda = torch.zeros( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad_cuda = torch.empty( + (1), dtype=torch.int32, device=topk_ids.device + ) + token_cnts_buffer = torch.zeros( + (num_experts + 1) * num_experts, dtype=torch.int32, device=topk_ids.device + ) + cumsum_buffer = torch.zeros( + num_experts + 1, dtype=torch.int32, device=topk_ids.device + ) + + sorted_ids_triton = torch.empty_like(sorted_ids_cuda) + sorted_ids_triton.fill_(topk_ids.numel()) + expert_ids_triton = torch.zeros_like(expert_ids_cuda) + num_tokens_post_pad_triton = torch.empty_like(num_tokens_post_pad_cuda) + + sorted_ids_vllm = torch.empty_like(sorted_ids_cuda) + sorted_ids_vllm.fill_(topk_ids.numel()) + expert_ids_vllm = torch.zeros_like(expert_ids_cuda) + num_tokens_post_pad_vllm = torch.empty_like(num_tokens_post_pad_cuda) + + # compare the performance of cuda, triton and vllm implementation + sgl_moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids_cuda, + expert_ids_cuda, + num_tokens_post_pad_cuda, + token_cnts_buffer, + cumsum_buffer, + ) + moe_align_block_size_triton( + topk_ids, + num_experts, + block_size, + sorted_ids_triton, + expert_ids_triton, + num_tokens_post_pad_triton, + ) + + try: + ops.moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids_vllm, + expert_ids_vllm, + num_tokens_post_pad_vllm, + ) + print(f"✅ VLLM implementation works with {num_experts} experts!") + vllm_works = True + except RuntimeError as e: + print(f"❌ VLLM implementation failed with {num_experts} experts: {e}") + vllm_works = False + + if torch.allclose(expert_ids_cuda, expert_ids_triton) and torch.allclose( + num_tokens_post_pad_cuda, num_tokens_post_pad_triton + ): + print("✅ SGL and Triton implementations match") + else: + print("❌ SGL and Triton implementations do not match") + print("SGL expert_ids:", expert_ids_cuda) + print("Triton expert_ids:", expert_ids_triton) + print("SGL num_tokens_post_pad:", num_tokens_post_pad_cuda) + print("Triton num_tokens_post_pad:", num_tokens_post_pad_triton) + + if ( + vllm_works + and torch.allclose(expert_ids_cuda, expert_ids_vllm) + and torch.allclose(num_tokens_post_pad_cuda, num_tokens_post_pad_vllm) + ): + print("✅ SGL and VLLM implementations match") + else: + if not vllm_works: + print("⚠️ VLLM comparison skipped due to failure") + else: + print("❌ SGL and VLLM implementations do not match") + print("SGL expert_ids:", expert_ids_cuda) + print("VLLM expert_ids:", expert_ids_vllm) + print("SGL num_tokens_post_pad:", num_tokens_post_pad_cuda) + print("VLLM num_tokens_post_pad:", num_tokens_post_pad_vllm) + + +# Test range +num_tokens_range = [16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] +num_experts_range = [8, 32, 64, 128, 256] +topk_range = [2, 4, 8] + +configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) + + +def get_topk_ids(num_tokens: int, num_experts: int, topk: int) -> torch.Tensor: + topk_ids = torch.zeros((num_tokens, topk), dtype=torch.int32, device="cuda") + for i in range(num_tokens): + topk_ids[i, :] = torch.randperm(num_experts, dtype=torch.int32, device="cuda")[ + :topk + ] + return topk_ids + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens", "num_experts", "topk"], + x_vals=configs, + line_arg="provider", + line_vals=["sgl", "triton", "vllm"], + line_names=["SGL", "Triton", "VLLM"], + styles=[("blue", "-"), ("red", "-"), ("green", "-")], + ylabel="us", + plot_name="moe-align-block-size-performance", + args={}, + ) +) +def benchmark(num_tokens, num_experts, topk, provider): + block_size = 128 + + if USE_RANDOM_PERM: + topk_ids = get_topk_ids(num_tokens, num_experts, topk) + else: + topk_ids = torch.randint( + 0, + num_experts, + (num_tokens, topk), + dtype=torch.int32, + device="cuda", + ) + + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) + sorted_ids = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + sorted_ids.fill_(topk_ids.numel()) + max_num_m_blocks = max_num_tokens_padded // block_size + expert_ids = torch.empty( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad = torch.empty((1), dtype=torch.int32, device=topk_ids.device) + token_cnts_buffer = torch.zeros( + (num_experts + 1) * num_experts, dtype=torch.int32, device=topk_ids.device + ) + cumsum_buffer = torch.zeros( + num_experts + 1, dtype=torch.int32, device=topk_ids.device + ) + + quantiles = [0.5, 0.2, 0.8] + if provider == "sgl": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: sgl_moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids.clone(), + expert_ids.clone(), + num_tokens_post_pad.clone(), + token_cnts_buffer, + cumsum_buffer, + ), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: moe_align_block_size_triton( + topk_ids, + num_experts, + block_size, + sorted_ids.clone(), + expert_ids.clone(), + num_tokens_post_pad.clone(), + ), + quantiles=quantiles, + ) + else: # vllm + try: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: ops.moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids.clone(), + expert_ids.clone(), + num_tokens_post_pad.clone(), + ), + quantiles=quantiles, + ) + except RuntimeError as e: + print(f"❌ VLLM benchmark failed with {num_experts} experts: {e}") + # Return extreme values to indicate failure in the chart + return float("inf"), float("inf"), float("inf") + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/moe_align_blocks/", + help="Path to save moe align benchmark results", + ) + parser.add_argument( + "--num_experts", + type=int, + default=256, + choices=[8, 16, 32, 64, 128, 256], + help="Number of experts for benchmark", + ) + parser.add_argument( + "--topk", + type=int, + default=8, + choices=[2, 4, 8], + help="Top-k value for benchmark", + ) + parser.add_argument( + "--skip_full_benchmark", + action="store_true", + help="Only run the calculate_diff function, skip full benchmarking", + ) + args = parser.parse_args() + + calculate_diff(num_tokens=1024, num_experts=args.num_experts, topk=args.topk) + + if not args.skip_full_benchmark: + print(f"\n📊 Running performance benchmark for {args.num_experts} experts...") + benchmark.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_moe_topk_softmax.py b/sgl-kernel/benchmark/bench_moe_topk_softmax.py new file mode 100644 index 0000000..5598cfb --- /dev/null +++ b/sgl-kernel/benchmark/bench_moe_topk_softmax.py @@ -0,0 +1,120 @@ +import itertools + +import pytest +import torch +import triton +from sgl_kernel import topk_softmax +from vllm import _custom_ops as vllm_custom_ops + + +def vllm_topk_softmax(gating_output, topk): + num_tokens, num_experts = gating_output.shape + + topk_weights = torch.empty( + (num_tokens, topk), device=gating_output.device, dtype=torch.float32 + ) + topk_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + token_expert_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + torch.ops._moe_C.topk_softmax( + topk_weights, topk_indices, token_expert_indices, gating_output + ) + return topk_weights, topk_indices + + +def sglang_topk_softmax(gating_output, topk): + num_tokens, num_experts = gating_output.shape + + topk_weights = torch.empty( + (num_tokens, topk), device=gating_output.device, dtype=torch.float32 + ) + topk_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + token_expert_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device=gating_output.device + ) + + topk_softmax( + topk_weights=topk_weights, + topk_ids=topk_indices, + token_expert_indices=token_expert_indices, + gating_output=gating_output, + ) + + return topk_weights, topk_indices + + +def calculate_diff(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), device="cuda", dtype=torch.float32 + ) + weights_vllm, indices_vllm = vllm_topk_softmax(gating_output.clone(), topk) + weights_sglang, indices_sglang = sglang_topk_softmax(gating_output.clone(), topk) + + weights_diff = torch.abs(weights_vllm - weights_sglang).mean().item() + indices_match = torch.equal(indices_vllm, indices_sglang) + + if ( + torch.allclose(weights_vllm, weights_sglang, atol=1e-3, rtol=1e-3) + and indices_match + ): + print("✅ VLLM and SGLang topk_softmax implementations match") + else: + print( + f"❌ Implementations differ: Weights diff={weights_diff}, Indices match={indices_match}" + ) + + +num_tokens_range = [128, 512, 1024, 2048, 4096, 8192, 16384, 32768] +num_experts_range = [32, 64, 128, 256, 12, 512] +topk_range = [1, 2, 4, 8] + +configs = list(itertools.product(num_tokens_range, num_experts_range, topk_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens", "num_experts", "topk"], + x_vals=configs, + line_arg="provider", + line_vals=["sglang", "vllm"], + line_names=["SGLang", "VLLM"], + styles=[("blue", "-"), ("green", "-")], + ylabel="Latency (us)", + plot_name="topk-softmax-performance", + args={}, + ) +) +def benchmark(num_tokens, num_experts, topk, provider): + + gating_output = torch.randn( + (num_tokens, num_experts), device="cuda", dtype=torch.float32 + ) + + if provider == "vllm" or provider == "vllm1": + fn = lambda: vllm_topk_softmax(gating_output, topk) + elif provider == "sglang" or provider == "sglang1": + fn = lambda: sglang_topk_softmax(gating_output, topk) + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + configs = [ + (20, 256, 4), + (20, 256, 8), + (20, 12, 4), + (20, 12, 1), + (20, 512, 4), + (20, 512, 1), + ] + for num_tokens, num_experts, topk in configs: + calculate_diff(num_tokens, num_experts, topk) + benchmark.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py b/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py new file mode 100644 index 0000000..7a07efd --- /dev/null +++ b/sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py @@ -0,0 +1,98 @@ +import itertools +import math +from typing import Any, Dict, List, Optional, Tuple + +import numpy as np +import torch +import triton +import triton.testing +from sgl_kernel import sgl_per_tensor_quant_fp8 +from vllm import _custom_ops as ops + +from sglang.srt.utils import is_hip + +is_hip_ = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + + +def vllm_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + return ops.scaled_fp8_quant(input, scale) + + +def sglang_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_type_: torch.dtype = torch.float8_e4m3fn + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + is_static = True + if scale is None: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + is_static = False + sgl_per_tensor_quant_fp8(input, output, scale, is_static) + + return output, scale + + +def calculate_diff(batch_size: int, seq_len: int): + """Calculate difference between VLLM and SGLang implementations.""" + device = torch.device("cuda") + x = torch.rand((batch_size, seq_len), dtype=torch.float16, device=device) + + vllm_out, vllm_scale = vllm_scaled_fp8_quant(x) + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x) + + scale_diff = torch.abs(vllm_scale - sglang_scale).item() + output_diff = torch.abs(vllm_out.float() - sglang_out.float()).mean().item() + + if torch.allclose( + vllm_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ) and torch.allclose(vllm_scale, sglang_scale, rtol=1e-3, atol=1e-5): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +batch_size_range = [16, 32, 64, 128] +seq_len_range = [64, 128, 256, 512, 1024, 2048] + +configs = list(itertools.product(batch_size_range, seq_len_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=configs, + line_arg="provider", + line_vals=["vllm", "sglang"], + line_names=["VLLM", "SGL Kernel"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="per-tensor-quant-fp8-performance", + args={}, + ) +) +def benchmark(batch_size, seq_len, provider): + dtype = torch.float16 + device = torch.device("cuda") + + x = torch.randn(batch_size * seq_len, 4096, device=device, dtype=dtype) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "vllm": + fn = lambda: vllm_scaled_fp8_quant(x.clone()) + elif provider == "sglang": + fn = lambda: sglang_scaled_fp8_quant(x.clone()) + + ms, min_ms, max_ms = triton.testing.do_bench(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + calculate_diff(batch_size=4, seq_len=4096) + benchmark.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py b/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py new file mode 100644 index 0000000..b03369c --- /dev/null +++ b/sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py @@ -0,0 +1,218 @@ +import itertools +from typing import Tuple + +import torch +import triton +import triton.language as tl +from sgl_kernel import sgl_per_token_group_quant_fp8, sgl_per_token_group_quant_int8 + +from sglang.srt.utils import is_hip + +is_hip_ = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + + +@triton.jit +def _per_token_group_quant_8bit( + # Pointers to inputs and output + y_ptr, + y_q_ptr, + y_s_ptr, + # Stride of input + y_stride, + # Collums of input + N, + # Avoid to divide zero + eps, + # Information for 8bit data type (int8 or fp8_type_) + max_8bit, + min_8bit, + # Meta-parameters + BLOCK: tl.constexpr, +): + """A Triton-accelerated function to perform per-token-group quantization on a + tensor. + This function converts the tensor values into 8bit values. + """ + # Map the program id to the row of X and Y it should compute. + g_id = tl.program_id(0) + y_ptr += g_id * y_stride + y_q_ptr += g_id * y_stride + y_s_ptr += g_id + + cols = tl.arange(0, BLOCK) # N <= BLOCK + mask = cols < N + + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32) + # Quant + _absmax = tl.maximum(tl.max(tl.abs(y)), eps) + y_s = _absmax / max_8bit + y_q = tl.clamp(y / y_s, min_8bit, max_8bit).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + cols, y_q, mask=mask) + tl.store(y_s_ptr, y_s) + + +def triton_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Function to perform per-token-group quantization on an input tensor `x`. + It converts the tensor values into signed float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Args: + x: The input tenosr with ndim >= 2. + group_size: The group size used for quantization. + eps: The minimum to avoid dividing zero. + dtype: The dype of output tensor. Note that only `torch.float8_e4m3fn` is supported for now. + Returns: + Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the scaling factor for quantization. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + if dst_dtype == torch.int8: + iinfo = torch.iinfo(dst_dtype) + max_8bit = iinfo.max + min_8bit = iinfo.min + else: + finfo = torch.finfo(dst_dtype) + max_8bit = finfo.max + min_8bit = finfo.min + + x_q = torch.empty_like(x, device=x.device, dtype=dst_dtype) + M = x.numel() // group_size + N = group_size + x_s = torch.empty( + x.shape[:-1] + (x.shape[-1] // group_size,), + device=x.device, + dtype=torch.float32, + ) + + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + _per_token_group_quant_8bit[(M,)]( + x, + x_q, + x_s, + group_size, + N, + eps, + max_8bit, + min_8bit, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + ) + + return x_q, x_s + + +def sglang_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, +): + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + x_q = torch.empty_like(x, device=x.device, dtype=dst_dtype) + x_s = torch.empty( + x.shape[:-1] + (x.shape[-1] // group_size,), + device=x.device, + dtype=torch.float32, + ) + + if dst_dtype == torch.int8: + iinfo = torch.iinfo(dst_dtype) + int8_max = iinfo.max + int8_min = iinfo.min + sgl_per_token_group_quant_int8(x, x_q, x_s, group_size, eps, int8_min, int8_max) + else: + f8_info = torch.finfo(dst_dtype) + fp8_max = f8_info.max + fp8_min = f8_info.min + sgl_per_token_group_quant_fp8(x, x_q, x_s, group_size, eps, fp8_min, fp8_max) + + return x_q, x_s + + +def calculate_diff(batch_size, seq_len, group_size, dst_dtype): + device = torch.device("cuda") + hidden_dim = group_size * 2 + + x = torch.randn(batch_size, seq_len, hidden_dim, device=device, dtype=torch.float16) + + x_q_triton, x_s_triton = triton_per_token_group_quant_8bit( + x.clone(), group_size, dst_dtype + ) + x_q_sglang, x_s_sglang = sglang_per_token_group_quant_8bit( + x.clone(), group_size, dst_dtype + ) + + if torch.allclose( + x_q_triton.to(torch.float32), x_q_sglang.to(torch.float32), rtol=1e-3, atol=1e-5 + ) and torch.allclose(x_s_triton, x_s_sglang, rtol=1e-3, atol=1e-5): + print(f"✅ {dst_dtype} implementations match") + else: + print("❌ Implementations differ") + + +batch_size_range = [1, 2, 4, 8, 16, 32, 64] +seq_len_range = [64, 128, 256, 512, 1024, 2048] +group_size_range = [128] # For DeepSeek V3/R1 +dst_dtype_range = [torch.int8, fp8_type_] + +configs = list( + itertools.product( + batch_size_range, seq_len_range, group_size_range, dst_dtype_range + ) +) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len", "group_size", "dst_dtype"], + x_vals=configs, + line_arg="provider", + line_vals=["triton", "sglang"], + line_names=["Triton", "SGL Kernel"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="per-token-group-quant-8bit-performance", + args={}, + ) +) +def benchmark(batch_size, seq_len, group_size, dst_dtype, provider): + device = torch.device("cuda") + hidden_dim = 7168 + + x = torch.randn(batch_size, seq_len, hidden_dim, device=device, dtype=torch.float16) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "triton": + fn = lambda: triton_per_token_group_quant_8bit(x.clone(), group_size, dst_dtype) + elif provider == "sglang": + fn = lambda: sglang_per_token_group_quant_8bit(x.clone(), group_size, dst_dtype) + + ms, min_ms, max_ms = triton.testing.do_bench(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + + calculate_diff(batch_size=4, seq_len=128, group_size=64, dst_dtype=torch.int8) + calculate_diff(batch_size=4, seq_len=128, group_size=64, dst_dtype=fp8_type_) + + benchmark.run(print_data=True) diff --git a/sgl-kernel/benchmark/bench_per_token_quant_fp8.py b/sgl-kernel/benchmark/bench_per_token_quant_fp8.py new file mode 100644 index 0000000..ed0bfc7 --- /dev/null +++ b/sgl-kernel/benchmark/bench_per_token_quant_fp8.py @@ -0,0 +1,90 @@ +import itertools +from typing import Optional, Tuple + +import torch +import triton +import triton.testing +from sgl_kernel import sgl_per_token_quant_fp8 +from vllm import _custom_ops as ops + +from sglang.srt.utils import is_hip + +is_hip_ = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + + +def vllm_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + return ops.scaled_fp8_quant(input, use_per_token_if_dynamic=True) + + +def sglang_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + scale = torch.zeros(input.size(0), device=input.device, dtype=torch.float32) + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + sgl_per_token_quant_fp8(input, output, scale) + + return output, scale + + +def calculate_diff(batch_size: int, seq_len: int): + """Calculate difference between VLLM and SGLang implementations.""" + device = torch.device("cuda") + x = torch.rand((batch_size, seq_len), dtype=torch.float16, device=device) + + vllm_out, vllm_scale = vllm_per_token_quant_fp8(x) + sglang_out, sglang_scale = sglang_per_token_quant_fp8(x) + + scale_diff = torch.abs(vllm_scale - sglang_scale).mean().item() + output_diff = torch.abs(vllm_out.float() - sglang_out.float()).mean().item() + + if torch.allclose( + vllm_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ) and torch.allclose(vllm_scale, sglang_scale, rtol=1e-3, atol=1e-5): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +batch_size_range = [16, 32, 64, 128] +seq_len_range = [64, 128, 256, 512, 1024, 2048, 4096] + +configs = list(itertools.product(batch_size_range, seq_len_range)) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=configs, + line_arg="provider", + line_vals=["vllm", "sglang"], + line_names=["VLLM", "SGL Kernel"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="per-token-dynamic-quant-fp8-performance", + args={}, + ) +) +def benchmark_quantization(batch_size, seq_len, provider): + dtype = torch.float16 + device = torch.device("cuda") + + x = torch.randn(batch_size * seq_len, 4096, device=device, dtype=dtype) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "vllm": + fn = lambda: vllm_per_token_quant_fp8(x.clone()) + elif provider == "sglang": + fn = lambda: sglang_per_token_quant_fp8(x.clone()) + + ms, min_ms, max_ms = triton.testing.do_bench(fn, quantiles=quantiles) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + +if __name__ == "__main__": + calculate_diff(batch_size=4, seq_len=4096) + benchmark_quantization.run(print_data=True) diff --git a/sgl-kernel/build.sh b/sgl-kernel/build.sh new file mode 100644 index 0000000..3936b4d --- /dev/null +++ b/sgl-kernel/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -ex +PYTHON_VERSION=$1 +CUDA_VERSION=$2 +PYTHON_ROOT_PATH=/opt/python/cp${PYTHON_VERSION//.}-cp${PYTHON_VERSION//.} + +if (( ${CUDA_VERSION%.*} < 12 )); then + ENABLE_SM90A=0 +else + ENABLE_SM90A=1 +fi + +docker run --rm \ + -v $(pwd):/sgl-kernel \ + pytorch/manylinux-builder:cuda${CUDA_VERSION} \ + bash -c " + ${PYTHON_ROOT_PATH}/bin/pip install --no-cache-dir torch==2.5.1 --index-url https://download.pytorch.org/whl/cu${CUDA_VERSION//.} && \ + ${PYTHON_ROOT_PATH}/bin/pip install --no-cache-dir ninja setuptools==75.0.0 wheel==0.41.0 numpy uv && \ + export TORCH_CUDA_ARCH_LIST='7.5 8.0 8.9 9.0+PTX' && \ + export CUDA_VERSION=${CUDA_VERSION} && \ + export SGL_KERNEL_ENABLE_BF16=1 && \ + export SGL_KERNEL_ENABLE_FP8=1 && \ + export SGL_KERNEL_ENABLE_SM90A=${ENABLE_SM90A} && \ + mkdir -p /usr/lib/x86_64-linux-gnu/ && \ + ln -s /usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib/stubs/libcuda.so /usr/lib/x86_64-linux-gnu/libcuda.so && \ + cd /sgl-kernel && \ + ls -la ${PYTHON_ROOT_PATH}/lib/python${PYTHON_VERSION}/site-packages/wheel/ && \ + PYTHONPATH=${PYTHON_ROOT_PATH}/lib/python${PYTHON_VERSION}/site-packages ${PYTHON_ROOT_PATH}/bin/python -m uv build --wheel -Cbuild-dir=build . --color=always && \ + ./rename_wheels.sh + " diff --git a/sgl-kernel/csrc/allreduce/custom_all_reduce.hip b/sgl-kernel/csrc/allreduce/custom_all_reduce.hip new file mode 100644 index 0000000..6c1ef0d --- /dev/null +++ b/sgl-kernel/csrc/allreduce/custom_all_reduce.hip @@ -0,0 +1,180 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include +#include +#include + +#include "custom_all_reduce_hip.cuh" + +// fake pointer type, must match fptr_t type in ops.h +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +fptr_t init_custom_ar(torch::Tensor& meta, torch::Tensor& rank_data, + const std::vector& handles, + const std::vector& offsets, int64_t rank, + bool full_nvlink) { + int world_size = offsets.size(); + if (world_size > 8) + throw std::invalid_argument("world size > 8 is not supported"); + if (world_size % 2 != 0) + throw std::invalid_argument("Odd num gpus is not supported for now"); + if (world_size != handles.size()) + throw std::invalid_argument( + "handles length should equal to offsets length"); + if (rank < 0 || rank >= world_size) + throw std::invalid_argument("invalid rank passed in"); + + hipIpcMemHandle_t ipc_handles[8]; + for (int i = 0; i < world_size; i++) { + std::memcpy(&ipc_handles[i], handles[i].data(), sizeof(hipIpcMemHandle_t)); + } + return (fptr_t) new vllm::CustomAllreduce( + reinterpret_cast(meta.data_ptr()), rank_data.data_ptr(), + rank_data.numel(), ipc_handles, offsets, rank, full_nvlink); +} + +/** + * Make sure tensor t's data lies completely within ((char)t.data_ptr()) + + * t.numel() * t.element_size(). This is slightly weaker than t.is_contiguous() + * because it allows transpose of contiguous slice (i.e. slicing the first + * dimension). Currently, we require this because stride information is not + * passed into the kernels and we treat input tensors as flat. + * + * Examples + * A = torch.zeros(3, 3, 3) + * 1. A: OK + * 2. A[1:]: OK + * 3. A.permute(2, 0, 1): OK + * 4. A[1:].permute(2, 0, 1): OK + * 5. A[None].expand(2, -1, -1, -1): Not OK + * 6. A[:, 1:, 1:]: Not OK + */ +bool _is_weak_contiguous(torch::Tensor& t) { + return t.is_contiguous() || + (t.storage().nbytes() - t.storage_offset() * t.element_size() == + t.numel() * t.element_size()); +} + +void _all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, + hipStream_t stream) { + auto fa = reinterpret_cast(_fa); + TORCH_CHECK(_is_weak_contiguous(out)); + switch (out.scalar_type()) { + case at::ScalarType::Float: { + fa->allreduce(stream, reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel()); + break; + } + case at::ScalarType::Half: { + fa->allreduce(stream, reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), out.numel()); + break; + } +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case at::ScalarType::BFloat16: { + fa->allreduce( + stream, reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), out.numel()); + break; + } +#endif + default: + throw std::runtime_error( + "custom allreduce only supports float32, float16 and bfloat16"); + } +} + +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) { + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(inp)); + auto stream = c10::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + _all_reduce(_fa, inp, out, stream); +} + +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer, + torch::Tensor& out) { + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(inp)); + auto stream = c10::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + + auto input_size = inp.numel() * inp.element_size(); + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK(input_size <= reg_buffer.numel() * reg_buffer.element_size(), + "registered buffer is too small to contain the input"); + AT_CUDA_CHECK(hipMemcpyAsync(reg_buffer.data_ptr(), inp.data_ptr(), + input_size, hipMemcpyDeviceToDevice, stream)); + _all_reduce(_fa, reg_buffer, out, stream); +} + +void dispose(fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + delete fa; +} + +int64_t meta_size() { return sizeof(vllm::Signal); } + +void register_buffer(fptr_t _fa, torch::Tensor& t, + const std::vector& handles, + const std::vector& offsets) { + auto fa = reinterpret_cast(_fa); + fa->register_buffer(handles, offsets, t.data_ptr()); +} + +std::tuple> get_graph_buffer_ipc_meta( + fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + auto [handle_bytes, offsets] = fa->get_graph_buffer_ipc_meta(); + auto options = + torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto handles = + torch::empty({static_cast(handle_bytes.size())}, options); + std::memcpy(handles.data_ptr(), handle_bytes.data(), handle_bytes.size()); + return {handles, std::move(offsets)}; +} + +void register_graph_buffers(fptr_t _fa, const std::vector& handles, + const std::vector>& offsets) { + auto fa = reinterpret_cast(_fa); + fa->register_graph_buffers(handles, offsets); +} + +void free_meta_buffer(void* buffer) { CUDACHECK(hipFree(buffer)); } + +torch::Tensor get_meta_buffer_ipc_handle(torch::Tensor& inp) { + auto options = + torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto data_handle = + torch::empty({static_cast(sizeof(hipIpcMemHandle_t))}, options); + CUDACHECK(hipIpcGetMemHandle((hipIpcMemHandle_t*)data_handle.data_ptr(), + inp.data_ptr())); + return data_handle; +} + +torch::Tensor allocate_meta_buffer(int64_t size) { + auto device_index = c10::hip::current_device(); + at::DeviceGuard device_guard(at::Device(at::DeviceType::CUDA, device_index)); + void* buffer; + hipStreamCaptureMode mode = hipStreamCaptureModeRelaxed; + auto stream = c10::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + AT_CUDA_CHECK(hipThreadExchangeStreamCaptureMode(&mode)); + AT_CUDA_CHECK( + hipExtMallocWithFlags((void**)&buffer, size, hipDeviceMallocUncached)); + AT_CUDA_CHECK(hipMemsetAsync(buffer, 0, size, stream)); + AT_CUDA_CHECK(hipStreamSynchronize(stream)); + AT_CUDA_CHECK(hipThreadExchangeStreamCaptureMode(&mode)); + auto options = torch::TensorOptions() + .dtype(torch::kI8) + .device(torch::kCUDA, device_index); + return torch::from_blob(buffer, {size}, free_meta_buffer, options); +} + +std::vector get_device_bdf(int dev) { + char busIdStr[] = "0000:00:00.0"; + std::vector bdf(sizeof(busIdStr), 0); + CUDACHECK(hipDeviceGetPCIBusId((char*)bdf.data(), sizeof(busIdStr), dev)); + bdf.resize(bdf.size() - 1); // remove trailing NULL + return bdf; +} diff --git a/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh b/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh new file mode 100644 index 0000000..7baf5f0 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh @@ -0,0 +1,582 @@ +// !!! This is a file automatically generated by hipify!!! +#pragma once + +#include +#ifdef USE_ROCM +#include +typedef __hip_bfloat16 nv_bfloat16; +#else +#include +#endif +#include +#include + +#include +#include +#include +#include +#include + +#define CUDACHECK(cmd) \ + do { \ + hipError_t e = cmd; \ + if (e != hipSuccess) { \ + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, hipGetErrorString(e)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + +namespace vllm { + +constexpr int kMaxBlocks = 64; +// note: we don't want to use atomics for signals because peer atomics are no +// supported on PCIe links +struct Signal { + alignas(128) uint32_t start[kMaxBlocks][8]; + alignas(128) uint32_t end[kMaxBlocks][8]; + alignas(128) uint32_t _flag[kMaxBlocks]; // incremental flags for each rank +}; + +#ifdef USE_ROCM +struct __align__(16) RankData { + const void* ptrs[8]; +}; +#else +struct __align__(16) RankData { + const void* __restrict__ ptrs[8]; +}; +#endif + +struct __align__(16) RankSignals { +#ifndef USE_ROCM + volatile +#endif + Signal* signals[8]; +}; + +// like std::array, but aligned +template +struct __align__(alignof(T) * sz) array_t { + T data[sz]; + using type = T; + static constexpr int size = sz; +}; + +// use packed type to maximize memory efficiency +// goal: generate ld.128 and st.128 instructions +template +struct packed_t { + // the (P)acked type for load/store + using P = array_t; + // the (A)ccumulator type for reduction + using A = array_t; +}; + +#define DINLINE __device__ __forceinline__ + +// scalar cast functions +DINLINE float upcast_s(half val) { + return __half2float(val); +} + +template +DINLINE T downcast_s(float val); +template <> +DINLINE half downcast_s(float val) { + return __float2half(val); +} + +// scalar add functions +// for some reason when compiling with Pytorch, the + operator for half and +// bfloat is disabled so we call the intrinsics directly +DINLINE half& assign_add(half& a, half b) { + a = __hadd(a, b); + return a; +} +DINLINE float& assign_add(float& a, float b) { + return a += b; +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +DINLINE float upcast_s(nv_bfloat16 val) { + return __bfloat162float(val); +} +template <> +DINLINE nv_bfloat16 downcast_s(float val) { + return __float2bfloat16(val); +} +DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) { + a = __hadd(a, b); + return a; +} +#endif + +template +DINLINE array_t& packed_assign_add(array_t& a, array_t b) { +#pragma unroll + for (int i = 0; i < N; i++) { + assign_add(a.data[i], b.data[i]); + } + return a; +} + +template +DINLINE array_t upcast(array_t val) { + if constexpr (std::is_same::value) { + return val; + } else { + array_t out; +#pragma unroll + for (int i = 0; i < N; i++) { + out.data[i] = upcast_s(val.data[i]); + } + return out; + } +} + +template +DINLINE O downcast(array_t val) { + if constexpr (std::is_same::value) { + return val; + } else { + O out; +#pragma unroll + for (int i = 0; i < O::size; i++) { + out.data[i] = downcast_s(val.data[i]); + } + return out; + } +} + +// This function is meant to be used as the first synchronization in the all +// reduce kernel. Thus, it doesn't need to make any visibility guarantees for +// prior memory accesses. Note: volatile writes will not be reordered against +// other volatile writes. +template +DINLINE void start_sync( + const RankSignals& sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + int rank) { +#ifdef USE_ROCM + uint32_t flag = self_sg->_flag[blockIdx.x] + 1; + if (threadIdx.x < ngpus) { + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + __scoped_atomic_store_n( + &sg.signals[threadIdx.x]->start[blockIdx.x][rank], flag, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + // wait until we got true from all ranks + while (__scoped_atomic_load_n(&self_sg->start[blockIdx.x][threadIdx.x], __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE) < + flag) + ; + } + __syncthreads(); + // use one thread to update flag + if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag; +#else + if (threadIdx.x < ngpus) { + // reset flag for next time + self_sg->end[blockIdx.x][threadIdx.x] = 0; + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + sg.signals[threadIdx.x]->start[blockIdx.x][rank] = 1; + // wait until we got true from all ranks + while (!self_sg->start[blockIdx.x][threadIdx.x]) + ; + } + __syncthreads(); +#endif +} + +// This function is meant to be used as the second or the final synchronization +// barrier in the all reduce kernel. If it's the final synchronization barrier, +// we don't need to make any visibility guarantees for prior memory accesses. +template +DINLINE void end_sync( + const RankSignals& sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + int rank) { +#ifdef USE_ROCM + __syncthreads(); + // eliminate the case that prior writes are not visible after signals become + // visible. Note that I did not managed to make this happen through a lot of + // testing. Might be the case that hardware provides stronger guarantee than + // the memory model. + uint32_t flag = self_sg->_flag[blockIdx.x] + 1; + if (threadIdx.x < ngpus) { + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + __scoped_atomic_store_n( + &sg.signals[threadIdx.x]->end[blockIdx.x][rank], + flag, + final_sync ? __ATOMIC_RELAXED : __ATOMIC_RELEASE, + __MEMORY_SCOPE_SYSTEM); + // wait until we got true from all ranks + while (__scoped_atomic_load_n( + &self_sg->end[blockIdx.x][threadIdx.x], + final_sync ? __ATOMIC_RELAXED : __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE) < flag) + ; + } + __syncthreads(); + // use one thread to update flag + if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag; +#else + __syncthreads(); + // eliminate the case that prior writes are not visible after signals become + // visible. Note that I did not managed to make this happen through a lot of + // testing. Might be the case that hardware provides stronger guarantee than + // the memory model. + if constexpr (!final_sync) __threadfence_system(); + if (threadIdx.x < ngpus) { + // reset flag for next time + self_sg->start[blockIdx.x][threadIdx.x] = 0; + // simultaneously write to the corresponding flag of all ranks. + // Latency = 1 p2p write + sg.signals[threadIdx.x]->end[blockIdx.x][rank] = 1; + // wait until we got true from all ranks + while (!self_sg->end[blockIdx.x][threadIdx.x]) + ; + } + if constexpr (!final_sync) __syncthreads(); +#endif +} + +template +DINLINE P packed_reduce(const P* ptrs[], int idx) { + A tmp = upcast(ptrs[0][idx]); +#pragma unroll + for (int i = 1; i < ngpus; i++) { + packed_assign_add(tmp, upcast(ptrs[i][idx])); + } + return downcast

(tmp); +} + +template +__global__ void __launch_bounds__(512, 1) cross_device_reduce_1stage( + RankData* _dp, + RankSignals sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + T* __restrict__ result, + int rank, + int size) { + using P = typename packed_t::P; + using A = typename packed_t::A; + // note: we don't reorder the address so the accumulation order is the same + // for all ranks, ensuring bitwise identical results + auto dp = *_dp; + start_sync(sg, self_sg, rank); + // do the actual reduction + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size; idx += gridDim.x * blockDim.x) { + ((P*)result)[idx] = packed_reduce((const P**)&dp.ptrs[0], idx); + } + end_sync(sg, self_sg, rank); +} + +template +#ifdef USE_ROCM +DINLINE P* get_tmp_buf(Signal* sg) { +#else +DINLINE P* get_tmp_buf(volatile Signal* sg) { +#endif + return (P*)(((Signal*)sg) + 1); +} + +template +__global__ void __launch_bounds__(512, 1) cross_device_reduce_2stage( + RankData* _dp, + RankSignals sg, +#ifndef USE_ROCM + volatile +#endif + Signal* self_sg, + T* __restrict__ result, + int rank, + int size) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int stride = gridDim.x * blockDim.x; + using P = typename packed_t::P; + using A = typename packed_t::A; + int part = size / ngpus; + int start = rank * part; + int end = rank == ngpus - 1 ? size : start + part; + int largest_part = part + size % ngpus; + const P* ptrs[ngpus]; + P* tmps[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int target = (rank + i) % ngpus; + ptrs[i] = (const P*)_dp->ptrs[target]; + tmps[i] = get_tmp_buf

(sg.signals[target]); + } + auto tmp_out = tmps[0]; + start_sync(sg, self_sg, rank); + // stage 1: reduce scatter + for (int idx = start + tid; idx < end; idx += stride) { + tmp_out[idx - start] = packed_reduce(ptrs, idx); + } + end_sync(sg, self_sg, rank); + + // stage 2: allgather. Note: it's important to match the tid between + // the two stages, because visibility across devices is only guaranteed + // between threads that have the same tid. If thread i computes the sum of + // start + i in the first stage, then thread i also gathers start + i from all + // ranks. + for (int idx = tid; idx < largest_part; idx += stride) { +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int gather_from_rank = ((rank + i) % ngpus); + if (gather_from_rank == ngpus - 1 || idx < part) { + int dst_idx = gather_from_rank * part + idx; + ((P*)result)[dst_idx] = tmps[i][idx]; + } + } + } +} + +using IPC_KEY = std::array; +static_assert(sizeof(IPC_KEY) == sizeof(hipIpcMemHandle_t)); +static_assert(alignof(IPC_KEY) == alignof(hipIpcMemHandle_t)); + +class CustomAllreduce { + public: + int rank_; + int world_size_; + bool full_nvlink_; + + // below are device pointers + RankSignals sg_; + std::unordered_map buffers_; + Signal* self_sg_; + + // stores the registered device pointers from all ranks + RankData *d_rank_data_base_, *d_rank_data_end_; + std::vector graph_unreg_buffers_; + // a map from IPC handles to opened IPC pointers + std::map ipc_handles_; + + /** + * meta is a pointer to device metadata and temporary buffer for allreduce. + * + * There's a total of sizeof(Signal) of prefix before the actual data, + * so meta + 1 points to actual temporary buffer. + * + * note: this class does not own any device memory. Any required buffers + * are passed in from the constructor + */ + CustomAllreduce( + Signal* meta, + void* rank_data, + size_t rank_data_sz, + const hipIpcMemHandle_t* handles, + const std::vector& offsets, + int rank, + bool full_nvlink = true) + : rank_(rank), + world_size_(offsets.size()), + full_nvlink_(full_nvlink), + self_sg_(meta), + d_rank_data_base_(reinterpret_cast(rank_data)), + d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) { + for (int i = 0; i < world_size_; i++) { + Signal* rank_sg; + if (i != rank_) { + char* handle = open_ipc_handle(&handles[i]); + handle += offsets[i]; + rank_sg = (Signal*)handle; + } else { + rank_sg = self_sg_; + } + sg_.signals[i] = rank_sg; + } + } + + char* open_ipc_handle(const void* ipc_handle) { + auto [it, new_handle] = ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr}); + if (new_handle) { + char* ipc_ptr; + CUDACHECK(hipIpcOpenMemHandle( + (void**)&ipc_ptr, *((const hipIpcMemHandle_t*)ipc_handle), hipIpcMemLazyEnablePeerAccess)); + it->second = ipc_ptr; + } + return it->second; + } + + std::pair, std::vector> get_graph_buffer_ipc_meta() { + auto num_buffers = graph_unreg_buffers_.size(); + auto handle_sz = sizeof(hipIpcMemHandle_t); + std::vector handles(handle_sz * num_buffers, 0); + std::vector offsets(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto ptr = graph_unreg_buffers_[i]; + void* base_ptr; + // note: must share the base address of each allocation, or we get wrong + // address + if (hipPointerGetAttribute( + &base_ptr, +#ifdef USE_ROCM + HIP_POINTER_ATTRIBUTE_RANGE_START_ADDR, +#else + CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, +#endif + (hipDeviceptr_t)ptr) != hipSuccess) + throw std::runtime_error("failed to get pointer attr"); + CUDACHECK(hipIpcGetMemHandle((hipIpcMemHandle_t*)&handles[i * handle_sz], base_ptr)); + offsets[i] = ((char*)ptr) - ((char*)base_ptr); + } + return std::make_pair(handles, offsets); + } + + void check_rank_data_capacity(size_t num = 1) { + if (d_rank_data_base_ + num > d_rank_data_end_) + throw std::runtime_error( + "Rank data buffer is overflowed by " + std::to_string(d_rank_data_base_ + num - d_rank_data_end_)); + } + + void register_buffer(const std::vector& handles, const std::vector& offsets, void* self) { + check_rank_data_capacity(); + RankData data; + for (int i = 0; i < world_size_; i++) { + if (i != rank_) { + char* handle = open_ipc_handle(handles[i].data()); + handle += offsets[i]; + data.ptrs[i] = handle; + } else { + data.ptrs[i] = self; + } + } + auto d_data = d_rank_data_base_++; + CUDACHECK(hipMemcpy(d_data, &data, sizeof(RankData), hipMemcpyHostToDevice)); + buffers_[self] = d_data; + } + + // note: when registering graph buffers, we intentionally choose to not + // deduplicate the addresses. That means if the allocator reuses some + // addresses, they will be registered again. This is to account for the remote + // possibility of different allocation patterns between ranks. For example, + // rank 1 may get the same input address for the second allreduce, but rank 2 + // got a different address. IPC handles have internal reference counting + // mechanism so overhead should be small. + void + register_graph_buffers(const std::vector& handles, const std::vector>& offsets) { + auto num_buffers = graph_unreg_buffers_.size(); + check_rank_data_capacity(num_buffers); + std::vector rank_data(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto self_ptr = graph_unreg_buffers_[i]; + auto& rd = rank_data[i]; + for (int j = 0; j < world_size_; j++) { + if (j != rank_) { + char* handle = open_ipc_handle(&handles[j][i * sizeof(hipIpcMemHandle_t)]); + handle += offsets[j][i]; + rd.ptrs[j] = handle; + } else { + rd.ptrs[j] = self_ptr; + } + } + } + CUDACHECK(hipMemcpy(d_rank_data_base_, rank_data.data(), sizeof(RankData) * num_buffers, hipMemcpyHostToDevice)); + d_rank_data_base_ += num_buffers; + graph_unreg_buffers_.clear(); + } + + /** + * This is the result after careful grid search. Using 36 blocks give the best + * or close to the best runtime on the devices I tried: A100, A10, A30, T4, + * V100. You'll notice that NCCL kernels also only take a small amount of SMs. + * Not quite sure the underlying reason, but my guess is that too many SMs + * will cause contention on NVLink bus. + */ + template + void allreduce( + hipStream_t stream, + T* input, + T* output, + int size, +#ifndef USE_ROCM + int threads = 512, + int block_limit = 36){ +#else + int threads = 512, + int block_limit = 16) { +#endif + auto d = packed_t::P::size; + if (size % d != 0) + throw std::runtime_error( + "custom allreduce currently requires input length to be multiple " + "of " + + std::to_string(d)); + if (block_limit > kMaxBlocks) + throw std::runtime_error( + "max supported block limit is " + std::to_string(kMaxBlocks) + ". Got " + std::to_string(block_limit)); + + RankData* ptrs; + hipStreamCaptureStatus status; + CUDACHECK(hipStreamIsCapturing(stream, &status)); + if (status == hipStreamCaptureStatusActive) { + ptrs = d_rank_data_base_ + graph_unreg_buffers_.size(); + graph_unreg_buffers_.push_back(input); + } else { + auto it = buffers_.find(input); + if (it == buffers_.end()) + throw std::runtime_error( + "buffer address " + std::to_string(reinterpret_cast(input)) + " is not registered!"); + ptrs = it->second; + } + + size /= d; + auto bytes = size * sizeof(typename packed_t::P); + int blocks = ::min(block_limit, (size + threads - 1) / threads); +#define KL(ngpus, name) \ + hipLaunchKernelGGL( \ + (name), dim3(blocks), dim3(threads), 0, stream, ptrs, sg_, self_sg_, output, rank_, size); +#define REDUCE_CASE(ngpus) \ + case ngpus: { \ + if (world_size_ == 2) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else if (full_nvlink_) { \ + if ((world_size_ <= 4 && bytes < 512 * 1024) || (world_size_ <= 8 && bytes < 256 * 1024)) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else { \ + KL(ngpus, cross_device_reduce_2stage); \ + } \ + } \ + break; \ + } + + switch (world_size_) { + REDUCE_CASE(2) + REDUCE_CASE(4) + REDUCE_CASE(6) + REDUCE_CASE(8) + default: + throw std::runtime_error( + "custom allreduce only supports num gpus in (2,4,6,8). Actual num " + "gpus = " + + std::to_string(world_size_)); + } +#undef REDUCE_CASE +#undef KL +} + +~CustomAllreduce() { + for (auto [_, ptr] : ipc_handles_) { + CUDACHECK(hipIpcCloseMemHandle(ptr)); + } +} +}; // namespace vllm +/** + * To inspect PTX/SASS, copy paste this header file to compiler explorer and add + a template instantiation: + * template void vllm::CustomAllreduce::allreduce(hipStream_t, half *, + half *, int, int, int); +*/ +} // namespace vllm diff --git a/sgl-kernel/csrc/allreduce/trt_reduce_internal.cu b/sgl-kernel/csrc/allreduce/trt_reduce_internal.cu new file mode 100644 index 0000000..283e1e8 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/trt_reduce_internal.cu @@ -0,0 +1,532 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// reference: +// https://github.com/NVIDIA/TensorRT-LLM/blob/release/0.14/cpp/tensorrt_llm/kernels/customAllReduceKernels.cu +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include "trt_reduce_internal.cuh" +#include "utils.h" + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ void st_flag_release(uint32_t const& flag, uint32_t* flag_addr) { + asm volatile("st.global.release.sys.b32 [%1], %0;" ::"r"(flag), "l"(flag_addr)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t ld_flag_acquire(uint32_t* flag_addr) { + uint32_t flag; + asm volatile("ld.global.acquire.sys.b32 %0, [%1];" : "=r"(flag) : "l"(flag_addr)); + return flag; +} + +static inline __device__ void st_flag_volatile(uint32_t const& flag, uint32_t* flag_addr) { + asm volatile("st.volatile.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr)); +} + +static inline __device__ uint32_t ld_flag_volatile(uint32_t* flag_addr) { + uint32_t flag; + asm volatile("ld.volatile.global.u32 %0, [%1];" : "=r"(flag) : "l"(flag_addr)); + return flag; +} + +namespace trt_llm { +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// Type Converter that packs data format to 128 bits data type +// +using PackedFloat = union { + int4 packed; + float unpacked[4]; +}; + +using PackedHalf = union { + int4 packed; + half2 unpacked[4]; +}; + +template +struct PackedOn16Bytes {}; + +template <> +struct PackedOn16Bytes { + using Type = PackedFloat; +}; + +template <> +struct PackedOn16Bytes { + using Type = PackedHalf; +}; + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +using PackedBFloat16 = union { + int4 packed; + __nv_bfloat162 unpacked[4]; +}; + +template <> +struct PackedOn16Bytes<__nv_bfloat16> { + using Type = PackedBFloat16; +}; +#endif + +// add two 128b data +template +inline __device__ int4 add128b(T& a, T& b) { + T c; + c.unpacked[0] = a.unpacked[0] + b.unpacked[0]; + c.unpacked[1] = a.unpacked[1] + b.unpacked[1]; + c.unpacked[2] = a.unpacked[2] + b.unpacked[2]; + c.unpacked[3] = a.unpacked[3] + b.unpacked[3]; + return c.packed; +} + +__inline__ __device__ void multi_gpu_barrier( + uint32_t** signals, + uint32_t const flag, + size_t const local_rank, + size_t const world_size, + int const tidx, + int const bidx) { + // After this function, at least one block in each GPU has reached the barrier + if (tidx < world_size) { + // we can think of signals having the shape [world_size, world_size] + // Dimension 0 is the "listening" dimension, dimension 1 is "emitting" dimension + + // Block 0 broadcasts its flag (local_rank on emitting dimension) to all receivers + size_t offset = (flag % 2) ? world_size : 0; + + if (bidx == 0) { + st_flag_release(flag, signals[tidx] + offset + local_rank); + } + + // All blocks check that corresponding block 0 on other GPUs have set the flag + // No deadlock because block #0 is always the first block started + uint32_t* peer_barrier_d = signals[local_rank] + offset + tidx; + while (ld_flag_acquire(peer_barrier_d) != flag) { + } + } + + __syncthreads(); +} + +template +__inline__ __device__ void block_barrier( + uint32_t** signals, + uint32_t const flag, + size_t const local_rank, + size_t const world_size, + int const tidx, + int const bidx, + int const grid_size) { + if constexpr (!start) { + __syncthreads(); + } + // After this function, the block of id == bidx of each GPU has reached the barrier + if (tidx < world_size) { + // we can think of signals having the shape [world_size, 2, num_blocks, world_size] + // (+ an offset on dim 2 to account for flags used in multi_gpu_barrier) + // Dimension 0 is the "listening" dimension, dimension 3 is "emitting" dimension + + // Block broadcast its flag (local_rank on emitting dimension) to all receivers + uint32_t flag_block_offset = world_size + bidx * world_size; + + flag_block_offset += (grid_size + 1) * world_size * (flag % 2); + + uint32_t* peer_barrier_d = signals[local_rank] + flag_block_offset + tidx; + // Blocks check that corresponding blocks on other GPUs have also set the flag + if constexpr (need_fence) { + st_flag_release(flag, signals[tidx] + flag_block_offset + local_rank); + while (ld_flag_acquire(peer_barrier_d) != flag) { + } + } else { + st_flag_volatile(flag, signals[tidx] + flag_block_offset + local_rank); + while (ld_flag_volatile(peer_barrier_d) != flag) { + } + } + } + if constexpr (start || need_fence) { + __syncthreads(); + } +} + +template +static __global__ void __launch_bounds__(512, 1) oneShotAllReduceKernel(AllReduceParams params) { + // Suppose that two GPUs participate in the AR exchange, and we start four blocks. + // The message is partitioned into chunks as detailed below: + // message + // |-------------------| + // GPU 0 | B0 | B1 | B2 | B3 | + // GPU 1 | B0 | B1 | B2 | B3 | + // + // Here the step-by-step behavior of one block: + // 1. B0 copies the chunk it is responsible for, from local_input to shareable buffer + // 2. B0 on GPU 0 and B0 on GPU 1 wait for each other (block_barrier) + // 3. B0 on GPU 0 pull and sum the chunk from GPU 1, writes the result to local_output + // + // With COPY_INPUT == false, skip step 1. and use gpu_barrier instead of block barrier during step 2. + // We only to know if the other GPU as arrived at the AR kernel, that would mean that data is ready + // + // With PUSH_MODE, we consider that the shared buffer is of size: + // params.peer_comm_buffer_ptrs: [world_size, world_size, message_size] + // + // Here the step-by-step behavior of one block: + // 1. B0 push the chunk is it responsible for into all other GPUs: + // params.peer_comm_buffer_ptrs[:, local_gpu, B0 slice] + // 2. block sync so the block is shared by other GPUs + // 3. Reduce along second dimension params.peer_comm_buffer_ptrs[local_gpu, :, B0 slice] + + int const bidx = blockIdx.x; + int const tidx = threadIdx.x; + int const grid_size = gridDim.x; + + // The number of elements packed into one for comms + static constexpr int NUM_ELTS = 16 / sizeof(T); + + // Packed data type for comms + using PackedStruct = typename PackedOn16Bytes::Type; + + // The source pointers. Distributed round-robin for the different warps. + auto peer_comm_buffer_ptrs = params.peer_comm_buffer_ptrs->ptrs; + T* local_shared_buffer = reinterpret_cast(peer_comm_buffer_ptrs[params.local_rank]); + // Start and end offsets of the thread + size_t chunk_start = bidx * params.elts_per_block + tidx * NUM_ELTS; + size_t chunk_end = std::min((bidx + 1) * params.elts_per_block, params.elts_per_rank); + + if constexpr (COPY_INPUT) { + T const* local_input_buffer = reinterpret_cast(params.local_input_buffer_ptr); + // Copy from local buffer to shareable buffer + for (size_t iter_offset = chunk_start; iter_offset < chunk_end; iter_offset += blockDim.x * NUM_ELTS) { + *reinterpret_cast(&local_shared_buffer[iter_offset]) = + *reinterpret_cast(&local_input_buffer[iter_offset]); + } + } + // wait for equivalent blocks of other GPUs to have copied data to their shareable buffer + block_barrier( + params.peer_barrier_ptrs_in, params.barrier_flag, params.local_rank, RANKS_PER_NODE, tidx, bidx, grid_size); + + // Each block accumulates the values from the different GPUs on the same node. + for (size_t iter_offset = chunk_start; iter_offset < chunk_end; iter_offset += blockDim.x * NUM_ELTS) { + // Iterate over the different ranks/devices on the node to load the values. + PackedStruct vals[RANKS_PER_NODE]; +#pragma unroll + for (int ii = 0; ii < RANKS_PER_NODE; ++ii) { + vals[ii].packed = *reinterpret_cast(&((T*)peer_comm_buffer_ptrs[ii])[iter_offset]); + } + + // Sum the values from the different ranks. + PackedStruct sums; + sums.packed = {0, 0, 0, 0}; +#pragma unroll + for (int rank = 0; rank < RANKS_PER_NODE; ++rank) { + // Always reduce from rank 0 to ensure stable reduce order. + sums.packed = add128b(sums, vals[rank]); + } + + // Store to the destination buffer. + *reinterpret_cast(&reinterpret_cast(params.local_output_buffer_ptr)[iter_offset]) = sums.packed; + } + block_barrier( + params.peer_barrier_ptrs_out, params.barrier_flag, params.local_rank, RANKS_PER_NODE, tidx, bidx, grid_size); +} + +template +static __global__ void __launch_bounds__(512, 1) twoShotAllReduceKernel(AllReduceParams params) { + // Suppose that two GPUs participate in the AR exchange, and we start two blocks. + // The message is partitioned into chunks as detailed below: + // message + // |-------------------| + // |--GPU 0--|--GPU 1--| (GPU responsibility parts) + // GPU 0 | B0 | B1 | B0 | B1 | + // GPU 1 | B0 | B1 | B0 | B1 | + // + // Here the step-by-step behavior of one block: + // 1. B0 copies all chunks is it responsible for, from local_input to shareable buffer + // 2. B0 on GPU 0 and B0 on GPU 1 wait for each other (block_barrier #0) + // 3. B0 on GPU 0 gather and sum the B0 chunks from GPU 1, that are in the GPU 0 responsibility + // part (the first half of the message, see GPU responsibility row above) + // 3bis. Likewise, B0 on GPU 1 copies and sum the chunks for GPU 0, + // where GPU 1 is responsible: the second half of the message. + // 4. B0 on GPU 0 and B0 on GPU 1 wait for each other (block_barrier #1) + // 5. B0 writes result to local_output. It gathers each chunk from its responsible GPU. + // For example, here it reads the first chunk from GPU 0 and second chunk from GPU 1. + // + // With COPY_INPUT == false, skip step 1. and use gpu_barrier instead of block barrier during step 2. + // We only to know if the other GPU as arrived at the AR kernel, that would mean that data is ready + // to be read. + // + // Note that compared to one-shot, one block (CTA) writes multiple input chunks and write multiple output chunks. + // However, it's only responsible for the summation of a single chunk. + // + // With PUSH_MODE, we consider that the shared buffer is of size: + // params.peer_comm_buffer_ptrs: [world_size, world_size, message_size / world_size] + // + // Here the step-by-step behavior of one block: + // 1. B0 push the chunks is it responsible for into the corresponding GPUs: + // params.peer_comm_buffer_ptrs[target_gpu, local_gpu, current B0 slice] + // 2. block sync so the blocks have been shared by other GPUs + // 3. Reduce along second dimension params.peer_comm_buffer_ptrs[local_gpu, :, B0 slice] + // 4. block barrier (corresponding blocks have finished reduction) + // 5. pull and write on local buffer, by reading params.peer_comm_buffer_ptrs[:, 0, B0 slice] (reduction result is + // written at index 0 of 2nd dim) + + int const bidx = blockIdx.x; + int const tidx = threadIdx.x; + int const grid_size = gridDim.x; + + // The number of elements packed into one for comms + static constexpr int PACKED_ELTS = 16 / sizeof(T); + using PackedType = typename PackedOn16Bytes::Type; + + T const* local_input_buffer = reinterpret_cast(params.local_input_buffer_ptr); + auto peer_comm_buffer_ptrs = params.peer_comm_buffer_ptrs->ptrs; + T* local_shared_buffer = reinterpret_cast(peer_comm_buffer_ptrs[params.local_rank]); + T* local_output_buffer = reinterpret_cast(params.local_output_buffer_ptr); + + size_t const chunk_start = bidx * params.elts_per_block + tidx * PACKED_ELTS; + size_t const chunk_end = min(chunk_start + params.elts_per_block, params.elts_per_rank); + + T* buffers[RANKS_PER_NODE]; + T* buffers_unorder[RANKS_PER_NODE]; + int ranks[RANKS_PER_NODE]; +#pragma unroll + for (int ii = 0; ii < RANKS_PER_NODE; ++ii) { + // A mapping of the ranks to scatter reads as much as possible + int rank = (params.local_rank + ii) % RANKS_PER_NODE; + ranks[ii] = rank; + buffers[ii] = reinterpret_cast(peer_comm_buffer_ptrs[rank]); + buffers_unorder[ii] = reinterpret_cast(peer_comm_buffer_ptrs[ii]); + } + +#if (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12)) +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); +#endif +#endif + + if constexpr (COPY_INPUT) { + // Copy all blocks from local buffer to shareable buffer + for (size_t local_offset = chunk_start; local_offset < chunk_end; local_offset += blockDim.x * PACKED_ELTS) { +#pragma unroll + for (int ii = 0; ii < RANKS_PER_NODE; ++ii) { + size_t offset_rank = ranks[ii] * params.elts_per_rank + local_offset; + if (offset_rank >= params.elts_total) { + continue; + } + *reinterpret_cast(&local_shared_buffer[offset_rank]) = + *reinterpret_cast(&local_input_buffer[offset_rank]); + } + } + } + block_barrier( + params.peer_barrier_ptrs_in, params.barrier_flag, params.local_rank, RANKS_PER_NODE, tidx, bidx, grid_size); + + // Each block accumulates the values from the different GPUs on the same node. + for (size_t local_offset = chunk_start; local_offset < chunk_end; local_offset += blockDim.x * PACKED_ELTS) { + size_t const responsible_block_offset = local_offset + params.rank_offset; + + // Iterate over the different ranks/devices on the node to load the values. + PackedType vals[RANKS_PER_NODE]; +#pragma unroll + for (int ii = 0; ii < RANKS_PER_NODE; ++ii) { + vals[ii].packed = *reinterpret_cast(&buffers_unorder[ii][responsible_block_offset]); + } + + // Sum the values from the different ranks. + PackedType sums; + sums.packed = {0, 0, 0, 0}; +#pragma unroll + for (int rank = 0; rank < RANKS_PER_NODE; ++rank) { + // Always reduce from rank 0 to ensure stable reduce order. + sums.packed = add128b(sums, vals[rank]); + } + + // Store to the local buffer or tmp buffer + if constexpr (COPY_INPUT) { + *reinterpret_cast(&local_shared_buffer[responsible_block_offset]) = sums.packed; + } else { + *reinterpret_cast(¶ms.tmp_result_buffers[params.local_rank][responsible_block_offset]) = sums.packed; + } + } + + block_barrier( + params.peer_barrier_ptrs_out, params.barrier_flag, params.local_rank, RANKS_PER_NODE, tidx, bidx, grid_size); + + // Gather all needed elts from other intra-node ranks + for (size_t local_offset = chunk_start; local_offset < chunk_end; local_offset += blockDim.x * PACKED_ELTS) { +#pragma unroll + for (int ii = 0; ii < RANKS_PER_NODE; ++ii) { + // use round-robin gathering from other ranks + size_t offset_rank = ranks[ii] * params.elts_per_rank + local_offset; + if (offset_rank >= params.elts_total) { + continue; + } + if constexpr (COPY_INPUT) { + *reinterpret_cast(&local_output_buffer[offset_rank]) = + *reinterpret_cast(&buffers[ii][offset_rank]); + } else { + *reinterpret_cast(&local_output_buffer[offset_rank]) = + *reinterpret_cast(¶ms.tmp_result_buffers[ranks[ii]][offset_rank]); + } + } + } +#if (defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 12)) +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaTriggerProgrammaticLaunchCompletion(); +#endif +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline int divUp(int a, int b) { + return (a + b - 1) / b; +} + +inline int roundUp(int a, int n) { + return divUp(a, n) * n; +} + +std::tuple kernelLaunchConfig(AllReduceStrategyType algo, AllReduceParams& params, size_t elts_per_thread) { + int blocks_per_grid = 1, threads_per_block = DEFAULT_BLOCK_SIZE; + switch (algo) { + case AllReduceStrategyType::ONESHOT: { + assert(params.elts_total % elts_per_thread == 0); + size_t const total_threads = roundUp(params.elts_total / elts_per_thread, WARP_SIZE); + threads_per_block = std::min(DEFAULT_BLOCK_SIZE, total_threads); + blocks_per_grid = std::min(static_cast(MAX_ALL_REDUCE_BLOCKS), divUp(total_threads, threads_per_block)); + params.elts_per_block = roundUp(divUp(params.elts_total, blocks_per_grid), elts_per_thread); + params.elts_per_rank = params.elts_total; + break; + } + case AllReduceStrategyType::TWOSHOT: { + assert(params.elts_total % (elts_per_thread * params.ranks_per_node) == 0); + size_t const total_threads = roundUp(params.elts_total / (elts_per_thread * params.ranks_per_node), WARP_SIZE); + + threads_per_block = std::min(DEFAULT_BLOCK_SIZE, total_threads); + blocks_per_grid = std::min(static_cast(MAX_ALL_REDUCE_BLOCKS), divUp(total_threads, threads_per_block)); + params.elts_per_rank = params.elts_total / params.ranks_per_node; + params.rank_offset = params.local_rank * params.elts_per_rank; + params.elts_per_block = roundUp(divUp(params.elts_per_rank, blocks_per_grid), elts_per_thread); + break; + } + default: + assert(false && "Algorithm not supported here."); + } + + return std::make_tuple(blocks_per_grid, threads_per_block); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +void dispatchARKernels( + AllReduceStrategyType algo, + AllReduceParams& param, + int blocks_per_grid, + int threads_per_block, + cudaStream_t stream) { + switch (algo) { + case AllReduceStrategyType::ONESHOT: { + oneShotAllReduceKernel<<>>(param); + break; + } + case AllReduceStrategyType::TWOSHOT: { + twoShotAllReduceKernel<<>>(param); + break; + } + } +} + +template +void dispatchARKernelsCopyInput(AllReduceStrategyType strat, AllReduceParams& param, cudaStream_t stream) { + size_t elts_per_thread = 16 / sizeof(T); + auto [blocks_per_grid, threads_per_block] = kernelLaunchConfig(strat, param, elts_per_thread); + switch (param.ranks_per_node) { + case 2: + dispatchARKernels(strat, param, blocks_per_grid, threads_per_block, stream); + break; + case 4: + dispatchARKernels(strat, param, blocks_per_grid, threads_per_block, stream); + break; + case 6: + dispatchARKernels(strat, param, blocks_per_grid, threads_per_block, stream); + break; + case 8: + dispatchARKernels(strat, param, blocks_per_grid, threads_per_block, stream); + break; + default: + break; + } +} + +template +void invokeOneOrTwoShotAllReduceKernel(AllReduceParams& param, AllReduceStrategyType strat, cudaStream_t stream) { + if (param.is_capturing) { + dispatchARKernelsCopyInput(strat, param, stream); + } else { + dispatchARKernelsCopyInput(strat, param, stream); + } + CHECK_CUDA_SUCCESS(cudaGetLastError()); +} + +void trtCustomAllReduce( + AllReduceParams& params, at::ScalarType data_type, AllReduceStrategyType strat, cudaStream_t stream) { + if (params.elts_total == 0) { + return; + } + + switch (data_type) { + case at::ScalarType::Float: + invokeOneOrTwoShotAllReduceKernel(params, strat, stream); + break; + case at::ScalarType::Half: + invokeOneOrTwoShotAllReduceKernel(params, strat, stream); + break; +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case at::ScalarType::BFloat16: + invokeOneOrTwoShotAllReduceKernel<__nv_bfloat16>(params, strat, stream); + break; +#endif + default: + assert(false && "Unsupported data type"); + } +} +} // namespace trt_llm diff --git a/sgl-kernel/csrc/allreduce/trt_reduce_kernel.cu b/sgl-kernel/csrc/allreduce/trt_reduce_kernel.cu new file mode 100644 index 0000000..5c87925 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/trt_reduce_kernel.cu @@ -0,0 +1,226 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// reference: https://github.com/NVIDIA/TensorRT-LLM/blob/release/0.14/cpp/tensorrt_llm/kernels/customAllReduceKernels.h + +#include + +#include + +#include "trt_reduce_internal.cuh" +#include "utils.h" + +using namespace trt_llm; + +using fptr_t = int64_t; +using IPC_KEY = std::array; + +class AllReduceMeta { + public: + AllReduceMeta( + int64_t rank_id, + int64_t world_size, + torch::Tensor& rank_data, + const std::vector& buffers, + const std::vector& tmp_result_buffers, + const std::vector& barrier_in, + const std::vector& barrier_out) { + this->rank_id = (int)rank_id; + this->world_size = (int)world_size; + this->barrier_in = barrier_in; + this->barrier_out = barrier_out; + this->tmp_result_buffers = tmp_result_buffers; + + this->rank_data_base = reinterpret_cast(rank_data.data_ptr()); + RankData data; + for (int i = 0; i < world_size; i++) { + data.ptrs[i] = (void*)buffers[i]; + } + auto d_data = this->rank_data_base++; + CHECK_CUDA_SUCCESS(cudaMemcpy(d_data, &data, sizeof(RankData), cudaMemcpyHostToDevice)); + this->buffers = d_data; + } + + ~AllReduceMeta() { + for (auto [_, ptr] : ipc_handles_) { + CHECK_CUDA_SUCCESS(cudaIpcCloseMemHandle(ptr)); + } + } + + public: + int world_size; + int rank_id; + std::vector barrier_in; + std::vector barrier_out; + std::vector tmp_result_buffers; + int barrier_flag = 1; + RankData* buffers; + RankData* rank_data_base; + std::vector graph_unreg_buffers; + std::map ipc_handles_; +}; + +// Get the number of bits for a given data type. +inline int get_bits(at::ScalarType dtype) { + switch (dtype) { + case at::ScalarType::Float: + return 32; + case at::ScalarType::Half: + case at::ScalarType::BFloat16: + return 16; + default: + assert(false && "Unsupported data type"); + } +} + +// Check if customized all-reduce kernels can be applied. +inline bool CanApplyCustomAllReduce(int64_t num_elements, at::ScalarType dtype) { + // The customized all-reduce kernel has the following requirement(s). + return num_elements % (16 / ((get_bits(dtype) + 7) / 8)) == 0; +} + +fptr_t init_custom_ar( + int64_t rank_id, + int64_t world_size, + torch::Tensor& rank_data, + const std::vector& buffers, + const std::vector& tmp_result_buffers, + const std::vector& barrier_in, + const std::vector& barrier_out) { + auto m = new AllReduceMeta(rank_id, world_size, rank_data, buffers, tmp_result_buffers, barrier_in, barrier_out); + return (fptr_t)m; +} + +void dispose(fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + delete fa; +} + +std::tuple, std::vector> get_graph_buffer_ipc_meta(fptr_t _fa) { + AllReduceMeta* m = reinterpret_cast(_fa); + auto num_buffers = m->graph_unreg_buffers.size(); + auto handle_sz = sizeof(cudaIpcMemHandle_t); + std::string handles(handle_sz * num_buffers, static_cast(0)); + std::vector offsets(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto ptr = m->graph_unreg_buffers[i]; + void* base_ptr; + // note: must share the base address of each allocation, or we get wrong + // address + if (cuPointerGetAttribute(&base_ptr, CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, (CUdeviceptr)ptr) != CUDA_SUCCESS) { + assert(false && "failed to get pointer attr"); + } + + CHECK_CUDA_SUCCESS(cudaIpcGetMemHandle((cudaIpcMemHandle_t*)&handles[i * handle_sz], base_ptr)); + offsets[i] = ((char*)ptr) - ((char*)base_ptr); + } + std::vector bytes(handles.begin(), handles.end()); + return std::make_pair(bytes, offsets); +} + +char* open_ipc_handle(AllReduceMeta* meta, const void* ipc_handle) { + auto [it, new_handle] = meta->ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr}); + if (new_handle) { + char* ipc_ptr; + CHECK_CUDA_SUCCESS(cudaIpcOpenMemHandle( + (void**)&ipc_ptr, *((const cudaIpcMemHandle_t*)ipc_handle), cudaIpcMemLazyEnablePeerAccess)); + it->second = ipc_ptr; + } + return it->second; +} + +// Note: when registering graph buffers, we intentionally choose to not +// deduplicate the addresses. That means if the allocator reuses some +// addresses, they will be registered again. This is to account for the remote +// possibility of different allocation patterns between ranks. For example, +// rank 1 may get the same input address for the second allreduce, but rank 2 +// got a different address. IPC handles have internal reference counting +// mechanism so overhead should be small. +void register_graph_buffers( + fptr_t _fa, const std::vector>& handles, const std::vector>& offsets) { + AllReduceMeta* m = reinterpret_cast(_fa); + std::vector handle_bytes; + handle_bytes.reserve(handles.size()); + for (int i = 0; i < handles.size(); i++) { + handle_bytes.emplace_back(handles[i].begin(), handles[i].end()); + } + auto num_buffers = m->graph_unreg_buffers.size(); + std::vector rank_data(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto self_ptr = m->graph_unreg_buffers[i]; + auto& rd = rank_data[i]; + for (int j = 0; j < m->world_size; j++) { + if (j != m->rank_id) { + char* handle = open_ipc_handle(m, &handle_bytes[j][i * sizeof(cudaIpcMemHandle_t)]); + handle += offsets[j][i]; + rd.ptrs[j] = handle; + } else { + rd.ptrs[j] = self_ptr; + } + } + } + CHECK_CUDA_SUCCESS( + cudaMemcpy(m->rank_data_base, rank_data.data(), sizeof(RankData) * num_buffers, cudaMemcpyHostToDevice)); + m->rank_data_base += num_buffers; + m->graph_unreg_buffers.clear(); +} + +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out) { + AllReduceMeta* m = reinterpret_cast(_fa); + auto stream = c10::cuda::getCurrentCUDAStream().stream(); + auto num_elements = inp.numel(); + auto dtype = inp.scalar_type(); + AllReduceStrategyType strategy = SelectImplementation(num_elements * ((get_bits(dtype) + 7) / 8), m->world_size); + + // should be gurantee in python code + assert(strategy == AllReduceStrategyType::ONESHOT || strategy == AllReduceStrategyType::TWOSHOT); + assert(CanApplyCustomAllReduce(num_elements, dtype)); + + // Initialize the all-reduce kernel arguments. + int world_size = m->world_size; + + AllReduceParams params; + params.ranks_per_node = world_size; + params.rank = m->rank_id; + params.local_rank = m->rank_id; + params.local_input_buffer_ptr = inp.data_ptr(); + params.local_output_buffer_ptr = out.data_ptr(); + params.elts_total = inp.numel(); + params.elts_size = inp.element_size(); + params.barrier_flag = ++(m->barrier_flag); + + cudaStreamCaptureStatus status; + CHECK_CUDA_SUCCESS(cudaStreamIsCapturing(stream, &status)); + params.is_capturing = (status == cudaStreamCaptureStatusActive); + if (params.is_capturing) { + params.peer_comm_buffer_ptrs = m->rank_data_base + m->graph_unreg_buffers.size(); + m->graph_unreg_buffers.push_back(params.local_input_buffer_ptr); + } else { + params.peer_comm_buffer_ptrs = m->buffers; + } + + for (int i = 0; i < world_size; ++i) { + params.tmp_result_buffers[i] = reinterpret_cast(m->tmp_result_buffers[i]); + } + for (int i = 0; i < world_size; ++i) { + params.peer_barrier_ptrs_in[i] = reinterpret_cast(m->barrier_in[i]); + } + for (int i = 0; i < world_size; ++i) { + params.peer_barrier_ptrs_out[i] = reinterpret_cast(m->barrier_out[i]); + } + + auto data_type = out.scalar_type(); + trtCustomAllReduce(params, data_type, strategy, stream); +} diff --git a/sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu b/sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu new file mode 100644 index 0000000..01bd479 --- /dev/null +++ b/sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu @@ -0,0 +1,154 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#define THREADS_PER_BLOCK 128 + +template +__global__ void lightning_attention_decode_kernel( + const T* __restrict__ q, // [b, h, 1, d] + const T* __restrict__ k, // [b, h, 1, d] + const T* __restrict__ v, // [b, h, 1, e] + const float* __restrict__ past_kv, // [b, h, d, e] + const float* __restrict__ slope, // [h, 1, 1] + T* __restrict__ output, // [b, h, 1, e] + float* __restrict__ new_kv, // [b, h, d, e] + const int batch_size, + const int num_heads, + const int qk_dim, + const int v_dim) { + extern __shared__ char smem[]; + T* __restrict__ q_shared = reinterpret_cast(smem); + T* __restrict__ k_shared = reinterpret_cast(smem + qk_dim * sizeof(T)); + T* __restrict__ v_shared = reinterpret_cast(smem + 2 * qk_dim * sizeof(T)); + float* __restrict__ new_kv_shared = reinterpret_cast(smem + (2 * qk_dim + v_dim) * sizeof(T)); + T* __restrict__ output_shared = + reinterpret_cast(smem + (2 * qk_dim + v_dim) * sizeof(T) + qk_dim * (v_dim + 1) * sizeof(float)); + + const int32_t tid = threadIdx.x; + const int32_t current_head = blockIdx.x; + const int32_t b = current_head / num_heads; + const int32_t h = current_head % num_heads; + + if (b >= batch_size) return; + + const int32_t qk_offset = b * num_heads * qk_dim + h * qk_dim; + const int32_t v_offset = b * num_heads * v_dim + h * v_dim; + const int32_t kv_offset = b * num_heads * qk_dim * v_dim + h * qk_dim * v_dim; + + // Load q, k, v into shared memory + for (int d = tid; d < qk_dim; d += blockDim.x) { + q_shared[d] = q[qk_offset + d]; + k_shared[d] = k[qk_offset + d]; + } + for (int e = tid; e < v_dim; e += blockDim.x) { + v_shared[e] = v[v_offset + e]; + } + + __syncthreads(); + + const float ratio = expf(-1.0f * slope[h]); + + // Compute new_kv + for (int d = tid; d < qk_dim; d += blockDim.x) { + const T k_val = k_shared[d]; + for (int e = 0; e < v_dim; ++e) { + const int past_kv_idx = kv_offset + d * v_dim + e; + const T v_val = v_shared[e]; + const float new_val = ratio * past_kv[past_kv_idx] + k_val * v_val; + const int shared_idx = d * (v_dim + 1) + e; + new_kv_shared[shared_idx] = new_val; + } + } + + __syncthreads(); + + // Store new_kv to global memory + for (int idx = tid; idx < qk_dim * v_dim; idx += blockDim.x) { + const int d = idx / v_dim; + const int e = idx % v_dim; + const int shared_idx = d * (v_dim + 1) + e; + const int global_idx = kv_offset + idx; + new_kv[global_idx] = new_kv_shared[shared_idx]; + } + + __syncthreads(); + + // Compute output + for (int e = tid; e < v_dim; e += blockDim.x) { + float sum = 0.0f; + for (int d = 0; d < qk_dim; ++d) { + const int shared_idx = d * (v_dim + 1) + e; + sum += q_shared[d] * new_kv_shared[shared_idx]; + } + output_shared[e] = static_cast(sum); + } + + __syncthreads(); + + // Store output to global memory + if (tid == 0) { + for (int e = 0; e < v_dim; ++e) { + output[v_offset + e] = output_shared[e]; + } + } +} + +void lightning_attention_decode( + const torch::Tensor& q, + const torch::Tensor& k, + const torch::Tensor& v, + const torch::Tensor& past_kv, + const torch::Tensor& slope, + torch::Tensor output, + torch::Tensor new_kv) { + TORCH_CHECK(q.is_contiguous(), "q must be contiguous"); + TORCH_CHECK(k.is_contiguous(), "k must be contiguous"); + TORCH_CHECK(v.is_contiguous(), "v must be contiguous"); + TORCH_CHECK(past_kv.is_contiguous(), "past_kv must be contiguous"); + + auto batch_size = q.size(0); + auto num_heads = q.size(1); + auto qk_dim = q.size(3); + auto v_dim = v.size(3); + + dim3 block(THREADS_PER_BLOCK); + dim3 grid(batch_size * num_heads); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, q.scalar_type(), "lightning_attention_decode_kernel", ([&] { + size_t smem_size = (2 * qk_dim + 2 * v_dim) * sizeof(scalar_t) + qk_dim * (v_dim + 1) * sizeof(float); + lightning_attention_decode_kernel<<>>( + q.data_ptr(), + k.data_ptr(), + v.data_ptr(), + past_kv.data_ptr(), + slope.data_ptr(), + output.data_ptr(), + new_kv.data_ptr(), + batch_size, + num_heads, + qk_dim, + v_dim); + })); +} diff --git a/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h b/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h new file mode 100644 index 0000000..9f85bee --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h @@ -0,0 +1,309 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/epilogue/threadblock/epilogue_per_row_per_col_scale.h + +#pragma once + +#include +#include + +namespace cutlass { +namespace epilogue { +namespace threadblock { + +template < + typename ThreadblockShape_, + int ThreadCount, + typename ScaleTileIterator_, + typename OutputTileIterator_, + typename ElementAccumulator_, + typename ElementCompute_, + typename ElementwiseFunctor_, + bool UseMasking_ = false> +class EpilogueVisitorPerRowPerCol { + public: + using ThreadblockShape = ThreadblockShape_; + static int const kThreadCount = ThreadCount; + + using ScaleTileIterator = ScaleTileIterator_; + using OutputTileIterator = OutputTileIterator_; + using ElementwiseFunctor = ElementwiseFunctor_; + + static int const kIterations = OutputTileIterator::kIterations; + static int const kElementsPerAccess = OutputTileIterator::kElementsPerAccess; + + using ElementOutput = typename OutputTileIterator::Element; + using LayoutOutput = cutlass::layout::RowMajor; + using ElementAccumulator = ElementAccumulator_; + + using AlphaScaleElementType = typename ScaleTileIterator::Element; + + using ElementCompute = ElementCompute_; + using AccumulatorFragment = Array; + using ComputeFragment = Array; + using OutputVector = Array; + + static int const kThreadsPerRow = OutputTileIterator::ThreadMap::Detail::kAccessWidth; + static bool const kHasMultiStepsInRow = (OutputTileIterator::ThreadMap::Iterations::kColumn > 1); + + /// Argument structure + struct Arguments { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + Arguments() : batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments(typename ElementwiseFunctor::Params elementwise_) + : elementwise(elementwise_), batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments( + typename ElementwiseFunctor::Params elementwise_, + int64_t batch_stride_alpha_, + int64_t batch_stride_C_, + int64_t batch_stride_D_) + : elementwise(elementwise_), + batch_stride_alpha(batch_stride_alpha_), + batch_stride_C(batch_stride_C_), + batch_stride_D(batch_stride_D_) {} + }; + + struct Params { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + CUTLASS_HOST_DEVICE + Params() {} + + CUTLASS_HOST_DEVICE + Params(Arguments const& args) + : elementwise(args.elementwise), + batch_stride_alpha(args.batch_stride_alpha), + batch_stride_C(args.batch_stride_C), + batch_stride_D(args.batch_stride_D) {} + }; + + /// Shared storage + struct SharedStorage {}; + + private: + Params const& params_; + SharedStorage& shared_storage_; + MatrixCoord extent_; + MatrixCoord extent_real_; + ElementwiseFunctor elementwise_; + + bool const with_bias_; + bool const per_token_quant_; + bool const per_channel_quant_; + + AlphaScaleElementType* ptr_alpha_row_; + AlphaScaleElementType* ptr_alpha_col_; + ScaleTileIterator iterator_alpha_col_; + OutputTileIterator iterator_C_; + OutputTileIterator iterator_D_; + + AlphaScaleElementType element_alpha_row_ = 1.0f; + AlphaScaleElementType element_alpha_col_ = 1.0f; + typename ScaleTileIterator::Fragment fragment_alpha_col_; + typename OutputTileIterator::Fragment fragment_C_; + typename OutputTileIterator::Fragment fragment_D_; + + ElementAccumulator beta_; + + int column_offset_; + + MatrixCoord thread_offset_; + + public: + CUTLASS_DEVICE + EpilogueVisitorPerRowPerCol( + Params const& params, + SharedStorage& shared_storage, + cutlass::MatrixCoord const& problem_size, + int thread_idx, + int warp_idx, + int lane_idx, + typename ScaleTileIterator::Params params_alpha_col, + typename OutputTileIterator::Params params_C, + typename OutputTileIterator::Params params_D, + bool with_bias, + bool per_token_quant, + bool per_channel_quant, + AlphaScaleElementType* ptr_alpha_row, + AlphaScaleElementType* ptr_alpha_col, + typename OutputTileIterator::Element* ptr_C, + typename OutputTileIterator::Element* ptr_D, + cutlass::MatrixCoord const& threadblock_offset = cutlass::MatrixCoord(0, 0), + int column_offset = 0, + cutlass::MatrixCoord const& problem_size_real = cutlass::MatrixCoord(0, 0)) + : params_(params), + shared_storage_(shared_storage), + extent_(problem_size), + elementwise_(params.elementwise), + with_bias_(with_bias), + per_token_quant_(per_token_quant), + per_channel_quant_(per_channel_quant), + ptr_alpha_row_(ptr_alpha_row), + ptr_alpha_col_(ptr_alpha_col), + iterator_alpha_col_(params_alpha_col, ptr_alpha_col, problem_size, thread_idx, threadblock_offset), + iterator_C_(params_C, ptr_C, problem_size, thread_idx, threadblock_offset), + iterator_D_(params_D, ptr_D, problem_size, thread_idx, threadblock_offset), + extent_real_(problem_size_real) { + if (!per_channel_quant_ && (ptr_alpha_col_ != nullptr)) { + element_alpha_col_ = *ptr_alpha_col_; + } + + if (!per_token_quant_ && (ptr_alpha_row_ != nullptr)) { + element_alpha_row_ = *ptr_alpha_row_; + } + } + + /// Helper to indicate split-K behavior + CUTLASS_DEVICE + void set_k_partition( + int split_k_index, ///< Index of this threadblock within split-K partitioned scheme + int split_k_slices) { ///< Total number of split-K slices + } + + /// Called to set the batch index + CUTLASS_DEVICE + void set_batch_index(int batch_idx) { + iterator_alpha_col_.add_pointer_offset(batch_idx * params_.batch_stride_alpha); + iterator_C_.add_pointer_offset(batch_idx * params_.batch_stride_C); + iterator_D_.add_pointer_offset(batch_idx * params_.batch_stride_D); + } + + /// Called at the start of the epilogue just before iterating over accumulator slices + CUTLASS_DEVICE + void begin_epilogue() { + if (per_channel_quant_) { + iterator_alpha_col_.load(fragment_alpha_col_); + } + + if (with_bias_) { + iterator_C_.load(fragment_C_); + } + } + + /// Called at the start of one step before starting accumulator exchange + CUTLASS_DEVICE + void begin_step(int step_idx) { + fragment_D_.clear(); + } + + /// Called at the start of a row + CUTLASS_DEVICE + void begin_row(int row_idx) { + // load alpha_row in begin_step only when per token(row) scaling is used + if (per_token_quant_) { + int thread_offset_row = + iterator_D_.thread_start_row() + OutputTileIterator::ThreadMap::iteration_offset(row_idx).row(); + + arch::global_load( + element_alpha_row_, ptr_alpha_row_ + thread_offset_row, thread_offset_row < extent_.row()); + } + } + + /// Called after accumulators have been exchanged for each accumulator vector + CUTLASS_DEVICE + void visit(int iter_idx, int row_idx, int column_idx, int frag_idx, AccumulatorFragment const& accum) { + NumericArrayConverter source_converter; + + ComputeFragment result = source_converter(accum); + if (per_channel_quant_) { + ComputeFragment alpha_col = reinterpret_cast(&fragment_alpha_col_)[column_idx]; + result = per_token_channel_scale_accumulator_(result, alpha_col, element_alpha_row_); + } else { + result = per_token_scale_accumulator_(result, element_alpha_col_, element_alpha_row_); + } + + if (with_bias_) { + NumericArrayConverter bias_converter; + OutputVector bias = reinterpret_cast(&fragment_C_)[column_idx]; + result = bias_accumulator_(result, bias_converter(bias)); + } + + // Convert to the output + NumericArrayConverter output_converter; + OutputVector& output = reinterpret_cast(&fragment_D_)[frag_idx]; + output = output_converter(result); + } + + /// Called at the end of a row + CUTLASS_DEVICE + void end_row(int row_idx) {} + + /// Called after all accumulator elements have been visited + CUTLASS_DEVICE + void end_step(int step_idx) { + iterator_D_.store(fragment_D_); + ++iterator_D_; + } + + /// Called after all steps have been completed + CUTLASS_DEVICE + void end_epilogue() {} + + private: + CUTLASS_DEVICE + ComputeFragment per_token_channel_scale_accumulator_( + ComputeFragment const& accum, ComputeFragment const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col[i] * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment per_token_scale_accumulator_( + ComputeFragment const& accum, AlphaScaleElementType const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment bias_accumulator_(ComputeFragment const& accum, ComputeFragment const& bias) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < OutputVector::kElements; ++i) { + result[i] = accum[i] + bias[i]; + } + return result; + } +}; + +} // namespace threadblock +} // namespace epilogue +} // namespace cutlass diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder.hpp new file mode 100644 index 0000000..6f2c0ec --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder.hpp @@ -0,0 +1,125 @@ +// Adapt from +// https://github.com/vllm-project/vllm/blob/v0.7.1/csrc/cutlass_extensions/gemm/collective/collective_buildler.hpp +// Modified from: cutlass/gemm/collective/builders/sm90_gmma_builder.inl +// clang-format off +#pragma once + +#include +#include "cutlass_extensions/gemm/dispatch_policy.hpp" +#include "cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp" + + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +// GMMA_TMA_WS_SS (BlockScaled Builders) +template < + class ElementA, + class GmemLayoutATag, + int AlignmentA, + class ElementB, + class GmemLayoutBTag, + int AlignmentB, + class ElementAccumulator, + class TileShape_MNK, + class ClusterShape_MNK, + class StageCountType, + int ScaleGranularityM +> +struct CollectiveBuilder< + arch::Sm90, + arch::OpClassTensorOp, + ElementA, + GmemLayoutATag, + AlignmentA, + ElementB, + GmemLayoutBTag, + AlignmentB, + ElementAccumulator, + TileShape_MNK, + ClusterShape_MNK, + StageCountType, + KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum, + cute::enable_if_t< + not detail::is_use_rmem_A()> +> { + using KernelScheduleType = KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum; + + static_assert(is_static::value); + static_assert(is_static::value); +#ifndef CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED + static_assert(cutlass::detail::dependent_false, "Unsupported Toolkit for SM90 Collective Builder\n"); +#endif + static_assert(detail::is_aligned(), + "Should meet TMA alignment requirement\n"); + + static constexpr bool IsArrayOfPointersGemm = (cute::is_any_of_v); + static constexpr bool IsFP8Input = detail::is_input_fp8(); + static_assert((!IsFP8Input || !IsArrayOfPointersGemm), + "KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum is only compatible with FP8 Blocked Scaled version right now."); + + // For fp32 types, map to tf32 MMA value type + using ElementAMma = cute::conditional_t, tfloat32_t, ElementA>; + using ElementBMma = cute::conditional_t, tfloat32_t, ElementB>; + + static constexpr cute::GMMA::Major GmmaMajorA = detail::gmma_ss_tag_to_major_A(); + static constexpr cute::GMMA::Major GmmaMajorB = detail::gmma_ss_tag_to_major_B(); + + static constexpr bool IsCooperative = cute::is_any_of_v>; + using AtomLayoutMNK = cute::conditional_t>, Layout>>; + + using TiledMma = decltype(cute::make_tiled_mma(cute::GMMA::ss_op_selector< + ElementAMma, ElementBMma, ElementAccumulator, TileShape_MNK, GmmaMajorA, GmmaMajorB>(), AtomLayoutMNK{})); + + using GmemTiledCopyA = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{}))); + using GmemTiledCopyB = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{}))); + + using SmemLayoutAtomA = decltype(detail::ss_smem_selector< + GmmaMajorA, ElementAMma, decltype(cute::get<0>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>()); + using SmemLayoutAtomB = decltype(detail::ss_smem_selector< + GmmaMajorB, ElementBMma, decltype(cute::get<1>(TileShape_MNK{})), decltype(cute::get<2>(TileShape_MNK{}))>()); + + static constexpr size_t TensorMapStorage = IsArrayOfPointersGemm ? sizeof(cute::TmaDescriptor) * 2 /* for A and B */ : 0; + static constexpr int KernelSmemCarveout = static_cast(TensorMapStorage); + + static constexpr int PipelineStages = detail::compute_stage_count_or_override(StageCountType{}); + using DispatchPolicy = MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8; + + using SmemCopyAtomA = void; + using SmemCopyAtomB = void; + + using CollectiveOp = CollectiveMma< + DispatchPolicy, + TileShape_MNK, + ElementA, + TagToStrideA_t, + ElementB, + TagToStrideB_t, + TiledMma, + GmemTiledCopyA, + SmemLayoutAtomA, + SmemCopyAtomA, + cute::identity, + GmemTiledCopyB, + SmemLayoutAtomB, + SmemCopyAtomB, + cute::identity + >; +}; + + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp new file mode 100644 index 0000000..a8c3bda --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp @@ -0,0 +1,733 @@ +// clang-format off +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.1/csrc/cutlass_extensions/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp +// Adapted (Heavily) from: https://github.com/soundOfDestiny/cutlass/blob/9d997ce0dea4c5fa1a617db6b7ff29aa9235822c/include/cutlass/gemm/collective/sm90_mma_tma_gmma_ss_warpspecialized_fp8_blockwise_scaling.hpp + +/*************************************************************************************************** + * Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + + #pragma once + + #include "cutlass/cutlass.h" + #include "cutlass/gemm/dispatch_policy.hpp" + #include "cutlass/trace.h" + #include "cutlass/numeric_types.h" + + #include "cute/arch/cluster_sm90.hpp" + #include "cute/arch/copy_sm80.hpp" + #include "cute/arch/copy_sm90.hpp" + #include "cute/algorithm/functional.hpp" + #include "cute/atom/mma_atom.hpp" + #include "cute/algorithm/gemm.hpp" + #include "cute/tensor_predicate.hpp" + #include "cute/numeric/arithmetic_tuple.hpp" + + ///////////////////////////////////////////////////////////////////////////////////////////////// + + namespace cutlass::gemm::collective { + using namespace cute; + + ///////////////////////////////////////////////////////////////////////////////////////////////// + + // WarpSpecialized Mainloop + template < + int Stages, + class ClusterShape, + class KernelSchedule, + int ScaleGranularityM_, + class TileShape_, + class ElementA_, + class StrideA_, + class ElementB_, + class StrideB_, + class TiledMma_, + class GmemTiledCopyA_, + class SmemLayoutAtomA_, + class SmemCopyAtomA_, + class TransformA_, + class GmemTiledCopyB_, + class SmemLayoutAtomB_, + class SmemCopyAtomB_, + class TransformB_> + struct CollectiveMma< + MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8, + TileShape_, + ElementA_, + StrideA_, + ElementB_, + StrideB_, + TiledMma_, + GmemTiledCopyA_, + SmemLayoutAtomA_, + SmemCopyAtomA_, + TransformA_, + GmemTiledCopyB_, + SmemLayoutAtomB_, + SmemCopyAtomB_, + TransformB_> + { + // + // Type Aliases + // + using DispatchPolicy = MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8; + using TileShape = TileShape_; + using ElementA = ElementA_; + using StrideA = StrideA_; + using ElementB = ElementB_; + using StrideB = StrideB_; + using TiledMma = TiledMma_; + using ElementAccumulator = typename TiledMma::ValTypeC; + using ElementBlockScale = ElementAccumulator; + using GmemTiledCopyA = GmemTiledCopyA_; + using GmemTiledCopyB = GmemTiledCopyB_; + using SmemLayoutAtomA = SmemLayoutAtomA_; + using SmemLayoutAtomB = SmemLayoutAtomB_; + using SmemCopyAtomA = SmemCopyAtomA_; + using SmemCopyAtomB = SmemCopyAtomB_; + using TransformA = TransformA_; + using TransformB = TransformB_; + using ArchTag = typename DispatchPolicy::ArchTag; + + using CtaShape_MNK = decltype(shape_div(TileShape{}, ClusterShape{})); + using MainloopPipeline = cutlass::PipelineTmaAsync; + using PipelineState = cutlass::PipelineState; + using PipelineParams = typename MainloopPipeline::Params; + + // Two threads per CTA are producers (1 for operand tile and 32 for scales) + static constexpr int NumProducerThreadEvents = 33; + + static constexpr int ScaleGranularityM = ScaleGranularityM_ == 0 ? size<0>(TileShape{}) : ScaleGranularityM_; + static constexpr int ScaleMsPerTile = size<0>(TileShape{}) / ScaleGranularityM; + + static_assert(cute::rank(SmemLayoutAtomA{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)"); + static_assert((size<0>(TileShape{}) % size<0>(SmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + + static_assert(cute::rank(SmemLayoutAtomB{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)"); + static_assert((size<1>(TileShape{}) % size<0>(SmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + static_assert((size<2>(TileShape{}) % size<1>(SmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + + static_assert((size<0>(TileShape{}) % ScaleGranularityM) == 0, "FP8 scaling granularity must evenly divide tile shape along M."); + + // Tile along modes in a way that maximizes the TMA box size. + using SmemLayoutA = decltype(tile_to_shape( + SmemLayoutAtomA{}, + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{}), Int{}), + cute::conditional_t< ::cutlass::gemm::detail::is_major<0,StrideA>(), Step<_2,_1,_3>, Step<_1,_2,_3>>{})); + using SmemLayoutB = decltype(tile_to_shape( + SmemLayoutAtomB{}, + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{}), Int{}), + cute::conditional_t< ::cutlass::gemm::detail::is_major<0,StrideB>(), Step<_2,_1,_3>, Step<_1,_2,_3>>{})); + + // Block scaling gmem-to-smem copy atom + using SmemBlockScalingCopyAtomA = Copy_Atom, ElementBlockScale>; + using SmemBlockScalingCopyAtomB = Copy_Atom, ElementBlockScale>; + + // Block scaling smem layout + using SmemLayoutScaleA = Layout, Int>>; + using SmemLayoutScaleB = Layout>, Stride<_1>>; // `ScaleNsPerTile` is always 1. + + static_assert(DispatchPolicy::Stages >= 2, "Specialization requires Stages set to value 1 or more."); + static_assert(cute::is_base_of::value && + cute::is_base_of::value, + "MMA atom must source both A and B operand from smem_desc for this mainloop."); + static_assert(cute::is_same_v || cute::is_same_v, + "GmemTiledCopy - invalid SM90 TMA copy atom specified."); + static_assert(cute::is_same_v || cute::is_same_v, + "GmemTiledCopy - invalid SM90 TMA copy atom specified."); + static_assert(cute::is_same_v, + "ElementAccumulator and ElementBlockScale should be same datatype"); + + struct SharedStorage + { + struct TensorStorage : cute::aligned_struct<128> { + cute::array_aligned> smem_A; // mxk + cute::array_aligned> smem_B; // nxk + cute::array_aligned> smem_scale_A; // ScaleMsPerTile x k + cute::array_aligned> smem_scale_B; // 1xk + } tensors; + + using PipelineStorage = typename MainloopPipeline::SharedStorage; + PipelineStorage pipeline; + }; + using TensorStorage = typename SharedStorage::TensorStorage; + using PipelineStorage = typename SharedStorage::PipelineStorage; + + // Host side kernel arguments + struct Arguments { + ElementA const* ptr_A; + StrideA dA; + ElementB const* ptr_B; + StrideB dB; + uint32_t mma_promotion_interval = 4; + ElementBlockScale const* ptr_scale_A; + ElementBlockScale const* ptr_scale_B; + }; + + // Device side kernel params + struct Params { + // Assumption: StrideA is congruent with Problem_MK + using TMA_A = decltype(make_tma_copy_A_sm90( + GmemTiledCopyA{}, + make_tensor(static_cast(nullptr), repeat_like(StrideA{}, int32_t(0)), StrideA{}), + SmemLayoutA{}(_,_,0), + TileShape{}, + ClusterShape{})); + // Assumption: StrideB is congruent with Problem_NK + using TMA_B = decltype(make_tma_copy_B_sm90( + GmemTiledCopyB{}, + make_tensor(static_cast(nullptr), repeat_like(StrideB{}, int32_t(0)), StrideB{}), + SmemLayoutB{}(_,_,0), + TileShape{}, + ClusterShape{})); + TMA_A tma_load_a; + TMA_B tma_load_b; + uint32_t tma_transaction_bytes = TmaTransactionBytes; + uint32_t tma_transaction_bytes_mk = TmaTransactionBytesMK; + uint32_t tma_transaction_bytes_nk = TmaTransactionBytesNK; + uint32_t mma_promotion_interval = 4; + // Block scaling factors for A and B + ElementBlockScale const* ptr_scale_A; + ElementBlockScale const* ptr_scale_B; + }; + + // + // Methods + // + + template + static constexpr Params + to_underlying_arguments(ProblemShape const& problem_shape, Arguments const& args, void* workspace) { + (void) workspace; + + // Optionally append 1s until problem shape is rank-4 (MNKL), in case it is only rank-3 (MNK) + auto problem_shape_MNKL = append<4>(problem_shape, 1); + auto [M,N,K,L] = problem_shape_MNKL; + + auto ptr_A = reinterpret_cast(args.ptr_A); + auto ptr_B = reinterpret_cast(args.ptr_B); + + Tensor tensor_a = make_tensor(ptr_A, make_layout(make_shape(M,K,L), args.dA)); + Tensor tensor_b = make_tensor(ptr_B, make_layout(make_shape(N,K,L), args.dB)); + typename Params::TMA_A tma_load_a = make_tma_copy_A_sm90( + GmemTiledCopyA{}, + tensor_a, + SmemLayoutA{}(_,_,cute::Int<0>{}), + TileShape{}, + ClusterShape{}); + typename Params::TMA_B tma_load_b = make_tma_copy_B_sm90( + GmemTiledCopyB{}, + tensor_b, + SmemLayoutB{}(_,_,cute::Int<0>{}), + TileShape{}, + ClusterShape{}); + uint32_t transaction_bytes_mk = TmaTransactionBytesMK; + uint32_t transaction_bytes_nk = TmaTransactionBytesNK; + uint32_t transaction_bytes = transaction_bytes_mk + transaction_bytes_nk; + + return { + tma_load_a, + tma_load_b, + transaction_bytes, + transaction_bytes_mk, + transaction_bytes_nk, + args.mma_promotion_interval, + args.ptr_scale_A, + args.ptr_scale_B + }; + } + + template + static bool + can_implement( + ProblemShape const& problem_shape, + [[maybe_unused]] Arguments const& args) { + constexpr int tma_alignment_bits = 128; + auto problem_shape_MNKL = append<4>(problem_shape, 1); + auto [M,N,K,L] = problem_shape_MNKL; + + bool implementable = true; + constexpr int min_tma_aligned_elements_A = tma_alignment_bits / cutlass::sizeof_bits::value; + implementable = implementable && cutlass::detail::check_alignment(cute::make_shape(M,K,L), StrideA{}); + constexpr int min_tma_aligned_elements_B = tma_alignment_bits / cutlass::sizeof_bits::value; + implementable = implementable && cutlass::detail::check_alignment(cute::make_shape(N,K,L), StrideB{}); + /* MMA promotion interval should be a multiple of 4, since each mainloop iteration would issue 4 MMA instructions. */ + implementable = implementable && (args.mma_promotion_interval % 4 == 0); + + if (!implementable) { + CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Problem Size doesn't meet the minimum alignment requirements for TMA.\n"); + } + return implementable; + } + + static constexpr int K_PIPE_MAX = DispatchPolicy::Stages; + static constexpr int K_PIPE_MMAS = 1; + static constexpr uint32_t TmaTransactionBytesMK = + cutlass::bits_to_bytes(size<0>(SmemLayoutA{}) * size<1>(SmemLayoutA{}) * static_cast(sizeof_bits::value)); + static constexpr uint32_t TmaTransactionBytesNK = + cutlass::bits_to_bytes(size<0>(SmemLayoutB{}) * size<1>(SmemLayoutB{}) * static_cast(sizeof_bits::value)); + static constexpr uint32_t TmaTransactionBytes = TmaTransactionBytesMK + TmaTransactionBytesNK; + + /// Issue Tma Descriptor Prefetch -- ideally from a single thread for best performance + CUTLASS_DEVICE + static void prefetch_tma_descriptors(Params const& mainloop_params) + { + cute::prefetch_tma_descriptor(mainloop_params.tma_load_a.get_tma_descriptor()); + cute::prefetch_tma_descriptor(mainloop_params.tma_load_b.get_tma_descriptor()); + } + + /// Set up the data needed by this collective for load and mma. + /// Returns a tuple of tensors. The collective and the kernel layer have the contract + /// Returned tuple must contain at least two elements, with the first two elements being: + /// gA_mkl - The tma tensor, A after a local tile so it has shape (BLK_M,BLK_K,m,k,l) + /// gB_nkl - The tma tensor, B after a local tile so it has shape (BLK_N,BLK_K,n,k,l) + template + CUTLASS_DEVICE auto + load_init(ProblemShape_MNKL const& problem_shape_MNKL, Params const& mainloop_params) const { + using X = Underscore; + // Separate out problem shape for convenience + auto [M,N,K,L] = problem_shape_MNKL; + + // TMA requires special handling of strides to deal with coord codomain mapping + // Represent the full tensors -- get these from TMA + Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor(make_shape(M,K,L)); // (m,k,l) + Tensor mB_nkl = mainloop_params.tma_load_b.get_tma_tensor(make_shape(N,K,L)); // (n,k,l) + + // Make tiled views, defer the slice + Tensor gA_mkl = local_tile(mA_mkl, TileShape{}, make_coord(_,_,_), Step<_1, X,_1>{}); // (BLK_M,BLK_K,m,k,l) + Tensor gB_nkl = local_tile(mB_nkl, TileShape{}, make_coord(_,_,_), Step< X,_1,_1>{}); // (BLK_N,BLK_K,n,k,l) + + constexpr auto scales_m = Int{}; + auto tM = get<2>(gA_mkl.shape()); + auto tN = get<2>(gB_nkl.shape()); + auto tK = get<3>(gA_mkl.shape()); + + // Make the tiled views of scale tensors + auto scaleA_shape = make_shape(M / ScaleGranularityM, tK, L); // (scale_m,k,l) + auto scaleA_layout = make_ordered_layout(scaleA_shape, Step<_0, _1, _2>{}); + auto scaleB_shape = make_shape(tN, tK, L); // (n,k,l) + auto scaleB_layout = make_ordered_layout(scaleB_shape, Step<_1, _0, _2>{}); + + // Note that mScaleA_mkl and mScaleB_nkl are already blocked tiled in the `m` host and + // gScaleA_mkl and gScaleB_nkl in `g` global memory are same as mScaleA_mkl and mScaleB_nkl. + Tensor mScaleA_mkl = make_tensor(make_gmem_ptr(mainloop_params.ptr_scale_A), scaleA_layout); // (scale_m,k,l) + Tensor mScaleB_nkl = make_tensor(make_gmem_ptr(mainloop_params.ptr_scale_B), scaleB_layout); // (n,k,l) + + return cute::make_tuple(gA_mkl, gB_nkl, mScaleA_mkl, mScaleB_nkl); + } + + /// Perform a collective-scoped matrix multiply-accumulate + /// Producer Perspective + template < + class TensorA, class TensorB, + class TensorScaleA, class TensorScaleB, + class KTileIterator, class BlockCoord + > + CUTLASS_DEVICE void + load( + Params const& mainloop_params, + MainloopPipeline pipeline, + PipelineState smem_pipe_write, + cute::tuple const& load_inputs, + BlockCoord const& blk_coord, + KTileIterator k_tile_iter, int k_tile_count, + int thread_idx, + uint32_t block_rank_in_cluster, + TensorStorage& shared_tensors) { + int lane_predicate = cute::elect_one_sync(); + + // Blockscaling: Tma loads for load_input and CpAsync for load_scale + Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.data()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE) + Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.data()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE) + Tensor sScaleA = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_A.data()), SmemLayoutScaleA{}); // (ScaleMsPerTile,k) + Tensor sScaleB = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_B.data()), SmemLayoutScaleB{}); // (k) + + // + // Prepare the TMA loads for A and B + // + + constexpr uint32_t cluster_shape_x = get<0>(ClusterShape()); + uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x}; + + Tensor gA_mkl = get<0>(load_inputs); + Tensor gB_nkl = get<1>(load_inputs); + + auto block_tma_a = mainloop_params.tma_load_a.get_slice(cluster_local_block_id.y); + auto block_tma_b = mainloop_params.tma_load_b.get_slice(cluster_local_block_id.x); + + // Partition the inputs based on the current block coordinates. + auto [m_coord, n_coord, k_coord, l_coord] = blk_coord; + Tensor gA = gA_mkl(_,_,m_coord,_,l_coord); // (BLK_M,BLK_K,k) + Tensor gB = gB_nkl(_,_,n_coord,_,l_coord); // (BLK_N,BLK_K,k) + + + // Block scaling: load_scale has scaling tensors in global memory which are not tiled + Tensor mScaleA_mkl = get<2>(load_inputs); + Tensor mScaleB_nkl = get<3>(load_inputs); + auto scales_m = get<0>(mScaleA_mkl.shape()); + + Tensor cScaleA_mkl = make_identity_tensor(mScaleA_mkl.shape()); + + Tensor gScaleA = local_tile( + mScaleA_mkl, make_tile(Int{}), + make_coord(m_coord,_,l_coord)); // (ScaleMsPerTile,k,1) + Tensor cScaleA = local_tile( + cScaleA_mkl, make_tile(Int{}), + make_coord(m_coord,_,l_coord)); + Tensor gScaleB = mScaleB_nkl(n_coord,_,l_coord); // (1,k,1) + + // TODO: test `scale_copy_a` with `ScaleMsPerTile` < 128 + TiledCopy scale_copy_a = make_tiled_copy(SmemBlockScalingCopyAtomA{}, + Layout>{}, Layout>{}); // (1,1,1) + TiledCopy scale_copy_b = make_tiled_copy(SmemBlockScalingCopyAtomB{}, + Layout>{}, Layout>{}); // (1,1,1) + ThrCopy thr_scale_copy_a = scale_copy_a.get_slice(threadIdx.x); + ThrCopy thr_scale_copy_b = scale_copy_b.get_slice(threadIdx.x); + + Tensor tAgA_ScaleA = thr_scale_copy_a.partition_S(gScaleA); + Tensor tAcA_ScaleA = thr_scale_copy_a.partition_S(cScaleA); + Tensor tAsA_ScaleA = thr_scale_copy_a.partition_D(sScaleA); + + Tensor tBgB_ScaleB = thr_scale_copy_b.partition_S(gScaleB); + Tensor tBsB_ScaleB = thr_scale_copy_b.partition_D(sScaleB); + + // Applies the mapping from block_tma_a + Tensor tAgA = block_tma_a.partition_S(gA); // (TMA,TMA_M,TMA_K,k) + Tensor tAsA = block_tma_a.partition_D(sA); // (TMA,TMA_M,TMA_K,PIPE) + + Tensor tBgB = block_tma_b.partition_S(gB); // (TMA,TMA_N,TMA_K,k) + Tensor tBsB = block_tma_b.partition_D(sB); // (TMA,TMA_N,TMA_K,PIPE) + + uint16_t mcast_mask_a = 0; + uint16_t mcast_mask_b = 0; + + // Issue TmaLoads for GEMM operands A/B and CpAsync for scale tensors + // Maps the tile -> block, value + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int n = 0; n < size<1>(block_layout); ++n) { + mcast_mask_a |= (uint16_t(1) << block_layout(cluster_local_block_id.x,n,Int<0>{})); + } + } + + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int m = 0; m < size<0>(block_layout); ++m) { + mcast_mask_b |= (uint16_t(1) << block_layout(m,cluster_local_block_id.y,Int<0>{})); + } + } + + // Allocate predicate tensors for a_scales (since we can't guarantee that + // all scales are valid, since we could have a partial tiles along M) + Tensor tApA_ScaleA = make_tensor(shape(tAsA_ScaleA(_,_,0))); + #pragma unroll + for (int i = 0; i < size(tApA_ScaleA); ++i) { + tApA_ScaleA(i) = get<0>(tAcA_ScaleA(i)) < scales_m; + } + + // Mainloop + CUTLASS_PRAGMA_NO_UNROLL + for ( ; k_tile_count > 0; --k_tile_count) { + // LOCK smem_pipe_write for _writing_ + pipeline.producer_acquire(smem_pipe_write); + + // + // Copy gmem to smem for *k_tile_iter + // + int write_stage = smem_pipe_write.index(); + using BarrierType = typename MainloopPipeline::ProducerBarrierType; + BarrierType* tma_barrier = pipeline.producer_get_barrier(smem_pipe_write); + + // Copy operands A and B from global memory to shared memory + if (lane_predicate) copy(mainloop_params.tma_load_a.with(*tma_barrier, mcast_mask_a), tAgA(_,_,_,*k_tile_iter), tAsA(_,_,_,write_stage)); + if (lane_predicate) copy(mainloop_params.tma_load_b.with(*tma_barrier, mcast_mask_b), tBgB(_,_,_,*k_tile_iter), tBsB(_,_,_,write_stage)); + + // Copy scale tensors from global memory to shared memory + copy_if(scale_copy_a, tApA_ScaleA, tAgA_ScaleA(_,_,*k_tile_iter), tAsA_ScaleA(_,_,write_stage)); + copy(scale_copy_b, tBgB_ScaleB(_,*k_tile_iter), tBsB_ScaleB(_,write_stage)); + pipeline.producer_commit(smem_pipe_write, cutlass::arch::cpasync_barrier_arrive_noinc); + + ++k_tile_iter; + + // Advance smem_pipe_write + ++smem_pipe_write; + } + } + + /// Perform a Producer Epilogue to prevent early exit of blocks in a Cluster + CUTLASS_DEVICE void + load_tail( + MainloopPipeline pipeline, + PipelineState smem_pipe_write) { + int lane_predicate = cute::elect_one_sync(); + + // Issue the epilogue waits + if (lane_predicate) { + /* This helps avoid early exit of blocks in Cluster + * Waits for all stages to either be released (all + * Consumer UNLOCKs), or if the stage was never used + * then would just be acquired since the phase was + * still inverted from make_producer_start_state + */ + pipeline.producer_tail(smem_pipe_write); + } + } + + /// Perform a collective-scoped matrix multiply-accumulate + /// Consumer Perspective + template < + class FrgTensorC + > + CUTLASS_DEVICE void + mma(MainloopPipeline pipeline, + PipelineState smem_pipe_read, + FrgTensorC& accum, + int k_tile_count, + int thread_idx, + TensorStorage& shared_tensors, + Params const& mainloop_params) { + + + static_assert(is_rmem::value, "C tensor must be rmem resident."); + static_assert(cute::rank(SmemLayoutA{}) == 3, "Smem layout must be rank 3."); + static_assert(cute::rank(SmemLayoutB{}) == 3, "Smem layout must be rank 3."); + static_assert(cute::is_void_v, + "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions."); + static_assert(cute::is_void_v, + "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions."); + + Tensor sA = make_tensor(make_smem_ptr(shared_tensors.smem_A.data()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE) + Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.data()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE) + + // Block scaling + Tensor sScaleAViewAsC = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_A.data()), + Layout< + Shape, Int>, cute::tuple_element_t<1, TileShape>, Int>, + Stride, _0, Int> + >{}); // ((ScaleGranularityM,ScaleMsPerTile),n,k) + Tensor sScaleB = make_tensor(cute::make_smem_ptr(shared_tensors.smem_scale_B.data()), SmemLayoutScaleB{}); // (k) + + // + // Define C accumulators and A/B partitioning + // + + // Layout of warp group to thread mapping + + static_assert(stride<0>(typename TiledMma::ALayout{}) == 0 and + stride<0>(typename TiledMma::BLayout{}) == 0 and + size<0>(typename TiledMma::ALayout{}) == NumThreadsPerWarpGroup and + size<0>(typename TiledMma::BLayout{}) == NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + + constexpr int MmaWarpGroups = size(TiledMma{}) / NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout(Int{}, + Int{}); + + int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / NumThreadsPerWarpGroup, 0); + + TiledMma tiled_mma; + auto thread_mma = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx)); + + Tensor tCsScaleAViewAsC = tiled_mma.get_slice(thread_idx).partition_C(sScaleAViewAsC); // (MMA,MMA_M,MMA_N,PIPE), `thread_mma` above is correct when partitioning A and B, but it is not correct when partitioning C. + + Tensor tCsA = thread_mma.partition_A(sA); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCsB = thread_mma.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE) + + // Allocate "fragments/descriptors" + Tensor tCrA = thread_mma.make_fragment_A(tCsA); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCrB = thread_mma.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE) + + CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(accum)); // M + CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<2>(accum)); // N + CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCsB)); // K + CUTE_STATIC_ASSERT_V(size<3>(tCsA) == size<3>(tCsB)); // PIPE + CUTE_STATIC_ASSERT_V(Int{} == size<2>(sA)); // PIPE + CUTE_STATIC_ASSERT_V(Int{} == size<2>(sB)); // PIPE + + // + // PIPELINED MAIN LOOP + // + static_assert((0 <= K_PIPE_MMAS) && (K_PIPE_MMAS < K_PIPE_MAX), + "ERROR : Incorrect number of MMAs in flight"); + + // We release buffers to producer warps(dma load) with some mmas in flight + PipelineState smem_pipe_release = smem_pipe_read; + + // Per block scale values for operand A and B + + using RegLayoutScaleAViewAsC = decltype(make_layout_like(tCsScaleAViewAsC(_, _, _, 0).layout())); // `make_layout_like` makes a compact layout. + using RegLayoutScaleAEssential = decltype(filter_zeros(RegLayoutScaleAViewAsC{}.stride(), RegLayoutScaleAViewAsC{}.shape())); // an interface to traverse the underlying storage for the compact layout mentioned above + + Tensor tCrScaleAViewAsC = make_tensor(RegLayoutScaleAViewAsC{}); // (MMA,MMA_M,MMA_N) + ElementBlockScale scale_b; + + // Prologue GMMAs + int prologue_mma_count = min(K_PIPE_MMAS, k_tile_count); + + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + GmmaFP8Accumulation accumulation(accum, mainloop_params.mma_promotion_interval, size<2>(tCrA)); + warpgroup_fence_operand(accumulation()); + CUTLASS_PRAGMA_UNROLL + for (int k_tile_prologue = prologue_mma_count; k_tile_prologue > 0; --k_tile_prologue) + { + // WAIT on smem_pipe_read until its data are available (phase bit flips from rdPhaseBit value) + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + if (accumulation.prepare_if_needed()) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + } + + int read_stage = smem_pipe_read.index(); + + // Load per block scale values from shared memory to registers. + scale_b = sScaleB[read_stage]; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) { + tCrScaleAViewAsC.data()[i] = tCsScaleAViewAsC(_, _, _, read_stage)(idx2crd(i, RegLayoutScaleAEssential{})); + } + if constexpr (ScaleMsPerTile == 1) { + static_assert(size(RegLayoutScaleAEssential{}) == 1); + tCrScaleAViewAsC.data()[0] = __shfl_sync(0xffffffff, tCrScaleAViewAsC.data()[0] * scale_b, 0); // `tCrScaleAViewAsC.data()[0]` are all same in a warp group when `ScaleMsPerTile == 1`. + } else { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) { + tCrScaleAViewAsC.data()[i] = tCrScaleAViewAsC.data()[i] * scale_b; + } + } + + warpgroup_arrive(); + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) { + // (V,M,K) x (V,N,K) => (V,M,N) + cute::gemm(tiled_mma, tCrA(_,_,k_block,read_stage), tCrB(_,_,k_block,read_stage), accumulation()); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + } + warpgroup_commit_batch(); + + // Block scale the accumulators with reg tensor `tCrScaleAViewAsC` + accumulation.scale_if_needed(tCrScaleAViewAsC); + + ++smem_pipe_read; + } + + warpgroup_fence_operand(accumulation()); + // Mainloop GMMAs + k_tile_count -= prologue_mma_count; + + CUTLASS_PRAGMA_NO_UNROLL + for ( ; k_tile_count > 0; --k_tile_count) + { + // WAIT on smem_pipe_read until its data are available (phase bit flips from rdPhaseBit value) + auto barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + // + // Compute on k_tile + // + + int read_stage = smem_pipe_read.index(); + + // Load per block scale values from shared memory to registers (at most twice per block along M and exactly once per block along N) + scale_b = sScaleB[read_stage]; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) { + tCrScaleAViewAsC.data()[i] = tCsScaleAViewAsC(_, _, _, read_stage)(idx2crd(i, RegLayoutScaleAEssential{})); + } + if constexpr (ScaleMsPerTile == 1) { + static_assert(size(RegLayoutScaleAEssential{}) == 1); + tCrScaleAViewAsC.data()[0] = __shfl_sync(0xffffffff, tCrScaleAViewAsC.data()[0] * scale_b, 0); // `tCrScaleAViewAsC.data()[0]` are all same in a warp group when `ScaleMsPerTile == 1`. + } else { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(RegLayoutScaleAEssential{}); i++) { + tCrScaleAViewAsC.data()[i] = tCrScaleAViewAsC.data()[i] * scale_b; + } + } + + if (accumulation.prepare_if_needed()) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + } + + warpgroup_fence_operand(accumulation()); + warpgroup_arrive(); + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tCrA); ++k_block) { + // (V,M,K) x (V,N,K) => (V,M,N) + cute::gemm(tiled_mma, tCrA(_,_,k_block,read_stage), tCrB(_,_,k_block,read_stage), accumulation()); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + } + warpgroup_commit_batch(); + + /// Wait on the GMMA barrier for K_PIPE_MMAS (or fewer) outstanding to ensure smem_pipe_write is consumed + warpgroup_wait(); + warpgroup_fence_operand(accumulation()); + + // Block scale the accumulators with reg tensor `tCrScaleAViewAsC` + accumulation.scale_if_needed(tCrScaleAViewAsC); + + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + + // Advance smem_pipe_read and smem_pipe_release + ++smem_pipe_read; + ++smem_pipe_release; + } + + accumulation.scale_residue_if_needed(tCrScaleAViewAsC); + + warpgroup_fence_operand(accumulation()); + } + + /// Perform a Consumer Epilogue to release all buffers + CUTLASS_DEVICE void + mma_tail(MainloopPipeline pipeline, PipelineState smem_pipe_release, int k_tile_count) { + // Prologue GMMAs + int prologue_mma_count = min(K_PIPE_MMAS, k_tile_count); + k_tile_count -= prologue_mma_count; + + smem_pipe_release.advance(k_tile_count); + + // Wait on all GMMAs to complete + warpgroup_wait<0>(); + + for (int count = 0; count < prologue_mma_count; ++count) { + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + } + }; + + ///////////////////////////////////////////////////////////////////////////////////////////////// + + } // namespace cutlass::gemm::collective + + ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp new file mode 100644 index 0000000..f62b51e --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp @@ -0,0 +1,37 @@ +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.1/csrc/cutlass_extensions/gemm/dispatch_policy.hpp +#pragma once + +#include + +namespace cutlass::gemm { + +////////////////////////////////////////////////////////////////////////////// + +// FP8 related policies (including Blocked Scaled Accumulation) +// `ScaleGranularityM` specifies scaling granularity along M, while zero-value +// `ScaleGranularityM` indicates that scaling granularity is +// `size<0>(TileShape_MNK{})` along M. +template +struct KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum : KernelTmaWarpSpecializedCooperative {}; + +// n-buffer in smem (Hopper TMA), pipelined with Hopper GMMA and TMA, Warp +// specialized dynamic schedule For FP8 kernels with Block Scaling +template < + int Stages_, + class ClusterShape_ = Shape<_1, _1, _1>, + class KernelSchedule = KernelTmaWarpSpecialized, + int ScaleGranularityM = 0 // `ScaleGranularityM` specifies scaling granularity along M, + // while zero-value `ScaleGranularityM` indicates that scaling + // granularity is `size<0>(TileShape_MNK{})` along M. + > +struct MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8 + : MainloopSm90TmaGmmaWarpSpecialized { + static_assert( + cute:: + is_same_v>, + "KernelSchedule must be one of the warp specialized policies"); +}; + +////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h new file mode 100644 index 0000000..b58d843 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h @@ -0,0 +1,356 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/gemm_universal_base_compat.h +#pragma once + +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace device { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +/* + This is the device layer from CUTLASS 2.10 (SHA - cc85b64cf676c45f98a17e3a47c0aafcf817f088) + It is replicated here since we needed to duplicate kernel level APIs for mixed dtype GEMMs + and SmoothQuant. The newer device layer is not compatible with these older kernel level APIs. + + Note: While CUTLASS 3.x supports stream-k, none of the kernels in the extensions folder support + that feature at the moment. + */ + +template +class GemmUniversalBaseCompat { + public: + using GemmKernel = GemmKernel_; + using ThreadblockShape = typename GemmKernel::Mma::Shape; + + using ElementA = typename GemmKernel::ElementA; + using LayoutA = typename GemmKernel::LayoutA; + using TensorRefA = TensorRef; + static ComplexTransform const kTransformA = GemmKernel::kTransformA; + + using ElementB = typename GemmKernel::ElementB; + using LayoutB = typename GemmKernel::LayoutB; + using TensorRefB = TensorRef; + static ComplexTransform const kTransformB = GemmKernel::kTransformB; + + using ElementC = typename GemmKernel::ElementC; + using LayoutC = typename GemmKernel::LayoutC; + using TensorRefC = TensorRef; + using TensorRefD = TensorRef; + + using ElementAccumulator = typename GemmKernel::Mma::Policy::Operator::ElementC; + + using EpilogueOutputOp = typename GemmKernel::EpilogueOutputOp; + using ThreadblockSwizzle = typename GemmKernel::ThreadblockSwizzle; + using Operator = typename GemmKernel::Operator; + + /// Argument structure + using Arguments = typename GemmKernel::Arguments; + + protected: + /// Kernel parameters object + typename GemmKernel::Params params_; + + protected: + /// Private helper to obtain the grid dimensions with fix-up for split-K + static void get_grid_shape_(gemm::GemmCoord& grid_tiled_shape, int& gemm_k_size, Arguments const& args) { + // Determine grid shape + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + gemm_k_size = args.problem_size.k(); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + } + + public: + /// Constructs the GEMM. + GemmUniversalBaseCompat() {} + + /// Determines whether the GEMM can execute the given problem. + static Status can_implement(Arguments const& args) { + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + ThreadblockSwizzle threadblock_swizzle; + dim3 grid = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + uint32_t const kGridYZMax = ((1 << (sizeof(uint16_t) * 8)) - 1); + + if (!(grid.y <= kGridYZMax && grid.z <= kGridYZMax)) { + return Status::kErrorInvalidProblem; + } + + return GemmKernel::can_implement(args); + } + + /// Gets the workspace size + static size_t get_workspace_size(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_workspace_size()"); + + size_t workspace_bytes = 0; + + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + if (args.mode == GemmUniversalMode::kGemmSplitKParallel) { + // Split-K parallel always requires a temporary workspace + workspace_bytes = sizeof(ElementC) * size_t(args.batch_stride_D) * size_t(grid_tiled_shape.k()); + } else if (args.mode == GemmUniversalMode::kGemm && grid_tiled_shape.k() > 1) { + // Serial split-K only requires a temporary workspace if the number of partitions along the + // GEMM K dimension is greater than one. + workspace_bytes = sizeof(int) * size_t(grid_tiled_shape.m()) * size_t(grid_tiled_shape.n()); + } + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + workspace_bytes += GemmKernel::get_extra_workspace_size(args, grid_tiled_shape); + + return workspace_bytes; + } + + /// Computes the grid shape + static dim3 get_grid_shape(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_grid_shape()"); + + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + dim3 result = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + CUTLASS_TRACE_HOST( + " grid_tiled_shape: " << grid_tiled_shape << "\n" + << " result = {" << result << "}"); + + return result; + } + + /// Computes the maximum number of active blocks per multiprocessor + static int maximum_active_blocks(int smem_capacity = -1) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::maximum_active_blocks()"); + + int max_active_blocks = -1; + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + CUTLASS_TRACE_HOST(" smem_size: " << smem_size << " bytes"); + + if (smem_size <= (48 << 10)) { + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, smem_size); + + if (result == cudaSuccess) { + CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks); + return max_active_blocks; + } + } else { + // Query assuming zero shared memory then compute occupancy limit based on SMEM + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, 0); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST( + " cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error " << cudaGetErrorString(result)); + + return -1; + } + + if (smem_capacity < 0) { + int device_idx = 0; + result = cudaGetDevice(&device_idx); + + if (result != cudaSuccess) { + return -1; + } + + cudaDeviceProp properties; + result = cudaGetDeviceProperties(&properties, device_idx); + + if (result != cudaSuccess) { + return -1; + } + + smem_capacity = static_cast(properties.sharedMemPerMultiprocessor); + } + + int occupancy = std::min(max_active_blocks, smem_capacity / smem_size); + + CUTLASS_TRACE_HOST(" occupancy: " << occupancy); + + return occupancy; + } + + CUTLASS_TRACE_HOST(" returning internal error"); + + return -1; + } + + /// Initializes GEMM state from arguments. + Status initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST( + "GemmUniversalBaseCompat::initialize() - workspace " << workspace + << ", stream: " << (stream ? "non-null" : "null")); + + size_t workspace_bytes = get_workspace_size(args); + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + if (workspace_bytes) { + if (!workspace) { + CUTLASS_TRACE_HOST(" error: device workspace must not be null"); + + return Status::kErrorWorkspaceNull; + } + + if (args.mode == GemmUniversalMode::kGemm) { + CUTLASS_TRACE_HOST(" clearing device workspace"); + cudaError_t result = cudaMemsetAsync(workspace, 0, workspace_bytes, stream); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" cudaMemsetAsync() returned error " << cudaGetErrorString(result)); + + return Status::kErrorInternal; + } + } + } + + // Get CUDA grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + // Initialize the Params structure + params_ = typename GemmKernel::Params(args, grid_tiled_shape, gemm_k_size, static_cast(workspace)); + + // Specify shared memory capacity for kernel. + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + if (smem_size >= (48 << 10)) { + cudaError_t result = + cudaFuncSetAttribute(Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size); + + if (result != cudaSuccess) { + return Status::kErrorInternal; + } + } + + return Status::kSuccess; + } + + /// Lightweight update given a subset of arguments + Status update(Arguments const& args, void* workspace = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat()::update() - workspace: " << workspace); + + size_t workspace_bytes = get_workspace_size(args); + + if (workspace_bytes && !workspace) { + return Status::kErrorWorkspaceNull; + } + + params_.update(args, workspace); + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status run(cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::run()"); + + // + // Configure grid and block dimensions + // + + ThreadblockSwizzle threadblock_swizzle; + + dim3 grid = threadblock_swizzle.get_grid_shape(params_.grid_tiled_shape); + dim3 block(GemmKernel::kThreadCount, 1, 1); + + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + // + // Launch kernel + // + + CUTLASS_TRACE_HOST(" grid: (" << grid << "), block: (" << block << "), SMEM: " << smem_size << " bytes"); + + // Launch + cutlass::Kernel<<>>(params_); + + // + // Query for errors + // + cudaError_t result = cudaGetLastError(); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" grid launch failed with error " << cudaGetErrorString(result)); + return Status::kErrorInternal; + } + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status operator()(cudaStream_t stream = nullptr) { + return run(stream); + } + + /// Runs the kernel using initialized state. + Status operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + Status status = initialize(args, workspace, stream); + + if (status == Status::kSuccess) { + status = run(stream); + } + + return status; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace device +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h new file mode 100644 index 0000000..905d11b --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h @@ -0,0 +1,492 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/gemm_with_epilogue_visitor.h + +#pragma once + +#include +#include +#include +#include +#include + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace kernel { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + typename Mma_, ///! Threadblock-scoped matrix multiply-accumulate + typename Epilogue_, ///! Epilogue + typename ThreadblockSwizzle_ ///! Threadblock swizzling function + > +struct GemmWithEpilogueVisitor { + public: + using Mma = Mma_; + using Epilogue = Epilogue_; + using EpilogueVisitor = typename Epilogue::Visitor; + using ThreadblockSwizzle = ThreadblockSwizzle_; + + using ElementA = typename Mma::IteratorA::Element; + using LayoutA = typename Mma::IteratorA::Layout; + using TensorRefA = TensorRef; + + using ElementB = typename Mma::IteratorB::Element; + using LayoutB = typename Mma::IteratorB::Layout; + using TensorRefB = TensorRef; + + using ElementCompute = typename EpilogueVisitor::ElementCompute; + using LayoutAlphaCol = cutlass::layout::RowMajor; + using LayoutAlphaRow = cutlass::layout::ColumnMajor; + using TensorRefAlphaCol = TensorRef; + using TensorRefAlphaRow = TensorRef; + + using ElementC = typename EpilogueVisitor::ElementOutput; + using LayoutC = typename Epilogue::Layout; + using TensorRefC = TensorRef; + + static ComplexTransform const kTransformA = Mma::kTransformA; + static ComplexTransform const kTransformB = Mma::kTransformB; + using Operator = typename Mma::Operator; + + using OperatorClass = typename Mma::Operator::OperatorClass; + using ThreadblockShape = typename Mma::Shape; + using WarpShape = typename Mma::Operator::Shape; + using InstructionShape = typename Mma::Policy::Operator::InstructionShape; + using ArchTag = typename Mma::ArchTag; + using EpilogueOutputOp = + typename Epilogue::Visitor::ElementwiseFunctor; // Define type so GemmUniversalBase doesn't complain + + static int const kStages = Mma::kStages; + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::kElementsPerAccess; + + /// Warp count (concept: GemmShape) + using WarpCount = typename Mma::WarpCount; + static int const kThreadCount = 32 * WarpCount::kCount; + + /// Split-K preserves splits that are 128b aligned + static int const kSplitKAlignment = const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value); + + // + // Structures + // + + /// Argument structure + struct Arguments { + // + // Data members + // + + GemmUniversalMode mode; + GemmCoord problem_size; + int batch_count; + + TensorRefA ref_A; + TensorRefB ref_B; + TensorRefAlphaCol ref_alpha_col; + TensorRefAlphaRow ref_alpha_row; + TensorRefC ref_C; + TensorRefC ref_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + int64_t batch_stride_D; + + typename EpilogueVisitor::Arguments epilogue_visitor; + + // + // Methods + // + + Arguments() : mode(GemmUniversalMode::kGemm), batch_count(1) {} + + /// constructs an arguments structure + Arguments( + GemmCoord problem_size_, + TensorRefA ref_A_, + TensorRefB ref_B_, + TensorRefAlphaCol ref_alpha_col_, + TensorRefAlphaRow ref_alpha_row_, + TensorRefC ref_C_, + TensorRefC ref_D_, + typename EpilogueVisitor::Arguments epilogue_visitor_) + : mode(GemmUniversalMode::kGemm), + problem_size(problem_size_), + batch_count(1), + ref_A(ref_A_), + ref_B(ref_B_), + ref_alpha_col(ref_alpha_col_), + ref_alpha_row(ref_alpha_row_), + ref_C(ref_C_), + ref_D(ref_D_), + batch_stride_A(0), + batch_stride_B(0), + batch_stride_D(0), + epilogue_visitor(epilogue_visitor_) {} + }; + + // + // Structure for precomputing values in host memory and passing to kernels + // + + /// Parameters structure + struct Params { + cutlass::gemm::GemmCoord problem_size; + cutlass::gemm::GemmCoord grid_tiled_shape; + int swizzle_log_tile; + + typename Mma::IteratorA::Params params_A; + typename Mma::IteratorB::Params params_B; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_row; + typename EpilogueVisitor::OutputTileIterator::Params params_C; + typename EpilogueVisitor::OutputTileIterator::Params params_D; + + GemmUniversalMode mode; + int batch_count; + int gemm_k_size; + + void* ptr_A; + void* ptr_B; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_row; + ElementC* ptr_C; + ElementC* ptr_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + + typename EpilogueVisitor::Params epilogue_visitor; + + // + // Methods + // + + CUTLASS_HOST_DEVICE + Params() + : swizzle_log_tile(0), + params_A(0), + params_B(0), + params_alpha_col(0), + params_C(0), + params_D(0), + batch_count(0), + gemm_k_size(0), + mode(cutlass::gemm::GemmUniversalMode::kGemm), + ptr_A(nullptr), + ptr_B(nullptr), + ptr_alpha_col(nullptr), + ptr_alpha_row(nullptr), + ptr_C(nullptr), + ptr_D(nullptr), + batch_stride_A(0), + batch_stride_B(0) {} + + Params(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape_, int gemm_k_size_, int* workspace_) + : problem_size(args.problem_size), + swizzle_log_tile(0), + params_A(args.ref_A.layout()), + params_B(args.ref_B.layout()), + params_alpha_col(args.ref_alpha_col.layout()), + params_alpha_row(args.ref_alpha_col.layout()), + params_C(args.ref_C.layout()), + params_D(args.ref_D.layout()), + mode(args.mode), + batch_count(args.batch_count), + gemm_k_size(args.problem_size.k()), + ptr_A(args.ref_A.data()), + ptr_B(args.ref_B.data()), + ptr_alpha_col(args.ref_alpha_col.data()), + ptr_alpha_row(args.ref_alpha_row.data()), + ptr_C(args.ref_C.data()), + ptr_D(args.ref_D.data()), + batch_stride_A(args.batch_stride_A), + batch_stride_B(args.batch_stride_B), + epilogue_visitor(args.epilogue_visitor) { + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + + swizzle_log_tile = threadblock_swizzle.get_log_tile(grid_tiled_shape); + } + }; + + /// Shared memory storage structure + union SharedStorage { + typename Mma::SharedStorage main_loop; + + struct { + typename Epilogue::SharedStorage epilogue; + typename EpilogueVisitor::SharedStorage visitor; + } epilogue; + }; + + public: + // + // Methods + // + + CUTLASS_DEVICE + GemmWithEpilogueVisitor() {} + + /// Determines whether kernel satisfies alignment + static Status can_implement(cutlass::gemm::GemmCoord const& problem_size) { + CUTLASS_TRACE_HOST("GemmWithEpilogueVisitor::can_implement()"); + + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::OutputTileIterator::kElementsPerAccess; + + bool isAMisaligned = false; + bool isBMisaligned = false; + bool isCMisaligned = false; + + if (platform::is_same::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } else if (platform::is_same::value) { + isAMisaligned = problem_size.m() % kAlignmentA; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } + + if (platform::is_same::value) { + isBMisaligned = problem_size.n() % kAlignmentB; + } else if (platform::is_same::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } + + if (platform::is_same::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } else if (platform::is_same::value) { + isCMisaligned = problem_size.m() % kAlignmentC; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } + + if (isAMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for A operand"); + return Status::kErrorMisalignedOperand; + } + + if (isBMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for B operand"); + return Status::kErrorMisalignedOperand; + } + + if (isCMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for C operand"); + return Status::kErrorMisalignedOperand; + } + + CUTLASS_TRACE_HOST(" returning kSuccess"); + + return Status::kSuccess; + } + + static Status can_implement(Arguments const& args) { + return can_implement(args.problem_size); + } + + static size_t get_extra_workspace_size(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape) { + return 0; + } + +#define SPLIT_K_ENABLED 1 + + /// Executes one GEMM + CUTLASS_DEVICE + void run_kernel_(Params const& params, SharedStorage& shared_storage) { + // Compute threadblock location + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // Early exit if CTA is out of range + if (params.grid_tiled_shape.m() <= threadblock_tile_offset.m() || + params.grid_tiled_shape.n() <= threadblock_tile_offset.n()) { + return; + } + + int offset_k = 0; + int problem_size_k = params.problem_size.k(); + + ElementA* ptr_A = static_cast(params.ptr_A); + ElementB* ptr_B = static_cast(params.ptr_B); + +#if SPLIT_K_ENABLED + // + // Fetch pointers based on mode. + // + if (params.mode == GemmUniversalMode::kGemm || params.mode == GemmUniversalMode::kGemmSplitKParallel) { + if (threadblock_tile_offset.k() + 1 < params.grid_tiled_shape.k()) { + problem_size_k = (threadblock_tile_offset.k() + 1) * params.gemm_k_size; + } + + offset_k = threadblock_tile_offset.k() * params.gemm_k_size; + } else if (params.mode == GemmUniversalMode::kBatched) { + ptr_A += threadblock_tile_offset.k() * params.batch_stride_A; + ptr_B += threadblock_tile_offset.k() * params.batch_stride_B; + } else if (params.mode == GemmUniversalMode::kArray) { + ptr_A = static_cast(params.ptr_A)[threadblock_tile_offset.k()]; + ptr_B = static_cast(params.ptr_B)[threadblock_tile_offset.k()]; + } +#endif + + // Compute initial location in logical coordinates + cutlass::MatrixCoord tb_offset_A{ + threadblock_tile_offset.m() * Mma::Shape::kM, + offset_k, + }; + + cutlass::MatrixCoord tb_offset_B{offset_k, threadblock_tile_offset.n() * Mma::Shape::kN}; + + // Compute position within threadblock + int thread_idx = threadIdx.x; + + // Construct iterators to A and B operands + typename Mma::IteratorA iterator_A( + params.params_A, ptr_A, {params.problem_size.m(), problem_size_k}, thread_idx, tb_offset_A); + + typename Mma::IteratorB iterator_B( + params.params_B, ptr_B, {problem_size_k, params.problem_size.n()}, thread_idx, tb_offset_B); + + // Broadcast the warp_id computed by lane 0 to ensure dependent code + // is compiled as warp-uniform. + int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); + + int lane_idx = threadIdx.x % 32; + + // + // Main loop + // + + // Construct thread-scoped matrix multiply + Mma mma(shared_storage.main_loop, thread_idx, warp_idx, lane_idx); + + typename Mma::FragmentC accumulators; + + accumulators.clear(); + + // Compute threadblock-scoped matrix multiply-add + int gemm_k_iterations = (problem_size_k - offset_k + Mma::Shape::kK - 1) / Mma::Shape::kK; + + // Compute threadblock-scoped matrix multiply-add + mma(gemm_k_iterations, accumulators, iterator_A, iterator_B, accumulators); + + // + // Masked tile iterators constructed from members + // + + threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // assume identity swizzle + MatrixCoord threadblock_offset( + threadblock_tile_offset.m() * Mma::Shape::kM, threadblock_tile_offset.n() * Mma::Shape::kN); + + int block_idx = threadblock_tile_offset.m() + threadblock_tile_offset.n() * params.grid_tiled_shape.m(); + + // + // Construct the epilogue visitor + // + + bool with_bias = true; + if (params.ptr_C == nullptr) { + with_bias = false; + } + + EpilogueVisitor epilogue_visitor( + params.epilogue_visitor, + shared_storage.epilogue.visitor, + params.problem_size.mn(), + thread_idx, + warp_idx, + lane_idx, + params.params_alpha_col, + params.params_C, + params.params_D, + with_bias, + true, + true, + params.ptr_alpha_row, + params.ptr_alpha_col, + params.ptr_C, + params.ptr_D, + threadblock_offset, + blockIdx.y * params.problem_size.m()); + + if (params.mode == GemmUniversalMode::kGemm) { + // Indicate which position in a serial reduction the output operator is currently updating + epilogue_visitor.set_k_partition(threadblock_tile_offset.k(), params.grid_tiled_shape.k()); + } else if (params.mode == GemmUniversalMode::kBatched || params.mode == GemmUniversalMode::kArray) { + epilogue_visitor.set_batch_index(threadblock_tile_offset.k()); + } + + // Construct the epilogue + Epilogue epilogue(shared_storage.epilogue.epilogue, thread_idx, warp_idx, lane_idx); + + // Execute the epilogue operator to update the destination tensor. + epilogue(epilogue_visitor, accumulators); + } + + template + CUTLASS_DEVICE void run_kernel(Params const& params, SharedStorage& shared_storage) { + if constexpr (platform::is_same::value) { + run_kernel_(params, shared_storage); + } else { + CUTLASS_NOT_IMPLEMENTED(); + } + } + + /// Executes one GEMM + CUTLASS_DEVICE + void operator()(Params const& params, SharedStorage& shared_storage) { + run_kernel(params, shared_storage); + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace kernel +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/elementwise/activation.cu b/sgl-kernel/csrc/elementwise/activation.cu new file mode 100644 index 0000000..242281f --- /dev/null +++ b/sgl-kernel/csrc/elementwise/activation.cu @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +__device__ __forceinline__ float silu(const float& val) { + return val / (1.0f + __expf(-val)); +} + +__device__ __forceinline__ float gelu(const float& val) { + constexpr float kAlpha = M_SQRT1_2; + return val * 0.5f * (1.0f + ::erf(val * kAlpha)); +} + +__device__ __forceinline__ float gelu_tanh(const float& val) { + const float cdf = 0.5f * (1.0f + math::tanh((0.7978845608028654f * (val + 0.044715f * val * val * val)))); + return val * cdf; +} + +void silu_and_mul(at::Tensor& out, at::Tensor& input, int64_t cuda_stream) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); + + return true; + }); +} + +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input, int64_t cuda_stream) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); + + return true; + }); +} + +void gelu_and_mul(at::Tensor& out, at::Tensor& input, int64_t cuda_stream) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); + + return true; + }); +} diff --git a/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu b/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu new file mode 100644 index 0000000..41f4d2e --- /dev/null +++ b/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu @@ -0,0 +1,55 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include + +#include "utils.h" + +using namespace flashinfer; + +void sgl_fused_add_rmsnorm(torch::Tensor input, torch::Tensor residual, torch::Tensor weight, double eps) { + CHECK_INPUT(input); + CHECK_INPUT(residual); + CHECK_INPUT(weight); + auto device = input.device(); + CHECK_EQ(residual.device(), device); + CHECK_EQ(weight.device(), device); + CHECK_DIM(2, input); // input: (batch_size, hidden_size) + CHECK_DIM(2, residual); // residual: (batch_size, hidden_size) + CHECK_DIM(1, weight); // weight: (hidden_size) + CHECK_EQ(input.size(0), residual.size(0)); + CHECK_EQ(input.size(1), residual.size(1)); + CHECK_EQ(input.size(1), weight.size(0)); + unsigned int batch_size = input.size(0); + unsigned int hidden_size = input.size(1); + + cudaStream_t torch_current_stream = at::cuda::getCurrentCUDAStream(); + // support float16, bfloat16 and float32 + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + cudaError_t status = norm::FusedAddRMSNorm( + static_cast(input.data_ptr()), + static_cast(residual.data_ptr()), + static_cast(weight.data_ptr()), + batch_size, + hidden_size, + eps, + torch_current_stream); + TORCH_CHECK( + status == cudaSuccess, "FusedAddRMSNorm failed with error code " + std::string(cudaGetErrorString(status))); + return true; + }); +} diff --git a/sgl-kernel/csrc/elementwise/rope.cu b/sgl-kernel/csrc/elementwise/rope.cu new file mode 100644 index 0000000..4274acf --- /dev/null +++ b/sgl-kernel/csrc/elementwise/rope.cu @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +void apply_rope_pos_ids_cos_sin_cache( + at::Tensor q, + at::Tensor k, + at::Tensor q_rope, + at::Tensor k_rope, + at::Tensor cos_sin_cache, + at::Tensor pos_ids, + bool interleave, + int64_t cuda_stream) { + CHECK_LAST_DIM_CONTIGUOUS(q); + CHECK_LAST_DIM_CONTIGUOUS(k); + CHECK_INPUT(cos_sin_cache); + CHECK_INPUT(pos_ids); + auto device = q.device(); + CHECK_EQ(k.device(), device); + CHECK_EQ(cos_sin_cache.device(), device); + CHECK_EQ(pos_ids.device(), device); + CHECK_DIM(3, q); // q: (nnz, H_Q, D) + CHECK_DIM(3, k); // k: (nnz, H_K, D) + // cos_sin_cache: (max_seq_len, R) + // First half of R is cos, second half is sin + CHECK_DIM(2, cos_sin_cache); + CHECK_EQ(q.size(0), k.size(0)); + CHECK_EQ(q.size(2), k.size(2)); + unsigned int rotary_dim = cos_sin_cache.size(1); + unsigned int num_qo_heads = q.size(1); + unsigned int num_kv_heads = k.size(1); + unsigned int head_dim = q.size(2); + unsigned int nnz = q.size(0); + size_t q_stride_n = q.stride(0); + size_t q_stride_h = q.stride(1); + size_t k_stride_n = k.stride(0); + size_t k_stride_h = k.stride(1); + size_t q_rope_stride_n = q_rope.stride(0); + size_t q_rope_stride_h = q_rope.stride(1); + size_t k_rope_stride_n = k_rope.stride(0); + size_t k_rope_stride_h = k_rope.stride(1); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(q.scalar_type(), c_type, [&] { + cudaError_t status = BatchQKApplyRotaryPosIdsCosSinCache( + static_cast(q.data_ptr()), + static_cast(k.data_ptr()), + static_cast(q_rope.data_ptr()), + static_cast(k_rope.data_ptr()), + static_cast(cos_sin_cache.data_ptr()), + static_cast(pos_ids.data_ptr()), + nnz, + num_qo_heads, + num_kv_heads, + rotary_dim, + head_dim, + q_stride_n, + q_stride_h, + k_stride_n, + k_stride_h, + q_rope_stride_n, + q_rope_stride_h, + k_rope_stride_n, + k_rope_stride_h, + interleave, + stream); + TORCH_CHECK( + status == cudaSuccess, + "BatchQKApplyRotaryPosIdsCosSinCache failed with error code " + std::string(cudaGetErrorString(status))); + return true; + }); +} diff --git a/sgl-kernel/csrc/gemm/awq_kernel.cu b/sgl-kernel/csrc/gemm/awq_kernel.cu new file mode 100644 index 0000000..188f0cb --- /dev/null +++ b/sgl-kernel/csrc/gemm/awq_kernel.cu @@ -0,0 +1,219 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/eb59b5a6cba6727d3727c0372258db9002f687c1/csrc/quantization/awq/gemm_kernels.cu#L350 +#include +#include +#include +#include +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +#include +#endif + +template +__device__ inline int lop3(int a, int b, int c) { + int res; + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(res) : "r"(a), "r"(b), "r"(c), "n"(lut)); + return res; +} + +__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750 + uint4 result; + + uint32_t* h = reinterpret_cast(&result); + uint32_t const i4s = reinterpret_cast(source); + + // First, we extract the i4s and construct an intermediate fp16 number. + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + static constexpr uint32_t BOTTOM_MASK = 0x000f000f; + static constexpr uint32_t TOP_MASK = 0x00f000f0; + static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400; + + // Note that the entire sequence only requires 1 shift instruction. This is + // thanks to the register packing format and the fact that we force our + // integers to be unsigned, and account for this in the fp16 subtractions. In + // addition, I exploit the fact that sub and fma have the same throughput in + // order to convert elt_23 and elt_67 to fp16 without having to shift them to + // the bottom bits before hand. + + // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW + // dependency if we issue immediately before required. + const uint32_t top_i4s = i4s >> 8; + // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[0]) + : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[1]) + : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[2]) + : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[3]) + : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + + // This is the half2 {1024, 1024} represented as an integer. + static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400; + // This is the half2 {1 / 16, 1 / 16} represented as an integer. + static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00; + // This is the half2 {-64, -64} represented as an integer. + static constexpr uint32_t NEG_64 = 0xd400d400; + + // Finally, we construct the output numbers. + // Convert elt_01 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_23 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64)); + // Convert elt_45 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_67 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64)); + + return result; +#else + assert(false); + return {}; +#endif +} + +__device__ uint4 dequantize_s4_to_bf16x2(uint32_t const& source) { +#if CUDA_VERSION >= 12000 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + uint4 result; + uint32_t* h = reinterpret_cast(&result); + uint32_t const i4s = source; + + // Define masks and constants + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t EX = 0x43004300; + static constexpr uint32_t MUL = 0x3F803F80; + static constexpr uint32_t ADD = 0xC300C300; + + int lo0 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s, MASK, EX); + int hi0 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 4, MASK, EX); + int lo1 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 8, MASK, EX); + int hi1 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 12, MASK, EX); + + nv_bfloat162* res = reinterpret_cast(h); + res[0] = __hfma2( + *reinterpret_cast(&lo0), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[1] = __hfma2( + *reinterpret_cast(&hi0), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[2] = __hfma2( + *reinterpret_cast(&lo1), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[3] = __hfma2( + *reinterpret_cast(&hi1), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + + return result; +#else + assert(false); + return {}; +#endif +#endif +} + +template +__global__ void __launch_bounds__(256) dequantize_weights( + int* __restrict__ qweight, + OutputT* __restrict__ scales, + int* __restrict__ qzeros, + OutputT* __restrict__ output, + int group_size, + int qweight_cols) { +#if CUDA_VERSION >= 12000 + int col = blockIdx.x * blockDim.x + threadIdx.x; + int row = blockIdx.y * blockDim.y + threadIdx.y; + + int group_idx = row / group_size; + int scale_offset = 8 * col + group_idx * qweight_cols * 8; + uint4 loaded_scale = *(uint4*)(scales + scale_offset); + + // Handle different data types + if constexpr (std::is_same::value) { + // FP16 path + uint4 zeros = dequantize_s4_to_fp16x2(qzeros[col + group_idx * qweight_cols]); + uint4 weight_fp16 = dequantize_s4_to_fp16x2(qweight[col + row * qweight_cols]); + + // Use PTX assembly for FP16 operations + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.x) : "r"(weight_fp16.x), "r"(zeros.x)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.x) : "r"(weight_fp16.x), "r"(loaded_scale.x)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.y) : "r"(weight_fp16.y), "r"(zeros.y)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.y) : "r"(weight_fp16.y), "r"(loaded_scale.y)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.z) : "r"(weight_fp16.z), "r"(zeros.z)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.z) : "r"(weight_fp16.z), "r"(loaded_scale.z)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.w) : "r"(weight_fp16.w), "r"(zeros.w)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.w) : "r"(weight_fp16.w), "r"(loaded_scale.w)); + + OutputT* output_ptr = output + 8 * col + 8 * row * qweight_cols; + *(uint4*)output_ptr = weight_fp16; + } else if constexpr (std::is_same::value) { + uint4 weight_raw = dequantize_s4_to_bf16x2(qweight[col + row * qweight_cols]); + uint4 zero_raw = dequantize_s4_to_bf16x2(qzeros[col + group_idx * qweight_cols]); + uint4 scale_raw = *reinterpret_cast(scales + scale_offset); + + // Vectorized processing (each uint4 contains 4 nv_bfloat162) + nv_bfloat162* weight_vec = reinterpret_cast(&weight_raw); + nv_bfloat162* zero_vec = reinterpret_cast(&zero_raw); + nv_bfloat162* scale_vec = reinterpret_cast(&scale_raw); + +// Single instruction dual-channel operation +#pragma unroll + for (int i = 0; i < 4; ++i) { // uint4 = 4 * nv_bfloat162 + weight_vec[i] = __hmul2(__hsub2(weight_vec[i], zero_vec[i]), scale_vec[i]); + } + + // Directly store to OutputT array (guaranteed contiguous memory) + OutputT* output_ptr = output + 8 * col + row * qweight_cols * 8; + static_assert(sizeof(uint4) == 8 * sizeof(OutputT), "Memory layout mismatch"); + *reinterpret_cast(output_ptr) = weight_raw; + } +#endif +} + +torch::Tensor awq_dequantize(torch::Tensor qweight, torch::Tensor scales, torch::Tensor qzeros) { + int qweight_rows = qweight.size(0); + int qweight_cols = qweight.size(1); + int group_size = qweight_rows / scales.size(0); + + int x_num_threads = 16; + int y_num_threads = 16; + int x_blocks = qweight_cols / x_num_threads; + int y_blocks = qweight_rows / y_num_threads; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(qweight)); + + auto output_tensor_options = torch::TensorOptions().dtype(scales.dtype()).device(scales.device()); + at::Tensor output = torch::empty({qweight_rows, qweight_cols * 8}, output_tensor_options); + + auto _qweight = reinterpret_cast(qweight.data_ptr()); + auto _zeros = reinterpret_cast(qzeros.data_ptr()); + + dim3 num_blocks(x_blocks, y_blocks); + dim3 threads_per_block(x_num_threads, y_num_threads); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (scales.scalar_type() == at::ScalarType::Half) { + auto _scales = reinterpret_cast(scales.data_ptr()); + auto _output = reinterpret_cast(output.data_ptr()); + dequantize_weights + <<>>(_qweight, _scales, _zeros, _output, group_size, qweight_cols); + } else { + auto _scales = reinterpret_cast<__nv_bfloat16*>(scales.data_ptr()); + auto _output = reinterpret_cast<__nv_bfloat16*>(output.data_ptr()); + dequantize_weights<__nv_bfloat16> + <<>>(_qweight, _scales, _zeros, _output, group_size, qweight_cols); + } + + return output; +} diff --git a/sgl-kernel/csrc/gemm/bmm_fp8.cu b/sgl-kernel/csrc/gemm/bmm_fp8.cu new file mode 100644 index 0000000..4a82b4b --- /dev/null +++ b/sgl-kernel/csrc/gemm/bmm_fp8.cu @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "pytorch_extension_utils.h" + +void bmm_fp8( + at::Tensor A, + at::Tensor B, + at::Tensor D, + at::Tensor A_scale, + at::Tensor B_scale, + at::Tensor workspace_buffer, + int64_t cublas_handle, + int64_t cuda_stream) { + TORCH_CHECK(A.is_cuda(), "A must be a CUDA tensor"); + TORCH_CHECK(B.is_cuda(), "B must be a CUDA tensor"); + TORCH_CHECK(D.is_cuda(), "D must be a CUDA tensor"); + TORCH_CHECK(A.dim() == 3, "Expected 3D tensor for A"); + TORCH_CHECK(B.dim() == 3, "Expected 3D tensor for B"); + TORCH_CHECK(D.dim() == 3, "Expected 3D tensor for D"); + TORCH_CHECK(A.size(0) == B.size(0) && A.size(0) == D.size(0), "Batch sizes must match"); + TORCH_CHECK(A.size(2) == B.size(1), "Incompatible matrix sizes"); + TORCH_CHECK(A.size(1) == D.size(1) && B.size(2) == D.size(2), "Result tensor has incorrect shape"); + + // PyTorch is row major by default. cuBLASLt is column major by default. + // We need row major D as expected. + // A ^ T * B = D, so D ^ T = B ^ T * A + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(B.scalar_type(), b_type, [&] { + return DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(A.scalar_type(), a_type, [&] { + return DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(D.scalar_type(), d_type, [&] { + auto batch_size = A.size(0); + auto m = A.size(1); + auto k = A.size(2); + auto n = B.size(2); + + auto lt_handle = reinterpret_cast(cublas_handle); + auto stream = reinterpret_cast(cuda_stream); + + auto status = flashinfer::bmm_fp8::bmm_fp8_internal_cublaslt( + workspace_buffer.data_ptr(), + workspace_buffer.numel(), + static_cast(B.data_ptr()), + static_cast(A.data_ptr()), + static_cast(D.data_ptr()), + batch_size, + n, + m, + k, + static_cast(B_scale.data_ptr()), + static_cast(A_scale.data_ptr()), + lt_handle, + stream); + TORCH_CHECK( + status == CUBLAS_STATUS_SUCCESS, "bmm_fp8_internal_cublaslt failed: ", cublasGetStatusString(status)); + return true; + }); + }); + }); +} diff --git a/sgl-kernel/csrc/gemm/cublas_grouped_gemm.cu b/sgl-kernel/csrc/gemm/cublas_grouped_gemm.cu new file mode 100644 index 0000000..ca7d131 --- /dev/null +++ b/sgl-kernel/csrc/gemm/cublas_grouped_gemm.cu @@ -0,0 +1,172 @@ +// References: +// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmgroupedbatchedex +// https://github.com/NVIDIA/CUDALibrarySamples/blob/master/cuBLAS/Extensions/GemmGroupedBatchedEx/cublas_GemmGroupedBatchedEx_example.cu +// https://github.com/zhihu/ZhiLight/blob/main/src/nn/linear/gemm_grouped.cpp + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "utils.h" + +static void check_group_count( + const std::vector& inputs, + const std::vector& weights, + const std::vector& outputs) { + TORCH_CHECK( + ((inputs.size() == weights.size()) && (inputs.size() == outputs.size())), + "The group count of inputs, weights and outputs should be the same."); +} + +static void check_device_dtype(const torch::Dtype& dtype, const std::vector& tensors) { + for (const auto& t : tensors) { + TORCH_CHECK(dtype == t.dtype(), "dtype of all the tensors should be the same"); + TORCH_CHECK(t.is_cuda(), "All tensors should be in Cuda memory"); + } +} + +static std::vector get_dims(const std::vector& tensors, int dim) { + std::vector results; + for (const auto& t : tensors) { + TORCH_CHECK(t.dim() == 2, "Should pass in 2D matrices"); + results.push_back(t.size(dim)); + } + return std::move(results); +} + +static std::vector get_strides(const std::vector& tensors, int dim) { + std::vector results; + for (const auto& t : tensors) { + results.push_back(t.stride(dim)); + } + return std::move(results); +} + +static void check_equal(const std::vector& a, const std::vector& b, const std::string& err_msg) { + for (int i = 0; i < a.size(); ++i) { + TORCH_CHECK(a[i] == b[i], err_msg); + } +} + +static std::vector get_tensor_ptrs(const std::vector& tensors) { + std::vector ptrs; + for (auto& t : tensors) { + ptrs.push_back(t.data_ptr()); + } + return std::move(ptrs); +} + +static torch::Tensor create_ptr_pointer(const std::vector& ptrs, cudaStream_t stream) { + auto options = torch::TensorOptions().dtype(torch::kDouble).device(torch::kCUDA); + torch::Tensor gpu_ptrs = torch::empty({static_cast(ptrs.size())}, options); + TORCH_CHECK( + cudaMemcpyAsync(gpu_ptrs.data_ptr(), ptrs.data(), sizeof(void*) * ptrs.size(), cudaMemcpyHostToDevice, stream) == + CUBLAS_STATUS_SUCCESS); + return gpu_ptrs; +} + +// We want compute input @ weight^T in row major +// This is equivalent to computing weight @ input^T in col major +// Cublas only accepts matrix in column major, so this arrangement is needed +void cublas_grouped_gemm( + const std::vector& inputs, // b: (m, k) row major = (k, m) col major + const std::vector& weights, // a: (n, k) row major = (n, k)^T col major + const std::vector& outputs, // c: (m, n) row major = (n, m) col major + const torch::Dtype& out_dtype, + int64_t cublas_handle, + int64_t cuda_stream) { + TORCH_CHECK( + out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, + "cublas grouped_gemm can" + "only be applied to float16 and bfloat16 dtype"); + + int group_count = inputs.size(); + check_group_count(inputs, weights, outputs); + std::vector group_size(group_count, 1); + + // Make sure all tensors are on cuda and use the same dtype + check_device_dtype(out_dtype, inputs); + check_device_dtype(out_dtype, weights); + check_device_dtype(out_dtype, outputs); + + // Weights should be transposed to (n, k) of column major + std::vector transa_array(group_count, CUBLAS_OP_T); + std::vector transb_array(group_count, CUBLAS_OP_N); + + // Get dim arrays + std::vector m_array = get_dims(weights, 0); + std::vector n_array = get_dims(inputs, 0); + std::vector k_array = get_dims(inputs, 1); + + // Make sure the dimensions in each group match + std::vector m_array1 = get_dims(outputs, 1); + std::vector n_array1 = get_dims(outputs, 0); + std::vector k_array1 = get_dims(weights, 1); + check_equal(m_array, m_array1, "sizes don't match on m dimension"); + check_equal(n_array, n_array1, "sizes don't match on n dimension"); + check_equal(k_array, k_array1, "sizes don't match on k dimension"); + + // Get leading dimensions + std::vector lda_array = get_strides(weights, 0); + std::vector ldb_array = get_strides(inputs, 0); + std::vector ldc_array = get_strides(outputs, 0); + + // Use default scaling factors + std::vector alpha_array(group_count, 1); + std::vector beta_array(group_count, 0); + + std::vector a_array = get_tensor_ptrs(weights); + std::vector b_array = get_tensor_ptrs(inputs); + std::vector c_array = get_tensor_ptrs(outputs); + + auto stream = reinterpret_cast(cuda_stream); + + // Should allocate tensors for storage of pointers + torch::Tensor d_a = create_ptr_pointer(a_array, stream); + torch::Tensor d_b = create_ptr_pointer(b_array, stream); + torch::Tensor d_c = create_ptr_pointer(c_array, stream); + +#if defined CUDA_VERSION && CUDA_VERSION >= 12050 + auto handle = reinterpret_cast(cublas_handle); + cudaDataType_t cuda_data_type = (out_dtype == torch::kHalf ? CUDA_R_16F : CUDA_R_16BF); + + auto status = cublasGemmGroupedBatchedEx( + handle, + transa_array.data(), + transb_array.data(), + m_array.data(), + n_array.data(), + k_array.data(), + alpha_array.data(), + (void**)d_a.data_ptr(), + cuda_data_type, + lda_array.data(), + (void**)d_b.data_ptr(), + cuda_data_type, + ldb_array.data(), + beta_array.data(), + (void**)d_c.data_ptr(), + cuda_data_type, + ldc_array.data(), + group_count, + group_size.data(), + CUBLAS_COMPUTE_32F); + TORCH_CHECK(status == CUBLAS_STATUS_SUCCESS, "cublas grouped gemm failed: ", cublasGetStatusString(status)); + TORCH_CHECK(cudaStreamSynchronize(stream) == cudaSuccess, "Failed when stream synchronization"); + return; +#endif + + TORCH_CHECK_NOT_IMPLEMENTED( + false, "Cublas GroupGemm is not implemented with current compute capability: ", getSMVersion()); +} diff --git a/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu b/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu new file mode 100644 index 0000000..c36fba5 --- /dev/null +++ b/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu @@ -0,0 +1,406 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/gemm/collective/collective_builder.hpp" +#include "cutlass_extensions/gemm/dispatch_policy.hpp" +#include "utils.h" + +using namespace cute; + +template +void launch_sm90_fp8_blockwise_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + using ElementAccumulator = float; + using ElementCompute = float; + using ElementBlockScale = float; + + using ElementA = cutlass::float_e4m3_t; + using LayoutA = cutlass::layout::RowMajor; + constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using ElementB = cutlass::float_e4m3_t; + using LayoutB = cutlass::layout::ColumnMajor; + constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementD = OutType; + using LayoutD = cutlass::layout::RowMajor; + constexpr int AlignmentD = AlignmentC; + + using ArchTag = cutlass::arch::Sm90; + using OperatorClass = cutlass::arch::OpClassTensorOp; + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative; + using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto; + using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT; + + using KernelSchedule = + cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum; + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + EpilogueTileType, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + ElementD, + LayoutD, + AlignmentD, + EpilogueSchedule, + StoreEpilogueCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutA, + AlignmentA, + ElementB, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + cutlass::gemm::PersistentScheduler>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(m, n, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{a_ptr, stride_a, b_ptr, stride_b, 4, a_s_ptr, b_s_ptr}; + typename GemmKernel::EpilogueArguments epilogue_args{{}, nullptr, stride_d, o_ptr, stride_d}; + + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + mainloop_args, + epilogue_args, + }; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess, cutlassGetStatusString(can_implement)) + + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, cutlassGetStatusString(status)) +} + +template < + typename OutType, + typename MmaTileShape, + typename PerSmTileShape, + typename EpilogueTileShape, + typename ScalesPerTile, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _1, _1>> +void launch_sm100_fp8_blockwise_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + static constexpr int ScaleMsPerTile = size<0>(ScalesPerTile{}); + static constexpr int ScaleGranularityM = size<0>(MmaTileShape{}) / ScaleMsPerTile; + static constexpr int ScaleGranularityN = size<1>(MmaTileShape{}) / size<1>(ScalesPerTile{}); + static constexpr int ScaleGranularityK = size<2>(MmaTileShape{}) / size<2>(ScalesPerTile{}); + + using ElementAB = cutlass::float_e4m3_t; + using ElementA = ElementAB; + using ElementB = ElementAB; + using ElementC = void; + using ElementD = OutType; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutD = cutlass::layout::RowMajor; + using LayoutC = LayoutD; + // This means both SFA and SFB are column-major. + using ScaleConfig = cutlass::detail::Sm100BlockwiseScaleConfig< + ScaleGranularityM, + ScaleGranularityN, + ScaleGranularityK, + cute::UMMA::Major::MN, + cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = AlignmentD; + + using ElementAccumulator = float; + using ElementBlockScale = float; + using ElementCompute = float; + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + cutlass::arch::OpClassTensorOp, + PerSmTileShape, + ClusterShape, + EpilogueTileShape, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + ElementD, + LayoutD, + AlignmentD, + cutlass::epilogue::TmaWarpSpecialized1Sm>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::KernelTmaWarpSpecializedBlockwise1SmSm100>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, + CollectiveMainloop, + CollectiveEpilogue, + cutlass::gemm::PersistentScheduler>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + auto scales_a_ptr = static_cast(scales_a.data_ptr()); + auto scales_b_ptr = static_cast(scales_b.data_ptr()); + auto c_ptr = static_cast(out.data_ptr()); + + using StrideA = typename GemmKernel::StrideA; + using StrideB = typename GemmKernel::StrideB; + using StrideD = typename GemmKernel::StrideD; + using StrideC = typename GemmKernel::StrideD; + + StrideA a_stride = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB b_stride = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC c_stride = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, a_stride, b_ptr, b_stride, scales_a_ptr, layout_SFA, scales_b_ptr, layout_SFB}; + + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, c_stride, c_ptr, c_stride}; + epilogue_args.thread.alpha = 1.0f; + + typename GemmKernel::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, {m, n, k, 1}, mainloop_args, epilogue_args}; + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess, cutlassGetStatusString(can_implement)) + + size_t workspace_size = gemm_op.get_workspace_size(args); + cutlass::device_memory::allocation workspace(workspace_size); + + auto init_status = gemm_op.initialize(args, workspace.get()); + TORCH_CHECK(init_status == cutlass::Status::kSuccess, cutlassGetStatusString(init_status)); + + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, cutlassGetStatusString(status)) +} + +template +void sm90_fp8_blockwise_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + using TileShape = Shape<_128, _128, _128>; + using ClusterShape = Shape<_1, _1, _1>; + launch_sm90_fp8_blockwise_scaled_mm(out, a, b, scales_a, scales_b); +} + +template +void sm100_fp8_blockwise_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + if (a.size(0) <= 128) { + using MmaTileShape = Shape<_64, _128, _128>; + using PerSmTileShape = Shape<_64, _128, _128>; + using EpilogueTileShape = Shape<_64, _64>; + using ScalesPerTile = Shape<_64, _1, _1>; + launch_sm100_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); + } else { + using MmaTileShape = Shape<_128, _128, _128>; + using PerSmTileShape = Shape<_128, _128, _128>; + using EpilogueTileShape = Shape<_128, _64>; + using ScalesPerTile = Shape<_128, _1, _1>; + launch_sm100_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); + } +} + +torch::Tensor fp8_blockwise_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_a must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + + TORCH_CHECK( + (mat_a.size(1) * mat_a.element_size()) % 16 == 0, "mat_a must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK( + (mat_b.size(0) * mat_b.element_size()) % 16 == 0, "mat_b must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK(mat_a.scalar_type() == torch::kFloat8_e4m3fn, "mat_a must be Float8_e4m3fn"); + TORCH_CHECK(mat_b.scalar_type() == torch::kFloat8_e4m3fn, "mat_b must be Float8_e4m3fn"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + auto is_contiguous_vector = [](const torch::Tensor& t) { + auto t_sizes = t.sizes(); + return t.is_contiguous() && + (t.dim() == 1 || (t.dim() == 2 && *std::min_element(t_sizes.begin(), t_sizes.end()) == 1)); + }; + + TORCH_CHECK(mat_a.size(0) == scales_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(mat_a.size(1) / 128 == scales_a.size(1), "size of scales_a is not matched"); + TORCH_CHECK(scales_a.stride(0) == 1 || is_contiguous_vector(scales_a), "scales_a must be M major"); + TORCH_CHECK(mat_b.size(0) / 128 == scales_b.size(0), "size of scales_b is not matched"); + TORCH_CHECK(mat_b.size(1) / 128 == scales_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_b.stride(0) == 1 || is_contiguous_vector(scales_b), "scales_b must be K major"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + TORCH_CHECK((out.size(1) * out.element_size()) % 16 == 0, "out must be multiple of 16 bytes for memory alignment"); + + auto sm_version = getSMVersion(); + +#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + if (sm_version == 90) { + if (out_dtype == torch::kBFloat16) { + sm90_fp8_blockwise_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b); + } else { + sm90_fp8_blockwise_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b); + } + return out; + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM100A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version == 100) { + int64_t original_rows = mat_a.size(0); + + torch::Tensor mat_a_padded = pad_tensor(mat_a, /*alignment=*/4); + torch::Tensor scales_a_padded = pad_tensor(scales_a, /*alignment=*/4, /*col_major=*/true); + torch::Tensor out_padded = torch::empty({mat_a_padded.size(0), mat_b.size(1)}, out.options()); + + if (out_dtype == torch::kBFloat16) { + sm100_fp8_blockwise_dispatch_shape( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } else { + sm100_fp8_blockwise_dispatch_shape(out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented fp8_blockwise_scaled_mm for current compute capability: ", sm_version); +} diff --git a/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu b/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu new file mode 100644 index 0000000..d3bc610 --- /dev/null +++ b/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu @@ -0,0 +1,1146 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_template.h +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_kernel_template_sm89.h +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_kernel_template_sm90.h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +using namespace cute; + +#if defined CUDA_VERSION && CUDA_VERSION >= 12040 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CtaShape, + typename WarpShape, + int Stages, + bool WithBias, + typename FP8MathOperator = cutlass::arch::OpMultiplyAdd, + template typename EpilogueVisitor = cutlass::epilogue::threadblock::Sm80EVT, + typename ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>> +struct DeviceGemmFp8RowwiseSm89 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + + using ElementA = ElementType; + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using ElementB = ElementType; + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = OutElementType; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementOutput = OutElementType; + using LayoutOutput = cutlass::layout::RowMajor; + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + using ElementAccumulator = AccumElementType; + using ElementComputeEpilogue = float; + using ArchTag = cutlass::arch::Sm89; + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>; + // Number of epilogue stages in EVT + static constexpr int EVTEpilogueStages = 1; + + using OutputTileThreadMap = cutlass::epilogue::threadblock:: + OutputTileThreadLayout; + + // Definition of EVT + using accSrc = cutlass::epilogue::threadblock::VisitorAccFetch; + + using ComputeBScale = cutlass::epilogue::threadblock::VisitorCompute< + cutlass::multiplies, + ElementComputeEpilogue, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using bScaleSrc = cutlass::epilogue::threadblock:: + VisitorRowBroadcast>; + using EpilogueBScale = cutlass::epilogue::threadblock::Sm80EVT; + + using ComputeAScale = cutlass::epilogue::threadblock:: + VisitorCompute; + using aScaleSrc = cutlass::epilogue::threadblock:: + VisitorColBroadcast>; + using EpilogueAScale = cutlass::epilogue::threadblock::Sm80EVT; + + // With bias + using biasSrc = + cutlass::epilogue::threadblock::VisitorRowBroadcast>; + using ComputeAScaleWithBias = cutlass::epilogue::threadblock::VisitorCompute< + cutlass::multiply_add, + ElementC, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using EpilogueAScaleWithBias = + cutlass::epilogue::threadblock::Sm80EVT; + + using dTar = cutlass::epilogue::threadblock::VisitorAuxStore< + OutputTileThreadMap, + ElementC, + cutlass::FloatRoundStyle::round_to_nearest, + Stride>; + using EpilogueStore = typename cutlass::platform::conditional< + WithBias, + cutlass::epilogue::threadblock::Sm80EVT, + cutlass::epilogue::threadblock::Sm80EVT>::type; + + using EpilogueOp = EpilogueStore; + + using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmWithVisitor< + ElementA, + LayoutA, + cutlass::ComplexTransform::kNone, + AlignmentA, + ElementB, + LayoutB, + cutlass::ComplexTransform::kNone, + AlignmentB, + ElementC, + LayoutC, + AlignmentC, + ElementAccumulator, + ElementComputeEpilogue, + OperatorClass, + ArchTag, + CtaShape, + WarpShape, + InstructionShape, + EpilogueOp, + ThreadblockSwizzle, + Stages, + FP8MathOperator, + EVTEpilogueStages>::GemmKernel; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename Gemm::Arguments prepare_sm89_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementT = typename Gemm::ElementA; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + int64_t lda = a.stride(0); + int64_t ldb = b.stride(1); + int64_t ldc = out.stride(0); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + ElementOutput const* ptr_bias = nullptr; + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value()) + ptr_bias = reinterpret_cast(bias.value().data_ptr()); + } + ElementOutput* ptr_d = reinterpret_cast(out.data_ptr()); + ElementComputeEpilogue const* ptr_scales_a = reinterpret_cast(scales_a.data_ptr()); + ElementComputeEpilogue const* ptr_scales_b = reinterpret_cast(scales_b.data_ptr()); + + typename Gemm::Arguments args( + cutlass::gemm::GemmUniversalMode::kGemm, // Mode + {m, n, k}, // Problem size + 1, // Split-k factor + {}, // Epilogue args + ptr_a, // a pointer + ptr_b, // b pointer + nullptr, // c pointer (unused) + nullptr, // d pointer (unused) + m * k, // batch stride a (unused) + n * k, // batch stride b (unused) + m * n, // batch stride c (unused) + m * n, // batch stride d (unused) + lda, // stride a + ldb, // stride b + ldc, // stride c (unused) + ldc); // stride d (unused) + if constexpr (WithBias) { + args.epilogue = { + { + { + {}, // Accumulator + {ptr_scales_b, ElementComputeEpilogue(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_scales_a, ElementComputeEpilogue(0), {_1{}, _0{}, _0{}}}, + {ptr_bias, ElementOutput(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_d, {n, _1{}, _0{}}}}; + } else { + args.epilogue = { + { + { + {}, // Accumulator + {ptr_scales_b, ElementComputeEpilogue(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_scales_a, ElementComputeEpilogue(0), {_1{}, _0{}, _0{}}}, + {} // Multiplies + }, + {ptr_d, {n, _1{}, _0{}}}}; + } + + return args; +} + +template +void launch_sm89_fp8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm89_fp8_args(out, a, b, scales_a, scales_b, bias); + Gemm gemm_op; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm89_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + if (bias) { + using Gemm = typename DeviceGemmFp8RowwiseSm89< + ElementInput, + ElementOutput, + AccumElementType, + CtaShape, + WarpShape, + Stages, + true>::Gemm; + return launch_sm89_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = typename DeviceGemmFp8RowwiseSm89< + ElementInput, + ElementOutput, + AccumElementType, + CtaShape, + WarpShape, + Stages, + false>::Gemm; + return launch_sm89_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm89_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + uint32_t const m = a.size(0); + uint32_t const n = out.size(1); + + if (m == 1) { + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 16) { + // M in (1, 16] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + // M in (16, 64] + if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + // M in (64, 128] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 256) { + // M in (128, 256] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 64, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 64, 128>, + cutlass::gemm::GemmShape<64, 32, 128>, + 4>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 512) { + // M in (256, 512) + if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 2>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } + } else { + // M in (512, inf) + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 3>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 2>(out, a, b, scales_a, scales_b, bias); + } + } +} +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm90 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + + // A matrix configuration + using ElementA = ElementType; // Element type for A matrix operand + using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand + static constexpr int AlignmentA = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A + // matrix in units of elements (up to 16 bytes) + + // B matrix configuration + using ElementB = ElementType; // Element type for B matrix operand + using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand + static constexpr int AlignmentB = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of B + // matrix in units of elements (up to 16 bytes) + + // C/D matrix configuration + using ElementC = void; // Element type for C matrix operands + using LayoutC = cutlass::layout::RowMajor; // Layout type for C matrix operands + static constexpr int AlignmentC = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrices in + // units of elements (up to 16 bytes) + + // Output matrix configuration + using ElementOutput = OutElementType; // Element type for output matrix operands + using LayoutOutput = cutlass::layout::RowMajor; // Layout type for output matrix operands + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + // // Auxiliary matrix configuration and other fusion types + // using ElementBias = float; + + // Multiply-accumulate blocking/pipelining details + using ElementAccumulator = AccumElementType; // Element type for internal accumulation + using ElementCompute = float; // Element type for compute + using ElementComputeEpilogue = float; + using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature + using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag + using TileShape = CTAShape; // Threadblock-level tile size + + static constexpr bool PONG = false; + static constexpr bool FAST_ACCUM = true; + static constexpr bool USE_BIAS = false; + + using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized + // based on the tile size + using KernelSchedule = cutlass::gemm::collective::KernelScheduleAuto; // Kernel to launch based on the default + // setting in the Collective Builder + // Implement rowwise scaling epilogue. + using XScale = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using WScale = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementOutput, + ElementOutput, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using Compute0 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, + ElementComputeEpilogue, // First stage output type. + ElementComputeEpilogue, // First stage input types. + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, + ElementOutput, + ElementComputeEpilogue, // Second stage input types. + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute1 = cutlass::epilogue::fusion::Sm90EVT; + + // With bias + using ComputeWithBias = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiply_add, + ElementOutput, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using EVTComputeWithBias = cutlass::epilogue::fusion::Sm90EVT; + + using EpilogueEVT = typename cutlass::platform::conditional::type; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm90, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementComputeEpilogue, + ElementC, + LayoutC, + AlignmentC, + ElementOutput, + LayoutOutput, + AlignmentOutput, + cutlass::epilogue::TmaWarpSpecialized, + EpilogueEVT>::CollectiveOp; + + using DefaultSchedule = cutlass::gemm::KernelTmaWarpSpecialized; + using PongSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong; + using FastDefaultSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum; + using FastPongSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum; + + using SlowAccum = DefaultSchedule; + using FastAccum = FastPongSchedule; // Default apply Pingpong + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutA, + AlignmentA, + ElementB, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + TileSchedulerType>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename Gemm::Arguments prepare_sm90_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementT = typename Gemm::ElementA; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + ElementOutput const* ptr_bias = nullptr; + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value()) + ptr_bias = reinterpret_cast(bias.value().data_ptr()); + } + ElementOutput* ptr_d = reinterpret_cast(out.data_ptr()); + ElementComputeEpilogue const* ptr_scales_a = reinterpret_cast(scales_a.data_ptr()); + ElementComputeEpilogue const* ptr_scales_b = reinterpret_cast(scales_b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, make_shape(m, n, 1)); + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {ptr_a, stride_a, ptr_b, stride_b}, + {{}, // epilogue.thread + nullptr, + stride_c, + ptr_d, + stride_d}}; + if constexpr (WithBias) { + args.epilogue.thread = { + {ptr_scales_a}, + { + {ptr_scales_b}, + {}, // Accumulator + {} // Multiplies + }, + {ptr_bias}, + {}, // Multiplies + }; + } else { + args.epilogue.thread = { + {ptr_scales_a}, + { + {ptr_scales_b}, + {}, // Accumulator + {} // Multiplies + }, + {}, // Multiplies + }; + } + + return args; +} + +template +void launch_sm90_fp8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm90_fp8_args(out, a, b, scales_a, scales_b, bias); + Gemm gemm_op; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template < + typename OutType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename TileSchedulerType> +void sm90_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias, + bool fast_accum = true, + bool use_persistent = false) { + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + using EpilogueScheduleType = cutlass::epilogue::TmaWarpSpecialized; + + if (bias) { + using Gemm = typename DeviceGemmFp8RowwiseSm90< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>::Gemm; + return launch_sm90_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = typename DeviceGemmFp8RowwiseSm90< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>::Gemm; + return launch_sm90_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm90_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + uint32_t const m = a.size(0); + using FastPingpongScheduler = cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum; + using FastBasicScheduler = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum; + using PersistentTileScheduler = cutlass::gemm::PersistentScheduler; + using BasicTileScheduler = void; + if (m <= 1) { + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _8, _1>, + FastBasicScheduler, + BasicTileScheduler>(out, a, b, scales_a, scales_b, bias); + } + if (m <= 64) { + // m in [1, 64] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _4, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else if (m <= 256) { + // m in (64, 256] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else if (m <= 1024) { + // m in (256, 1024] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_128, _128, _128>, + Shape<_1, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else { + // m in (1024, inf) + return sm90_fp8_dispatch_bias< + OutType, + Shape<_128, _128, _128>, + Shape<_2, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } +} +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm100 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + using TileShape = CTAShape; + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using ElementComputeEpilogue = float; + using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + OutElementType, + OutElementType, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using LayoutD = cutlass::layout::RowMajor; + static constexpr int AlignmentD = AlignmentC; + + using Compute1MulAdd = cutlass::epilogue::fusion:: + Sm90Compute; + using Compute1Mul = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute = typename std::conditional_t< + WithBias, + cutlass::epilogue::fusion::Sm90EVT, + cutlass::epilogue::fusion::Sm90EVT>; + using ArgumentType = typename EVTCompute::Arguments; + // MMA type + using ElementAccumulator = AccumElementType; + + // Epilogue types + using ElementCompute = float; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + OutElementType, + LayoutD, + AlignmentD, + EpilogueScheduleType, + EVTCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + ElementType, + LayoutA, + AlignmentA, + ElementType, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + template + static auto args_from_tensor(torch::Tensor const& tensor) { + using Arguments = typename Descriptor::Arguments; + auto* data_ptr = static_cast(tensor.data_ptr()); + static_assert( + std::is_same_v || std::is_same_v || std::is_same_v); + return Arguments{data_ptr}; + } + + public: + static ArgumentType prepare_args( + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + std::optional const& bias = std::nullopt) { + auto a_args = args_from_tensor(a_scales); + auto b_args = args_from_tensor(b_scales); + + typename EVTCompute0::Arguments evt0_args{b_args, {}, {}}; + + if constexpr (WithBias) { + auto bias_args = args_from_tensor(bias.value()); + return ArgumentType{a_args, evt0_args, bias_args, {}}; + } else { + return ArgumentType{a_args, evt0_args, {}}; + } + } +}; + +template +typename GemmType::Gemm::Arguments prepare_sm100_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using Gemm = typename GemmType::Gemm; + using ElementT = typename Gemm::ElementA; + using ElementC = typename Gemm::ElementC; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using GemmKernel = typename Gemm::GemmKernel; + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = StrideC; + using StrideAux = StrideC; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(m, n, 1)); + StrideAux aux_stride = stride_d; + + typename GemmKernel::MainloopArguments mainloop_args{ptr_a, stride_a, ptr_b, stride_b}; + + typename GemmKernel::ProblemShape prob_shape = {m, n, k, 1}; + cutlass::KernelHardwareInfo hw_info; + typename GemmKernel::TileSchedulerArguments scheduler = {}; + + auto ptr_c = static_cast(out.data_ptr()); + + auto prepare_epilogue_args = [&](const c10::optional& bias = c10::nullopt) { + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value(), "Bias tensor is required but not provided."); + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b, bias.value()), ptr_c, stride_c, ptr_c, stride_d}; + } else { + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b), ptr_c, stride_c, ptr_c, stride_d}; + } + }; + + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, + prob_shape, + mainloop_args, + prepare_epilogue_args(bias), + hw_info, + scheduler}; + return args; +} + +template +void launch_sm100_fp8_scaled_mm( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm100_fp8_args(out, a, b, scales_a, scales_b, bias); + + typename Gemm::Gemm gemm_op; + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm100_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using CTAShape = Shape<_256, _128, _64>; + using ClusterShape = Shape<_2, _2, _1>; + using MainloopScheduleType = cutlass::gemm::collective::KernelScheduleAuto; + using EpilogueScheduleType = cutlass::epilogue::collective::EpilogueScheduleAuto; + using TileSchedulerType = void; + + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + + if (bias) { + using Gemm = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm100_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + return sm100_fp8_dispatch_bias(out, a, b, scales_a, scales_b, bias); +} +#endif + +torch::Tensor fp8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_a must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + + TORCH_CHECK( + (mat_a.size(1) * mat_a.element_size()) % 16 == 0, "mat_a must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK( + (mat_b.size(0) * mat_b.element_size()) % 16 == 0, "mat_b must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK(mat_a.scalar_type() == torch::kFloat8_e4m3fn, "mat_a must be Float8_e4m3fn"); + TORCH_CHECK(mat_b.scalar_type() == torch::kFloat8_e4m3fn, "mat_b must be Float8_e4m3fn"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + TORCH_CHECK(scales_a.numel() == mat_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(scales_b.numel() == mat_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_a.is_contiguous(), "scales_a must be contiguous"); + TORCH_CHECK(scales_b.is_contiguous(), "scales_b msut be contiguous"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + if (bias) { + TORCH_CHECK(bias->numel() == mat_b.size(1), "size of bias is not matched"); + TORCH_CHECK(bias->is_contiguous(), "bias must be contiguous"); + TORCH_CHECK(bias->dtype() == out_dtype, "bias dtype must match output dtype"); + } + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + TORCH_CHECK((out.size(1) * out.element_size()) % 16 == 0, "out must be multiple of 16 bytes for memory alignment"); + + auto sm_version = getSMVersion(); + +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version >= 100) { + if (out_dtype == torch::kBFloat16) { + sm100_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm100_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + if (sm_version >= 90) { + if (out_dtype == torch::kBFloat16) { + sm90_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm90_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12040 + if (sm_version == 89) { + if (out_dtype == torch::kBFloat16) { + sm89_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm89_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + + TORCH_CHECK_NOT_IMPLEMENTED(false, "No implemented fp8_scaled_mm for current compute capability: ", sm_version); +} diff --git a/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu b/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu new file mode 100644 index 0000000..a81dba3 --- /dev/null +++ b/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu @@ -0,0 +1,747 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h" +#include "cutlass_extensions/gemm/gemm_universal_base_compat.h" +#include "cutlass_extensions/gemm/gemm_with_epilogue_visitor.h" +#include "utils.h" + +using namespace cute; + +template < + typename ElementOutput, + typename ArchTag, + typename ThreadblockShape, + typename WarpShape, + typename InstructionShape, + int NumStages> +void cutlass_int8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementAccumulator = int32_t; + using ElementCompute = float; + using ElementInputA = int8_t; + using ElementInputB = int8_t; + + using OperatorClass = cutlass::arch::OpClassTensorOp; + using ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>; + + using DefaultGemmConf = cutlass::gemm::device:: + DefaultGemmConfiguration; + using EpilogueOutputOp = typename DefaultGemmConf::EpilogueOutputOp; + + using GemmKernel_ = typename cutlass::gemm::kernel::DefaultGemm< + ElementInputA, + cutlass::layout::RowMajor, + DefaultGemmConf::kAlignmentA, + ElementInputB, + cutlass::layout::ColumnMajor, + DefaultGemmConf::kAlignmentB, + ElementOutput, + cutlass::layout::RowMajor, + ElementAccumulator, + OperatorClass, + ArchTag, + ThreadblockShape, + WarpShape, + InstructionShape, + EpilogueOutputOp, + ThreadblockSwizzle, + NumStages, + true, + typename DefaultGemmConf::Operator>::GemmKernel; + + using AlphaColTileIterator = cutlass::epilogue::threadblock::PredicatedTileIterator< + cutlass::epilogue::threadblock::OutputTileOptimalThreadMap< + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Shape, + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Count, + GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::kThreads, + GemmKernel_::Epilogue::OutputTileIterator::kElementsPerAccess, + cutlass::sizeof_bits::value>, + ElementCompute>; + + using EpilogueVisitor = typename cutlass::epilogue::threadblock::EpilogueVisitorPerRowPerCol< + ThreadblockShape, + GemmKernel_::kThreadCount, + AlphaColTileIterator, + typename GemmKernel_::Epilogue::OutputTileIterator, + ElementAccumulator, + ElementCompute, + EpilogueOutputOp>; + + using Epilogue = typename cutlass::epilogue::threadblock:: + EpilogueWithVisitorFromExistingEpilogue::Epilogue; + + using GemmKernel = + cutlass::gemm::kernel::GemmWithEpilogueVisitor; + + using Gemm = cutlass::gemm::device::GemmUniversalBaseCompat; + + Gemm gemm_op; + + int m = mat_a.size(0); + int k = mat_a.size(1); + int n = mat_b.size(1); + + auto a_ptr = static_cast(mat_a.data_ptr()); + auto b_ptr = static_cast(mat_b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + int64_t lda = mat_a.stride(0); + int64_t ldb = mat_b.stride(1); + int64_t ldd = out.stride(0); + + ElementOutput* bias_ptr = nullptr; + int64_t ldc = 0; + if (bias) { + bias_ptr = static_cast(bias->data_ptr()); + } + + typename EpilogueOutputOp::Params linearScalingParams; + typename EpilogueVisitor::Arguments visitor_args{linearScalingParams}; + + typename Gemm::Arguments args{ + {m, n, k}, {a_ptr, lda}, {b_ptr, ldb}, {b_s_ptr, 0}, {a_s_ptr, 0}, {bias_ptr, ldc}, {o_ptr, ldd}, visitor_args}; + + auto workspace = torch::empty( + gemm_op.get_workspace_size(args), torch::TensorOptions().dtype(torch::kUInt8).device(mat_a.device())); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK( + can_implement == cutlass::Status::kSuccess, + "gemm cannot implement, error: ", + cutlassGetStatusString(can_implement)); + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "gemm executioin failed, error: ", cutlassGetStatusString(status)); +} + +template +void sm75_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + if (m <= 32) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 128, 64>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (m <= 64) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (m <= 256) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template +void sm80_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 16) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 6>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 32) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 6>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128 && n < 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +// Dispatch shape for sm89 (L40S, L20, RTX 4090), according to: +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_int8_dispatch.cuh +template +void sm89_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 16) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 128, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 4>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 32) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 128, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 4>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 16384) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 256) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 16384) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<256, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template < + typename ElementOutput, + typename TileShape, + typename ClusterShape, + typename MainloopScheduleType, + bool WithBias> +void cutlass_int8_scaled_mm_sm90( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ArchTag = cutlass::arch::Sm90; + + using ElementAccumulator = int32_t; + using ElementCompute = float; + using ElementInputA = int8_t; + using ElementInputB = int8_t; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using EpilogueScheduleType = cutlass::epilogue::TmaWarpSpecialized; + using TileSchedulerType = cutlass::gemm::PersistentScheduler; + + using XScale = cutlass::epilogue::fusion:: + Sm90ColBroadcast<0, TileShape, ElementCompute, ElementCompute, Stride, Int<0>, Int<0>>>; + + using WScale = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, TileShape, ElementCompute, ElementCompute, Stride, Int<1>, Int<0>>>; + + using Bias = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, TileShape, ElementOutput, ElementOutput, Stride, Int<1>, Int<0>>>; + + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + // Scale + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute1 = cutlass::epilogue::fusion::Sm90EVT; + + // With bias + using ComputeWithBias = cutlass::epilogue::fusion:: + Sm90Compute; + using EVTComputeWithBias = cutlass::epilogue::fusion::Sm90EVT; + + using EpilogueEVT = typename cutlass::platform::conditional::type; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementOutput, + cutlass::layout::RowMajor, + AlignmentC, + ElementOutput, + cutlass::layout::RowMajor, + AlignmentOutput, + EpilogueScheduleType, + EpilogueEVT>::CollectiveOp; + + using Stages = cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementInputA, + cutlass::layout::RowMajor, + AlignmentA, + ElementInputB, + cutlass::layout::ColumnMajor, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + Stages, + MainloopScheduleType>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + TileSchedulerType>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = mat_a.size(0); + int k = mat_a.size(1); + int n = mat_b.size(1); + + auto a_ptr = static_cast(mat_a.data_ptr()); + auto b_ptr = static_cast(mat_b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, make_shape(m, n, 1)); + + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {a_ptr, stride_a, b_ptr, stride_b}, + {{}, // epilogue.thread + nullptr, + stride_c, + o_ptr, + stride_d}}; + + if constexpr (WithBias) { + ElementOutput* bias_ptr = static_cast(bias->data_ptr()); + args.epilogue.thread = { + {a_s_ptr}, + {{b_s_ptr}, {}, {}}, + {bias_ptr}, + {}, + }; + } else { + args.epilogue.thread = { + {a_s_ptr}, + {{b_s_ptr}, {}, {}}, + {}, + }; + } + + auto workspace = torch::empty( + gemm_op.get_workspace_size(args), torch::TensorOptions().dtype(torch::kUInt8).device(mat_a.device())); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK( + can_implement == cutlass::Status::kSuccess, + "gemm cannot implement, error: ", + cutlassGetStatusString(can_implement)); + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "gemm executioin failed, error: ", cutlassGetStatusString(status)); +} + +template +void sm90_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + if (bias) { + cutlass_int8_scaled_mm_sm90( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm_sm90( + out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template +void sm90_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 32) { + if (n < 8192) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_1, _8, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _128, _128>, + Shape<_1, _8, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n < 8192) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_1, _4, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _256>, + Shape<_1, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + if (n <= 4096) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _128, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_128, _128, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecializedPingpong>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +torch::Tensor int8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_a must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + TORCH_CHECK(mat_a.size(1) % 16 == 0, "mat_a.size(1) must be multiple of 16 for memory alignment"); + TORCH_CHECK(mat_b.size(0) % 16 == 0, "mat_b.size(0) must be multiple of 16 for memory alignment"); + TORCH_CHECK(mat_b.size(1) % 8 == 0, "mat_b.size(1) must be multiple of 8 for memory alignment"); // out.stride(0) + TORCH_CHECK(mat_a.scalar_type() == torch::kInt8, "mat_a must be Int8"); + TORCH_CHECK(mat_b.scalar_type() == torch::kInt8, "mat_b must be Int8"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + TORCH_CHECK(scales_a.numel() == mat_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(scales_b.numel() == mat_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_a.is_contiguous(), "scales_a must be contiguous"); + TORCH_CHECK(scales_b.is_contiguous(), "scales_b msut be contiguous"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + if (bias) { + TORCH_CHECK(bias->numel() == mat_b.size(1), "size of bias is not matched"); + TORCH_CHECK(bias->is_contiguous(), "bias must be contiguous"); + TORCH_CHECK(bias->dtype() == out_dtype, "bias dtype must match output dtype"); + } + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + + auto sm_version = getSMVersion(); + + if (sm_version >= 75 && sm_version < 80) { + TORCH_CHECK(out_dtype == torch::kHalf, "out_dtype must be Half for SM75"); + sm75_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (sm_version >= 80 && sm_version < 90) { + // sm86/sm89 has a much smaller shared memory size (100K) than sm80 (160K) + if (sm_version == 86 || sm_version == 89) { + if (out_dtype == torch::kBFloat16) { + sm89_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm89_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + if (out_dtype == torch::kBFloat16) { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } + } + } else if (sm_version == 90) { +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + // cutlass 3.x + if (out_dtype == torch::kBFloat16) { + sm90_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm90_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } +#else + // fallback to cutlass 2.x + if (out_dtype == torch::kBFloat16) { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } +#endif + } else { + TORCH_CHECK_NOT_IMPLEMENTED(false, "No implemented int8_scaled_mm for current compute capability."); + } + + return out; +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu b/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu new file mode 100644 index 0000000..60fda7d --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu @@ -0,0 +1,29 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 +void scaled_fp4_quant_sm100a( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf); +#endif + +void scaled_fp4_quant( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return scaled_fp4_quant_sm100a(output, input, output_sf, input_sf); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 quantization"); +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu b/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu new file mode 100644 index 0000000..5024d20 --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu @@ -0,0 +1,399 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "utils.h" + +// Get type2 from type or vice versa (applied to half and bfloat16) +template +struct TypeConverter { + using Type = half2; +}; // keep for generality + +template <> +struct TypeConverter { + using Type = half; +}; + +template <> +struct TypeConverter { + using Type = half2; +}; + +template <> +struct TypeConverter<__nv_bfloat162> { + using Type = __nv_bfloat16; +}; + +template <> +struct TypeConverter<__nv_bfloat16> { + using Type = __nv_bfloat162; +}; + +#define ELTS_PER_THREAD 8 + +constexpr int CVT_FP4_ELTS_PER_THREAD = 8; +constexpr int CVT_FP4_SF_VEC_SIZE = 16; + +// Convert 8 float32 values into 8 e2m1 values (represented as one uint32_t). +inline __device__ uint32_t fp32_vec_to_e2m1(float (&array)[8]) { + // PTX instructions used here requires sm100a. +#if CUDA_VERSION >= 12080 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) && __CUDA_ARCH_HAS_FEATURE__(SM100_ALL) + uint32_t val; + asm volatile( + "{\n" + ".reg .b8 byte0;\n" + ".reg .b8 byte1;\n" + ".reg .b8 byte2;\n" + ".reg .b8 byte3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n" + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n" + "}" + : "=r"(val) + : "f"(array[0]), + "f"(array[1]), + "f"(array[2]), + "f"(array[3]), + "f"(array[4]), + "f"(array[5]), + "f"(array[6]), + "f"(array[7])); + return val; +#else + return 0; +#endif +#endif +} + +// Convert 4 float2 values into 8 e2m1 values (represented as one uint32_t). +inline __device__ uint32_t fp32_vec_to_e2m1(float2 (&array)[4]) { + // PTX instructions used here requires sm100a. +#if CUDA_VERSION >= 12080 +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) && __CUDA_ARCH_HAS_FEATURE__(SM100_ALL) + uint32_t val; + asm volatile( + "{\n" + ".reg .b8 byte0;\n" + ".reg .b8 byte1;\n" + ".reg .b8 byte2;\n" + ".reg .b8 byte3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n" + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n" + "}" + : "=r"(val) + : "f"(array[0].x), + "f"(array[0].y), + "f"(array[1].x), + "f"(array[1].y), + "f"(array[2].x), + "f"(array[2].y), + "f"(array[3].x), + "f"(array[3].y)); + return val; +#else + return 0; +#endif +#endif +} + +// Fast reciprocal. +inline __device__ float reciprocal_approximate_ftz(float a) { + float b; + asm volatile("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(b) : "f"(a)); + return b; +} + +template +__device__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset(int rowIdx, int colIdx, int numCols, SFType* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + static_assert(CVT_FP4_NUM_THREADS_PER_SF == 1 || CVT_FP4_NUM_THREADS_PER_SF == 2); + + // One pair of threads write one SF to global memory. + // TODO: stage through smem for packed STG.32 + // is it better than STG.8 from 4 threads ? + if (threadIdx.x % CVT_FP4_NUM_THREADS_PER_SF == 0) { + // SF vector index (16 elements share one SF in the K dimension). + int32_t kIdx = colIdx / CVT_FP4_NUM_THREADS_PER_SF; + int32_t mIdx = rowIdx; + + // SF layout [numMTiles, numKTiles, 32 (mTile), 4 (mTile), 4(kTile)] + // --> index [mTileIdx, kTileIdx, outerMIdx, innerMIdx, innerKIdx] + + int32_t mTileIdx = mIdx / (32 * 4); + // SF vector size 16. + int factor = CVT_FP4_SF_VEC_SIZE * 4; + int32_t numKTiles = (numCols + factor - 1) / factor; + int64_t mTileStride = numKTiles * 32 * 4 * 4; + + int32_t kTileIdx = (kIdx / 4); + int64_t kTileStride = 32 * 4 * 4; + + // M tile layout [32, 4] is column-major. + int32_t outerMIdx = (mIdx % 32); + int64_t outerMStride = 4 * 4; + + int32_t innerMIdx = (mIdx % (32 * 4)) / 32; + int64_t innerMStride = 4; + + int32_t innerKIdx = (kIdx % 4); + int64_t innerKStride = 1; + + // Compute the global offset. + int64_t SFOffset = mTileIdx * mTileStride + kTileIdx * kTileStride + outerMIdx * outerMStride + + innerMIdx * innerMStride + innerKIdx * innerKStride; + + return reinterpret_cast(SFout) + SFOffset; + } +#endif + return nullptr; +} + +// Define a 16 bytes packed data type. +template +struct PackedVec { + typename TypeConverter::Type elts[4]; +}; + +template <> +struct PackedVec<__nv_fp8_e4m3> { + __nv_fp8x2_e4m3 elts[8]; +}; + +// Quantizes the provided PackedVec into the uint32_t output +template +__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec& vec, float SFScaleVal, uint8_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + // Get absolute maximum values among the local 8 values. + auto localMax = __habs2(vec.elts[0]); + +// Local maximum value. +#pragma unroll + for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + localMax = __hmax2(localMax, __habs2(vec.elts[i])); + } + + // Get the absolute maximum among all 16 values (two threads). + localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax); + // Get the final absolute maximum values. + float vecMax = float(__hmax(localMax.x, localMax.y)); + + // Get the SF (max value of the vector / max value of e2m1). + // maximum value of e2m1 = 6.0. + // TODO: use half as compute data type. + float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f)); + // 8 bits representation of the SF. + uint8_t fp8SFVal; + // Write the SF to global memory (STG.8). + if constexpr (UE8M0_SF) { + __nv_fp8_e8m0 tmp; + tmp.__x = __nv_cvt_float_to_e8m0(SFValue, __NV_SATFINITE, cudaRoundPosInf); + SFValue = static_cast(tmp); + fp8SFVal = tmp.__x; + } else { + // Here SFValue is always positive, so E4M3 is the same as UE4M3. + __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue); + fp8SFVal = tmp.__x; + SFValue = static_cast(tmp); + } + // Get the output scale. + // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) * + // reciprocal(SFScaleVal)) + float outputScale = + SFValue != 0 ? reciprocal_approximate_ftz(SFValue * reciprocal_approximate_ftz(SFScaleVal)) : 0.0f; + + if (SFout) { + // Write the SF to global memory (STG.8). + *SFout = fp8SFVal; + } + + // Convert the input to float. + float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + fp2Vals[i] = __half22float2(vec.elts[i]); + } else { + fp2Vals[i] = __bfloat1622float2(vec.elts[i]); + } + fp2Vals[i].x *= outputScale; + fp2Vals[i].y *= outputScale; + } + + // Convert to e2m1 values. + uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals); + + // Write the e2m1 values to global memory. + return e2m1Vec; +#else + return 0; +#endif +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, int32_t numCols, Type const* in, float const* SFScale, uint32_t* out, uint32_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[0]; + + // Input tensor row/col loops. + for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) { + for (int colIdx = threadIdx.x; colIdx < numCols / CVT_FP4_ELTS_PER_THREAD; colIdx += blockDim.x) { + int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = inOffset; + auto& out_pos = out[outOffset]; + + auto sf_out = + cvt_quant_to_fp4_get_sf_out_offset(rowIdx, colIdx, numCols, SFout); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } + } +#endif +} + +template +void invokeFP4Quantization( + int m, + int n, + T const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream) { + // Grid, Block size. + // Each thread converts 8 values. + dim3 block(std::min(int(n / ELTS_PER_THREAD), 512)); + // Get number of blocks per SM (assume we can fully utilize the SM). + int const numBlocksPerSM = 2048 / block.x; + dim3 grid(std::min(int(m), multiProcessorCount * numBlocksPerSM)); + + // Launch the cvt kernel. + if (useUE8M0) { + cvt_fp16_to_fp4<<>>( + m, n, input, SFScale, reinterpret_cast(output), reinterpret_cast(SFOuput)); + } else { + cvt_fp16_to_fp4<<>>( + m, n, input, SFScale, reinterpret_cast(output), reinterpret_cast(SFOuput)); + } +} + +// Instantiate the function. +template void invokeFP4Quantization( + int m, + int n, + half const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream); + +template void invokeFP4Quantization( + int m, + int n, + __nv_bfloat16 const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream); + +inline int getMultiProcessorCount() { + static int multi_processor_count = []() { + int device_id = 0; + int count = 0; + + // Get the current CUDA device ID + CHECK_CUDA_SUCCESS(cudaGetDevice(&device_id)); + + // Get the number of multiprocessors for the current device + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&count, cudaDevAttrMultiProcessorCount, device_id)); + + return count; // Initialize the static variable + }(); + + return multi_processor_count; // Return the cached value on subsequent calls +} + +void scaled_fp4_quant_sm100a( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf) { + int32_t m = input.size(0); + int32_t n = input.size(1); + + TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16."); + + int multiProcessorCount = getMultiProcessorCount(); + + auto input_sf_ptr = static_cast(input_sf.data_ptr()); + auto sf_out = static_cast(output_sf.data_ptr()); + auto output_ptr = static_cast(output.data_ptr()); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + + // We don't support e8m0 scales at this moment. + bool useUE8M0 = false; + + switch (input.scalar_type()) { + case torch::kHalf: { + auto input_ptr = reinterpret_cast(input.data_ptr()); + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out, useUE8M0, multiProcessorCount, stream); + break; + } + case torch::kBFloat16: { + auto input_ptr = reinterpret_cast<__nv_bfloat16 const*>(input.data_ptr()); + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out, useUE8M0, multiProcessorCount, stream); + break; + } + default: { + std::cerr << "Observing: " << input.scalar_type() << " for the input datatype which is invalid"; + throw std::runtime_error("Unsupported input data type for quantize_to_fp4."); + } + } +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu new file mode 100644 index 0000000..8bcd8c5 --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu @@ -0,0 +1,39 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 +void cutlass_scaled_fp4_mm_sm100a( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha); +#endif + +void cutlass_scaled_fp4_mm( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return cutlass_scaled_fp4_mm_sm100a(D, A, B, A_sf, B_sf, alpha); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 mm kernel."); +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu new file mode 100644 index 0000000..c3da779 --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu @@ -0,0 +1,365 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +// clang-format off +#include "cutlass/cutlass.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/util/packed_stride.hpp" +// clang-format on + +/** + * Helper function for checking CUTLASS errors + */ +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +using namespace cute; + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +// Kernel Perf config +template +struct KernelTraits; + +template <> +struct KernelTraits { + using MmaTileShape = Shape<_128, _128, _256>; + using ClusterShape = Shape<_1, _1, _1>; + using PerSmTileShape_MNK = Shape<_128, _128, _256>; +}; + +template <> +struct KernelTraits { + using MmaTileShape = Shape<_256, _256, _256>; + using ClusterShape = Shape<_4, _4, _1>; + using PerSmTileShape_MNK = Shape<_128, _256, _256>; +}; + +template <> +struct KernelTraits { + using MmaTileShape = Shape<_256, _256, _256>; + using ClusterShape = Shape<_4, _4, _1>; + using PerSmTileShape_MNK = Shape<_128, _256, _256>; +}; + +template +struct Fp4GemmSm100 { + // A matrix configuration + using ElementA = cutlass::nv_float4_t; + using LayoutATag = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 32; + + // B matrix configuration + using ElementB = cutlass::nv_float4_t; + using LayoutBTag = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 32; + + // C/D matrix configuration + using ElementD = T; + using ElementC = T; + using LayoutCTag = cutlass::layout::RowMajor; + using LayoutDTag = cutlass::layout::RowMajor; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + // Kernel functional config + using ElementAccumulator = float; + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; + + // Kernel Perf config + using MmaTileShape = typename KernelTraits::MmaTileShape; + using ClusterShape = typename KernelTraits::ClusterShape; + using PerSmTileShape_MNK = typename KernelTraits::PerSmTileShape_MNK; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + PerSmTileShape_MNK, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutCTag, + AlignmentC, + ElementD, + LayoutDTag, + AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutATag, + AlignmentA, + ElementB, + LayoutBTag, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::collective::KernelScheduleAuto>::CollectiveOp; + + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using StrideA = typename Gemm::GemmKernel::StrideA; + using LayoutA = decltype(cute::make_layout(make_shape(0, 0, 0), StrideA{})); + using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using LayoutB = decltype(cute::make_layout(make_shape(0, 0, 0), StrideB{})); + using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using LayoutC = decltype(cute::make_layout(make_shape(0, 0, 0), StrideC{})); + using StrideD = typename Gemm::GemmKernel::StrideD; + using LayoutD = decltype(cute::make_layout(make_shape(0, 0, 0), StrideD{})); +}; + +template +typename T::Gemm::Arguments args_from_options( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t M, + int64_t N, + int64_t K) { + using ElementA = typename T::Gemm::ElementA; + using ElementB = typename T::Gemm::ElementB; + using ElementSFA = cutlass::float_ue4m3_t; + using ElementSFB = cutlass::float_ue4m3_t; + using ElementD = typename T::Gemm::ElementD; + using ElementCompute = float; + using StrideA = typename T::StrideA; + using StrideB = typename T::StrideB; + using StrideD = typename T::StrideD; + using Sm100BlkScaledConfig = typename T::Gemm::GemmKernel::CollectiveMainloop::Sm100BlkScaledConfig; + + int m = static_cast(M); + int n = static_cast(N); + int k = static_cast(K); + auto stride_A = cutlass::make_cute_packed_stride(StrideA{}, {m, k, 1}); + auto stride_B = cutlass::make_cute_packed_stride(StrideB{}, {n, k, 1}); + auto stride_D = cutlass::make_cute_packed_stride(StrideD{}, {m, n, 1}); + + auto layout_SFA = Sm100BlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1)); + auto layout_SFB = Sm100BlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1)); + + typename T::Gemm::Arguments arguments{ + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {// Mainloop arguments + static_cast(A.data_ptr()), + stride_A, + static_cast(B.data_ptr()), + stride_B, + static_cast(A_sf.data_ptr()), + layout_SFA, + static_cast(B_sf.data_ptr()), + layout_SFB}, + { // Epilogue arguments + {}, // epilogue.thread + static_cast(D.data_ptr()), + stride_D, + static_cast(D.data_ptr()), + stride_D}}; + auto& fusion_args = arguments.epilogue.thread; + fusion_args.alpha_ptr = static_cast(alpha.data_ptr()); + return arguments; +} + +template +void runGemm( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + typename Fp4GemmSm100::Gemm gemm; + + auto arguments = args_from_options>(D, A, B, A_sf, B_sf, alpha, m, n, k); + + size_t workspace_size = Fp4GemmSm100::Gemm::get_workspace_size(arguments); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(A.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + CUTLASS_CHECK(gemm.can_implement(arguments)); + + CUTLASS_CHECK(gemm.initialize(arguments, workspace.data_ptr(), stream)); + + CUTLASS_CHECK(gemm.run(arguments, workspace.data_ptr(), stream)); +} +#else +template +void runGemm( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + TORCH_CHECK( + false, + "Unsupported CUTLASS version. Set VLLM_CUTLASS_SRC_DIR to " + "a CUTLASS 3.8 source directory to enable support."); +} +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +#define CHECK_TYPE(x, st, m) TORCH_CHECK(x.scalar_type() == st, "Inconsistency of Tensor type:", m) +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, "must be contiguous") +#define CHECK_INPUT(x, st, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); \ + CHECK_TYPE(x, st, m) + +constexpr auto FLOAT4_E2M1X2 = at::ScalarType::Byte; +constexpr auto SF_DTYPE = at::ScalarType::Float8_e4m3fn; + +void cutlass_scaled_fp4_mm_sm100a( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha) { + CHECK_INPUT(A, FLOAT4_E2M1X2, "a"); + CHECK_INPUT(B, FLOAT4_E2M1X2, "b"); + + CHECK_INPUT(A_sf, SF_DTYPE, "scale_a"); + CHECK_INPUT(B_sf, SF_DTYPE, "scale_b"); + + CHECK_INPUT(alpha, at::ScalarType::Float, "alpha"); + + TORCH_CHECK(A.dim() == 2, "a must be a matrix"); + TORCH_CHECK(B.dim() == 2, "b must be a matrix"); + TORCH_CHECK( + A.sizes()[1] == B.sizes()[1], + "a and b shapes cannot be multiplied (", + A.sizes()[0], + "x", + A.sizes()[1], + " and ", + B.sizes()[0], + "x", + B.sizes()[1], + ")"); + + auto const m = A.sizes()[0]; + auto const n = B.sizes()[0]; + auto const k = A.sizes()[1] * 2; + + constexpr int alignment = 32; + TORCH_CHECK( + k % alignment == 0, + "Expected k to be divisible by ", + alignment, + ", but got a shape: (", + A.sizes()[0], + "x", + A.sizes()[1], + "), k: ", + k, + "."); + TORCH_CHECK( + n % alignment == 0, + "Expected n to be divisible by ", + alignment, + ", but got b shape: (", + B.sizes()[0], + "x", + B.sizes()[1], + ")."); + + auto round_up = [](int x, int y) { return (x + y - 1) / y * y; }; + int rounded_m = round_up(m, 128); + int rounded_n = round_up(n, 128); + // Since k is divisible by 32 (alignment), k / 16 is guaranteed to be an + // integer. + int rounded_k = round_up(k / 16, 4); + + TORCH_CHECK(A_sf.dim() == 2, "scale_a must be a matrix"); + TORCH_CHECK(B_sf.dim() == 2, "scale_b must be a matrix"); + TORCH_CHECK( + A_sf.sizes()[1] == B_sf.sizes()[1], + "scale_a and scale_b shapes cannot be multiplied (", + A_sf.sizes()[0], + "x", + A_sf.sizes()[1], + " and ", + B_sf.sizes()[0], + "x", + B_sf.sizes()[1], + ")"); + TORCH_CHECK( + A_sf.sizes()[0] == rounded_m && A_sf.sizes()[1] == rounded_k, + "scale_a must be padded and swizzled to a shape (", + rounded_m, + "x", + rounded_k, + "), but got a shape (", + A_sf.sizes()[0], + "x", + A_sf.sizes()[1], + ")"); + TORCH_CHECK( + B_sf.sizes()[0] == rounded_n && B_sf.sizes()[1] == rounded_k, + "scale_b must be padded and swizzled to a shape (", + rounded_n, + "x", + rounded_k, + "), but got a shape (", + B_sf.sizes()[0], + "x", + B_sf.sizes()[1], + ")"); + + auto out_dtype = D.dtype(); + at::cuda::CUDAGuard device_guard{(char)A.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(A.get_device()); + + if (out_dtype == at::ScalarType::Half) { + runGemm(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::BFloat16) { + runGemm(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::Float) { + runGemm(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + TORCH_CHECK(false, "Unsupported output data type of nvfp4 mm"); + } +} diff --git a/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu b/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu new file mode 100644 index 0000000..b10dd96 --- /dev/null +++ b/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu @@ -0,0 +1,123 @@ +#include +#include + +#include +#include +#include + +#include "utils.h" + +template +__global__ void +per_tensor_absmax_kernel(const T* __restrict__ input, float* __restrict__ output_s, const int64_t num_elements) { + float max_value = 0.0f; + unsigned int tid = threadIdx.x; + unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int grid_size = blockDim.x * gridDim.x; + + constexpr uint32_t vec_size = 16 / sizeof(T); + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = num_elements / vec_size; + + for (int32_t i = gid; i < num_vec_elems; i += grid_size) { + vec_t input_vec; + input_vec.cast_load(input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + max_value = fmaxf(max_value, fabsf(val)); + } + } + + const int32_t remaining_start = num_vec_elems * vec_size; + for (int32_t idx = remaining_start + gid; idx < num_elements; idx += grid_size) { + float val = static_cast(input[idx]); + max_value = fmaxf(max_value, fabsf(val)); + } + + max_value = blockReduceMax(max_value); + + if (tid == 0) { + atomicMaxFloat(output_s, max_value / FP8_E4M3_MAX); + } +} + +template +__global__ void per_tensor_quant_fp8_kernel( + const T* __restrict__ input, + FP8_TYPE* __restrict__ output, + const float* __restrict__ scale, + const int64_t num_elements) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int grid_size = blockDim.x * gridDim.x; + const float scale_val = 1.0f / (*scale); + + // We want to store 128 bits of data at a time. 16 = 128 / 8 bits + // Load is already vectorized, so 16 elements work for T. + const uint32_t VEC_SIZE = 16; + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = num_elements / VEC_SIZE; + + for (int32_t i = gid; i < num_vec_elems; i += grid_size) { + vec_t input_vec; + input_vec.cast_load(input + i * VEC_SIZE); + + FP8_TYPE output_arr[VEC_SIZE]; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + float val = fmax(fmin(static_cast(input_vec[j]) * scale_val, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#ifndef USE_ROCM + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + *(uint4*)(output + i * VEC_SIZE) = *(uint4*)output_arr; + } + + const int32_t remaining_start = num_vec_elems * VEC_SIZE; + for (int32_t idx = remaining_start + gid; idx < num_elements; idx += grid_size) { + float val = fmax(-FP8_E4M3_MAX, fmin(static_cast(input[idx]) * scale_val, FP8_E4M3_MAX)); +#ifndef USE_ROCM + output[idx] = static_cast(val); +#else + output[idx] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } +} + +void sgl_per_tensor_quant_fp8(torch::Tensor input, torch::Tensor output_q, torch::Tensor output_s, bool is_static) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + + const int block_size = 256; + const int num_elements = input.numel(); + const int num_blocks = min((num_elements + block_size - 1) / block_size, 1024); + + dim3 grid(num_blocks); + dim3 block(block_size); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (is_static == false) { + per_tensor_absmax_kernel<<>>( + static_cast(input.data_ptr()), static_cast(output_s.data_ptr()), num_elements); + } + + per_tensor_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + num_elements); + return true; + }); +} diff --git a/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu b/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu new file mode 100644 index 0000000..57a5ab8 --- /dev/null +++ b/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu @@ -0,0 +1,167 @@ +#include +#include + +#include +#include + +#include "utils.h" + +__device__ __forceinline__ float GroupReduceMax(float val, const int tid) { + unsigned mask = 0xffff; + + val = fmaxf(val, __shfl_xor_sync(mask, val, 8)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 4)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 2)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 1)); + return val; +} + +template +__global__ void per_token_group_quant_8bit_kernel( + const T* __restrict__ input, + void* __restrict__ output_q, + float* __restrict__ output_s, + const int group_size, + const int num_groups, + const int groups_per_block, + const float eps, + const float min_8bit, + const float max_8bit) { + const int threads_per_group = 16; + const int local_group_id = threadIdx.x / threads_per_group; + const int lane_id = threadIdx.x % threads_per_group; + + const int block_group_id = blockIdx.x * groups_per_block; + const int block_group_offset = (block_group_id + local_group_id) * group_size; + + float local_absmax = eps; + + const T* group_input = input + block_group_offset; + DST_DTYPE* group_output = static_cast(output_q) + block_group_offset; + float* scale_output = output_s + (block_group_id + local_group_id); + + constexpr uint32_t vec_size = 16 / sizeof(T); + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = group_size / vec_size; + + for (int32_t i = lane_id; i < num_vec_elems; i += 16) { + vec_t input_vec; + input_vec.cast_load(group_input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + float abs_val = fabsf(val); + local_absmax = fmaxf(local_absmax, abs_val); + } + } + + local_absmax = GroupReduceMax(local_absmax, lane_id); + + const float y_s = local_absmax / max_8bit; + + if (lane_id == 0) { + *scale_output = y_s; + } + + for (int32_t i = lane_id; i < num_vec_elems; i += 16) { + vec_t input_vec; + input_vec.cast_load(group_input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + float q_val = fminf(fmaxf(val / y_s, min_8bit), max_8bit); + group_output[i * vec_size + j] = DST_DTYPE(q_val); + } + } +} + +void sgl_per_token_group_quant_8bit( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double min_8bit, + double max_8bit) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + + const int num_groups = input.numel() / group_size; + + CHECK_EQ(input.numel() % group_size, 0); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + constexpr int THREADS_PER_GROUP = 16; + + int groups_per_block = 1; + + if (num_groups % 16 == 0) { + groups_per_block = 16; + } else if (num_groups % 8 == 0) { + groups_per_block = 8; + } else if (num_groups % 4 == 0) { + groups_per_block = 4; + } else if (num_groups % 2 == 0) { + groups_per_block = 2; + } + + auto dst_type = output_q.scalar_type(); + const int num_blocks = num_groups / groups_per_block; + const int num_threads = groups_per_block * THREADS_PER_GROUP; + +#define LAUNCH_KERNEL(T, DST_DTYPE) \ + do { \ + dim3 grid(num_blocks); \ + dim3 block(num_threads); \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit); \ + } while (0) + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (dst_type == at::ScalarType::Char) { + LAUNCH_KERNEL(scalar_t, int8_t); + return true; + } else if (dst_type == at::ScalarType::Float8_e4m3fn) { + LAUNCH_KERNEL(scalar_t, c10::Float8_e4m3fn); + return true; + } + return false; + }); + +#undef LAUNCH_KERNEL +} + +void sgl_per_token_group_quant_int8( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double int8_min, + double int8_max) { + sgl_per_token_group_quant_8bit(input, output_q, output_s, group_size, eps, int8_min, int8_max); +} + +void sgl_per_token_group_quant_fp8( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double fp8_min, + double fp8_max) { + sgl_per_token_group_quant_8bit(input, output_q, output_s, group_size, eps, fp8_min, fp8_max); +} diff --git a/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu b/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu new file mode 100644 index 0000000..db09483 --- /dev/null +++ b/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu @@ -0,0 +1,106 @@ +#include + +#include +#include +#include + +#include "utils.h" + +template +__global__ void per_token_quant_fp8_kernel( + const T* __restrict__ input, + FP8_TYPE* __restrict__ output_q, + float* __restrict__ output_s, + const int64_t hidden_dim, + const int64_t num_tokens) { + const int token_idx = blockIdx.x; + if (token_idx >= num_tokens) return; + + const int tid = threadIdx.x; + const int block_dim = blockDim.x; + + const T* token_input = input + token_idx * hidden_dim; + FP8_TYPE* token_output = output_q + token_idx * hidden_dim; + + float max_value = 0.0f; + + // We want to store 128 bits of data at a time. 16 = 128 / 8 bits + // Load is already vectorized, so 16 elements work for T. + const uint32_t VEC_SIZE = 16; + using vec_t = flashinfer::vec_t; + const int32_t num_vec_elems = hidden_dim / VEC_SIZE; + + // Find max using vectorized loads + for (int32_t i = tid; i < num_vec_elems; i += block_dim) { + vec_t input_vec; + input_vec.cast_load(token_input + i * VEC_SIZE); + +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + float val = static_cast(input_vec[j]); + max_value = fmaxf(max_value, fabsf(val)); + } + } + + max_value = blockReduceMax(max_value); + + __shared__ float scale; + if (tid == 0) { + scale = max_value / FP8_E4M3_MAX; + output_s[token_idx] = scale; + } + __syncthreads(); + + const float scale_inv = 1.0f / scale; + + // Quantize using vectorized loads + for (int32_t i = tid; i < num_vec_elems; i += block_dim) { + vec_t input_vec; + input_vec.cast_load(token_input + i * VEC_SIZE); + + FP8_TYPE output_arr[VEC_SIZE]; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + float val = fmaxf(fminf(static_cast(input_vec[j]) * scale_inv, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#ifndef USE_ROCM + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + + *(uint4*)(token_output + i * VEC_SIZE) = *(uint4*)output_arr; + } +} + +void sgl_per_token_quant_fp8(torch::Tensor input, torch::Tensor output_q, torch::Tensor output_s) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + + const auto input_sizes = input.sizes(); + const int64_t num_tokens = input_sizes[0]; + const int64_t hidden_dim = input_sizes[1]; + + TORCH_CHECK(hidden_dim % 16 == 0, "Hidden dimension must be divisible by 16, but got ", hidden_dim); + + const int block_size = 256; + const int num_blocks = num_tokens; + + dim3 grid(num_blocks); + dim3 block(block_size); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + per_token_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + return true; + }); +} diff --git a/sgl-kernel/csrc/moe/moe_align_kernel.cu b/sgl-kernel/csrc/moe/moe_align_kernel.cu new file mode 100644 index 0000000..6ffa739 --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_align_kernel.cu @@ -0,0 +1,160 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include + +#include "utils.h" + +#define WARP_SIZE 32 + +template +__global__ void count_and_sort_expert_tokens_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ cumsum_buffer, + size_t numel) { + const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const size_t stride = blockDim.x * gridDim.x; + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i]; + int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1); + sorted_token_ids[rank_post_pad] = i; + } +} + +template +__global__ void moe_align_block_size_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t padded_num_experts, + int32_t experts_per_warp, + int32_t block_size, + size_t numel, + int32_t* __restrict__ cumsum) { + extern __shared__ int32_t shared_counts[]; + + const int warp_id = threadIdx.x / WARP_SIZE; + const int my_expert_start = warp_id * experts_per_warp; + + for (int i = 0; i < experts_per_warp; ++i) { + if (my_expert_start + i < padded_num_experts) { + shared_counts[warp_id * experts_per_warp + i] = 0; + } + } + + __syncthreads(); + + const size_t tokens_per_thread = CEILDIV(numel, blockDim.x); + const size_t start_idx = threadIdx.x * tokens_per_thread; + + for (int i = start_idx; i < numel && i < start_idx + tokens_per_thread; ++i) { + int expert_id = topk_ids[i]; + int warp_idx = expert_id / experts_per_warp; + int expert_offset = expert_id % experts_per_warp; + atomicAdd(&shared_counts[warp_idx * experts_per_warp + expert_offset], 1); + } + + __syncthreads(); + + if (threadIdx.x == 0) { + cumsum[0] = 0; + for (int i = 1; i <= num_experts; ++i) { + int expert_count = 0; + int warp_idx = (i - 1) / experts_per_warp; + int expert_offset = (i - 1) % experts_per_warp; + expert_count = shared_counts[warp_idx * experts_per_warp + expert_offset]; + + cumsum[i] = cumsum[i - 1] + CEILDIV(expert_count, block_size) * block_size; + } + *total_tokens_post_pad = cumsum[num_experts]; + } + + __syncthreads(); + + if (threadIdx.x < num_experts) { + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1]; i += block_size) { + expert_ids[i / block_size] = threadIdx.x; + } + } +} + +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor token_cnts_buffer, + torch::Tensor cumsum_buffer) { + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + int64_t padded_num_experts = ((num_experts + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; + + int experts_per_warp; + int threads; + + if (num_experts <= 8) { + experts_per_warp = 8; + threads = 256; + } else if (num_experts <= 16) { + experts_per_warp = 16; + threads = 512; + } else { + experts_per_warp = WARP_SIZE; + threads = 1024; + } + + threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; + + DISPATCH_INTEGRAL_TYPES(topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] { + auto align_kernel = moe_align_block_size_kernel; + + size_t num_warps = CEILDIV(padded_num_experts, experts_per_warp); + size_t shared_mem_size = num_warps * experts_per_warp * sizeof(int32_t); + + align_kernel<<<1, threads, shared_mem_size, stream>>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + padded_num_experts, + experts_per_warp, + block_size, + topk_ids.numel(), + cumsum_buffer.data_ptr()); + + const int block_threads = std::min(256, (int)threads); + const int num_blocks = (topk_ids.numel() + block_threads - 1) / block_threads; + const int max_blocks = 65535; + const int actual_blocks = std::min(num_blocks, max_blocks); + + auto sort_kernel = count_and_sort_expert_tokens_kernel; + sort_kernel<<>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + cumsum_buffer.data_ptr(), + topk_ids.numel()); + }); +} diff --git a/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu new file mode 100644 index 0000000..ac78ebb --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu @@ -0,0 +1,505 @@ +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu +// which is originally adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#ifndef USE_ROCM +#include +#include +#else +#include +#include +#endif + +#include "utils.h" + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +/// Aligned array type +template < + typename T, + /// Number of elements in the array + int N, + /// Alignment requirement in bytes + int Alignment = sizeof(T) * N> +class alignas(Alignment) AlignedArray { + float data[N]; +}; + +// ====================== Softmax things =============================== +// We have our own implementation of softmax here so we can support transposing the output +// in the softmax kernel when we extend this module to support expert-choice routing. +template +__launch_bounds__(TPB) __global__ + void moeSoftmax(const float* input, const bool* finished, float* output, const int num_cols) { + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + __shared__ float normalizing_factor; + __shared__ float float_max; + + const int thread_row_offset = blockIdx.x * num_cols; + + cub::Sum sum; + float threadData(-FLT_MAX); + + // Don't touch finished rows. + if ((finished != nullptr) && finished[blockIdx.x]) { + return; + } + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData = max(static_cast(input[idx]), threadData); + } + + const float maxElem = BlockReduce(tmpStorage).Reduce(threadData, cub::Max()); + + if (threadIdx.x == 0) { + float_max = maxElem; + } + __syncthreads(); + + threadData = 0; + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData += exp((static_cast(input[idx]) - float_max)); + } + + const auto Z = BlockReduce(tmpStorage).Reduce(threadData, sum); + + if (threadIdx.x == 0) { + normalizing_factor = 1.f / Z; + } + __syncthreads(); + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + const float val = exp((static_cast(input[idx]) - float_max)) * normalizing_factor; + output[idx] = val; + } +} + +template +__launch_bounds__(TPB) __global__ void moeTopK( + const float* inputs_after_softmax, + const bool* finished, + float* output, + int* indices, + int* source_rows, + const int num_experts, + const int k, + const int start_expert, + const int end_expert) { + using cub_kvp = cub::KeyValuePair; + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + cub_kvp thread_kvp; + cub::ArgMax arg_max; + + const int num_rows = gridDim.x; + const int block_row = blockIdx.x; + + const bool row_is_active = finished ? !finished[block_row] : true; + const int thread_read_offset = blockIdx.x * num_experts; + for (int k_idx = 0; k_idx < k; ++k_idx) { + thread_kvp.key = 0; + thread_kvp.value = -1.f; // This is OK because inputs are probabilities + + cub_kvp inp_kvp; + for (int expert = threadIdx.x; expert < num_experts; expert += TPB) { + const int idx = thread_read_offset + expert; + inp_kvp.key = expert; + inp_kvp.value = inputs_after_softmax[idx]; + + for (int prior_k = 0; prior_k < k_idx; ++prior_k) { + const int prior_winning_expert = indices[k * block_row + prior_k]; + + if (prior_winning_expert == expert) { + inp_kvp = thread_kvp; + } + } + + thread_kvp = arg_max(inp_kvp, thread_kvp); + } + + const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max); + if (threadIdx.x == 0) { + // Ignore experts the node isn't responsible for with expert parallelism + const int expert = result_kvp.key; + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + const int idx = k * block_row + k_idx; + output[idx] = result_kvp.value; + indices[idx] = should_process_row ? (expert - start_expert) : num_experts; + assert(indices[idx] >= 0); + source_rows[idx] = k_idx * num_rows + block_row; + } + __syncthreads(); + } +} + +// ====================== TopK softmax things =============================== + +/* + A Top-K gating softmax written to exploit when the number of experts in the MoE layers + are a small power of 2. This allows us to cleanly share the rows among the threads in + a single warp and eliminate communication between warps (so no need to use shared mem). + + It fuses the softmax, max and argmax into a single kernel. + + Limitations: + 1) This implementation is intended for when the number of experts is a small power of 2. + 2) This implementation assumes k is small, but will work for any k. +*/ + +template +__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSoftmax( + const float* input, + const bool* finished, + float* output, + const int num_rows, + int* indices, + int* source_rows, + const int k, + const int start_expert, + const int end_expert) { + // We begin by enforcing compile time assertions and setting up compile time constants. + static_assert(VPT == (VPT & -VPT), "VPT must be power of 2"); + static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2"); + static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2"); + static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16"); + + // Number of bytes each thread pulls in per load + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(float); + static constexpr int ELTS_PER_ROW = NUM_EXPERTS; + static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT; + static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG; + + // Restrictions based on previous section. + static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg"); + static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp"); + static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2"); + static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size"); + + // We have NUM_EXPERTS elements per row. We specialize for small #experts + static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT; + static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW; + static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; + + // Restrictions for previous section. + static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp"); + + // ===================== From this point, we finally start computing run-time variables. ======================== + + // Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps. + // This, each block processes a chunk of rows. We start by computing the start row for each block. + const int cta_base_row = blockIdx.x * ROWS_PER_CTA; + + // Now, using the base row per thread block, we compute the base row per warp. + const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP; + + // The threads in a warp are split into sub-groups that will work on a row. + // We compute row offset for each thread sub-group + const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW; + const int thread_row = warp_base_row + thread_row_in_warp; + + // Threads with indices out of bounds should early exit here. + if (thread_row >= num_rows) { + return; + } + const bool row_is_active = finished ? !finished[thread_row] : true; + + // We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the + // row it will read. + const float* thread_row_ptr = input + thread_row * ELTS_PER_ROW; + + // Now, we compute the group each thread belong to in order to determine the first column to start loads. + const int thread_group_idx = threadIdx.x % THREADS_PER_ROW; + const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG; + const float* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + + // Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory, + // this can support all powers of 2 up to 16. + // NOTE(woosuk): The original implementation uses CUTLASS aligned array here. + // We defined our own aligned array and use it here to avoid the dependency on CUTLASS. + using AccessType = AlignedArray; + + // Finally, we pull in the data from global mem + float row_chunk[VPT]; + AccessType* row_chunk_vec_ptr = reinterpret_cast(&row_chunk); + const AccessType* vec_thread_read_ptr = reinterpret_cast(thread_read_ptr); +#pragma unroll + for (int ii = 0; ii < LDG_PER_THREAD; ++ii) { + row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW]; + } + + // First, we perform a max reduce within the thread. We can do the max in fp16 safely (I think) and just + // convert to float afterwards for the exp + sum reduction. + float thread_max = row_chunk[0]; +#pragma unroll + for (int ii = 1; ii < VPT; ++ii) { + thread_max = max(thread_max, row_chunk[ii]); + } + +// Now, we find the max within the thread group and distribute among the threads. We use a butterfly reduce. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + thread_max = max(thread_max, SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, thread_max, mask, THREADS_PER_ROW)); + } + + // From this point, thread max in all the threads have the max within the row. + // Now, we subtract the max from each element in the thread and take the exp. We also compute the thread local sum. + float row_sum = 0; +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = expf(row_chunk[ii] - thread_max); + row_sum += row_chunk[ii]; + } + +// Now, we perform the sum reduce within each thread group. Similar to the max reduce, we use a bufferfly pattern. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + row_sum += SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, row_sum, mask, THREADS_PER_ROW); + } + + // From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables + // respectively. Finally, we can scale the rows for the softmax. Technically, for top-k gating we don't need to + // compute the entire softmax row. We can likely look at the maxes and only compute for the top-k values in the row. + // However, this kernel will likely not be a bottle neck and it seems better to closer match torch and find the + // argmax after computing the softmax. + const float reciprocal_row_sum = 1.f / row_sum; + +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = row_chunk[ii] * reciprocal_row_sum; + } + + // Now, softmax_res contains the softmax of the row chunk. Now, I want to find the topk elements in each row, along + // with the max index. + int start_col = first_elt_read_by_thread; + static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW; + + for (int k_idx = 0; k_idx < k; ++k_idx) { + // First, each thread does the local argmax + float max_val = row_chunk[0]; + int expert = start_col; +#pragma unroll + for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) { +#pragma unroll + for (int ii = 0; ii < ELTS_PER_LDG; ++ii) { + float val = row_chunk[ldg * ELTS_PER_LDG + ii]; + + // No check on the experts here since columns with the smallest index are processed first and only + // updated if > (not >=) + if (val > max_val) { + max_val = val; + expert = col + ii; + } + } + } + +// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max. +// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can +// then blank out their max with -inf and the warp can run more iterations... +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW); + int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW); + + // We want lower indices to "win" in every thread so we break ties this way + if (other_max > max_val || (other_max == max_val && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + // Write the max for this k iteration to global memory. + if (thread_group_idx == 0) { + // Add a guard to ignore experts not included by this node + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + // The lead thread from each sub-group will write out the final results to global memory. (This will be a + // single) thread per row of the input/output matrices. + const int idx = k * thread_row + k_idx; + output[idx] = max_val; + indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS; + source_rows[idx] = k_idx * num_rows + thread_row; + } + + // Finally, we clear the value in the thread with the current max if there is another iteration to run. + if (k_idx + 1 < k) { + const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG; + const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW; + + // Only the thread in the group which produced the max will reset the "winning" value to -inf. + if (thread_group_idx == thread_to_clear_in_group) { + const int offset_for_expert = expert % ELTS_PER_LDG; + // Safe to set to any negative value since row_chunk values must be between 0 and 1. + row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f; + } + } + } +} + +namespace detail { +// Constructs some constants needed to partition the work across threads at compile time. +template +struct TopkConstants { + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(float); + static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, ""); + static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE)); + static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG; + static constexpr int THREADS_PER_ROW = EXPERTS / VPT; + static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW; +}; +} // namespace detail + +template +void topkGatingSoftmaxLauncherHelper( + const float* input, + const bool* finished, + float* output, + int* indices, + int* source_row, + const int num_rows, + const int k, + const int start_expert, + const int end_expert, + cudaStream_t stream) { + static constexpr std::size_t MAX_BYTES_PER_LDG = 16; + + static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(float) * EXPERTS); + using Constants = detail::TopkConstants; + static constexpr int VPT = Constants::VPT; + static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP; + const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP; + const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB; + + dim3 block_dim(WARP_SIZE, WARPS_PER_TB); + topkGatingSoftmax<<>>( + input, finished, output, num_rows, indices, source_row, k, start_expert, end_expert); +} + +#define LAUNCH_SOFTMAX(NUM_EXPERTS, WARPS_PER_TB) \ + topkGatingSoftmaxLauncherHelper( \ + gating_output, \ + nullptr, \ + topk_weights, \ + topk_indices, \ + token_expert_indices, \ + num_tokens, \ + topk, \ + 0, \ + num_experts, \ + stream); + +void topkGatingSoftmaxKernelLauncher( + const float* gating_output, + float* topk_weights, + int* topk_indices, + int* token_expert_indices, + float* softmax_workspace, + const int num_tokens, + const int num_experts, + const int topk, + cudaStream_t stream) { + static constexpr int WARPS_PER_TB = 4; + switch (num_experts) { + case 1: + LAUNCH_SOFTMAX(1, WARPS_PER_TB); + break; + case 2: + LAUNCH_SOFTMAX(2, WARPS_PER_TB); + break; + case 4: + LAUNCH_SOFTMAX(4, WARPS_PER_TB); + break; + case 8: + LAUNCH_SOFTMAX(8, WARPS_PER_TB); + break; + case 16: + LAUNCH_SOFTMAX(16, WARPS_PER_TB); + break; + case 32: + LAUNCH_SOFTMAX(32, WARPS_PER_TB); + break; + case 64: + LAUNCH_SOFTMAX(64, WARPS_PER_TB); + break; + case 128: + LAUNCH_SOFTMAX(128, WARPS_PER_TB); + break; + case 256: + LAUNCH_SOFTMAX(256, WARPS_PER_TB); + break; + default: { + TORCH_CHECK( + softmax_workspace != nullptr, + "softmax_workspace must be provided for num_experts that are not a power of 2."); + static constexpr int TPB = 256; + moeSoftmax<<>>(gating_output, nullptr, softmax_workspace, num_experts); + moeTopK<<>>( + softmax_workspace, + nullptr, + topk_weights, + topk_indices, + token_expert_indices, + num_experts, + topk, + 0, + num_experts); + } + } +} + +void topk_softmax( + torch::Tensor& topk_weights, // [num_tokens, topk] + torch::Tensor& topk_indices, // [num_tokens, topk] + torch::Tensor& token_expert_indices, // [num_tokens, topk] + torch::Tensor& gating_output) // [num_tokens, num_experts] +{ + const int num_experts = gating_output.size(-1); + const int num_tokens = gating_output.numel() / num_experts; + const int topk = topk_weights.size(-1); + + const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0); + const bool needs_workspace = !is_pow_2 || num_experts > 256; + const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + torch::Tensor softmax_workspace = torch::empty({workspace_size}, gating_output.options()); + topkGatingSoftmaxKernelLauncher( + gating_output.data_ptr(), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + token_expert_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + stream); +} diff --git a/sgl-kernel/csrc/speculative/eagle_utils.cu b/sgl-kernel/csrc/speculative/eagle_utils.cu new file mode 100644 index 0000000..aeb6b84 --- /dev/null +++ b/sgl-kernel/csrc/speculative/eagle_utils.cu @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2025 by SGLang team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#ifndef USE_ROCM +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#endif + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask [draft_token*(seq_len[0]+draft_token) | draft_token*(seq_len[1]+draft_token) | ..] = +// [sum(verified_seq_len)*draft_token+bs*draft_token*draft_token] positions [bs * draft_token] retrive_index [b, +// draft_token] retrive_next_token [b, draft_token] retrive_next_sibling [b, draft_token] +__global__ void build_tree_efficient( + int64_t* parent_list, + int64_t* selected_index, + int32_t* verified_seq_len, + bool* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_tree_idx = draft_token_num * draft_token_num * bid; + for (int i = 0; i < bid; i++) { + seq_tree_idx += verified_seq_len[i] * draft_token_num; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx = seq_tree_idx + (seq_len + draft_token_num) * tid + seq_len + 1; + for (int i = 0; i < draft_token_num - 1; i++) { + tree_mask[token_tree_idx + i] = false; + } + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + tree_mask[token_tree_idx + cur_position] = true; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num) { + // TODO (ying) check shape + // TODO (ying) check type + int bs = parent_list.size(0); + dim3 grid(bs); + dim3 block(draft_token_num); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + build_tree_efficient<<>>( + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num)); +} + +template +__global__ void VerifyTreeGreedy( + IdType* predicts, + IdType* accept_index, + IdType* accept_token_num, // mutable + IdType* candidates, + IdType* retrive_index, + IdType* retrive_next_token, + IdType* retrive_next_sibling, + IdType* target_predict, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens) { + uint32_t bx = blockIdx.x; + + IdType last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + IdType target_token_id = target_predict[last_accepted_retrive_idx]; + + if (draft_token_id == target_token_id) { + // accept token + predicts[last_accepted_retrive_idx] = target_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + predicts[last_accepted_retrive_idx] = target_predict[last_accepted_retrive_idx]; +} + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// target_predict: [bs, num_draft_tokens] +void verify_tree_greedy( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict, + int64_t cuda_stream = 0) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(target_predict); + auto device = target_predict.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(target_predict.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, target_predict); + unsigned int batch_size = candidates.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_predict.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, target_predict.size(1)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'candidates' to be of type int (torch.int32)."); + } + if (retrive_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'retrive_index' to be of type int (torch.int32)."); + } + if (retrive_next_token.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type int (torch.int32)."); + } + if (retrive_next_sibling.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type int (torch.int32)."); + } + if (target_predict.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'target_predict' to be of type int (torch.int32)."); + } + + cudaStream_t stream = reinterpret_cast(cuda_stream); + dim3 grid(batch_size); + dim3 block(1); + + VerifyTreeGreedy<<>>( + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(target_predict.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens); +} diff --git a/sgl-kernel/csrc/speculative/packbit.cu b/sgl-kernel/csrc/speculative/packbit.cu new file mode 100644 index 0000000..c65ba45 --- /dev/null +++ b/sgl-kernel/csrc/speculative/packbit.cu @@ -0,0 +1,47 @@ +// This is only a pluggin used for flashinfer 0.1.6. The new version does not need it. +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +// bitorder = "little" +void segment_packbits( + at::Tensor x, at::Tensor input_indptr, at::Tensor output_indptr, at::Tensor y, int64_t cuda_stream) { + CHECK_INPUT(x); + CHECK_INPUT(input_indptr); + CHECK_INPUT(output_indptr); + auto device = x.device(); + CHECK_EQ(input_indptr.device(), device); + CHECK_EQ(output_indptr.device(), device); + CHECK_EQ(y.device(), device); + unsigned int batch_size = input_indptr.size(0) - 1; + CHECK_EQ(output_indptr.size(0), batch_size + 1); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + cudaError_t status = quantization::SegmentPackBits( + static_cast(x.data_ptr()), + static_cast(y.data_ptr()), + static_cast(input_indptr.data_ptr()), + static_cast(output_indptr.data_ptr()), + batch_size, + quantization::BitOrder::kLittle, + stream); +} diff --git a/sgl-kernel/csrc/speculative/pytorch_extension_utils_rocm.h b/sgl-kernel/csrc/speculative/pytorch_extension_utils_rocm.h new file mode 100644 index 0000000..fa5fd12 --- /dev/null +++ b/sgl-kernel/csrc/speculative/pytorch_extension_utils_rocm.h @@ -0,0 +1,20 @@ +#include + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) diff --git a/sgl-kernel/csrc/speculative/speculative_sampling.cu b/sgl-kernel/csrc/speculative/speculative_sampling.cu new file mode 100644 index 0000000..c03e1d7 --- /dev/null +++ b/sgl-kernel/csrc/speculative/speculative_sampling.cu @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pytorch_extension_utils.h" +#include "speculative_sampling.cuh" + +using namespace flashinfer; + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// uniform_samples: [bs, num_draft_tokens] +// target_probs: [bs, num_draft_tokens, vocab_size] +void tree_speculative_sampling_target_only( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor uniform_samples, + at::Tensor target_probs, + at::Tensor draft_probs, + double threshold_single, + double threshold_acc, + bool deterministic = true, + int64_t cuda_stream = 0) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(uniform_samples); + CHECK_INPUT(target_probs); + auto device = target_probs.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(uniform_samples.device(), device); + CHECK_EQ(target_probs.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, uniform_samples); + CHECK_DIM(3, target_probs); + CHECK_DIM(3, draft_probs); + unsigned int batch_size = uniform_samples.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + unsigned int vocab_size = target_probs.size(2); + CHECK_EQ(batch_size, candidates.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_probs.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, uniform_samples.size(1)); + CHECK_EQ(num_draft_tokens, target_probs.size(1)); + CHECK_EQ(vocab_size, target_probs.size(2)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'candidates' to be of type int (torch.int32)."); + } + if (retrive_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'retrive_index' to be of type int (torch.int32)."); + } + if (retrive_next_token.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type int (torch.int32)."); + } + if (retrive_next_sibling.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type int (torch.int32)."); + } + if (uniform_samples.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'uniform_samples' to be of type float (torch.float32)."); + } + if (target_probs.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'target_probs' to be of type float (torch.float32)."); + } + if (draft_probs.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'target_probs' to be of type float (torch.float32)."); + } + CHECK_GE(threshold_single, 0); + CHECK_GE(1, threshold_single); + CHECK_GE(threshold_acc, 0); + CHECK_GE(1, threshold_acc); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + cudaError_t status = sampling::TreeSpeculativeSamplingTargetOnly( + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(uniform_samples.data_ptr()), + static_cast(target_probs.data_ptr()), + static_cast(draft_probs.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens, + vocab_size, + static_cast(threshold_single), + static_cast(threshold_acc), + deterministic, + stream); + + TORCH_CHECK( + status == cudaSuccess, + "TreeSpeculativeSamplingTargetOnly failed with error code " + std::string(cudaGetErrorString(status))); +} diff --git a/sgl-kernel/csrc/speculative/speculative_sampling.cuh b/sgl-kernel/csrc/speculative/speculative_sampling.cuh new file mode 100644 index 0000000..158dfa4 --- /dev/null +++ b/sgl-kernel/csrc/speculative/speculative_sampling.cuh @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2024-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SPECULATIVE_SAMPLING_CUH_ +#define SPECULATIVE_SAMPLING_CUH_ + +#include + +#include + +namespace flashinfer { + +namespace sampling { + +using namespace cub; + +template < + uint32_t BLOCK_THREADS, + BlockScanAlgorithm SCAN_ALGORITHM, + BlockReduceAlgorithm REDUCE_ALGORITHM, + uint32_t VEC_SIZE, + bool DETERMINISTIC, + typename DType, + typename IdType> +__global__ void TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* accept_index, // mutable + IdType* accept_token_num, // mutable + IdType* candidates, + IdType* retrive_index, + IdType* retrive_next_token, + IdType* retrive_next_sibling, + DType* uniform_samples, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + const uint32_t bx = blockIdx.x, tx = threadIdx.x; + + extern __shared__ __align__(alignof(SamplingTempStorage)) + uint8_t smem_sampling[]; + auto& temp_storage = + reinterpret_cast&>(smem_sampling); + + DType prob_acc = 0.0; + uint32_t cur_prob_offset = bx * num_draft_tokens * d; + DType coin = uniform_samples[bx * num_draft_tokens]; + IdType last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + DType target_prob_single = target_probs[cur_prob_offset + draft_token_id]; + prob_acc += target_prob_single; + + if (coin <= prob_acc / threshold_acc || target_prob_single >= threshold_single) { + // accept token + prob_acc = 0.; + cur_prob_offset = (bx * num_draft_tokens + cur_index) * d; + coin = uniform_samples[bx * num_draft_tokens + cur_index]; + predicts[last_accepted_retrive_idx] = draft_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + // FIXME: leverage draft probs + draft_probs[cur_prob_offset + draft_token_id] = target_probs[cur_prob_offset + draft_token_id]; + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + + // sample from relu(target_probs - draft_probs) + DType sum_relu_q_minus_p(0); + vec_t q_vec, p_vec; + DType relu_q_minus_p[VEC_SIZE]; + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + sum_relu_q_minus_p += BlockReduce(temp_storage.block_prim.reduce) + .Sum(relu_q_minus_p); + __syncthreads(); + } + if (tx == 0) { + temp_storage.block_aggregate.value = sum_relu_q_minus_p; + } + // init the first rejected token to (d - 1) + temp_storage.sampled_id = d - 1; + __syncthreads(); + sum_relu_q_minus_p = temp_storage.block_aggregate.value; + DType u = coin * sum_relu_q_minus_p; + + DType aggregate_relu_q_minus_p(0); + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } + + vec_t relu_q_minus_p_vec; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p_vec[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + + DeviceSamplingFromProb( + i, d, [&](DType x) { return x > 0; }, u, relu_q_minus_p_vec, aggregate_relu_q_minus_p, &temp_storage); + if (aggregate_relu_q_minus_p > u) { + break; + } + } + __syncthreads(); + // set the first rejected token + predicts[last_accepted_retrive_idx] = temp_storage.sampled_id; + // value at not used indices are undefined +} + +template +cudaError_t TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* output_token_ids, // mutable + IdType* output_accepted_token_num, // mutable + IdType* candidates, + IdType* retrive_index, + IdType* retrive_next_token, + IdType* retrive_next_sibling, + DType* uniform_samples, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single = 1, + DType threshold_acc = 1, + bool deterministic = true, + cudaStream_t stream = 0) { + constexpr uint32_t BLOCK_THREADS = 1024; + const uint32_t vec_size = std::gcd(16 / sizeof(DType), d); + + const uint32_t smem_size = sizeof(SamplingTempStorage); + dim3 nblks(batch_size); + dim3 nthrs(BLOCK_THREADS); + float capped_threshold_acc = fmaxf(threshold_acc, 1e-9f); + void* args[] = { + &predicts, + &output_token_ids, + &output_accepted_token_num, + &candidates, + &retrive_index, + &retrive_next_token, + &retrive_next_sibling, + &uniform_samples, + &target_probs, + &draft_probs, + &batch_size, + &num_speculative_tokens, + &num_draft_tokens, + &d, + &threshold_single, + &capped_threshold_acc}; + DISPATCH_ALIGNED_VEC_SIZE( + vec_size, VEC_SIZE, {DISPATCH_DETERMINISTIC(deterministic, DETERMINISTIC, { + auto kernel = TreeSpeculativeSamplingTargetOnly< + BLOCK_THREADS, + SCAN_ALGO, + REDUCE_ALGO, + VEC_SIZE, + DETERMINISTIC, + DType, + IdType>; + FLASHINFER_CUDA_CALL(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + FLASHINFER_CUDA_CALL(cudaLaunchKernel((void*)kernel, nblks, nthrs, args, smem_size, stream)); + })}); + return cudaSuccess; +} + +} // namespace sampling + +} // namespace flashinfer + +#endif // SPECULATIVE_SAMPLING_CUH_ diff --git a/sgl-kernel/csrc/torch_extension.cc b/sgl-kernel/csrc/torch_extension.cc new file mode 100644 index 0000000..16d9adb --- /dev/null +++ b/sgl-kernel/csrc/torch_extension.cc @@ -0,0 +1,202 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_EXPAND(sgl_kernel, m) { + /* + * From csrc/allreduce + */ + m.def( + "init_custom_ar(int rank_id, int world_size, Tensor rank_data, int[] buffers, int[] tmp_result_buffers, int[] " + "barrier_in, int[] barrier_out) -> int"); + m.impl("init_custom_ar", torch::kCUDA, &init_custom_ar); + + m.def("dispose", &dispose); + + m.def("all_reduce(int fa, Tensor inp, Tensor! out) -> ()"); + m.impl("all_reduce", torch::kCUDA, &all_reduce); + + m.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta); + m.def("register_graph_buffers", ®ister_graph_buffers); + + /* + * From csrc/attention + */ + m.def( + "lightning_attention_decode(Tensor q, Tensor k, Tensor v, Tensor past_kv, Tensor slope, Tensor! output, Tensor! " + "new_kv) -> ()"); + m.impl("lightning_attention_decode", torch::kCUDA, &lightning_attention_decode); + + /* + * From csrc/elementwise + */ + m.def("rmsnorm(Tensor! output, Tensor input, Tensor weight, float eps, int cuda_stream) -> ()"); + m.impl("rmsnorm", torch::kCUDA, &rmsnorm); + + m.def("fused_add_rmsnorm(Tensor! input, Tensor! residual, Tensor weight, float eps) -> ()"); + m.impl("fused_add_rmsnorm", torch::kCUDA, &sgl_fused_add_rmsnorm); + + m.def("gemma_rmsnorm(Tensor! output, Tensor input, Tensor weight, float eps, int cuda_stream) -> ()"); + m.impl("gemma_rmsnorm", torch::kCUDA, &gemma_rmsnorm); + + m.def("gemma_fused_add_rmsnorm(Tensor! input, Tensor! residual, Tensor weight, float eps, int cuda_stream) -> ()"); + m.impl("gemma_fused_add_rmsnorm", torch::kCUDA, &gemma_fused_add_rmsnorm); + + m.def("silu_and_mul(Tensor! out, Tensor input, int cuda_stream) -> ()"); + m.impl("silu_and_mul", torch::kCUDA, &silu_and_mul); + + m.def("gelu_tanh_and_mul(Tensor! out, Tensor input, int cuda_stream) -> ()"); + m.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul); + + m.def("gelu_and_mul(Tensor! out, Tensor input, int cuda_stream) -> ()"); + m.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul); + + m.def( + "apply_rope_pos_ids_cos_sin_cache(Tensor q, Tensor k, Tensor! q_rope, Tensor! k_rope, Tensor cos_sin_cache, " + "Tensor pos_ids, bool interleave, int cuda_stream) -> ()"); + m.impl("apply_rope_pos_ids_cos_sin_cache", torch::kCUDA, &apply_rope_pos_ids_cos_sin_cache); + + /* + * From csrc/gemm + */ + m.def("awq_dequantize(Tensor qweight, Tensor scales, Tensor qzeros) -> Tensor"); + m.impl("awq_dequantize", torch::kCUDA, &awq_dequantize); + + m.def( + "int8_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype, Tensor? " + "bias) -> Tensor"); + m.impl("int8_scaled_mm", torch::kCUDA, &int8_scaled_mm); + + m.def( + "fp8_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype, Tensor? " + "bias) -> Tensor"); + m.impl("fp8_scaled_mm", torch::kCUDA, &fp8_scaled_mm); + + m.def( + "fp8_blockwise_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype) -> " + "Tensor"); + m.impl("fp8_blockwise_scaled_mm", torch::kCUDA, &fp8_blockwise_scaled_mm); + + m.def( + "sgl_per_token_group_quant_fp8(Tensor input, Tensor output_q, Tensor output_s, int group_size," + " float eps, float fp8_min, float fp8_max) -> ()"); + m.impl("sgl_per_token_group_quant_fp8", torch::kCUDA, &sgl_per_token_group_quant_fp8); + + m.def( + "sgl_per_token_group_quant_int8(Tensor input, Tensor output_q, Tensor output_s, int group_size," + " float eps, float int8_min, float int8_max) -> ()"); + m.impl("sgl_per_token_group_quant_int8", torch::kCUDA, &sgl_per_token_group_quant_int8); + + m.def("sgl_per_tensor_quant_fp8(Tensor input, Tensor output_q, Tensor output_s, bool is_static) -> ()"); + m.impl("sgl_per_tensor_quant_fp8", torch::kCUDA, &sgl_per_tensor_quant_fp8); + + m.def("sgl_per_token_quant_fp8(Tensor input, Tensor output_q, Tensor output_s) -> ()"); + m.impl("sgl_per_token_quant_fp8", torch::kCUDA, &sgl_per_token_quant_fp8); + + m.def( + "cublas_grouped_gemm(Tensor[] inputs, Tensor[] weights, Tensor[] outputs," + " ScalarType out_dtype, int cublas_handle, int cuda_stream) -> ()"); + m.impl("cublas_grouped_gemm", torch::kCUDA, &cublas_grouped_gemm); + + m.def( + "cutlass_scaled_fp4_mm(Tensor! out, Tensor a, Tensor b," + " Tensor block_scale_a, Tensor block_scale_b," + " Tensor alpha) -> ()"); + m.impl("cutlass_scaled_fp4_mm", torch::kCUDA, &cutlass_scaled_fp4_mm); + + m.def( + "scaled_fp4_quant(Tensor! output, Tensor! input," + " Tensor! output_scale, Tensor! input_scale) -> ()"); + m.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant); + + /* + * From csrc/moe + */ + m.def( + "moe_align_block_size(Tensor topk_ids, int num_experts, int block_size, Tensor! sorted_token_ids, Tensor! " + "experts_ids, Tensor! num_tokens_post_pad, Tensor! token_cnts_buffer, Tensor! cumsum_buffer) -> ()"); + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); + + m.def( + "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! " + "token_expert_indices, Tensor gating_output) -> ()"); + m.impl("topk_softmax", torch::kCUDA, &topk_softmax); + + /* + * From csrc/speculative + */ + m.def( + "tree_speculative_sampling_target_only(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor uniform_samples, Tensor target_probs, Tensor draft_probs, " + "float threshold_single, float threshold_acc, " + "bool deterministic, int cuda_stream) -> ()"); + m.impl("tree_speculative_sampling_target_only", torch::kCUDA, &tree_speculative_sampling_target_only); + + m.def( + "verify_tree_greedy(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor target_predict, int cuda_stream) -> ()"); + m.impl("verify_tree_greedy", torch::kCUDA, &verify_tree_greedy); + + m.def( + "build_tree_kernel_efficient(Tensor parent_list, Tensor selected_index, Tensor verified_seq_len, " + "Tensor! tree_mask, Tensor! positions, Tensor! retrive_index, Tensor! retrive_next_token, " + "Tensor! retrive_next_sibling, int topk, int depth, int draft_token_num) -> ()"); + m.impl("build_tree_kernel_efficient", torch::kCUDA, &build_tree_kernel_efficient); + + m.def("segment_packbits(Tensor x, Tensor input_indptr, Tensor output_indptr, Tensor! y, int cuda_stream) -> ()"); + m.impl("segment_packbits", torch::kCUDA, &segment_packbits); + + /* + * From FlashInfer + */ + m.def( + "bmm_fp8(Tensor A, Tensor B, Tensor! D, Tensor A_scale, Tensor B_scale, Tensor workspace_buffer, int " + "cublas_handle, int cuda_stream) -> ()"); + m.impl("bmm_fp8", torch::kCUDA, &bmm_fp8); + + m.def( + "min_p_sampling_from_probs(Tensor probs, Tensor uniform_samples, Tensor! samples, Tensor? maybe_min_p_arr, float " + "min_p_val, bool deterministic, int cuda_stream) -> ()"); + m.impl("min_p_sampling_from_probs", torch::kCUDA, &min_p_sampling_from_probs); + + m.def( + "top_k_renorm_probs(Tensor probs, Tensor! renorm_probs, Tensor? maybe_top_k_arr, int top_k_val, int " + "cuda_stream) -> ()"); + m.impl("top_k_renorm_probs", torch::kCUDA, &top_k_renorm_probs); + + m.def( + "top_p_renorm_probs(Tensor probs, Tensor! renorm_probs, Tensor? maybe_top_p_arr, float top_p_val, int " + "cuda_stream) -> ()"); + m.impl("top_p_renorm_probs", torch::kCUDA, &top_p_renorm_probs); + + m.def( + "top_k_top_p_sampling_from_probs(Tensor probs, Tensor uniform_samples, Tensor! samples, Tensor! success, Tensor? " + "maybe_top_k_arr, float top_k_val, Tensor? maybe_top_p_arr, float top_p_val, bool deterministic, int " + "cuda_stream) -> ()"); + m.impl("top_k_top_p_sampling_from_probs", torch::kCUDA, &top_k_top_p_sampling_from_probs); + + m.def( + "top_p_sampling_from_probs(Tensor probs, Tensor uniform_samples, Tensor! samples, Tensor! success, Tensor? " + "maybe_top_p_arr, float top_p_val, bool deterministic, int cuda_stream) -> ()"); + m.impl("top_p_sampling_from_probs", torch::kCUDA, &top_p_sampling_from_probs); +} + +REGISTER_EXTENSION(common_ops) diff --git a/sgl-kernel/csrc/torch_extension_rocm.cc b/sgl-kernel/csrc/torch_extension_rocm.cc new file mode 100644 index 0000000..6da15a3 --- /dev/null +++ b/sgl-kernel/csrc/torch_extension_rocm.cc @@ -0,0 +1,82 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_EXPAND(sgl_kernel, m) { + /* + * From csrc/allreduce + */ + m.def( + "init_custom_ar(Tensor meta, Tensor rank_data, " + "str[] handles, int[] offsets, int rank, " + "bool full_nvlink) -> int"); + m.impl("init_custom_ar", torch::kCUDA, &init_custom_ar); + + m.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()"); + m.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg); + + m.def( + "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> " + "()"); + m.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg); + + m.def("dispose", &dispose); + + m.def("meta_size", &meta_size); + + m.def( + "register_buffer(int fa, Tensor t, str[] handles, " + "int[] offsets) -> ()"); + m.impl("register_buffer", torch::kCUDA, ®ister_buffer); + + m.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta); + m.def("register_graph_buffers", ®ister_graph_buffers); + + m.def("allocate_meta_buffer", &allocate_meta_buffer); + m.impl("allocate_meta_buffer", torch::kCUDA, &allocate_meta_buffer); + + m.def("get_meta_buffer_ipc_handle", &get_meta_buffer_ipc_handle); + m.impl("get_meta_buffer_ipc_handle", torch::kCPU, &get_meta_buffer_ipc_handle); + + /* + * From csrc/moe + */ + m.def( + "moe_align_block_size(Tensor topk_ids, int num_experts, int block_size, Tensor! sorted_token_ids, Tensor! " + "experts_ids, Tensor! num_tokens_post_pad, Tensor! token_cnts_buffer, Tensor! cumsum_buffer) -> ()"); + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); + m.def( + "topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor! " + "token_expert_indices, Tensor gating_output) -> ()"); + m.impl("topk_softmax", torch::kCUDA, &topk_softmax); + + m.def( + "verify_tree_greedy(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor target_predict, int cuda_stream) -> ()"); + m.impl("verify_tree_greedy", torch::kCUDA, &verify_tree_greedy); + + m.def( + "build_tree_kernel_efficient(Tensor parent_list, Tensor selected_index, Tensor verified_seq_len, " + "Tensor! tree_mask, Tensor! positions, Tensor! retrive_index, Tensor! retrive_next_token, " + "Tensor! retrive_next_sibling, int topk, int depth, int draft_token_num) -> ()"); + m.impl("build_tree_kernel_efficient", torch::kCUDA, &build_tree_kernel_efficient); +} + +REGISTER_EXTENSION(common_ops) diff --git a/sgl-kernel/include/sgl_kernel_ops.h b/sgl-kernel/include/sgl_kernel_ops.h new file mode 100644 index 0000000..f4961ab --- /dev/null +++ b/sgl-kernel/include/sgl_kernel_ops.h @@ -0,0 +1,290 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#define _CONCAT(A, B) A##B +#define CONCAT(A, B) _CONCAT(A, B) + +#define _STRINGIFY(A) #A +#define STRINGIFY(A) _STRINGIFY(A) + +#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE) + +#define REGISTER_EXTENSION(NAME) \ + PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \ + static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, STRINGIFY(NAME), nullptr, 0, nullptr}; \ + return PyModule_Create(&module); \ + } + +using fptr_t = int64_t; + +/* + * From csrc/allreduce + */ +#ifdef USE_ROCM +// ROCM custom allreduce +fptr_t init_custom_ar( + torch::Tensor& meta, + torch::Tensor& rank_data, + const std::vector& handles, + const std::vector& offsets, + int64_t rank, + bool full_nvlink); +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out); +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer, torch::Tensor& out); +void dispose(fptr_t _fa); +int64_t meta_size(); +void register_buffer( + fptr_t _fa, torch::Tensor& t, const std::vector& handles, const std::vector& offsets); +std::tuple> get_graph_buffer_ipc_meta(fptr_t _fa); +void register_graph_buffers( + fptr_t _fa, const std::vector& handles, const std::vector>& offsets); +torch::Tensor allocate_meta_buffer(int64_t size); +torch::Tensor get_meta_buffer_ipc_handle(torch::Tensor& inp); +#else +// TRTLLM custom allreduce +fptr_t init_custom_ar( + int64_t rank_id, + int64_t world_size, + torch::Tensor& rank_data, + const std::vector& buffers, + const std::vector& tmp_result_buffers, + const std::vector& barrier_in, + const std::vector& barrier_out); +void dispose(fptr_t _fa); +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out); +std::tuple, std::vector> get_graph_buffer_ipc_meta(fptr_t _fa); +void register_graph_buffers( + fptr_t _fa, const std::vector>& handles, const std::vector>& offsets); +#endif + +/* + * From csrc/attention + */ +void lightning_attention_decode( + const torch::Tensor& q, + const torch::Tensor& k, + const torch::Tensor& v, + const torch::Tensor& past_kv, + const torch::Tensor& slope, + torch::Tensor output, + torch::Tensor new_kv); + +/* + * From csrc/elementwise + */ +void rmsnorm(at::Tensor& output, at::Tensor& input, at::Tensor& weight, double eps, int64_t cuda_stream); +void sgl_fused_add_rmsnorm(torch::Tensor input, torch::Tensor residual, torch::Tensor weight, double eps); +void gemma_rmsnorm(at::Tensor& output, at::Tensor& input, at::Tensor& weight, double eps, int64_t cuda_stream); +void gemma_fused_add_rmsnorm( + at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps, int64_t cuda_stream); +void silu_and_mul(at::Tensor& out, at::Tensor& input, int64_t cuda_stream); +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input, int64_t cuda_stream); +void gelu_and_mul(at::Tensor& out, at::Tensor& input, int64_t cuda_stream); +void apply_rope_pos_ids_cos_sin_cache( + at::Tensor q, + at::Tensor k, + at::Tensor q_rope, + at::Tensor k_rope, + at::Tensor cos_sin_cache, + at::Tensor pos_ids, + bool interleave, + int64_t cuda_stream); + +/* + * From csrc/gemm + */ +torch::Tensor awq_dequantize(torch::Tensor qweight, torch::Tensor scales, torch::Tensor qzeros); +void cutlass_scaled_fp4_mm( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha); +torch::Tensor int8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias); +torch::Tensor fp8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias); +torch::Tensor fp8_blockwise_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype); +void scaled_fp4_quant( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_scale, torch::Tensor const& input_scale); +void sgl_per_token_group_quant_fp8( + at::Tensor input, + at::Tensor output_q, + at::Tensor output_s, + int64_t group_size, + double eps, + double fp8_min, + double fp8_max); +void sgl_per_token_group_quant_int8( + at::Tensor input, + at::Tensor output_q, + at::Tensor output_s, + int64_t group_size, + double eps, + double int8_min, + double int8_max); +void sgl_per_tensor_quant_fp8(at::Tensor input, at::Tensor output_q, at::Tensor output_s, bool is_static); +void sgl_per_token_quant_fp8(at::Tensor input, at::Tensor output_q, at::Tensor output_s); +void cublas_grouped_gemm( + const std::vector& inputs, + const std::vector& weights, + const std::vector& outputs, + const torch::Dtype& out_dtype, + int64_t cublas_handle, + int64_t cuda_stream); +void bmm_fp8( + at::Tensor A, + at::Tensor B, + at::Tensor D, + at::Tensor A_scale, + at::Tensor B_scale, + at::Tensor workspace_buffer, + int64_t cublas_handle, + int64_t cuda_stream); + +/* + * From csrc/moe + */ +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor token_cnts_buffer, + torch::Tensor cumsum_buffer); + +void topk_softmax( + torch::Tensor& topk_weights, + torch::Tensor& topk_indices, + torch::Tensor& token_expert_indices, + torch::Tensor& gating_output); + +/* + * From csrc/speculative + */ +void tree_speculative_sampling_target_only( + at::Tensor predicts, // mutable + at::Tensor accept_index, // mutable + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor uniform_samples, + at::Tensor target_probs, + at::Tensor draft_probs, + double threshold_single = 1, + double threshold_acc = 1, + bool deterministic = true, + int64_t cuda_stream = 0); + +void verify_tree_greedy( + at::Tensor predicts, // mutable + at::Tensor accept_index, // mutable + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict, + int64_t cuda_stream = 0); + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num); + +void segment_packbits( + at::Tensor x, at::Tensor input_indptr, at::Tensor output_indptr, at::Tensor y, int64_t cuda_stream); + +/* + * From FlashInfer + */ +void min_p_sampling_from_probs( + at::Tensor probs, + at::Tensor uniform_samples, + at::Tensor samples, + std::optional maybe_min_p_arr, + double min_p_val, + bool deterministic, + int64_t cuda_stream); +void top_k_renorm_probs( + at::Tensor probs, + at::Tensor renorm_probs, + std::optional maybe_top_k_arr, + int64_t top_k_val, + int64_t cuda_stream); +void top_p_renorm_probs( + at::Tensor probs, + at::Tensor renorm_probs, + std::optional maybe_top_p_arr, + double top_p_val, + int64_t cuda_stream); +void top_k_top_p_sampling_from_probs( + at::Tensor probs, + at::Tensor uniform_samples, + at::Tensor samples, + at::Tensor success, + std::optional maybe_top_k_arr, + double top_k_val, + std::optional maybe_top_p_arr, + double top_p_val, + bool deterministic, + int64_t cuda_stream); +void top_p_sampling_from_probs( + at::Tensor probs, + at::Tensor uniform_samples, + at::Tensor samples, + at::Tensor success, + std::optional maybe_top_p_arr, + double top_p_val, + bool deterministic, + int64_t cuda_stream); diff --git a/sgl-kernel/include/trt_reduce_internal.cuh b/sgl-kernel/include/trt_reduce_internal.cuh new file mode 100644 index 0000000..9fec59b --- /dev/null +++ b/sgl-kernel/include/trt_reduce_internal.cuh @@ -0,0 +1,109 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// reference: +// https://github.com/NVIDIA/TensorRT-LLM/blob/release/0.14/cpp/tensorrt_llm/plugins/ncclPlugin/allreducePlugin.cpp +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace trt_llm { +constexpr size_t WARP_SIZE = 32; +constexpr size_t MAX_ALL_REDUCE_BLOCKS = 32; +constexpr size_t MAX_RANKS_PER_NODE = 8; +constexpr size_t DEFAULT_BLOCK_SIZE = 512; + +enum class AllReduceStrategyType : int8_t { + RING = 0, + ONESHOT = 1, + TWOSHOT = 2, + AUTO = 3, +}; + +struct RankData { + void* ptrs[MAX_RANKS_PER_NODE]; +}; + +struct AllReduceParams { + size_t elts_size; + size_t elts_total; + size_t elts_per_rank; + size_t elts_per_block; + size_t rank_offset; + size_t ranks_per_node, rank, local_rank; + uint32_t barrier_flag; + uint32_t* peer_barrier_ptrs_in[MAX_RANKS_PER_NODE]; + uint32_t* peer_barrier_ptrs_out[MAX_RANKS_PER_NODE]; + uint32_t* tmp_result_buffers[MAX_RANKS_PER_NODE]; + RankData* peer_comm_buffer_ptrs; + void* local_input_buffer_ptr; + void* local_output_buffer_ptr; + bool is_capturing; +}; + +inline size_t GetMaxRequiredWorkspaceSize(int world_size) { + if (world_size <= 2) { + return 16 * 1024 * 1024; + } + return 8 * 1024 * 1024; +} + +inline AllReduceStrategyType SelectImplementation(size_t message_size, int world_size) { + const size_t maxWorkspaceSize = GetMaxRequiredWorkspaceSize(world_size); + + if (message_size > maxWorkspaceSize) { + assert(false && "Custom allreduce do not ring currently"); + return AllReduceStrategyType::RING; + } + + if (world_size <= 2) { + return AllReduceStrategyType::ONESHOT; + } + + if (world_size <= 4) { + if (message_size < 1 * 1024 * 1024) { + return AllReduceStrategyType::ONESHOT; + } + return AllReduceStrategyType::TWOSHOT; + } + + if (message_size < 512 * 1024) { + return AllReduceStrategyType::ONESHOT; + } + return AllReduceStrategyType::TWOSHOT; +} + +void trtCustomAllReduce( + AllReduceParams& params, at::ScalarType data_type, AllReduceStrategyType strat, cudaStream_t stream); + +} // namespace trt_llm diff --git a/sgl-kernel/include/utils.h b/sgl-kernel/include/utils.h new file mode 100644 index 0000000..229c6e9 --- /dev/null +++ b/sgl-kernel/include/utils.h @@ -0,0 +1,348 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include + +#include + +#ifndef USE_ROCM +// Adapt from FlashInfer +#ifdef FLASHINFER_ENABLE_F16 +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = nv_half; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_F16(c_type, ...) +#endif + +#ifdef FLASHINFER_ENABLE_BF16 +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = nv_bfloat16; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_BF16(c_type, ...) +#endif + +#ifdef FLASHINFER_ENABLE_FP8_E4M3 +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) \ + case at::ScalarType::Float8_e4m3fn: { \ + using c_type = __nv_fp8_e4m3; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) +#endif + +#ifdef FLASHINFER_ENABLE_FP8_E5M2 +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) \ + case at::ScalarType::Float8_e5m2: { \ + using c_type = __nv_fp8_e5m2; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) +#endif + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch fp8 data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH(var_name, cond, ...) \ + [&]() -> bool { \ + switch (cond) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch " var_name " " << int(cond); \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH_U16x2(var1_name, var2_name, cond1, cond2, ...) \ + [&]() -> bool { \ + switch (pack_u16(cond1, cond2)) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch (" var1_name ", " var2_name "): (" << int(cond1) << ", " \ + << int(cond2) << ")"; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_CASE(case_expr, case_var, ...) \ + case case_expr: { \ + constexpr auto case_var = case_expr; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_U16x2(case_expr1, case_expr2, case_var1, case_var2, ...) \ + case pack_u16(case_expr1, case_expr2): { \ + constexpr auto case_var1 = case_expr1; \ + constexpr auto case_var2 = case_expr2; \ + return __VA_ARGS__(); \ + } + +#define DISPATCH_BOOL(expr, const_expr, ...) \ + [&]() -> bool { \ + if (expr) { \ + constexpr bool const_expr = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool const_expr = false; \ + return __VA_ARGS__(); \ + } \ + }() + +inline void check_shape(const at::Tensor& a, const at::Tensor& b, const char* a_name, const char* b_name) { + TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ", a.dim(), " vs ", b.dim()); + for (int i = 0; i < a.dim(); ++i) { + TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name, ".size(", i, ")"); + } +} + +inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) { + return (uint32_t(a) << 16) | uint32_t(b); +} + +#define CHECK_GQA_HEAD_DIVISIBLE(num_qo_heads, num_kv_heads) \ + TORCH_CHECK( \ + num_qo_heads % num_kv_heads == 0, \ + "num_qo_heads(", \ + num_qo_heads, \ + ") must be divisible by num_kv_heads(", \ + num_kv_heads, \ + ")") + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b) + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) + +inline bool is_float8_tensor(const at::Tensor& tensor) { + return tensor.scalar_type() == at::ScalarType::Float8_e4m3fn || tensor.scalar_type() == at::ScalarType::Float8_e5m2; +} +#endif + +struct cuda_error : public std::runtime_error { + /** + * @brief Constructs a `cuda_error` object with the given `message`. + * + * @param message The error char array used to construct `cuda_error` + */ + cuda_error(const char* message) : std::runtime_error(message) {} + /** + * @brief Constructs a `cuda_error` object with the given `message` string. + * + * @param message The `std::string` used to construct `cuda_error` + */ + cuda_error(std::string const& message) : cuda_error{message.c_str()} {} +}; + +#define CHECK_CUDA_SUCCESS(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + std::stringstream _message; \ + auto s = cudaGetErrorString(e); \ + _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__; \ + throw cuda_error(_message.str()); \ + } \ + } while (0) + +#define CHECK_IS_CUDA(x) TORCH_CHECK(x.device().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_IS_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_CUDA_INPUT(x) \ + CHECK_IS_CUDA(x); \ + CHECK_IS_CONTIGUOUS(x) + +inline int getSMVersion() { + int device{-1}; + CHECK_CUDA_SUCCESS(cudaGetDevice(&device)); + int sm_major = 0; + int sm_minor = 0; + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&sm_major, cudaDevAttrComputeCapabilityMajor, device)); + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&sm_minor, cudaDevAttrComputeCapabilityMinor, device)); + return sm_major * 10 + sm_minor; +} + +// SGLANG_SHFL_XOR_* adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/cuda_compat.h#L19-L28 +#ifndef USE_ROCM +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor_sync((mask), (var), (lane_mask), (width)) +#else +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor((var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor((var), (lane_mask), (width)) +#endif + +#ifndef USE_ROCM +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + case at::ScalarType::Float: { \ + using c_type = float; \ + return __VA_ARGS__(); \ + } \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() +#endif + +#define DISPATCH_CASE_INTEGRAL_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__) + +#define DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) + +#define CEILDIV(x, y) (((x) + (y) - 1) / (y)) +#define WARP_SIZE 32 + +#ifndef USE_ROCM +#include +using FP8_TYPE = c10::Float8_e4m3fn; +C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +#else +#include + +using FP8_TYPE = c10::Float8_e4m3fnuz; +constexpr auto FP8_E4M3_MAX = 224.0f; +#endif + +#ifndef USE_ROCM +__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) { + float old; + old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) + : __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value))); + return old; +} + +__device__ __forceinline__ float warpReduceMax(float max_value) { + max_value = fmaxf(max_value, SGLANG_SHFL_XOR_SYNC(0xffffffff, max_value, 16)); + max_value = fmaxf(max_value, SGLANG_SHFL_XOR_SYNC(0xffffffff, max_value, 8)); + max_value = fmaxf(max_value, SGLANG_SHFL_XOR_SYNC(0xffffffff, max_value, 4)); + max_value = fmaxf(max_value, SGLANG_SHFL_XOR_SYNC(0xffffffff, max_value, 2)); + max_value = fmaxf(max_value, SGLANG_SHFL_XOR_SYNC(0xffffffff, max_value, 1)); + return max_value; +} + +__device__ __forceinline__ float blockReduceMax(float max_value) { + static __shared__ float warpLevelMaxs[WARP_SIZE]; + const int laneId = threadIdx.x % WARP_SIZE; + const int warpId = threadIdx.x / WARP_SIZE; + + max_value = warpReduceMax(max_value); + + if (laneId == 0) warpLevelMaxs[warpId] = max_value; + __syncthreads(); + + max_value = (threadIdx.x < blockDim.x / WARP_SIZE) ? warpLevelMaxs[laneId] : 0; + if (warpId == 0) max_value = warpReduceMax(max_value); + + return max_value; +} +#endif + +// Pads to a multiple of `alignment` rows. +inline torch::Tensor pad_tensor(const torch::Tensor& tensor, int64_t alignment = 4, bool is_column_major = false) { + int64_t rows = tensor.size(0); + int64_t cols = tensor.size(1); + int64_t pad_rows = (alignment - (rows % alignment)) % alignment; // Compute padding size + + if (pad_rows == 0) { + return tensor; // Already aligned + } + + torch::Tensor padding = torch::zeros({pad_rows, cols}, tensor.options()); + torch::Tensor tensor_padded = torch::cat({tensor, padding}, 0); // Pad along rows + + // Ensure column-major layout + if (is_column_major) { + return tensor_padded.t().contiguous().t(); + } + return tensor_padded; +} diff --git a/sgl-kernel/pyproject.toml b/sgl-kernel/pyproject.toml new file mode 100644 index 0000000..393dc6d --- /dev/null +++ b/sgl-kernel/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = [ + "scikit-build-core>=0.10", + "torch==2.5.1", + "wheel", +] +build-backend = "scikit_build_core.build" + +[project] +name = "sgl-kernel" +version = "0.0.5.post4" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.9" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: GPU :: NVIDIA CUDA" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] + +[tool.scikit-build] +cmake.build-type = "Release" +minimum-version = "build-system.requires" + +wheel.py-api = "cp39" +wheel.license-files = [] +wheel.packages = ["python/sgl_kernel"] diff --git a/sgl-kernel/pyproject_rocm.toml b/sgl-kernel/pyproject_rocm.toml new file mode 100644 index 0000000..db9c4a5 --- /dev/null +++ b/sgl-kernel/pyproject_rocm.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ + "setuptools>=75.0", + "scikit-build-core>=0.10", + "torch==2.5.1", + "wheel", +] +build-backend = "setuptools.build_meta" + +[project] +name = "sgl-kernel" +version = "0.0.5.post4" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.9" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: GPU :: NVIDIA CUDA" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] diff --git a/sgl-kernel/python/sgl_kernel/__init__.py b/sgl-kernel/python/sgl_kernel/__init__.py new file mode 100644 index 0000000..1a668fe --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/__init__.py @@ -0,0 +1,57 @@ +import ctypes +import os + +import torch + +if os.path.exists("/usr/local/cuda/targets/x86_64-linux/lib/libcudart.so.12"): + ctypes.CDLL( + "/usr/local/cuda/targets/x86_64-linux/lib/libcudart.so.12", + mode=ctypes.RTLD_GLOBAL, + ) + +from sgl_kernel import common_ops +from sgl_kernel.allreduce import * +from sgl_kernel.attention import lightning_attention_decode +from sgl_kernel.elementwise import ( + apply_rope_with_cos_sin_cache_inplace, + fused_add_rmsnorm, + gelu_and_mul, + gelu_tanh_and_mul, + gemma_fused_add_rmsnorm, + gemma_rmsnorm, + rmsnorm, + silu_and_mul, +) +from sgl_kernel.gemm import ( + awq_dequantize, + bmm_fp8, + cublas_grouped_gemm, + cutlass_scaled_fp4_mm, + fp8_blockwise_scaled_mm, + fp8_scaled_mm, + int8_scaled_mm, + scaled_fp4_quant, + sgl_per_tensor_quant_fp8, + sgl_per_token_group_quant_fp8, + sgl_per_token_group_quant_int8, + sgl_per_token_quant_fp8, +) +from sgl_kernel.moe import moe_align_block_size, topk_softmax +from sgl_kernel.sampling import ( + min_p_sampling_from_probs, + top_k_renorm_prob, + top_k_top_p_sampling_from_probs, + top_p_renorm_prob, + top_p_sampling_from_probs, +) +from sgl_kernel.speculative import ( + build_tree_kernel_efficient, + segment_packbits, + tree_speculative_sampling_target_only, + verify_tree_greedy, +) +from sgl_kernel.version import __version__ + +build_tree_kernel = ( + None # TODO(ying): remove this after updating the sglang python code. +) diff --git a/sgl-kernel/python/sgl_kernel/allreduce.py b/sgl-kernel/python/sgl_kernel/allreduce.py new file mode 100644 index 0000000..3a6e58d --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/allreduce.py @@ -0,0 +1,77 @@ +from typing import List, Tuple + +import torch + +if torch.version.hip is not None: + # ROCM custom allreduce + def init_custom_ar( + meta: torch.Tensor, + rank_data: torch.Tensor, + handles: List[str], + offsets: List[int], + rank: int, + full_nvlink: bool, + ) -> int: + return torch.ops.sgl_kernel.init_custom_ar.default( + meta, rank_data, handles, offsets, rank, full_nvlink + ) + + def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + torch.ops.sgl_kernel.all_reduce_reg.default(fa, inp, out) + + def all_reduce_unreg( + fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, out: torch.Tensor + ) -> None: + torch.ops.sgl_kernel.all_reduce_unreg.default(fa, inp, reg_buffer, out) + + def dispose(fa: int) -> None: + torch.ops.sgl_kernel.dispose.default(fa) + + def meta_size() -> int: + return torch.ops.sgl_kernel.meta_size.default() + + def register_buffer( + fa: int, t: torch.Tensor, handles: List[str], offsets: List[int] + ) -> None: + return torch.ops.sgl_kernel.register_buffer.default(fa, t, handles, offsets) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[torch.Tensor, List[int]]: + return torch.ops.sgl_kernel.get_graph_buffer_ipc_meta.default(fa) + + def register_graph_buffers( + fa: int, handles: List[str], offsets: List[List[int]] + ) -> None: + torch.ops.sgl_kernel.register_graph_buffers.default(fa, handles, offsets) + + def allocate_meta_buffer(size: int) -> torch.Tensor: + return torch.ops.sgl_kernel.allocate_meta_buffer.default(size) + + def get_meta_buffer_ipc_handle(inp: torch.Tensor) -> torch.Tensor: + return torch.ops.sgl_kernel.get_meta_buffer_ipc_handle.default(inp) + +else: + # TRTLLM custom allreduce + def init_custom_reduce( + rank_id, num_devices, rank_data, buffers, tmp_buffers, barrier_in, barrier_out + ): + return torch.ops.sgl_kernel.init_custom_ar.default( + rank_id, + num_devices, + rank_data, + buffers, + tmp_buffers, + barrier_in, + barrier_out, + ) + + def custom_dispose(fa): + torch.ops.sgl_kernel.dispose.default(fa) + + def custom_reduce(fa, inp, out): + torch.ops.sgl_kernel.all_reduce.default(fa, inp, out) + + def get_graph_buffer_ipc_meta(fa): + return torch.ops.sgl_kernel.get_graph_buffer_ipc_meta.default(fa) + + def register_graph_buffers(fa, handles, offsets): + torch.ops.sgl_kernel.register_graph_buffers.default(fa, handles, offsets) diff --git a/sgl-kernel/python/sgl_kernel/attention.py b/sgl-kernel/python/sgl_kernel/attention.py new file mode 100644 index 0000000..6ad1d34 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/attention.py @@ -0,0 +1,7 @@ +import torch + + +def lightning_attention_decode(q, k, v, past_kv, slope, output, new_kv): + torch.ops.sgl_kernel.lightning_attention_decode.default( + q, k, v, past_kv, slope, output, new_kv + ) diff --git a/sgl-kernel/python/sgl_kernel/elementwise.py b/sgl-kernel/python/sgl_kernel/elementwise.py new file mode 100644 index 0000000..3db1571 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/elementwise.py @@ -0,0 +1,153 @@ +from typing import Optional + +import torch +from sgl_kernel.utils import get_cuda_stream + + +# These implementations extensively draw from and build upon the FlashInfer project https://github.com/flashinfer-ai/flashinfer +# Kudos to @yzh119 +def rmsnorm( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out is None: + out = torch.empty_like(input) + torch.ops.sgl_kernel.rmsnorm.default(out, input, weight, eps, get_cuda_stream()) + return out + + +def fused_add_rmsnorm( + input: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6 +) -> None: + torch.ops.sgl_kernel.fused_add_rmsnorm.default(input, residual, weight, eps) + + +def gemma_rmsnorm( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out is None: + out = torch.empty_like(input) + torch.ops.sgl_kernel.gemma_rmsnorm.default( + out, input, weight, eps, get_cuda_stream() + ) + return out + + +def gemma_fused_add_rmsnorm( + input: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6 +) -> None: + torch.ops.sgl_kernel.gemma_fused_add_rmsnorm.default( + input, residual, weight, eps, get_cuda_stream() + ) + + +def _check_shape(input: torch.Tensor, output: torch.Tensor) -> None: + assert input.ndim == output.ndim, f"{input.ndim} != {output.ndim}" + assert ( + input.shape[:-1] == output.shape[:-1] + ), f"{input.shape[:-1]} != {output.shape[:-1]}" + assert ( + input.shape[-1] == 2 * output.shape[-1] + ), f"{input.shape[-1]} != {2 * output.shape[-1]}" + + +def silu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.silu_and_mul.default(out, input, get_cuda_stream()) + return out + + +def gelu_tanh_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.gelu_tanh_and_mul.default(out, input, get_cuda_stream()) + return out + + +def gelu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.gelu_and_mul.default(out, input, get_cuda_stream()) + return out + + +def apply_rope_with_cos_sin_cache_inplace( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + head_size: int, + cos_sin_cache: torch.Tensor, + is_neox: bool = True, +) -> None: + r""" + Apply rotary embedding to keys and queries with precomputed cos/sin values. + This is designed to be compatible with the SGL/vLLM implementation. + The result is inplace applied to the input tensors. + + Parameters + ---------- + positions : torch.Tensor + Position indices, shape: ``(nnz)``. + query : torch.Tensor + Query tensor, shape: ``(nnz, num_q_heads * head_size)``. + key : torch.Tensor + Key tensor, shape: ``(nnz, num_k_heads * head_size)``. + cos_sin_cache : torch.Tensor + Cosine and Sine cache tensor, shape: ``(max_seq_len, rotary_dim)``. + Cosine is the first half and Sine is the second half on rotary_dim. + is_neox : bool + Whether to use Neox style RoPE, default: ``True``. + + * If ``True``, the last dimension of the query/key tensor is not interleaved, i.e., + we rorate the first half dimensions ``([..., :head_dim//2])`` and the second half + dimensions ``([..., head_dim//2:])``. + + * If ``False``, the last dimension of the query/key tensor is interleaved, i.e., + we rotate the even dimensions ``([..., ::2])`` and odd dimensions ``([..., 1::2])``. + Note + ---- + The rotary dimension is determined by the cosine cache and sine cache. + """ + if cos_sin_cache.dtype != torch.float32: + raise ValueError("cos_sin_cache should be float32") + + torch.ops.sgl_kernel.apply_rope_pos_ids_cos_sin_cache.default( + q=query.view(query.shape[0], -1, head_size), + k=key.view(key.shape[0], -1, head_size), + q_rope=query.view(query.shape[0], -1, head_size), + k_rope=key.view(key.shape[0], -1, head_size), + cos_sin_cache=cos_sin_cache, + pos_ids=positions.long(), + interleave=(not is_neox), + cuda_stream=get_cuda_stream(), + ) diff --git a/sgl-kernel/python/sgl_kernel/gemm.py b/sgl-kernel/python/sgl_kernel/gemm.py new file mode 100644 index 0000000..b1ef345 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/gemm.py @@ -0,0 +1,219 @@ +from typing import List, Optional, Tuple + +import torch +from sgl_kernel.utils import _get_cache_buf, get_cuda_stream + + +def awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.ByteTensor: + return torch.ops.sgl_kernel.awq_dequantize.default(qweight, scales, qzeros) + + +def int8_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): + return torch.ops.sgl_kernel.int8_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + bias, + ) + + +def fp8_blockwise_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype): + return torch.ops.sgl_kernel.fp8_blockwise_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + ) + + +def fp8_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): + return torch.ops.sgl_kernel.fp8_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + bias, + ) + + +def _bmm_fp8_internal( + workspace_buffer: torch.Tensor, + A: torch.Tensor, + B: torch.Tensor, + D: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, +) -> None: + cublas_handle = torch.cuda.current_blas_handle() + torch.ops.sgl_kernel.bmm_fp8.default( + A, + B, + D, + A_scale, + B_scale, + workspace_buffer, + cublas_handle, + get_cuda_stream(), + ) + + +def bmm_fp8( + A: torch.Tensor, + B: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, + dtype: torch.dtype, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out is None: + out = torch.empty( + (A.shape[0], A.shape[1], B.shape[2]), + device=A.device, + dtype=dtype, + ) + workspace_buffer = _get_cache_buf("bmm_fp8_workspace", 32 * 1024 * 1024, A.device) + _bmm_fp8_internal(workspace_buffer, A, B, out, A_scale, B_scale) + return out + + +def sgl_per_token_group_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + group_size: int, + eps: float, + fp8_min: float, + fp8_max: float, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_group_quant_fp8.default( + input, output_q, output_s, group_size, eps, fp8_min, fp8_max + ) + + +def sgl_per_token_group_quant_int8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + group_size: int, + eps: float, + int8_min: float, + int8_max: float, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_group_quant_int8.default( + input, output_q, output_s, group_size, eps, int8_min, int8_max + ) + + +def sgl_per_tensor_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + is_static: bool, +) -> None: + torch.ops.sgl_kernel.sgl_per_tensor_quant_fp8.default( + input, output_q, output_s, is_static + ) + + +def cublas_grouped_gemm( + inputs: List[torch.Tensor], + weights: List[torch.Tensor], + outputs: List[torch.Tensor], + out_dtype: torch.dtype, +) -> None: + assert ( + len(inputs) > 0 and len(weights) > 0 and len(outputs) > 0 + ), "Inputs/weights/outputs should not be empty!" + cublas_handle = torch.cuda.current_blas_handle() + torch.ops.sgl_kernel.cublas_grouped_gemm.default( + inputs, + weights, + outputs, + out_dtype, + cublas_handle, + get_cuda_stream(), + ) + + +def sgl_per_token_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_quant_fp8.default(input, output_q, output_s) + + +def cutlass_scaled_fp4_mm( + a: torch.Tensor, + b: torch.Tensor, + block_scale_a: torch.Tensor, + block_scale_b: torch.Tensor, + alpha: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + assert a.ndim == 2 and b.ndim == 2 + m, n = a.shape[0], b.shape[0] + out = torch.empty((m, n), dtype=out_dtype, device=a.device) + torch.ops.sgl_kernel.cutlass_scaled_fp4_mm.default( + out, a, b, block_scale_a, block_scale_b, alpha + ) + return out + + +def scaled_fp4_quant( + input: torch.Tensor, input_global_scale: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP4 and return quantized tensor and scale. + + This function quantizes the last dimension of the given tensor `input`. For + every 16 consecutive elements, a single dynamically computed scaling factor + is shared. This scaling factor is quantized using the `input_global_scale` + and is stored in a swizzled layout (see + https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x). + + Args: + input: The input tensor to be quantized to FP4 + input_global_scale: A scalar scaling factor for the entire tensor. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every + two values are packed into a uint8 and float8_e4m3 scaling factors + in a sizzled layout. + """ + assert input.ndim >= 1, f"input.ndim needs to be >= 1, but got {input.ndim}." + other_dims = 1 if input.ndim == 1 else -1 + input = input.reshape(other_dims, input.shape[-1]) + m, n = input.shape + block_size = 16 + device = input.device + + assert n % block_size == 0, f"last dim has to be multiple of 16, but got {n}." + assert input.dtype in ( + torch.float16, + torch.bfloat16, + ), f"input.dtype needs to be fp16 or bf16 but got {input.dtype}." + + # Two fp4 values will be packed into an uint8. + output = torch.empty((m, n // 2), device=device, dtype=torch.uint8) + + # We use the rounded values to store the swizzled values. Then, the scaling + # factors in float8_e4m3fn are packed into an int32 for every 4 values. + rounded_m = ((m + 128 - 1) // 128) * 128 + scale_n = n // block_size + rounded_n = ((scale_n + 4 - 1) // 4) * 4 + output_scale = torch.empty( + (rounded_m, rounded_n // 4), device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.scaled_fp4_quant.default( + output, input, output_scale, input_global_scale + ) + output_scale = output_scale.view(torch.float8_e4m3fn) + return output, output_scale diff --git a/sgl-kernel/python/sgl_kernel/moe.py b/sgl-kernel/python/sgl_kernel/moe.py new file mode 100644 index 0000000..84c79fd --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/moe.py @@ -0,0 +1,34 @@ +import torch + + +def moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_token_ids, + experts_ids, + num_tokens_post_pad, + token_cnts_buffer, + cumsum_buffer, +): + torch.ops.sgl_kernel.moe_align_block_size.default( + topk_ids, + num_experts, + block_size, + sorted_token_ids, + experts_ids, + num_tokens_post_pad, + token_cnts_buffer, + cumsum_buffer, + ) + + +def topk_softmax( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + token_expert_indices: torch.Tensor, + gating_output: float, +) -> None: + torch.ops.sgl_kernel.topk_softmax.default( + topk_weights, topk_ids, token_expert_indices, gating_output + ) diff --git a/sgl-kernel/python/sgl_kernel/sampling.py b/sgl-kernel/python/sgl_kernel/sampling.py new file mode 100644 index 0000000..7c94e9e --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/sampling.py @@ -0,0 +1,210 @@ +from typing import Optional, Tuple, Union + +import torch +from sgl_kernel.utils import _to_tensor_scalar_tuple, get_cuda_stream + + +def _top_k_renorm_probs_internal( + probs: torch.Tensor, + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, +) -> torch.Tensor: + probs = probs.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + renorm_probs = torch.empty_like(probs) + torch.ops.sgl_kernel.top_k_renorm_probs.default( + probs, + renorm_probs, + maybe_top_k_arr, + top_k_val, + get_cuda_stream(), + ) + return renorm_probs + + +def top_k_renorm_probs( + probs: torch.Tensor, + top_k: Union[torch.Tensor, int], +) -> torch.Tensor: + return _top_k_renorm_probs_internal(probs, *_to_tensor_scalar_tuple(top_k)) + + +top_k_renorm_prob = top_k_renorm_probs + + +def _top_p_renorm_probs_internal( + probs: torch.Tensor, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, +) -> torch.Tensor: + probs = probs.float() + maybe_top_p_arr = maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + renorm_probs = torch.empty_like(probs) + torch.ops.sgl_kernel.top_p_renorm_probs.default( + probs, + renorm_probs, + maybe_top_p_arr, + top_p_val, + get_cuda_stream(), + ) + return renorm_probs + + +def top_p_renorm_probs( + probs: torch.Tensor, + top_p: Union[torch.Tensor, float], +) -> torch.Tensor: + return _top_p_renorm_probs_internal(probs, *_to_tensor_scalar_tuple(top_p)) + + +top_p_renorm_prob = top_p_renorm_probs + + +def _top_p_sampling_from_probs_internal( + probs: torch.Tensor, + uniform_samples: torch.Tensor, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, + deterministic: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + with probs.device as device: + probs = probs.float() + uniform_samples = uniform_samples.float() + maybe_top_p_arr = ( + maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + success = torch.empty(probs.size(0), dtype=torch.bool, device=device) + torch.ops.sgl_kernel.top_p_sampling_from_probs.default( + probs, + uniform_samples, + samples, + success, + maybe_top_p_arr, + top_p_val, + deterministic, + get_cuda_stream(), + ) + return samples, success + + +def top_p_sampling_from_probs( + probs: torch.Tensor, + uniform_samples: torch.Tensor, + top_p: Union[torch.Tensor, float], + deterministic: bool = True, + check_nan: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_p_sampling_from_probs_internal( + probs, uniform_samples, *_to_tensor_scalar_tuple(top_p), deterministic + ) + + +def _top_k_top_p_sampling_from_probs_internal( + probs: torch.Tensor, + uniform_samples: torch.Tensor, + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, + deterministic: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + with probs.device as device: + probs = probs.float() + uniform_samples = uniform_samples.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + maybe_top_p_arr = ( + maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + success = torch.empty(probs.size(0), dtype=torch.bool, device=device) + torch.ops.sgl_kernel.top_k_top_p_sampling_from_probs.default( + probs, + uniform_samples, + samples, + success, + maybe_top_k_arr, + top_k_val, + maybe_top_p_arr, + top_p_val, + deterministic, + get_cuda_stream(), + ) + return samples, success + + +def top_k_top_p_sampling_from_probs( + probs: torch.Tensor, + uniform_samples: torch.Tensor, + top_k: Union[torch.Tensor, int], + top_p: Union[torch.Tensor, float], + filter_apply_order: str = "top_k_first", + deterministic: bool = True, + check_nan: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + if filter_apply_order == "top_k_first": + renorm_probs = top_k_renorm_probs(probs, top_k) + return top_p_sampling_from_probs( + renorm_probs, uniform_samples, top_p, deterministic, check_nan=check_nan + ) + elif filter_apply_order == "joint": + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_k_top_p_sampling_from_probs_internal( + probs, + uniform_samples, + *_to_tensor_scalar_tuple(top_k), + *_to_tensor_scalar_tuple(top_p), + deterministic, + ) + else: + raise ValueError(f"Invalid filter_apply_order: {filter_apply_order}") + + +def _min_p_sampling_from_probs_internal( + probs: torch.Tensor, + uniform_samples: torch.Tensor, + maybe_min_p_arr: Optional[torch.Tensor], + min_p_val: float, + deterministic: bool, +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + uniform_samples = uniform_samples.float() + maybe_min_p_arr = ( + maybe_min_p_arr.float() if maybe_min_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.min_p_sampling_from_probs.default( + probs, + uniform_samples, + samples, + maybe_min_p_arr, + min_p_val, + deterministic, + get_cuda_stream(), + ) + return samples + + +def min_p_sampling_from_probs( + probs: torch.Tensor, + uniform_samples: torch.Tensor, + min_p: Union[torch.Tensor, float], + deterministic: bool = True, + check_nan: bool = False, +) -> torch.Tensor: + if uniform_samples.dim() == 2: + # Take the first row (round) of uniform_samples + uniform_samples = uniform_samples[0] + + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _min_p_sampling_from_probs_internal( + probs, uniform_samples, *_to_tensor_scalar_tuple(min_p), deterministic + ) diff --git a/sgl-kernel/python/sgl_kernel/speculative.py b/sgl-kernel/python/sgl_kernel/speculative.py new file mode 100644 index 0000000..6eee583 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/speculative.py @@ -0,0 +1,101 @@ +import torch +from sgl_kernel.utils import get_cuda_stream + + +def tree_speculative_sampling_target_only( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + uniform_samples: torch.Tensor, + target_probs: torch.Tensor, + draft_probs: torch.Tensor, + threshold_single: float = 1.0, + threshold_acc: float = 1.0, + deterministic: bool = True, +) -> None: + torch.ops.sgl_kernel.tree_speculative_sampling_target_only.default( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + target_probs, + draft_probs, + threshold_single, + threshold_acc, + deterministic, + get_cuda_stream(), + ) + + +def verify_tree_greedy( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + target_predict: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.verify_tree_greedy.default( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + target_predict, + get_cuda_stream(), + ) + + +def build_tree_kernel_efficient( + parent_list: torch.Tensor, + selected_index: torch.Tensor, + verified_seq_len: torch.Tensor, + tree_mask: torch.Tensor, + positions: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + topk: int, + depth: int, + draft_token_num: int, +) -> None: + torch.ops.sgl_kernel.build_tree_kernel_efficient.default( + parent_list, + selected_index, + verified_seq_len, + tree_mask, + positions, + retrive_index, + retrive_next_token, + retrive_next_sibling, + topk, + depth, + draft_token_num, + ) + + +def segment_packbits( + x: torch.Tensor, + input_indptr: torch.Tensor, + output_indptr: torch.Tensor, + y: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.segment_packbits.default( + x, + input_indptr, + output_indptr, + y, + torch.cuda.current_stream().cuda_stream, + ) diff --git a/sgl-kernel/python/sgl_kernel/utils.py b/sgl-kernel/python/sgl_kernel/utils.py new file mode 100644 index 0000000..d930678 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/utils.py @@ -0,0 +1,41 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from typing import Dict, Tuple + +import torch + + +def get_cuda_stream() -> int: + return torch.cuda.current_stream().cuda_stream + + +_cache_buf: Dict[Tuple[str, torch.device], torch.Tensor] = {} + + +def _get_cache_buf(name: str, bytes: int, device: torch.device) -> torch.Tensor: + key = (name, device) + buf = _cache_buf.get(key) + if buf is None: + buf = torch.empty(bytes, dtype=torch.uint8, device=device) + _cache_buf[key] = buf + return buf + + +def _to_tensor_scalar_tuple(x): + if isinstance(x, torch.Tensor): + return (x, 0) + else: + return (None, x) diff --git a/sgl-kernel/python/sgl_kernel/version.py b/sgl-kernel/python/sgl_kernel/version.py new file mode 100644 index 0000000..15dc849 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/version.py @@ -0,0 +1 @@ +__version__ = "0.0.5.post4" diff --git a/sgl-kernel/rename_wheels.sh b/sgl-kernel/rename_wheels.sh new file mode 100644 index 0000000..95c2418 --- /dev/null +++ b/sgl-kernel/rename_wheels.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -ex + +WHEEL_DIR="dist" + +wheel_files=($WHEEL_DIR/*.whl) +for wheel in "${wheel_files[@]}"; do + new_wheel="${wheel/linux/manylinux2014}" + + if [[ "$wheel" != "$new_wheel" ]]; then + echo "Renaming $wheel to $new_wheel" + mv -- "$wheel" "$new_wheel" + + fi +done + +echo "Wheel renaming completed." diff --git a/sgl-kernel/setup.py b/sgl-kernel/setup.py new file mode 100644 index 0000000..7b21ba5 --- /dev/null +++ b/sgl-kernel/setup.py @@ -0,0 +1,249 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import os +import shutil +import sys +from pathlib import Path + +import torch +from setuptools import find_packages, setup +from setuptools.command.build_py import build_py +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +root = Path(__file__).parent.resolve() + + +if "bdist_wheel" in sys.argv and "--plat-name" not in sys.argv: + sys.argv.extend(["--plat-name", "manylinux2014_x86_64"]) + + +def _get_cuda_version(): + if torch.version.cuda: + return tuple(map(int, torch.version.cuda.split("."))) + return (0, 0) + + +def _get_device_sm(): + if torch.cuda.is_available(): + major, minor = torch.cuda.get_device_capability() + return major * 10 + minor + return 0 + + +def _get_version(): + with open(root / "pyproject.toml") as f: + for line in f: + if line.startswith("version"): + return line.split("=")[1].strip().strip('"') + + +operator_namespace = "sgl_kernel" +cutlass_default = root / "3rdparty" / "cutlass" +cutlass = Path(os.environ.get("CUSTOM_CUTLASS_SRC_DIR", default=cutlass_default)) +flashinfer = root / "3rdparty" / "flashinfer" +deepgemm = root / "3rdparty" / "deepgemm" +include_dirs = [ + root / "include", + root / "csrc", + cutlass.resolve() / "include", + cutlass.resolve() / "tools" / "util" / "include", + flashinfer.resolve() / "include", + flashinfer.resolve() / "include" / "gemm", + flashinfer.resolve() / "csrc", + "cublas", +] + + +class CustomBuildPy(build_py): + def run(self): + self.copy_deepgemm_to_build_lib() + self.make_jit_include_symlinks() + build_py.run(self) + + def make_jit_include_symlinks(self): + # Make symbolic links of third-party include directories + build_include_dir = os.path.join(self.build_lib, "deep_gemm/include") + os.makedirs(build_include_dir, exist_ok=True) + + third_party_include_dirs = [ + cutlass.resolve() / "include" / "cute", + cutlass.resolve() / "include" / "cutlass", + ] + + for d in third_party_include_dirs: + dirname = str(d).split("/")[-1] + src_dir = d + dst_dir = f"{build_include_dir}/{dirname}" + assert os.path.exists(src_dir) + if os.path.exists(dst_dir): + assert os.path.islink(dst_dir) + os.unlink(dst_dir) + os.symlink(src_dir, dst_dir, target_is_directory=True) + + def copy_deepgemm_to_build_lib(self): + """ + This function copies DeepGemm to python's site-packages + """ + dst_dir = os.path.join(self.build_lib, "deep_gemm") + os.makedirs(dst_dir, exist_ok=True) + + # Copy deepgemm/deep_gemm to the build directory + src_dir = os.path.join(str(deepgemm.resolve()), "deep_gemm") + + # Remove existing directory if it exists + if os.path.exists(dst_dir): + shutil.rmtree(dst_dir) + + # Copy the directory + shutil.copytree(src_dir, dst_dir) + + +nvcc_flags = [ + "-DNDEBUG", + f"-DOPERATOR_NAMESPACE={operator_namespace}", + "-O3", + "-Xcompiler", + "-fPIC", + "-gencode=arch=compute_75,code=sm_75", + "-gencode=arch=compute_80,code=sm_80", + "-gencode=arch=compute_89,code=sm_89", + "-gencode=arch=compute_90,code=sm_90", + "-std=c++17", + "-DFLASHINFER_ENABLE_F16", + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1", + "-DCUTLASS_VERSIONS_GENERATED", + "-DCUTE_USE_PACKED_TUPLE=1", + "-DCUTLASS_TEST_LEVEL=0", + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1", + "-DCUTLASS_DEBUG_TRACE_LEVEL=0", + "--ptxas-options=-v", + "--expt-relaxed-constexpr", + "-Xcompiler=-Wconversion", + "-Xcompiler=-fno-strict-aliasing", +] +nvcc_flags_fp8 = [ + "-DFLASHINFER_ENABLE_FP8", + "-DFLASHINFER_ENABLE_FP8_E4M3", + "-DFLASHINFER_ENABLE_FP8_E5M2", +] + +sources = [ + "csrc/allreduce/trt_reduce_internal.cu", + "csrc/allreduce/trt_reduce_kernel.cu", + "csrc/attention/lightning_attention_decode_kernel.cu", + "csrc/elementwise/activation.cu", + "csrc/elementwise/fused_add_rms_norm_kernel.cu", + "csrc/elementwise/rope.cu", + "csrc/gemm/bmm_fp8.cu", + "csrc/gemm/cublas_grouped_gemm.cu", + "csrc/gemm/awq_kernel.cu", + "csrc/gemm/fp8_gemm_kernel.cu", + "csrc/gemm/fp8_blockwise_gemm_kernel.cu", + "csrc/gemm/int8_gemm_kernel.cu", + "csrc/gemm/nvfp4_quant_entry.cu", + "csrc/gemm/nvfp4_quant_kernels.cu", + "csrc/gemm/nvfp4_scaled_mm_entry.cu", + "csrc/gemm/nvfp4_scaled_mm_kernels.cu", + "csrc/gemm/per_token_group_quant_8bit.cu", + "csrc/gemm/per_token_quant_fp8.cu", + "csrc/gemm/per_tensor_quant_fp8.cu", + "csrc/moe/moe_align_kernel.cu", + "csrc/moe/moe_topk_softmax_kernels.cu", + "csrc/speculative/eagle_utils.cu", + "csrc/speculative/speculative_sampling.cu", + "csrc/speculative/packbit.cu", + "csrc/torch_extension.cc", + "3rdparty/flashinfer/csrc/norm.cu", + "3rdparty/flashinfer/csrc/renorm.cu", + "3rdparty/flashinfer/csrc/sampling.cu", +] + +enable_bf16 = os.getenv("SGL_KERNEL_ENABLE_BF16", "0") == "1" +enable_fp8 = os.getenv("SGL_KERNEL_ENABLE_FP8", "0") == "1" +enable_fp4 = os.getenv("SGL_KERNEL_ENABLE_FP4", "0") == "1" +enable_sm90a = os.getenv("SGL_KERNEL_ENABLE_SM90A", "0") == "1" +enable_sm100a = os.getenv("SGL_KERNEL_ENABLE_SM100A", "0") == "1" +cuda_version = _get_cuda_version() +sm_version = _get_device_sm() + +if torch.cuda.is_available(): + if cuda_version >= (12, 0) and sm_version >= 90: + nvcc_flags.append("-gencode=arch=compute_90a,code=sm_90a") + if cuda_version >= (12, 8) and sm_version >= 100: + nvcc_flags.append("-gencode=arch=compute_100,code=sm_100") + nvcc_flags.append("-gencode=arch=compute_100a,code=sm_100a") + nvcc_flags.append("-DENABLE_NVFP4=1") + else: + nvcc_flags.append("-use_fast_math") + if sm_version >= 90: + nvcc_flags.extend(nvcc_flags_fp8) + if sm_version >= 80: + nvcc_flags.append("-DFLASHINFER_ENABLE_BF16") +else: + # compilation environment without GPU + if enable_sm100a: + nvcc_flags.append("-gencode=arch=compute_100a,code=sm_100a") + if enable_sm90a: + nvcc_flags.append("-gencode=arch=compute_90a,code=sm_90a") + if enable_fp4: + nvcc_flags.append("-DENABLE_NVFP4=1") + if enable_fp8: + nvcc_flags.extend(nvcc_flags_fp8) + if enable_bf16: + nvcc_flags.append("-DFLASHINFER_ENABLE_BF16") + +for flag in [ + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_BFLOAT16_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", +]: + try: + torch.utils.cpp_extension.COMMON_NVCC_FLAGS.remove(flag) + except ValueError: + pass + +cxx_flags = ["-O3"] +libraries = ["c10", "torch", "torch_python", "cuda", "cublas"] +extra_link_args = ["-Wl,-rpath,$ORIGIN/../../torch/lib", "-L/usr/lib/x86_64-linux-gnu"] + +ext_modules = [ + CUDAExtension( + name="sgl_kernel.common_ops", + sources=sources, + include_dirs=include_dirs, + extra_compile_args={ + "nvcc": nvcc_flags, + "cxx": cxx_flags, + }, + libraries=libraries, + extra_link_args=extra_link_args, + py_limited_api=True, + ), +] + +setup( + name="sgl-kernel", + version=_get_version(), + packages=find_packages(where="python"), + package_dir={"": "python"}, + ext_modules=ext_modules, + cmdclass={ + "build_ext": BuildExtension.with_options(use_ninja=True), + "build_py": CustomBuildPy, + }, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/sgl-kernel/setup_rocm.py b/sgl-kernel/setup_rocm.py new file mode 100644 index 0000000..66a2d0a --- /dev/null +++ b/sgl-kernel/setup_rocm.py @@ -0,0 +1,86 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import sys +from pathlib import Path + +from setuptools import find_packages, setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +root = Path(__file__).parent.resolve() + + +def _get_version(): + with open(root / "pyproject.toml") as f: + for line in f: + if line.startswith("version"): + return line.split("=")[1].strip().strip('"') + + +operator_namespace = "sgl_kernel" +include_dirs = [ + root / "include", + root / "csrc", +] + +sources = [ + "csrc/allreduce/custom_all_reduce.hip", + "csrc/moe/moe_align_kernel.cu", + "csrc/moe/moe_topk_softmax_kernels.cu", + "csrc/torch_extension_rocm.cc", + "csrc/speculative/eagle_utils.cu", +] + +cxx_flags = ["-O3"] +libraries = ["hiprtc", "amdhip64", "c10", "torch", "torch_python"] +extra_link_args = ["-Wl,-rpath,$ORIGIN/../../torch/lib", "-L/usr/lib/x86_64-linux-gnu"] + +hipcc_flags = [ + "-DNDEBUG", + f"-DOPERATOR_NAMESPACE={operator_namespace}", + "-O3", + "-Xcompiler", + "-fPIC", + "-std=c++17", + "-D__HIP_PLATFORM_AMD__=1", + "--amdgpu-target=gfx942", + "-DENABLE_BF16", + "-DENABLE_FP8", +] + +ext_modules = [ + CUDAExtension( + name="sgl_kernel.common_ops", + sources=sources, + include_dirs=include_dirs, + extra_compile_args={ + "nvcc": hipcc_flags, + "cxx": cxx_flags, + }, + libraries=libraries, + extra_link_args=extra_link_args, + py_limited_api=True, + ), +] + +setup( + name="sgl-kernel", + version=_get_version(), + packages=find_packages(where="python"), + package_dir={"": "python"}, + ext_modules=ext_modules, + cmdclass={"build_ext": BuildExtension.with_options(use_ninja=True)}, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/sgl-kernel/tests/speculative/test_eagle_utils.py b/sgl-kernel/tests/speculative/test_eagle_utils.py new file mode 100644 index 0000000..1514029 --- /dev/null +++ b/sgl-kernel/tests/speculative/test_eagle_utils.py @@ -0,0 +1,98 @@ +import torch +import torch.nn.functional as F +from sgl_kernel import verify_tree_greedy + + +def test_verify_tree_greedy(): + candidates = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [7, 8, 9, 10, 11, 12], + ], + dtype=torch.int32, + device="cuda", + ) + retrive_index = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + ], + dtype=torch.int32, + device="cuda", + ) + retrive_next_token = torch.tensor( + [ + [1, 2, -1, 4, 5, -1], + [4, 2, 3, -1, 5, -1], + ], + dtype=torch.int32, + device="cuda", + ) + retrive_next_sibling = torch.tensor( + [ + [-1, 3, -1, -1, -1, -1], + [-1, -1, -1, -1, 1, -1], + ], + dtype=torch.int32, + device="cuda", + ) + + target_logits = torch.full((2, 6, 20), 1, dtype=torch.float32, device="cuda") + target_logits[0, 0, 3] = 10 + target_logits[0, 3, 4] = 10 + target_logits[0, 4, 5] = 10 + target_logits[1, 0, 11] = 10 + target_logits[1, 4, 12] = 10 + for i in range(target_logits.shape[0]): + for j in range(target_logits.shape[1]): + if torch.max(target_logits[i][j]) < 10: + target_logits[i][j][18] = 10 + + print(f"{target_logits=}") + target_predict = torch.argmax(target_logits, dim=-1).to(torch.int32) + predict_shape = (12,) + + bs = candidates.shape[0] + num_spec_step = 4 + num_draft_tokens = candidates.shape[1] + + predicts = torch.full( + predict_shape, -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_index = torch.full( + (bs, num_spec_step), -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_token_num = torch.full((bs,), 0, dtype=torch.int32, device="cuda") # mutable + + print(f"{candidates=}") + print(f"{retrive_index=}") + print(f"{retrive_next_token=}") + print(f"{retrive_next_sibling=}") + print(f"{target_predict=}") + + verify_tree_greedy( + predicts=predicts, + accept_index=accept_index, + accept_token_num=accept_token_num, + candidates=candidates, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + target_predict=target_predict, + ) + + print(f"{predicts=}") + print(f"{accept_index=}") + print(f"{accept_token_num=}") + + return predicts, accept_index, accept_token_num + + +if __name__ == "__main__": + predicts, accept_index, accept_token_num = test_verify_tree_greedy() + assert predicts.tolist() == [3, -1, -1, 4, 5, 18, 11, -1, -1, -1, 12, 18] + assert accept_index.tolist() == [ + [0, 3, 4, 5], + [6, 10, 11, -1], + ] + assert accept_token_num.tolist() == [3, 2] diff --git a/sgl-kernel/tests/speculative/test_speculative_sampling.py b/sgl-kernel/tests/speculative/test_speculative_sampling.py new file mode 100644 index 0000000..2d45db2 --- /dev/null +++ b/sgl-kernel/tests/speculative/test_speculative_sampling.py @@ -0,0 +1,122 @@ +import torch +import torch.nn.functional as F +from sgl_kernel import tree_speculative_sampling_target_only + + +def test_tree_speculative_sampling_target_only(threshold_single=1, threshold_acc=1): + print( + f"\n============= run test: {threshold_single=} {threshold_acc=} ==============\n" + ) + candidates = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [7, 8, 9, 10, 11, 12], + ], + dtype=torch.int32, + device="cuda", + ) + retrive_index = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + ], + dtype=torch.int32, + device="cuda", + ) + retrive_next_token = torch.tensor( + [ + [1, 2, -1, 4, 5, -1], + [4, 2, 3, -1, 5, -1], + ], + dtype=torch.int32, + device="cuda", + ) + retrive_next_sibling = torch.tensor( + [ + [-1, 3, -1, -1, -1, -1], + [-1, -1, -1, -1, 1, -1], + ], + dtype=torch.int32, + device="cuda", + ) + + target_logits = torch.full((2, 6, 20), 1, dtype=torch.float32, device="cuda") + target_logits[0, 0, 3] = 10 + target_logits[0, 3, 4] = 10 + target_logits[0, 4, 5] = 10 + target_logits[1, 0, 11] = 10 + target_logits[1, 4, 12] = 10 + for i in range(target_logits.shape[0]): + for j in range(target_logits.shape[1]): + if torch.max(target_logits[i][j]) < 10: + target_logits[i][j][18] = 10 + + temperatures = torch.tensor([0.01, 0.01], dtype=torch.float32, device="cuda") + predict_shape = (12,) + + bs = candidates.shape[0] + num_spec_step = 4 + num_draft_tokens = candidates.shape[1] + + predicts = torch.full( + predict_shape, -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_index = torch.full( + (bs, num_spec_step), -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_token_num = torch.full((bs,), 0, dtype=torch.int32, device="cuda") # mutable + + expanded_temperature = temperatures.unsqueeze(1).unsqueeze(1) + target_probs = F.softmax(target_logits / expanded_temperature, dim=-1) + draft_probs = torch.full_like(target_probs, 0, dtype=torch.float32, device="cuda") + + coins = torch.rand(bs, num_draft_tokens, device="cuda").to(torch.float32) + print(f"{candidates=}") + print(f"{retrive_index=}") + print(f"{retrive_next_token=}") + print(f"{retrive_next_sibling=}") + print(f"{coins=}") + + tree_speculative_sampling_target_only( + predicts=predicts, + accept_index=accept_index, + accept_token_num=accept_token_num, + candidates=candidates, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + uniform_samples=coins, + target_probs=target_probs, + draft_probs=draft_probs, + threshold_single=threshold_single, + threshold_acc=threshold_acc, + deterministic=True, + ) + + print(f"{predicts=}") + print(f"{accept_index=}") + print(f"{accept_token_num=}") + + return predicts, accept_index, accept_token_num + + +if __name__ == "__main__": + predicts, accept_index, accept_token_num = ( + test_tree_speculative_sampling_target_only(threshold_single=1, threshold_acc=1) + ) + assert predicts.tolist() == [3, -1, -1, 4, 5, 18, 11, -1, -1, -1, 12, 18] + assert accept_index.tolist() == [ + [0, 3, 4, 5], + [6, 10, 11, -1], + ] + assert accept_token_num.tolist() == [3, 2] + + predicts, accept_index, accept_token_num = ( + test_tree_speculative_sampling_target_only(threshold_single=0, threshold_acc=0) + ) + assert predicts.tolist() == [1, 2, 18, -1, -1, -1, 11, -1, -1, -1, 12, 18] + assert accept_index.tolist() == [ + [0, 1, 2, -1], + [6, 10, 11, -1], + ] + assert accept_token_num.tolist() == [2, 2] diff --git a/sgl-kernel/tests/test_activation.py b/sgl-kernel/tests/test_activation.py new file mode 100644 index 0000000..4359344 --- /dev/null +++ b/sgl-kernel/tests/test_activation.py @@ -0,0 +1,39 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_activation.py + +import pytest +import sgl_kernel +import torch + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_silu_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.silu(x[..., :dim]) + y = sgl_kernel.silu_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_gelu_tanh_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.gelu(x[..., :dim], approximate="tanh") + y = sgl_kernel.gelu_tanh_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_gelu_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.gelu(x[..., :dim], approximate="none") + y = sgl_kernel.gelu_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_awq_dequant.py b/sgl-kernel/tests/test_awq_dequant.py new file mode 100644 index 0000000..bad3e2c --- /dev/null +++ b/sgl-kernel/tests/test_awq_dequant.py @@ -0,0 +1,132 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import awq_dequantize +from vllm import _custom_ops as ops + + +def reverse_awq_order(t: torch.Tensor): + bits = 4 + AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + reverse_order_tensor = torch.arange( + t.shape[-1], + dtype=torch.int32, + device=t.device, + ) + reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits) + reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER] + reverse_order_tensor = reverse_order_tensor.view(-1) + + t = t[:, reverse_order_tensor] & 0xF + return t + + +# qweights - [R , C // 8], int32 +# scales - [R // G, C ], float16 +# zeros - [R // G, C // 8], int32 +def awq_dequantize_torch( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor, group_size: int +) -> torch.Tensor: + + if group_size == -1: + group_size = qweight.shape[0] + + bits = 4 + shifts = torch.arange(0, 32, bits, device=qzeros.device) + + iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + + iweights = iweights.view(iweights.shape[0], -1) + + zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + zeros = zeros.view(qzeros.shape[0], -1) + zeros = reverse_awq_order(zeros) + + iweights = reverse_awq_order(iweights) + + iweights = torch.bitwise_and(iweights, (2**bits) - 1) + zeros = torch.bitwise_and(zeros, (2**bits) - 1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def vllm_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.Tensor: + return ops.awq_dequantize(qweight, scales, qzeros, 0, 0, 0) + + +def sglang_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.Tensor: + return awq_dequantize(qweight, scales, qzeros) + + +@pytest.mark.parametrize( + "qweight_row,qweight_col,is_bf16_act", + list( + itertools.product( + [3584, 18944, 128, 256, 512, 1024], + [448, 576, 4736, 16, 32, 64, 128], + [True, False], + ) + ), +) +def test_awq_dequant_compare_implementations( + qweight_row: int, qweight_col: int, is_bf16_act: bool +): + device = torch.device("cuda") + + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + + if is_bf16_act: + scales = torch.rand(scales_row, scales_col, dtype=torch.bfloat16, device=device) + else: + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + # Run both implementations + vllm_out = vllm_awq_dequantize(qweight, scales.to(torch.float16), qzeros) + torch_out = awq_dequantize_torch(qweight, scales, qzeros, group_size) + sglang_out = sglang_awq_dequantize(qweight, scales, qzeros) + + # Compare results + torch.testing.assert_close( + torch_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ) + if not is_bf16_act: + torch.testing.assert_close( + vllm_out.to(torch.float32), + sglang_out.to(torch.float32), + rtol=1e-3, + atol=1e-5, + ) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_bmm_fp8.py b/sgl-kernel/tests/test_bmm_fp8.py new file mode 100644 index 0000000..e0be928 --- /dev/null +++ b/sgl-kernel/tests/test_bmm_fp8.py @@ -0,0 +1,43 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_bmm_fp8.py + +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import bmm_fp8 + + +def to_float8(x, dtype=torch.float8_e4m3fn): + finfo = torch.finfo(dtype) + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12) + scale = finfo.max / amax + x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max) + return x_scl_sat.to(dtype), scale.float().reciprocal() + + +@pytest.mark.parametrize("input_dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +@pytest.mark.parametrize("mat2_dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +@pytest.mark.parametrize("res_dtype", [torch.bfloat16, torch.float16]) +def test_bmm_fp8(input_dtype, mat2_dtype, res_dtype): + if input_dtype == torch.float8_e5m2 and mat2_dtype == torch.float8_e5m2: + pytest.skip("Invalid combination: both input and mat2 are e5m2") + + input = torch.randn([16, 48, 64], device="cuda", dtype=torch.bfloat16) + input_fp8, input_inv_s = to_float8(input, dtype=input_dtype) + + # mat2 row major -> column major + mat2 = torch.randn([16, 80, 64], device="cuda", dtype=torch.bfloat16).transpose( + -2, -1 + ) + mat2_fp8, mat2_inv_s = to_float8(mat2, dtype=mat2_dtype) + + res = torch.empty([16, 48, 80], device="cuda", dtype=res_dtype) + bmm_fp8(input_fp8, mat2_fp8, input_inv_s, mat2_inv_s, res_dtype, res) + + reference = torch.bmm(input, mat2) + cos_sim = F.cosine_similarity(reference.reshape(-1), res.reshape(-1), dim=0) + assert cos_sim > 0.99 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_cublas_grouped_gemm.py b/sgl-kernel/tests/test_cublas_grouped_gemm.py new file mode 100644 index 0000000..9aac569 --- /dev/null +++ b/sgl-kernel/tests/test_cublas_grouped_gemm.py @@ -0,0 +1,49 @@ +import unittest + +import torch +from sgl_kernel import cublas_grouped_gemm + + +def torch_grouped_gemm(a_array, b_array, out_dtype): + c_array = [] + for a, b in zip(a_array, b_array): + c_array.append(torch.matmul(a, b.t()).to(out_dtype)) + return c_array + + +class TestGroupedGemm(unittest.TestCase): + def _test_accuracy(self, Ms, Ns, Ks, out_dtype): + group_count = len(Ms) + a_array = [] + b_array = [] + c_array_cublas = [] + for i in range(group_count): + M, N, K = Ms[i], Ns[i], Ks[i] + a_array.append(torch.randn((M, K), device="cuda", dtype=out_dtype) * 5) + b_array.append(torch.randn((N, K), device="cuda", dtype=out_dtype) * 5) + c_array_cublas.append(torch.empty((M, N), device="cuda", dtype=out_dtype)) + + c_array_torch = torch_grouped_gemm(a_array, b_array, out_dtype) + cublas_grouped_gemm(a_array, b_array, c_array_cublas, out_dtype) + + for i in range(group_count): + M, N, K = Ms[i], Ns[i], Ks[i] + torch.testing.assert_close(c_array_torch[i], c_array_cublas[i]) + print(f"M={M}, N={N}, K={K}, out_dtype={out_dtype}: OK") + + def test_accuracy(self): + Ms = [1, 16, 32, 256, 1024] + Ns = [2, 16, 128, 256, 4096] + Ks = [3, 16, 32, 512, 8192] + out_dtypes = [torch.float16, torch.bfloat16] + for out_dtype in out_dtypes: + self._test_accuracy(Ms, Ns, Ks, out_dtype) + + +if __name__ == "__main__": + if torch.cuda.is_available(): + cuda_version = tuple(map(int, torch.version.cuda.split("."))) + if cuda_version >= (12, 5): + unittest.main() + else: + print(f"Cuda version {cuda_version} lower than 12.5, not executing tests.") diff --git a/sgl-kernel/tests/test_deep_gemm.py b/sgl-kernel/tests/test_deep_gemm.py new file mode 100644 index 0000000..bb56849 --- /dev/null +++ b/sgl-kernel/tests/test_deep_gemm.py @@ -0,0 +1,263 @@ +import os +import random +import unittest +from typing import Any, Tuple + +import deep_gemm +import torch +from deep_gemm import calc_diff, ceil_div, get_col_major_tma_aligned_tensor, jit + +""" +fork deepgemm/tests/test_core.py +""" + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def construct(m: int, k: int, n: int) -> Tuple[ + Tuple[torch.Tensor, torch.Tensor], + Tuple[torch.Tensor, torch.Tensor], + torch.Tensor, + torch.Tensor, +]: + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + ref_out = x @ y.t() + + x_fp8, y_fp8 = per_token_cast_to_fp8(x), per_block_cast_to_fp8(y) + # Transpose earlier so that the testing will not trigger transposing kernels + x_fp8 = (x_fp8[0], get_col_major_tma_aligned_tensor(x_fp8[1])) + return x_fp8, y_fp8, out, ref_out + + +def construct_grouped( + num_groups: int, m: int, k: int, n: int, is_masked: bool +) -> Tuple[ + Tuple[torch.Tensor, torch.Tensor], + Tuple[torch.Tensor, torch.Tensor], + torch.Tensor, + torch.Tensor, +]: + x = torch.randn((num_groups, m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((num_groups, n, k), device="cuda", dtype=torch.bfloat16) + out = torch.empty((num_groups, m, n), device="cuda", dtype=torch.bfloat16) + ref_out = torch.einsum("gmk,gnk->gmn", x, y) + + assert m % 4 == 0, f"TMA alignment error: {m}" + x_fp8 = ( + torch.empty_like(x, dtype=torch.float8_e4m3fn), + torch.empty((num_groups, m, k // 128), device="cuda", dtype=torch.float), + ) + y_fp8 = ( + torch.empty_like(y, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, (n + 127) // 128, k // 128), device="cuda", dtype=torch.float + ), + ) + for i in range(num_groups): + x_fp8[0][i], x_fp8[1][i] = per_token_cast_to_fp8(x[i]) + y_fp8[0][i], y_fp8[1][i] = per_block_cast_to_fp8(y[i]) + + # For non-masked input, we must merge the group and M dims + if not is_masked: + x_fp8 = (x_fp8[0].view(-1, k), per_token_cast_to_fp8(x.view(-1, k))[1]) + out, ref_out = out.view(-1, n), ref_out.view(-1, n) + + # Transpose earlier so that the testing will not trigger transposing kernels + x_fp8 = (x_fp8[0], get_col_major_tma_aligned_tensor(x_fp8[1])) + return x_fp8, y_fp8, out, ref_out + + +class TestDeepGemmCore(unittest.TestCase): + @classmethod + def setUpClass(cls): + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + torch.manual_seed(0) + random.seed(0) + + print("Library path:") + print(f" > {deep_gemm.__path__}\n") + + def test_gemm(self): + print("Testing GEMM:") + for m in (64, 128, 4096): + for k, n in [ + (7168, 2112), + (1536, 24576), + (512, 32768), + (16384, 7168), + (7168, 4096), + (2048, 7168), + ]: + x_fp8, y_fp8, out, ref_out = construct(m, k, n) + deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + diff = calc_diff(out, ref_out) + self.assertTrue(diff < 0.001, f"{m=}, {k=}, {n=}, {diff:.5f}") + + def test_m_grouped_gemm_contiguous(self): + print("Testing grouped contiguous GEMM:") + + for num_groups, m, k, n in ( + (4, 8192, 7168, 4096), + (4, 8192, 2048, 7168), + (8, 4096, 7168, 4096), + (8, 4096, 2048, 7168), + ): + # TODO: make a stronger test + x_fp8, y_fp8, out, ref_out = construct_grouped( + num_groups, m, k, n, is_masked=False + ) + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1).expand(num_groups, m).contiguous().view(-1) + ) + deep_gemm.m_grouped_gemm_fp8_fp8_bf16_nt_contiguous( + x_fp8, y_fp8, out, m_indices + ) + diff = calc_diff(out, ref_out) + self.assertTrue(diff < 0.001, f"m={m * num_groups}, {k=}, {n=}, {diff:.5f}") + + def test_m_grouped_gemm_masked(self): + print("Testing grouped masked GEMM:") + + for num_groups, m in ((1, 1024), (2, 512), (4, 256)): + for k, n in ( + (7168, 4096), + (2048, 7168), + ): + # Test correctness + masked_m_candidates = list( + filter( + lambda candidate: candidate <= m, (64, 128, 192, 256, 320, 384) + ) + ) + for i in range(10): + x_fp8, y_fp8, out, ref_out = construct_grouped( + num_groups, m, k, n, is_masked=True + ) + masked_m = torch.empty( + (num_groups,), device="cuda", dtype=torch.int + ) + for j in range(num_groups): + masked_m[j] = random.choice(masked_m_candidates) + expected_m = min(int(masked_m.float().mean()) + 1, m) + deep_gemm.m_grouped_gemm_fp8_fp8_bf16_nt_masked( + x_fp8, y_fp8, out, masked_m, expected_m + ) + for j in range(num_groups): + diff = calc_diff( + out[j, : masked_m[j].item()], + ref_out[j, : masked_m[j].item()], + ) + self.assertTrue( + diff < 0.001, + f"{m=}, {k=}, {n=}, {j=}, masked_m={masked_m[j]}, {num_groups=}, {diff:.5f}", + ) + + +""" +fork deepgemm/tests/test_jit.py +""" + + +class Capture: + def __init__(self) -> None: + self.read_fd = None + self.write_fd = None + self.saved_stdout = None + self.captured = None + + def __enter__(self) -> Any: + self.read_fd, self.write_fd = os.pipe() + self.saved_stdout = os.dup(1) + os.dup2(self.write_fd, 1) + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + os.dup2(self.saved_stdout, 1) + os.close(self.write_fd) + with os.fdopen(self.read_fd, "r") as f: + self.captured = f.read() + + def capture(self) -> str: + return self.captured + + +class TestDeepGemmJIT(unittest.TestCase): + def test_jit(self): + # Runtime + print(f"NVCC compiler: {jit.get_nvcc_compiler()}\n") + + # Templates + print("Generated code:") + args = ( + ("lhs", torch.float8_e4m3fn), + ("rhs", torch.float8_e4m3fn), + ("scale", torch.float), + ("out", torch.bfloat16), + ("enable_double_streams", bool), + ("stream", torch.cuda.Stream), + ) + body = "\n" + body += "std::cout << reinterpret_cast(lhs) << std::endl;\n" + body += "std::cout << reinterpret_cast(rhs) << std::endl;\n" + body += "std::cout << reinterpret_cast(scale) << std::endl;\n" + body += "std::cout << reinterpret_cast(out) << std::endl;\n" + body += "std::cout << enable_double_streams << std::endl;\n" + body += "std::cout << reinterpret_cast(stream) << std::endl;\n" + code = jit.generate((), args, body) + print(code) + + # Build + print("Building ...") + func = jit.build("test_func", args, code) + + # Test correctness + print("Running ...") + fp8_tensor = torch.empty((1,), dtype=torch.float8_e4m3fn, device="cuda") + fp32_tensor = torch.empty((1,), dtype=torch.float, device="cuda") + bf16_tensor = torch.empty((1,), dtype=torch.bfloat16, device="cuda") + with Capture() as capture: + self.assertTrue( + func( + fp8_tensor, + fp8_tensor, + fp32_tensor, + bf16_tensor, + True, + torch.cuda.current_stream(), + ) + == 0 + ) + output = capture.capture() + ref_output = f"{fp8_tensor.data_ptr()}\n{fp8_tensor.data_ptr()}\n{fp32_tensor.data_ptr()}\n{bf16_tensor.data_ptr()}\n1\n{torch.cuda.current_stream().cuda_stream}\n" + self.assertTrue(output == ref_output, f"{output=}, {ref_output=}") + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-kernel/tests/test_fp4_gemm.py b/sgl-kernel/tests/test_fp4_gemm.py new file mode 100644 index 0000000..5c092bd --- /dev/null +++ b/sgl-kernel/tests/test_fp4_gemm.py @@ -0,0 +1,151 @@ +import pytest +import torch +from sgl_kernel import cutlass_scaled_fp4_mm, scaled_fp4_quant + +if torch.cuda.get_device_capability() < (10, 0): + pytest.skip( + reason="Nvfp4 Requires compute capability of 10 or above.", + allow_module_level=True, + ) + +DTYPES = [torch.float16, torch.bfloat16] +# m, n, k +SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)] +PAD_SHAPES = [(150, 128, 64), (128, 128, 96)] +SHAPES.extend(PAD_SHAPES) + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +kE2M1ToFloatArray = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, +] + + +def e2m1_to_fp32(int4_value): + signBit = int4_value & 0x8 + int4_absValue = int4_value & 0x7 + float_result = kE2M1ToFloatArray[int4_absValue] + if signBit: + float_result = -float_result + return float_result + + +def break_fp4_bytes(a, dtype): + assert a.dtype == torch.uint8 + m, n = a.shape + a = a.flatten() + # Get upper 4 bits + highHalfByte = (a & 0xF0) >> 4 + # Get lower 4 bits + lowHalfByte = a & 0x0F + fH = torch.tensor([e2m1_to_fp32(x) for x in highHalfByte]).to(a.device) + fL = torch.tensor([e2m1_to_fp32(x) for x in lowHalfByte]).to(a.device) + # [0xAB, 0xCD] -> [0xB, 0xA, 0xD, 0xC] + out = torch.stack((fL, fH), dim=-1).reshape(m, n * 2) + return out + + +def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size): + sf_m, sf_k = a_sf_swizzled.shape + m_tiles = (m + 128 - 1) // 128 + f = block_size * 4 + k_tiles = (k + f - 1) // f + tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size) + return out[0:m, 0:k] + + +def dequantize_to_dtype( + tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16 +): + """Dequantize the fp4 tensor back to high precision.""" + # Two fp4 values are packed into one uint8. + assert tensor_fp4.dtype == torch.uint8 + m, packed_k = tensor_fp4.shape + k = packed_k * 2 + tensor_f32 = break_fp4_bytes(tensor_fp4, dtype) + tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size) + tensor_sf = tensor_sf.view(torch.float8_e4m3fn) + tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size) + tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale + + # scale the tensor + out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k) + return out + + +def get_ref_results( + a_fp4, + b_fp4, + a_sf, + b_sf, + a_global_scale, + b_global_scale, + m, + n, + dtype, + block_size, + device, +): + _, m_k = a_fp4.shape + _, n_k = b_fp4.shape + assert m_k == n_k + a_in_dtype = dequantize_to_dtype( + a_fp4, a_sf, a_global_scale, dtype=dtype, device=device, block_size=block_size + ) + b_in_dtype = dequantize_to_dtype( + b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size + ) + return torch.matmul(a_in_dtype, b_in_dtype.t()) + + +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("shape", SHAPES) +@torch.inference_mode() +def test_nvfp4_gemm( + dtype: torch.dtype, + shape: tuple[int, int], +) -> None: + m, n, packed_k = shape + k = packed_k * 2 + block_size = 16 + a_dtype = torch.randn((m, k), dtype=dtype, device="cuda") + b_dtype = torch.randn((n, k), dtype=dtype, device="cuda") + + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1) + ).to(torch.float32) + b_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1) + ).to(torch.float32) + alpha = 1.0 / (a_global_scale * b_global_scale) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a_dtype, a_global_scale) + b_fp4, b_scale_interleaved = scaled_fp4_quant(b_dtype, b_global_scale) + + expected_out = get_ref_results( + a_fp4, + b_fp4, + a_scale_interleaved, + b_scale_interleaved, + a_global_scale, + b_global_scale, + m, + n, + dtype, + block_size, + "cuda", + ) + out = cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ) + + torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1) diff --git a/sgl-kernel/tests/test_fp4_quantize.py b/sgl-kernel/tests/test_fp4_quantize.py new file mode 100644 index 0000000..6b24893 --- /dev/null +++ b/sgl-kernel/tests/test_fp4_quantize.py @@ -0,0 +1,164 @@ +import pytest +import torch +from sgl_kernel import scaled_fp4_quant + +if torch.cuda.get_device_capability() < (10, 0): + pytest.skip( + reason="Nvfp4 Requires compute capability of 10 or above.", + allow_module_level=True, + ) + +DTYPES = [torch.float16, torch.bfloat16] +SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)] +PAD_SHAPES = [ + (90, 64), + (150, 64), + (128, 48), + (128, 80), + (150, 80), + (90, 48), + (90, 128), + (150, 128), + (150, 48), + (90, 80), +] + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +# E2M1 to float +# 0111 -> 6 +# 0110 -> 4 +# 0101 -> 3 +# 0100 -> 2 +# 0011 -> 1.5 +# 0010 -> 1 +# 0001 -> 0.5 +# 0000 -> 0 +E2M1_TO_FLOAT32 = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + 0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, +] +BLOCK_SIZE = 16 + + +def cast_from_fp4(x, m, n): + # The fp4 values are packed in uint8 as [v_1st | v_2nd] + v_2nd = x & 0xF + v_1st = (x >> 4) & 0xF + c = torch.stack((v_2nd, v_1st), dim=-1) + out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()]) + out = out.reshape(m, n).to(torch.float32) + return out + + +def cast_to_fp4(x): + sign = torch.sign(x) + x = torch.abs(x) + x[(x >= 0.0) & (x <= 0.25)] = 0.0 + x[(x > 0.25) & (x < 0.75)] = 0.5 + x[(x >= 0.75) & (x <= 1.25)] = 1.0 + x[(x > 1.25) & (x < 1.75)] = 1.5 + x[(x >= 1.75) & (x <= 2.5)] = 2.0 + x[(x > 2.5) & (x < 3.5)] = 3.0 + x[(x >= 3.5) & (x <= 5.0)] = 4.0 + x[x > 5.0] = 6.0 + return x * sign + + +def get_reciprocal(x): + if isinstance(x, torch.Tensor): + return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x) + elif isinstance(x, (float, int)): + return 0.0 if x == 0 else 1.0 / x + else: + raise TypeError("Input must be a float, int, or a torch.Tensor.") + + +def ref_nvfp4_quant(x, global_scale): + assert global_scale.dtype == torch.float32 + assert x.ndim == 2 + m, n = x.shape + x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE)) + vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) + scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX)) + scale = scale.to(torch.float8_e4m3fn).to(torch.float32) + output_scale = get_reciprocal(scale * get_reciprocal(global_scale)) + + scaled_x = x.to(torch.float32) * output_scale + clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n) + return cast_to_fp4(clipped_x), scale.squeeze(-1) + + +def recover_swizzled_scales(scale, m, n): + rounded_m = ((m + 128 - 1) // 128) * 128 + scale_n = n // BLOCK_SIZE + rounded_n = ((scale_n + 4 - 1) // 4) * 4 + # Recover the swizzled scaling factor to linear layout + tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32) + return result[:m, :scale_n] + + +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("shape", SHAPES) +@torch.inference_mode() +def test_quantize_to_fp4( + dtype: torch.dtype, + shape: tuple[int, int], +) -> None: + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + m, n = shape + + x = torch.randn((m, n), dtype=dtype) + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = scaled_fp4_quant(x, global_scale) + scale_ans = recover_swizzled_scales(out_scale, m, n) + out_ans = cast_from_fp4(out, m, n) + + torch.testing.assert_close(out_ans, out_ref) + torch.testing.assert_close(scale_ans, scale_ref) + + +@pytest.mark.parametrize("pad_shape", PAD_SHAPES) +@torch.inference_mode() +def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None: + torch.manual_seed(42) + dtype = torch.float16 + torch.set_default_device("cuda:0") + + m, n = pad_shape + + x = torch.randn((m, n), dtype=dtype) + + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = scaled_fp4_quant(x, global_scale) + + scale_ans = recover_swizzled_scales(out_scale, m, n) + out_ans = cast_from_fp4(out, m, n) + + torch.testing.assert_close(out_ans, out_ref) + torch.testing.assert_close(scale_ans, scale_ref) diff --git a/sgl-kernel/tests/test_fp8_blockwise_gemm.py b/sgl-kernel/tests/test_fp8_blockwise_gemm.py new file mode 100644 index 0000000..4ae7ae0 --- /dev/null +++ b/sgl-kernel/tests/test_fp8_blockwise_gemm.py @@ -0,0 +1,112 @@ +import unittest +from typing import Optional, Type + +import torch +from sgl_kernel import fp8_blockwise_scaled_mm + + +def cdiv(a: int, b: int) -> int: + """Ceiling division.""" + return -(a // -b) + + +def scale_shape(shape, group_shape): + assert len(shape) == len(group_shape) + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +def baseline_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: Type[torch.dtype], + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + + # We treat N-dimensional group scaling as extended numpy-style broadcasting + # in numpy simply stretches dimensions with an extent of 1 to match the + # the target shape by repeating the data along that dimension (broadcasting) + # , we extend these semantics to say if the extent of a dimension in the + # source shape is not 1 and does not match the target shape we repeat each + # element along that dimension src_shape[dim] // target_shape[dim] times + # example if we have: + # a = [[1, 2], and target_shape = (2, 4) + # [3, 4]] + # then we would expand a to: + # a = [[1, 1, 2, 2], + # [3, 3, 4, 4]] + # NOTE this function this function does not explicitly broadcast dimensions + # with an extent of 1, since this can be done implicitly by pytorch + def group_broadcast(t, shape): + for i, s in enumerate(shape): + if t.shape[i] != s and t.shape[i] != 1: + assert s % t.shape[i] == 0 + t = ( + t.unsqueeze(i + 1) + .expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :]) + .flatten(i, i + 1) + ) + return t + + scale_a = group_broadcast(scale_a, a.shape) + scale_b = group_broadcast(scale_b, b.shape) + + output = torch.mm( + (scale_a * a.to(dtype=torch.float32)), (scale_b * b.to(dtype=torch.float32)) + ).to(out_dtype) + + if bias is not None: + output = output + bias + + return output + + +class TestFp8Gemm(unittest.TestCase): + def _test_accuracy_once(self, M, N, K, out_dtype, device): + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + ) + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + b_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + ) + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn).t() + + scale_a_group_shape = (1, 128) + scale_b_group_shape = (128, 128) + scale_a_shape = scale_shape(a_fp8.shape, scale_a_group_shape) + scale_b_shape = scale_shape(b_fp8.shape, scale_b_group_shape) + + scale_a = torch.randn(scale_a_shape, device=device, dtype=torch.float32) * 0.001 + scale_b = torch.randn(scale_b_shape, device=device, dtype=torch.float32) * 0.001 + scale_a = scale_a.t().contiguous().t() + scale_b = scale_b.t().contiguous().t() + + o1 = torch.empty((M, N), device=device, dtype=torch.bfloat16) + o = baseline_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype) + o1 = fp8_blockwise_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype) + + rtol = 0.02 + atol = 1 + torch.testing.assert_close(o, o1, rtol=rtol, atol=atol) + print(f"M={M}, N={N}, K={K}, out_dtype={out_dtype}: OK") + + def test_accuracy(self): + Ms = [1, 128, 512, 1024, 4096] + Ns = [128, 512, 1024, 4096] + Ks = [512, 1024, 4096, 8192, 16384] + out_dtypes = [torch.bfloat16, torch.float16] + for M in Ms: + for N in Ns: + for K in Ks: + for out_dtype in out_dtypes: + self._test_accuracy_once(M, N, K, out_dtype, "cuda") + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-kernel/tests/test_fp8_gemm.py b/sgl-kernel/tests/test_fp8_gemm.py new file mode 100644 index 0000000..1a73186 --- /dev/null +++ b/sgl-kernel/tests/test_fp8_gemm.py @@ -0,0 +1,67 @@ +import unittest + +import torch +from sgl_kernel import fp8_scaled_mm + + +def torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias): + o = torch.matmul(a.to(torch.float32), b.to(torch.float32)) + + o = o.to(torch.float32) + temp1 = o * scale_a.view(-1, 1) + temp2 = temp1 * scale_b.view(1, -1) + final = temp2.to(out_dtype) + if bias is not None: + final = final + bias.view(1, -1) + + return final + + +class TestFp8Gemm(unittest.TestCase): + def _test_accuracy_once(self, M, N, K, with_bias, out_dtype, device): + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + ) + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + b_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + ) + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + scale_a = torch.randn((M,), device=device, dtype=torch.float32) * 0.001 + scale_b = torch.randn((N,), device=device, dtype=torch.float32) * 0.001 + if with_bias: + bias = torch.randn((N,), device=device, dtype=out_dtype) + else: + bias = None + o1 = torch.empty((M, N), device=device, dtype=torch.bfloat16) + b_fp8 = b_fp8.t() + o = torch_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype, bias) + o1 = fp8_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype, bias) + rtol = 0.02 + atol = 1 + torch.testing.assert_close(o, o1, rtol=rtol, atol=atol) + print(f"M={M}, N={N}, K={K}, with_bias={with_bias}, out_dtype={out_dtype}: OK") + + def test_accuracy(self): + Ms = [1, 128, 512, 1024, 4096] + Ns = [16, 128, 512, 1024, 4096] + Ks = [512, 1024, 4096, 8192, 16384] + bias_opts = [True, False] + out_dtypes = [torch.bfloat16, torch.float16] + for M in Ms: + for N in Ns: + for K in Ks: + for with_bias in bias_opts: + for out_dtype in out_dtypes: + self._test_accuracy_once( + M, N, K, with_bias, out_dtype, "cuda" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-kernel/tests/test_int8_gemm.py b/sgl-kernel/tests/test_int8_gemm.py new file mode 100644 index 0000000..951de31 --- /dev/null +++ b/sgl-kernel/tests/test_int8_gemm.py @@ -0,0 +1,56 @@ +import unittest + +import torch +from sgl_kernel import int8_scaled_mm +from vllm._custom_ops import cutlass_scaled_mm as vllm_scaled_mm + + +def to_int8(tensor: torch.Tensor) -> torch.Tensor: + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) + + +def torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias): + o = torch.matmul(a.to(torch.float32), b.to(torch.float32)) + if bias is not None: + o = o.to(torch.float32) * scale_a.view(-1, 1) * scale_b.view(1, -1) + bias + else: + o = o.to(torch.float32) * scale_a.view(-1, 1) * scale_b.view(1, -1) + return o.to(out_dtype) + + +class TestInt8Gemm(unittest.TestCase): + def _test_accuracy_once(self, M, N, K, with_bias, out_dtype, device): + a = to_int8(torch.randn((M, K), device=device) * 5) + b = to_int8(torch.randn((N, K), device=device).t() * 5) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + if with_bias: + bias = torch.randn((N,), device="cuda", dtype=out_dtype) * 10 + else: + bias = None + + o = int8_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + o1 = torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + o2 = vllm_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + torch.testing.assert_close(o, o1) + torch.testing.assert_close(o, o2) + print(f"M={M}, N={N}, K={K}, with_bias={with_bias}, out_dtype={out_dtype}: OK") + + def test_accuracy(self): + Ms = [1, 16, 32, 64, 128, 512, 1024, 4096, 8192] + Ns = [16, 128, 512, 1024, 4096, 8192, 16384] + Ks = [512, 1024, 4096, 8192, 16384] + bias_opts = [True, False] + out_dtypes = [torch.float16, torch.bfloat16] + for M in Ms: + for N in Ns: + for K in Ks: + for with_bias in bias_opts: + for out_dtype in out_dtypes: + self._test_accuracy_once( + M, N, K, with_bias, out_dtype, "cuda" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-kernel/tests/test_lightning_attention_decode.py b/sgl-kernel/tests/test_lightning_attention_decode.py new file mode 100644 index 0000000..8f5d4bb --- /dev/null +++ b/sgl-kernel/tests/test_lightning_attention_decode.py @@ -0,0 +1,88 @@ +import pytest +import torch +from sgl_kernel import lightning_attention_decode + + +def naive_lightning_attention_decode(q, k, v, past_kv, slope): + """Naive implementation of lightning attention decode""" + original_dtype = q.dtype + ratio = torch.exp(-slope) # [h, 1, 1] + + kv = past_kv + b, h, n, d = q.shape + + output = [] + for i in range(n): + kv = ratio * kv.to(torch.float32) + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + qkv = torch.einsum( + "... n e, ... e d -> ... n d", + q[:, :, i : i + 1].to(torch.float32), + kv.to(torch.float32), + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + + return output.to(original_dtype), kv + + +configs = [ + # (batch_size, num_heads, dim, embed_dim) + (1, 8, 64, 64), + (2, 8, 64, 64), + (1, 32, 32, 64), + (2, 32, 32, 64), + (4, 32, 64, 64), + (4, 32, 64, 64), + (16, 64, 96, 96), + (64, 64, 96, 96), +] + +dtypes = [torch.float32, torch.float16, torch.bfloat16] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.parametrize("dtype", dtypes) +@pytest.mark.parametrize("batch_size,num_heads,dim,embed_dim", configs) +def test_lightning_attention_decode(dtype, batch_size, num_heads, dim, embed_dim): + device = torch.device("cuda") + + q = torch.randn(batch_size, num_heads, 1, dim, device=device, dtype=dtype) + k = torch.randn(batch_size, num_heads, 1, dim, device=device, dtype=dtype) + v = torch.randn(batch_size, num_heads, 1, embed_dim, device=device, dtype=dtype) + past_kv = torch.randn(batch_size, num_heads, dim, embed_dim, device=device) + slope = torch.randn(num_heads, 1, 1, device=device) + + ref_output, ref_new_kv = naive_lightning_attention_decode(q, k, v, past_kv, slope) + + output = torch.empty_like(ref_output) + new_kv = torch.empty_like(ref_new_kv) + lightning_attention_decode(q, k, v, past_kv, slope, output, new_kv) + + rtol = 1e-2 + atol = 1e-2 + + torch.testing.assert_close( + output, + ref_output, + rtol=rtol, + atol=atol, + msg=f"Output mismatch for batch_size={batch_size}, num_heads={num_heads}, " + f"dim={dim}, embed_dim={embed_dim}, dtype={dtype}", + ) + + torch.testing.assert_close( + new_kv, + ref_new_kv, + rtol=rtol, + atol=atol, + msg=f"New KV mismatch for batch_size={batch_size}, num_heads={num_heads}, " + f"dim={dim}, embed_dim={embed_dim}, dtype={dtype}", + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_moe_align.py b/sgl-kernel/tests/test_moe_align.py new file mode 100644 index 0000000..3d89c34 --- /dev/null +++ b/sgl-kernel/tests/test_moe_align.py @@ -0,0 +1,226 @@ +import itertools + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_align_block_size + + +def ceil_div(a, b): + return (a + b - 1) // b + + +@triton.jit +def moe_align_block_size_stage1( + topk_ids_ptr, + tokens_cnts_ptr, + num_experts: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = pid * tokens_per_thread + off_c = (pid + 1) * num_experts + + for i in range(tokens_per_thread): + if start_idx + i < numel: + idx = tl.load(topk_ids_ptr + start_idx + i) + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) + + +@triton.jit +def moe_align_block_size_stage2( + tokens_cnts_ptr, + num_experts: tl.constexpr, +): + pid = tl.program_id(0) + last_cnt = 0 + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) + last_cnt = last_cnt + token_cnt + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) + + +@triton.jit +def moe_align_block_size_stage3( + total_tokens_post_pad_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, +): + last_cumsum = 0 + off_cnt = num_experts * num_experts + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size + tl.store(cumsum_ptr + i, last_cumsum) + tl.store(total_tokens_post_pad_ptr, last_cumsum) + + +@triton.jit +def moe_align_block_size_stage4( + topk_ids_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = tl.load(cumsum_ptr + pid) + end_idx = tl.load(cumsum_ptr + pid + 1) + + for i in range(start_idx, end_idx, block_size): + tl.store(expert_ids_ptr + i // block_size, pid) + + start_idx = pid * tokens_per_thread + off_t = pid * num_experts + + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): + expert_id = tl.load(topk_ids_ptr + i) + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) + tl.store(sorted_token_ids_ptr + rank_post_pad, i) + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) + + +def moe_align_block_size_triton( + topk_ids: torch.Tensor, + num_experts: int, + block_size: int, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor, +) -> None: + numel = topk_ids.numel() + grid = (num_experts,) + tokens_cnts = torch.zeros( + (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device + ) + cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) + tokens_per_thread = ceil_div(numel, num_experts) + + moe_align_block_size_stage1[grid]( + topk_ids, + tokens_cnts, + num_experts, + numel, + tokens_per_thread, + ) + moe_align_block_size_stage2[grid]( + tokens_cnts, + num_experts, + ) + moe_align_block_size_stage3[(1,)]( + num_tokens_post_pad, + tokens_cnts, + cumsum, + num_experts, + block_size, + ) + moe_align_block_size_stage4[grid]( + topk_ids, + sorted_token_ids, + expert_ids, + tokens_cnts, + cumsum, + num_experts, + block_size, + numel, + tokens_per_thread, + ) + + +@pytest.mark.parametrize( + "block_size,num_tokens,topk,num_experts", + list( + itertools.product( + [32, 64, 128, 256], # block_size + [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096], # num_tokens + [1, 2, 4, 8, 16, 32, 64], # topk + [64, 160, 256], # num_experts + ) + ), +) +def test_moe_align_block_size_compare_implementations( + block_size, num_tokens, topk, num_experts +): + # For DeepSeek V3, we have 256 experts + + topk_ids = torch.stack( + [ + torch.randperm(num_experts, dtype=torch.int32, device="cuda")[:topk] + for _ in range(num_tokens) + ] + ) + + max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) + + sorted_ids_cuda = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + sorted_ids_cuda.fill_(topk_ids.numel()) + max_num_m_blocks = max_num_tokens_padded // block_size + expert_ids_cuda = torch.zeros( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad_cuda = torch.empty( + (1), dtype=torch.int32, device=topk_ids.device + ) + token_cnts_buffer = torch.empty( + (num_experts + 1) * num_experts, + dtype=torch.int32, + device=topk_ids.device, + ) + cumsum_buffer = torch.empty( + num_experts + 1, dtype=torch.int32, device=topk_ids.device + ) + + sorted_ids_triton = torch.empty_like(sorted_ids_cuda) + sorted_ids_triton.fill_(topk_ids.numel()) + expert_ids_triton = torch.zeros_like(expert_ids_cuda) + num_tokens_post_pad_triton = torch.empty_like(num_tokens_post_pad_cuda) + + moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_ids_cuda, + expert_ids_cuda, + num_tokens_post_pad_cuda, + token_cnts_buffer, + cumsum_buffer, + ) + + moe_align_block_size_triton( + topk_ids, + num_experts, + block_size, + sorted_ids_triton, + expert_ids_triton, + num_tokens_post_pad_triton, + ) + + assert torch.allclose(expert_ids_cuda, expert_ids_triton), ( + f"Expert IDs mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA expert_ids: {expert_ids_cuda}\n" + f"Triton expert_ids: {expert_ids_triton}" + ) + + assert torch.allclose(num_tokens_post_pad_cuda, num_tokens_post_pad_triton), ( + f"Num tokens post pad mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA num_tokens_post_pad: {num_tokens_post_pad_cuda}\n" + f"Triton num_tokens_post_pad: {num_tokens_post_pad_triton}" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_moe_topk_softmax.py b/sgl-kernel/tests/test_moe_topk_softmax.py new file mode 100644 index 0000000..09acde5 --- /dev/null +++ b/sgl-kernel/tests/test_moe_topk_softmax.py @@ -0,0 +1,53 @@ +import itertools + +import pytest +import torch +from sgl_kernel import topk_softmax + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_softmax(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + token_expert_indices = torch.empty( + (num_tokens, topk), dtype=torch.int32, device="cuda" + ) + + topk_softmax( + topk_weights, + topk_indices, + token_expert_indices, + gating_output, + ) + + # Native torch implementation + softmax_output = torch.softmax(gating_output, dim=-1) + topk_weights_ref, topk_indices_ref = torch.topk(softmax_output, topk, dim=-1) + + # Verify the top-k weights and indices match the torch native ones + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: torch={topk_indices_ref} vs SGLang={topk_weights}" + + assert torch.equal( + topk_indices_ref, topk_indices + ), f"Indices mismatch: torch={topk_indices_ref}, SGLang={topk_indices}" + + print("✅ Native torch and custom kernel implementations match.") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_norm.py b/sgl-kernel/tests/test_norm.py new file mode 100644 index 0000000..d22da93 --- /dev/null +++ b/sgl-kernel/tests/test_norm.py @@ -0,0 +1,133 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_norm.py + +import pytest +import sgl_kernel +import torch + + +def llama_rms_norm(x, w, eps=1e-6): + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * w.float() + x = x.to(orig_dtype) + return x + + +def gemma_rms_norm(x, w, eps=1e-6): + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * (1.0 + w.float()) + x = x.to(orig_dtype) + return x + + +def gemma_fused_add_rms_norm(x, residual, w, eps=1e-6): + orig_dtype = x.dtype + x = x + residual + residual = x + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * (1.0 + w.float()) + x = x.to(orig_dtype) + return x, residual + + +def fused_add_rms_norm(x, residual, weight, eps): + orig_dtype = x.dtype + x = x.to(torch.float32) + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = (x * weight.float()).to(orig_dtype) + return x, residual + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +@pytest.mark.parametrize("specify_out", [True, False]) +def test_norm(batch_size, hidden_size, dtype, specify_out): + x = torch.randn(batch_size, hidden_size).to(0).to(dtype) + w = torch.randn(hidden_size).to(0).to(dtype) + + y_ref = llama_rms_norm(x, w) + if specify_out: + y = torch.empty_like(x) + sgl_kernel.rmsnorm(x, w, out=y) + else: + y = sgl_kernel.rmsnorm(x, w) + + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +def test_fused_add_rmsnorm(batch_size, hidden_size, dtype): + eps = 1e-6 + + x = torch.randn(batch_size, hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) + weight = torch.randn(hidden_size, dtype=dtype, device="cuda") + + x_native, residual_native = fused_add_rms_norm( + x.clone(), residual.clone(), weight, eps + ) + + x_fused = x.clone() + residual_fused = residual.clone() + sgl_kernel.fused_add_rmsnorm(x_fused, residual_fused, weight, eps) + + torch.testing.assert_close(x_fused, x_native, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(residual_fused, residual_native, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +@pytest.mark.parametrize("specify_out", [True, False]) +def test_gemma_norm(batch_size, hidden_size, dtype, specify_out): + x = torch.randn(batch_size, hidden_size).to(0).to(dtype) + w = torch.randn(hidden_size).to(0).to(dtype) + + y_ref = gemma_rms_norm(x, w) + if specify_out: + y = torch.empty_like(x) + sgl_kernel.gemma_rmsnorm(x, w, out=y) + else: + y = sgl_kernel.gemma_rmsnorm(x, w) + + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +def test_gemma_fused_add_rmsnorm(batch_size, hidden_size, dtype): + eps = 1e-6 + + x = torch.randn(batch_size, hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) + weight = torch.randn(hidden_size, dtype=dtype, device="cuda") + + x_native, residual_native = gemma_fused_add_rms_norm( + x.clone(), residual.clone(), weight, eps + ) + + x_fused = x.clone() + residual_fused = residual.clone() + sgl_kernel.gemma_fused_add_rmsnorm(x_fused, residual_fused, weight, eps) + + torch.testing.assert_close(x_fused, x_native, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(residual_fused, residual_native, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_per_tensor_quant_fp8.py b/sgl-kernel/tests/test_per_tensor_quant_fp8.py new file mode 100644 index 0000000..70b05af --- /dev/null +++ b/sgl-kernel/tests/test_per_tensor_quant_fp8.py @@ -0,0 +1,67 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import sgl_per_tensor_quant_fp8 +from vllm import _custom_ops as ops + +from sglang.srt.utils import is_hip + +is_hip_ = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + + +def vllm_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + return ops.scaled_fp8_quant(input, scale) + + +def sglang_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_type_: torch.dtype = torch.float8_e4m3fn + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + is_static = True + if scale is None: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + is_static = False + sgl_per_tensor_quant_fp8(input, output, scale, is_static) + + return output, scale + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim", + list(itertools.product([128, 256, 512], [512, 2048, 4096])), +) +def test_per_tensor_quant_compare_implementations( + num_tokens: int, + hidden_dim: int, +): + device = torch.device("cuda") + x = torch.rand((num_tokens, hidden_dim), dtype=torch.float16, device=device) + + vllm_out, vllm_scale = vllm_scaled_fp8_quant(x) + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x) + + torch.testing.assert_close(vllm_scale, sglang_scale, rtol=1e-3, atol=1e-3) + torch.testing.assert_close( + vllm_out.float(), sglang_out.float(), rtol=1e-3, atol=1e-3 + ) + + scale = torch.rand(1, dtype=torch.float32, device=device) + vllm_out, vllm_scale = vllm_scaled_fp8_quant(x, scale) + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x, scale) + + torch.testing.assert_close(vllm_scale, sglang_scale, rtol=1e-3, atol=1e-3) + torch.testing.assert_close( + vllm_out.float(), sglang_out.float(), rtol=1e-3, atol=1e-3 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_per_token_group_quant_8bit.py b/sgl-kernel/tests/test_per_token_group_quant_8bit.py new file mode 100644 index 0000000..b628a6a --- /dev/null +++ b/sgl-kernel/tests/test_per_token_group_quant_8bit.py @@ -0,0 +1,178 @@ +import itertools +from typing import Tuple + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import sgl_per_token_group_quant_fp8, sgl_per_token_group_quant_int8 + +from sglang.srt.utils import is_hip + +is_hip_ = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + + +@triton.jit +def _per_token_group_quant_8bit( + # Pointers to inputs and output + y_ptr, + y_q_ptr, + y_s_ptr, + # Stride of input + y_stride, + # Collums of input + N, + # Avoid to divide zero + eps, + # Information for 8bit data type (int8 or fp8_type_) + max_8bit, + min_8bit, + # Meta-parameters + BLOCK: tl.constexpr, +): + """A Triton-accelerated function to perform per-token-group quantization on a + tensor. + This function converts the tensor values into 8bit values. + """ + # Map the program id to the row of X and Y it should compute. + g_id = tl.program_id(0) + y_ptr += g_id * y_stride + y_q_ptr += g_id * y_stride + y_s_ptr += g_id + + cols = tl.arange(0, BLOCK) # N <= BLOCK + mask = cols < N + + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32) + # Quant + _absmax = tl.maximum(tl.max(tl.abs(y)), eps) + y_s = _absmax / max_8bit + y_q = tl.clamp(y / y_s, min_8bit, max_8bit).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + cols, y_q, mask=mask) + tl.store(y_s_ptr, y_s) + + +def triton_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Function to perform per-token-group quantization on an input tensor `x`. + It converts the tensor values into signed float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Args: + x: The input tenosr with ndim >= 2. + group_size: The group size used for quantization. + eps: The minimum to avoid dividing zero. + dtype: The dype of output tensor. Note that only `torch.float8_e4m3fn` is supported for now. + Returns: + Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the scaling factor for quantization. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + if dst_dtype == torch.int8: + iinfo = torch.iinfo(dst_dtype) + max_8bit = iinfo.max + min_8bit = iinfo.min + else: + finfo = torch.finfo(dst_dtype) + max_8bit = finfo.max + min_8bit = finfo.min + + x_q = torch.empty_like(x, device=x.device, dtype=dst_dtype) + M = x.numel() // group_size + N = group_size + x_s = torch.empty( + x.shape[:-1] + (x.shape[-1] // group_size,), + device=x.device, + dtype=torch.float32, + ) + + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + _per_token_group_quant_8bit[(M,)]( + x, + x_q, + x_s, + group_size, + N, + eps, + max_8bit, + min_8bit, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + ) + + return x_q, x_s + + +def sglang_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, +): + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + x_q = torch.empty_like(x, device=x.device, dtype=dst_dtype) + x_s = torch.empty( + x.shape[:-1] + (x.shape[-1] // group_size,), + device=x.device, + dtype=torch.float32, + ) + + if dst_dtype == torch.int8: + iinfo = torch.iinfo(dst_dtype) + int8_max = iinfo.max + int8_min = iinfo.min + sgl_per_token_group_quant_int8(x, x_q, x_s, group_size, eps, int8_min, int8_max) + else: + f8_info = torch.finfo(dst_dtype) + fp8_max = f8_info.max + fp8_min = f8_info.min + sgl_per_token_group_quant_fp8(x, x_q, x_s, group_size, eps, fp8_min, fp8_max) + + return x_q, x_s + + +@pytest.mark.parametrize( + "batch_size, seq_len, group_size, dst_dtype", + list( + itertools.product( + [1, 2, 4, 8, 16, 32, 64, 128], # batch_size + [64, 128, 256, 512, 1024, 2048], # seq_len + [16, 32, 64, 128, 256], # group_size + [torch.int8, fp8_type_], # dtype + ) + ), +) +def test_per_token_group_quant_compare_implementations( + batch_size, seq_len, group_size, dst_dtype +): + x = torch.randn( + (batch_size, seq_len, group_size * 2), device="cuda", dtype=torch.float16 + ) + + x_q_triton, x_s_triton = triton_per_token_group_quant_8bit(x, group_size, dst_dtype) + x_q_sglang, x_s_sglang = sglang_per_token_group_quant_8bit(x, group_size, dst_dtype) + + assert torch.allclose( + x_q_triton.to(torch.float32), x_q_sglang.to(torch.float32), rtol=1e-3, atol=1e-5 + ) + assert torch.allclose(x_s_triton, x_s_sglang, rtol=1e-3, atol=1e-5) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_per_token_quant_fp8.py b/sgl-kernel/tests/test_per_token_quant_fp8.py new file mode 100644 index 0000000..20b2722 --- /dev/null +++ b/sgl-kernel/tests/test_per_token_quant_fp8.py @@ -0,0 +1,55 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import sgl_per_token_quant_fp8 +from vllm import _custom_ops as ops + +from sglang.srt.utils import is_hip + +is_hip_ = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if is_hip_ else torch.float8_e4m3fn + + +def vllm_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + return ops.scaled_fp8_quant(input, use_per_token_if_dynamic=True) + + +def sglang_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + scale = torch.zeros(input.size(0), device=input.device, dtype=torch.float32) + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + + sgl_per_token_quant_fp8(input, output, scale) + scale = scale.reshape(-1, 1) + + return output, scale + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim", + list(itertools.product([128, 256, 512], [512, 2048, 4096])), +) +def test_per_token_quant_compare_implementations( + num_tokens: int, + hidden_dim: int, +): + device = torch.device("cuda") + x = torch.rand((num_tokens, hidden_dim), dtype=torch.float16, device=device) + + vllm_out, vllm_scale = vllm_per_token_quant_fp8(x) + sglang_out, sglang_scale = sglang_per_token_quant_fp8(x) + + torch.testing.assert_close(vllm_scale, sglang_scale, rtol=1e-3, atol=1e-3) + torch.testing.assert_close( + vllm_out.float(), sglang_out.float(), rtol=1e-3, atol=1e-3 + ) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_rotary_embedding.py b/sgl-kernel/tests/test_rotary_embedding.py new file mode 100644 index 0000000..fa937a6 --- /dev/null +++ b/sgl-kernel/tests/test_rotary_embedding.py @@ -0,0 +1,200 @@ +from typing import Any, Dict, List, Optional, Tuple, Union + +import pytest +import torch +from sgl_kernel import apply_rope_with_cos_sin_cache_inplace + + +# vLLM torch native +def _apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + """ + Args: + x: [num_tokens, num_heads, head_size] + cos: [num_tokens, head_size // 2] + sin: [num_tokens, head_size // 2] + is_neox_style: Whether to use the Neox-style or GPT-J-style rotary + positional embeddings. + """ + cos = cos.unsqueeze(-2).to(x.dtype) + sin = sin.unsqueeze(-2).to(x.dtype) + if is_neox_style: + x1, x2 = torch.chunk(x, 2, dim=-1) + else: + x1 = x[..., ::2] + x2 = x[..., 1::2] + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_neox_style: + return torch.cat((o1, o2), dim=-1) + else: + return torch.stack((o1, o2), dim=-1).flatten(-2) + + +class RotaryEmbedding(torch.nn.Module): + # Reference: https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/rotary_embedding.py + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cache = self._compute_cos_sin_cache() + self.cos_sin_cache: torch.Tensor + self.register_buffer("cos_sin_cache", cache, persistent=False) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-native implementation of forward().""" + if offsets is not None: + positions = positions + offsets + + positions = positions.flatten() + num_tokens = positions.shape[0] + cos_sin = self.cos_sin_cache.index_select(0, positions) + + # Modification: float32 is required for the rotary embedding to work correctly + query = query.to(torch.float32) + key = key.to(torch.float32) + + cos, sin = cos_sin.chunk(2, dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + + # Modification: convert to the correct dtype + query = query.to(self.dtype) + key = key.to(self.dtype) + return query, key + + +class FlashInferRotaryEmbedding(RotaryEmbedding): + def forward_cuda( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + apply_rope_with_cos_sin_cache_inplace( + positions=positions, + query=query, + key=key, + head_size=self.head_size, + cos_sin_cache=self.cos_sin_cache, + is_neox=self.is_neox_style, + ) + + return query, key + + +@pytest.mark.parametrize( + "head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype, device, batch_size, seq_len, num_q_heads, num_kv_heads", + [ + (64, 64, 32, 8000, True, torch.bfloat16, "cuda", 32, 32, 1, 1), + (256, 128, 4096, 10000, True, torch.bfloat16, "cuda", 2, 512, 4, 2), + (512, 128, 311, 10000, True, torch.bfloat16, "cuda", 3, 39, 4, 2), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 32, 8), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 16, 4), + (512, 128, 311, 10000, False, torch.bfloat16, "cuda", 3, 39, 4, 2), + ], +) +def test_correctness( + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q_heads: int, + num_kv_heads: int, +): + rope_ref = RotaryEmbedding( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ).to(device) + rope_flashinfer = FlashInferRotaryEmbedding( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ).to(device) + + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q_heads * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv_heads * head_size, dtype=dtype, device=device + ) + + query_ref, key_ref = query.clone(), key.clone() + query_flashinfer, key_flashinfer = query.clone(), key.clone() + + query_ref_out, key_ref_out = rope_ref.forward_native(pos_ids, query_ref, key_ref) + query_flashinfer_out, key_flashinfer_out = rope_flashinfer.forward_cuda( + pos_ids, query_flashinfer, key_flashinfer + ) + + print(query_ref_out) + print(query_flashinfer_out) + + torch.testing.assert_close( + query_ref_out, query_flashinfer_out, atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(key_ref_out, key_flashinfer_out, atol=1e-2, rtol=1e-2) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_sampling.py b/sgl-kernel/tests/test_sampling.py new file mode 100644 index 0000000..7d3bc50 --- /dev/null +++ b/sgl-kernel/tests/test_sampling.py @@ -0,0 +1,141 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/93e1a2634e22355b0856246b032b285ad1d1da6b/tests/test_sampling.py + +import pytest +import sgl_kernel +import torch + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 500, 32000, 128256]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_joint_sampling_from_probs(batch_size, vocab_size, p): + torch.manual_seed(42) + if p == 0.1: + k = int(vocab_size * 0.5) + elif p == 0.5: + k = int(vocab_size * 0.1) + else: + raise ValueError("p not recognized") + max_top_k_trails = 32 + eps = 1e-4 + pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + # top-p mask + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask_top_p = torch.zeros(batch_size, vocab_size, dtype=torch.int32).to(0) + mask_top_p.scatter_add_(1, indices, (cdf > (1 - p) - eps).int()) + # top-k mask + sorted_prob, _ = torch.sort(normalized_prob, descending=True) + pivot = sorted_prob[:, k - 1] + mask_top_k = (normalized_prob >= pivot.unsqueeze(-1)).int() + # overall mask + mask = torch.minimum(mask_top_p, mask_top_k) + uniform_samples = torch.empty(max_top_k_trails, batch_size, dtype=torch.float32).to( + 0 + ) + top_p_tensor = torch.full((batch_size,), p).to(0) + top_k_tensor = torch.full((batch_size,), k).to(0) + + num_trails = 1000 + for _ in range(num_trails): + uniform_samples.uniform_() + samples, success = sgl_kernel.top_k_top_p_sampling_from_probs( + normalized_prob, + uniform_samples, + top_k_tensor, + top_p_tensor, + filter_apply_order="joint", + ) + assert torch.all(success) + assert torch.all(samples < vocab_size) and torch.all(samples >= 0) + assert torch.all(mask[torch.arange(batch_size), samples] == 1), normalized_prob[ + torch.arange(batch_size), samples + ] + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 500, 32000, 128256]) +@pytest.mark.parametrize("p", [0.1, 0.5, 0.9]) +def test_top_p_renorm_probs(batch_size, vocab_size, p): + pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask = torch.zeros(batch_size, vocab_size, dtype=torch.int32).to(0) + mask.scatter_add_(1, indices, (cdf >= (1 - p)).int()) + renorm_prob_ground_truth = normalized_prob + renorm_prob_ground_truth[mask == 0] = 0 + renorm_prob_ground_truth = renorm_prob_ground_truth / renorm_prob_ground_truth.sum( + dim=-1, keepdim=True + ) + + renorm_prob = sgl_kernel.top_p_renorm_prob(normalized_prob, p) + torch.testing.assert_close( + renorm_prob_ground_truth, + renorm_prob, + rtol=1e-3, + atol=1e-3, + ) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 500, 32000, 128256]) +@pytest.mark.parametrize("k", [10, 100, 500]) +def test_top_k_renorm_probs(batch_size, vocab_size, k): + if k > vocab_size: + pytest.skip("k should be less than vocab_size") + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, _ = torch.sort(normalized_prob, descending=True) + pivot = sorted_prob[:, k - 1] + mask = (normalized_prob >= pivot.unsqueeze(-1)).int() + renorm_prob_ground_truth = normalized_prob + renorm_prob_ground_truth[mask == 0] = 0 + renorm_prob_ground_truth = renorm_prob_ground_truth / renorm_prob_ground_truth.sum( + dim=-1, keepdim=True + ) + + renorm_prob = sgl_kernel.top_k_renorm_prob(normalized_prob, k) + torch.testing.assert_close( + renorm_prob_ground_truth, + renorm_prob, + rtol=1e-3, + atol=1e-3, + ) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 500, 32000, 128256]) +@pytest.mark.parametrize("p", [0.05, 0.1, 0.2, 0.7, 1]) +def test_min_p_sampling(batch_size, vocab_size, p): + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size).to(0) + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + # scale min-p + top_probs = sorted_prob[:, -1].unsqueeze(-1) + scaled_p = p * top_probs + # min-p mask + mask = torch.zeros(batch_size, vocab_size, dtype=torch.int32).to(0) + mask.scatter_add_(1, indices, (sorted_prob >= scaled_p).int()) + uniform_samples = torch.empty(batch_size, dtype=torch.float32).to(0) + min_p_tensor = torch.full((batch_size,), p).to(0) + + num_trails = 1000 + for _ in range(num_trails): + uniform_samples.uniform_() + samples = sgl_kernel.min_p_sampling_from_probs( + normalized_prob, + uniform_samples, + min_p_tensor, + ) + + assert torch.all(mask[torch.arange(batch_size), samples] == 1), samples[ + torch.nonzero(mask[torch.arange(batch_size), samples] == 0) + ] + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_trt_allreduce.py b/sgl-kernel/tests/test_trt_allreduce.py new file mode 100644 index 0000000..9bbc4e7 --- /dev/null +++ b/sgl-kernel/tests/test_trt_allreduce.py @@ -0,0 +1,244 @@ +import ctypes +import logging +import random +import socket +import time +import unittest +from typing import Any, List, Optional + +import ray +import sgl_kernel.allreduce as custom_ops +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup +from vllm import _custom_ops as vllm_ops + +from sglang.srt.distributed.device_communicators.cuda_wrapper import CudaRTLibrary + +logger = logging.getLogger(__name__) + + +def get_open_port() -> int: + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, + cls: Any, + test_target: Any, +) -> None: + # Using ray helps debugging the error when it failed + # as compared to multiprocessing. + # NOTE: We need to set working_dir for distributed tests, + # otherwise we may get import errors on ray workers + ray.init(log_to_driver=True) + + distributed_init_port = get_open_port() + refs = [] + for rank in range(world_size): + refs.append(test_target.remote(cls, world_size, rank, distributed_init_port)) + ray.get(refs) + + ray.shutdown() + + +class TestCustomAllReduce(unittest.TestCase): + @classmethod + def setUpClass(cls): + random.seed(42) + cls.test_sizes = [512, 4096, 32768, 262144, 524288, 1048576, 2097152] + cls.world_sizes = [2, 4, 8] + + @staticmethod + def create_shared_buffer( + size_in_bytes: int, group: Optional[ProcessGroup] = None + ) -> List[int]: + """ + Creates a shared buffer and returns a list of pointers + representing the buffer on all processes in the group. + """ + lib = CudaRTLibrary() + pointer = lib.cudaMalloc(size_in_bytes) + handle = lib.cudaIpcGetMemHandle(pointer) + world_size = dist.get_world_size(group=group) + rank = dist.get_rank(group=group) + handles = [None] * world_size + dist.all_gather_object(handles, handle, group=group) + + pointers: List[int] = [] + for i, h in enumerate(handles): + if i == rank: + pointers.append(pointer.value) # type: ignore + else: + pointers.append(lib.cudaIpcOpenMemHandle(h).value) # type: ignore + + return pointers + + @staticmethod + def free_shared_buffer( + pointers: List[int], group: Optional[ProcessGroup] = None + ) -> None: + rank = dist.get_rank(group=group) + lib = CudaRTLibrary() + lib.cudaFree(ctypes.c_void_p(pointers[rank])) + + def test_correctness(self): + for world_size in self.world_sizes: + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.correctness) + + def test_performance(self): + for world_size in self.world_sizes: + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.performance) + + def init_custom_allreduce(self, rank, world_size, group): + buffer_max_size = 8 * 1024 * 1024 + barrier_max_size = 8 * (24 + 2) * 8 + + self.buffer_ptrs = self.create_shared_buffer(buffer_max_size, group=group) + self.tmp_result_buffer_ptrs = self.create_shared_buffer( + buffer_max_size, group=group + ) + self.barrier_in_ptrs = self.create_shared_buffer(barrier_max_size, group=group) + self.barrier_out_ptrs = self.create_shared_buffer(barrier_max_size, group=group) + self.rank_data = torch.empty( + 8 * 1024 * 1024, dtype=torch.uint8, device=torch.device("cuda:0") + ) + + self.custom_ptr = custom_ops.init_custom_reduce( + rank, + world_size, + self.rank_data, + self.buffer_ptrs, + self.tmp_result_buffer_ptrs, + self.barrier_in_ptrs, + self.barrier_out_ptrs, + ) + + def custom_allreduce(self, inp, out): + custom_ops.custom_reduce(self.custom_ptr, inp, out) + + def free_custom_allreduce(self, group): + self.free_shared_buffer(self.buffer_ptrs, group) + self.free_shared_buffer(self.tmp_result_buffer_ptrs, group) + self.free_shared_buffer(self.barrier_in_ptrs, group) + self.free_shared_buffer(self.barrier_out_ptrs, group) + custom_ops.custom_dispose(self.custom_ptr) + + def init_vllm_allreduce(self, rank, group): + self.vllm_rank = rank + self.vllm_max_size = 8 * 1024 * 1024 + self.vllm_meta_ptrs = self.create_shared_buffer( + vllm_ops.meta_size() + self.vllm_max_size, group=group + ) + self.vllm_buffer_ptrs = self.create_shared_buffer( + self.vllm_max_size, group=group + ) + self.vllm_rank_data = torch.empty( + 8 * 1024 * 1024, dtype=torch.uint8, device=torch.device("cuda:0") + ) + self.vllm_ptr = vllm_ops.init_custom_ar( + self.vllm_meta_ptrs, self.vllm_rank_data, rank, True + ) + vllm_ops.register_buffer(self.vllm_ptr, self.vllm_buffer_ptrs) + + def vllm_allreduce(self, inp, out): + vllm_ops.all_reduce( + self.vllm_ptr, + inp, + out, + self.vllm_buffer_ptrs[self.vllm_rank], + self.vllm_max_size, + ) + + def free_vllm_allreduce(self, group): + vllm_ops.dispose(self.vllm_ptr) + self.free_shared_buffer(self.vllm_meta_ptrs, group) + self.free_shared_buffer(self.vllm_buffer_ptrs, group) + + @staticmethod + def init_distributed_env(world_size, rank, distributed_init_port): + device = torch.device("cuda:0") + torch.cuda.set_device(device) + ranks = [i for i in range(world_size)] + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + dist.init_process_group( + backend="nccl", + init_method=distributed_init_method, + rank=rank, + world_size=world_size, + ) + group = torch.distributed.new_group(ranks, backend="gloo") + return group + + # compare result with torch.distributed + @ray.remote(num_gpus=1, max_calls=1) + def correctness(self, world_size, rank, distributed_init_port): + group = self.init_distributed_env(world_size, rank, distributed_init_port) + + self.init_custom_allreduce(rank=rank, world_size=world_size, group=group) + + test_loop = 10 + for sz in self.test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(test_loop): + inp1 = torch.randint( + 1, 16, (sz,), dtype=dtype, device=torch.cuda.current_device() + ) + out1 = torch.empty_like(inp1) + self.custom_allreduce(inp1, out1) + + dist.all_reduce(inp1, group=group) + torch.testing.assert_close(out1, inp1) + + self.free_custom_allreduce(group) + + # compare performance with vllm + @ray.remote(num_gpus=1, max_calls=1) + def performance(self, world_size, rank, distributed_init_port): + group = self.init_distributed_env(world_size, rank, distributed_init_port) + + self.init_vllm_allreduce(rank, group) + self.init_custom_allreduce(rank=rank, world_size=world_size, group=group) + + for sz in self.test_sizes: + inp1 = torch.randint( + 1, 16, (sz,), dtype=torch.float32, device=torch.cuda.current_device() + ) + out1 = torch.empty_like(inp1) + test_loop = 5000 + start = time.time() + for _ in range(test_loop): + self.custom_allreduce(inp1, out1) + elapse_custom = time.time() - start + + start = time.time() + for _ in range(test_loop): + self.vllm_allreduce(inp1, out1) + elapse_vllm = time.time() - start + + if rank == 0: + logger.warning( + f"test_size = {sz}, world_size = {world_size}, " + f"vllm time = {elapse_vllm * 1000 / test_loop:.4f}ms, " + f"custom time = {elapse_custom * 1000 / test_loop:.4f}ms " + ) + + self.free_custom_allreduce(group) + self.free_vllm_allreduce(group) + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-router/Cargo.lock b/sgl-router/Cargo.lock new file mode 100644 index 0000000..d0c59ae --- /dev/null +++ b/sgl-router/Cargo.lock @@ -0,0 +1,2557 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "actix-codec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "actix-http" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48f96fc3003717aeb9856ca3d02a8c7de502667ad76eeacd830b48d2e91fac4" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "ahash", + "base64", + "bitflags", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "h2 0.3.26", + "http 0.2.12", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "mime", + "percent-encoding", + "pin-project-lite", + "rand", + "sha1", + "smallvec", + "tokio", + "tokio-util", + "tracing", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d324164c51f63867b57e73ba5936ea151b8a41a1d23d1031eeb9f70d0236f8" +dependencies = [ + "bytestring", + "cfg-if", + "http 0.2.12", + "regex", + "regex-lite", + "serde", + "tracing", +] + +[[package]] +name = "actix-rt" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eda4e2a6e042aa4e55ac438a2ae052d3b5da0ecf83d7411e1a368946925208" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca2549781d8dd6d75c40cf6b6051260a2cc2f3c62343d761a969a0640646894" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9180d76e5cc7ccbc4d60a506f2c727730b154010262df5b910eb17dbe4b8cb38" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "ahash", + "bytes", + "bytestring", + "cfg-if", + "cookie", + "derive_more", + "encoding_rs", + "futures-core", + "futures-util", + "impl-more", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "regex-lite", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" + +[[package]] +name = "bytestring" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +dependencies = [ + "bytes", +] + +[[package]] +name = "cc" +version = "1.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "clap" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cookie" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_more" +version = "0.99.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.6", + "http 1.1.0", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-more" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae21c3177a27788957044151cc2800043d127acaa460a47ebb9b84dfa2c6aa0" + +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown 0.15.1", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "ipnet" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + +[[package]] +name = "libc" +version = "0.2.161" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + +[[package]] +name = "local-channel" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" +dependencies = [ + "futures-core", + "futures-sink", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d873d7c67ce09b42110d801813efbc9364414e356be9935700d368351657487" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi", + "libc", + "log", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "reqwest" +version = "0.12.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.4.6", + "http 1.1.0", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "windows-registry", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schannel" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "serde" +version = "1.0.214" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.214" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sglang_router_rs" +version = "0.0.0" +dependencies = [ + "actix-web", + "bytes", + "chrono", + "clap", + "dashmap", + "env_logger", + "futures-util", + "http 1.1.0", + "log", + "pyo3", + "rand", + "reqwest", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "tempfile" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/sgl-router/Cargo.toml b/sgl-router/Cargo.toml new file mode 100644 index 0000000..be7373f --- /dev/null +++ b/sgl-router/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "sglang_router_rs" +version = "0.0.0" +edition = "2021" + +[lib] +name = "sglang_router_rs" +# Pure Rust library: Just omit crate-type (defaults to rlib) +# Python/C binding + Rust library: Use ["cdylib", "rlib"] +crate-type = ["cdylib", "rlib"] + +[dependencies] +actix-web = "4.0" +serde = { version = "1.0", features = ["derive"] } +clap = { version = "4.4", features = ["derive"] } +bytes = "1.8.0" +rand = "0.8.5" +reqwest = { version = "0.12.8", features = ["stream", "blocking"] } +futures-util = "0.3" +serde_json = "1.0" +pyo3 = { version = "0.22.5", features = ["extension-module"] } +dashmap = "6.1.0" +http = "1.1.0" +env_logger = "0.11.5" +log = "0.4.22" +chrono = "0.4.38" +tokio = "1.42.0" + +[profile.release] +lto = "thin" +codegen-units = 1 diff --git a/sgl-router/MANIFEST.in b/sgl-router/MANIFEST.in new file mode 100644 index 0000000..e1d6e7a --- /dev/null +++ b/sgl-router/MANIFEST.in @@ -0,0 +1,3 @@ +# Must include: +include Cargo.toml # Rust project configuration +recursive-include src *.rs # Rust source files diff --git a/sgl-router/README.md b/sgl-router/README.md new file mode 100644 index 0000000..61c9e69 --- /dev/null +++ b/sgl-router/README.md @@ -0,0 +1,97 @@ +# SGLang Router + +SGLang router is a standalone module implemented in Rust to achieve data parallelism across SGLang instances. + +## User docs + +Please check https://docs.sglang.ai/router/router.html + +## Developer docs + +### Prerequisites + +- Rust and Cargo installed + +```bash +# Install rustup (Rust installer and version manager) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +# Follow the installation prompts, then reload your shell +source $HOME/.cargo/env + +# Verify installation +rustc --version +cargo --version +``` + +- Python with pip installed + + +### Build Process + +#### 1. Build Rust Project + +```bash +$ cargo build +``` + +#### 2. Build Python Binding + +##### Option A: Build and Install Wheel +1. Build the wheel package: +```bash +$ pip install setuptools-rust wheel build +$ python -m build +``` + +2. Install the generated wheel: +```bash +$ pip install +``` + +If you want one handy command to do build + install for every change you make: + +```bash +$ python -m build && pip install --force-reinstall dist/*.whl +``` + +##### Option B: Development Mode + +For development purposes, you can install the package in editable mode: + +Warning: Using editable python binding can suffer from performance degradation!! Please build a fresh wheel for every update if you want to test performance. + +```bash +$ pip install -e . +``` + +**Note:** When modifying Rust code, you must rebuild the wheel for changes to take effect. + +### Troubleshooting + +1. If rust analyzer is not working in VSCode, set `rust-analyzer.linkedProjects` to the absolute path of `Cargo.toml` in your repo. For example: + +```json +{ + "rust-analyzer.linkedProjects": ["/workspaces/sglang/sgl-router/Cargo.toml"] +} +``` + +### CI/CD Setup + +The continuous integration pipeline consists of three main steps: + +#### 1. Build Wheels +- Uses `cibuildwheel` to create manylinux x86_64 packages +- Compatible with major Linux distributions (Ubuntu, CentOS, etc.) +- Additional configurations can be added to support other OS/architectures +- Reference: [cibuildwheel documentation](https://cibuildwheel.pypa.io/en/stable/) + +#### 2. Build Source Distribution +- Creates a source distribution containing the raw, unbuilt code +- Enables `pip` to build the package from source when prebuilt wheels are unavailable + +#### 3. Publish to PyPI +- Uploads both wheels and source distribution to PyPI + +The CI configuration is based on the [tiktoken workflow](https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/.github/workflows/build_wheels.yml#L1). diff --git a/sgl-router/py_src/sglang_router/__init__.py b/sgl-router/py_src/sglang_router/__init__.py new file mode 100644 index 0000000..0817404 --- /dev/null +++ b/sgl-router/py_src/sglang_router/__init__.py @@ -0,0 +1,7 @@ +# a lightweihgt wrapper on router with argument type and comments +# no wrapper on policy type => direct export +from sglang_router.router import Router +from sglang_router.version import __version__ +from sglang_router_rs import PolicyType + +__all__ = ["Router", "PolicyType", "__version__"] diff --git a/sgl-router/py_src/sglang_router/launch_router.py b/sgl-router/py_src/sglang_router/launch_router.py new file mode 100644 index 0000000..38f1fbb --- /dev/null +++ b/sgl-router/py_src/sglang_router/launch_router.py @@ -0,0 +1,268 @@ +import argparse +import dataclasses +import logging +import sys +from typing import List, Optional + +from sglang_router import Router +from sglang_router_rs import PolicyType + + +def setup_logger(): + logger = logging.getLogger("router") + logger.setLevel(logging.INFO) + + formatter = logging.Formatter( + "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + +@dataclasses.dataclass +class RouterArgs: + # Worker configuration + worker_urls: List[str] = dataclasses.field(default_factory=list) + host: str = "127.0.0.1" + port: int = 30000 + + # Routing policy + policy: str = "cache_aware" + worker_startup_timeout_secs: int = 300 + worker_startup_check_interval: int = 10 + cache_threshold: float = 0.5 + balance_abs_threshold: int = 32 + balance_rel_threshold: float = 1.0001 + eviction_interval: int = 60 + max_tree_size: int = 2**24 + max_payload_size: int = 4 * 1024 * 1024 # 4MB + verbose: bool = False + + @staticmethod + def add_cli_args( + parser: argparse.ArgumentParser, + use_router_prefix: bool = False, + exclude_host_port: bool = False, + ): + """ + Add router-specific arguments to an argument parser. + + Args: + parser: The argument parser to add arguments to + use_router_prefix: If True, prefix all arguments with 'router-' to avoid conflicts + exclude_host_port: If True, don't add host and port arguments (used when inheriting from server) + """ + prefix = "router-" if use_router_prefix else "" + + # Worker configuration + if not exclude_host_port: + parser.add_argument( + "--host", + type=str, + default=RouterArgs.host, + help="Host address to bind the router server", + ) + parser.add_argument( + "--port", + type=int, + default=RouterArgs.port, + help="Port number to bind the router server", + ) + + parser.add_argument( + "--worker-urls", + type=str, + nargs="+", + help="List of worker URLs (e.g., http://worker1:8000 http://worker2:8000)", + ) + + # Routing policy configuration + parser.add_argument( + f"--{prefix}policy", + type=str, + default=RouterArgs.policy, + choices=["random", "round_robin", "cache_aware"], + help="Load balancing policy to use", + ) + parser.add_argument( + f"--{prefix}worker-startup-timeout-secs", + type=int, + default=RouterArgs.worker_startup_timeout_secs, + help="Timeout in seconds for worker startup", + ) + parser.add_argument( + f"--{prefix}worker-startup-check-interval", + type=int, + default=RouterArgs.worker_startup_check_interval, + help="Interval in seconds between checks for worker startup", + ) + parser.add_argument( + f"--{prefix}cache-threshold", + type=float, + default=RouterArgs.cache_threshold, + help="Cache threshold (0.0-1.0) for cache-aware routing", + ) + parser.add_argument( + f"--{prefix}balance-abs-threshold", + type=int, + default=RouterArgs.balance_abs_threshold, + help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware", + ) + parser.add_argument( + f"--{prefix}balance-rel-threshold", + type=float, + default=RouterArgs.balance_rel_threshold, + help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware", + ) + parser.add_argument( + f"--{prefix}eviction-interval", + type=int, + default=RouterArgs.eviction_interval, + help="Interval in seconds between cache eviction operations", + ) + parser.add_argument( + f"--{prefix}max-tree-size", + type=int, + default=RouterArgs.max_tree_size, + help="Maximum size of the approximation tree for cache-aware routing", + ) + parser.add_argument( + f"--{prefix}max-payload-size", + type=int, + default=RouterArgs.max_payload_size, + help="Maximum payload size in bytes", + ) + parser.add_argument( + f"--{prefix}verbose", + action="store_true", + help="Enable verbose logging", + ) + + @classmethod + def from_cli_args( + cls, args: argparse.Namespace, use_router_prefix: bool = False + ) -> "RouterArgs": + """ + Create RouterArgs instance from parsed command line arguments. + + Args: + args: Parsed command line arguments + use_router_prefix: If True, look for arguments with 'router-' prefix + """ + prefix = "router_" if use_router_prefix else "" + worker_urls = args.worker_urls if args.worker_urls is not None else [] + return cls( + worker_urls=worker_urls, + host=args.host, + port=args.port, + policy=getattr(args, f"{prefix}policy"), + worker_startup_timeout_secs=getattr( + args, f"{prefix}worker_startup_timeout_secs" + ), + worker_startup_check_interval=getattr( + args, f"{prefix}worker_startup_check_interval" + ), + cache_threshold=getattr(args, f"{prefix}cache_threshold"), + balance_abs_threshold=getattr(args, f"{prefix}balance_abs_threshold"), + balance_rel_threshold=getattr(args, f"{prefix}balance_rel_threshold"), + eviction_interval=getattr(args, f"{prefix}eviction_interval"), + max_tree_size=getattr(args, f"{prefix}max_tree_size"), + max_payload_size=getattr(args, f"{prefix}max_payload_size"), + verbose=getattr(args, f"{prefix}verbose", False), + ) + + +def policy_from_str(policy_str: str) -> PolicyType: + """Convert policy string to PolicyType enum.""" + policy_map = { + "random": PolicyType.Random, + "round_robin": PolicyType.RoundRobin, + "cache_aware": PolicyType.CacheAware, + } + return policy_map[policy_str] + + +def launch_router(args: argparse.Namespace) -> Optional[Router]: + """ + Launch the SGLang router with the configuration from parsed arguments. + + Args: + args: Namespace object containing router configuration + Can be either raw argparse.Namespace or converted RouterArgs + + Returns: + Router instance if successful, None if failed + """ + logger = logging.getLogger("router") + try: + # Convert to RouterArgs if needed + if not isinstance(args, RouterArgs): + router_args = RouterArgs.from_cli_args(args) + else: + router_args = args + + router = Router( + worker_urls=router_args.worker_urls, + host=router_args.host, + port=router_args.port, + policy=policy_from_str(router_args.policy), + worker_startup_timeout_secs=router_args.worker_startup_timeout_secs, + worker_startup_check_interval=router_args.worker_startup_check_interval, + cache_threshold=router_args.cache_threshold, + balance_abs_threshold=router_args.balance_abs_threshold, + balance_rel_threshold=router_args.balance_rel_threshold, + eviction_interval_secs=router_args.eviction_interval, + max_tree_size=router_args.max_tree_size, + max_payload_size=router_args.max_payload_size, + verbose=router_args.verbose, + ) + + router.start() + return router + + except Exception as e: + logger.error(f"Error starting router: {e}") + raise e + + +class CustomHelpFormatter( + argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter +): + """Custom formatter that preserves both description formatting and shows defaults""" + + pass + + +def parse_router_args(args: List[str]) -> RouterArgs: + """Parse command line arguments and return RouterArgs instance.""" + parser = argparse.ArgumentParser( + description="""SGLang Router - High-performance request distribution across worker nodes + +Usage: +This launcher enables starting a router with individual worker instances. It is useful for +multi-node setups or when you want to start workers and router separately. + +Examples: + python -m sglang_router.launch_router --worker-urls http://worker1:8000 http://worker2:8000 + python -m sglang_router.launch_router --worker-urls http://worker1:8000 http://worker2:8000 --cache-threshold 0.7 --balance-abs-threshold 64 --balance-rel-threshold 1.2 + + """, + formatter_class=CustomHelpFormatter, + ) + + RouterArgs.add_cli_args(parser, use_router_prefix=False) + return RouterArgs.from_cli_args(parser.parse_args(args), use_router_prefix=False) + + +def main() -> None: + router_args = parse_router_args(sys.argv[1:]) + launch_router(router_args) + + +if __name__ == "__main__": + main() diff --git a/sgl-router/py_src/sglang_router/launch_server.py b/sgl-router/py_src/sglang_router/launch_server.py new file mode 100644 index 0000000..74353c2 --- /dev/null +++ b/sgl-router/py_src/sglang_router/launch_server.py @@ -0,0 +1,202 @@ +import argparse +import copy +import logging +import multiprocessing as mp +import os +import random +import signal +import sys +import time +from typing import List + +import requests +from setproctitle import setproctitle +from sglang_router.launch_router import RouterArgs, launch_router + +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_port_available + + +def setup_logger(): + logger = logging.getLogger("router") + logger.setLevel(logging.INFO) + + formatter = logging.Formatter( + "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s - %(filename)s:%(lineno)d", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + +logger = setup_logger() + + +# Create new process group +def run_server(server_args, dp_rank): + """ + Note: + + 1. Without os.setpgrp(), all processes share the same PGID. When you press Ctrl+C, the terminal sends SIGINT to all processes in the group simultaneously. + This can cause leaf processes to terminate first, which messes up the cleaning order and produces orphaned processes. + + Terminal (PGID=100) + └── Main Python Process (PGID=100) + └── Server Process 1 (PGID=100) + └── Scheduler 1 + └── Detokenizer 1 + └── Server Process 2 (PGID=100) + └── Scheduler 2 + └── Detokenizer 2 + + 2. With os.setpgrp(), the main Python process and its children are in a separate group. Now: + + Terminal (PGID=100) + └── Main Python Process (PGID=200) + └── Server Process 1 (PGID=300) + └── Scheduler 1 + └── Detokenizer 1 + └── Server Process 2 (PGID=400) + └── Scheduler 2 + └── Detokenizer 2 + """ + # create new process group + os.setpgrp() + + setproctitle("sglang::server") + # Set SGLANG_DP_RANK environment variable + os.environ["SGLANG_DP_RANK"] = str(dp_rank) + + launch_server(server_args) + + +def launch_server_process( + server_args: ServerArgs, worker_port: int, dp_id: int +) -> mp.Process: + """Launch a single server process with the given args and port.""" + server_args = copy.deepcopy(server_args) + server_args.port = worker_port + server_args.base_gpu_id = dp_id * server_args.tp_size + server_args.dp_size = 1 + + proc = mp.Process(target=run_server, args=(server_args, dp_id)) + proc.start() + return proc + + +def wait_for_server_health(host: str, port: int, timeout: int = 300) -> bool: + """Wait for server to be healthy by checking /health endpoint.""" + start_time = time.time() + url = f"http://{host}:{port}/health" + + while time.time() - start_time < timeout: + try: + response = requests.get(url, timeout=5) + if response.status_code == 200: + return True + except requests.exceptions.RequestException: + pass + time.sleep(1) + return False + + +def find_available_ports(base_port: int, count: int) -> List[int]: + """Find consecutive available ports starting from base_port.""" + available_ports = [] + current_port = base_port + + while len(available_ports) < count: + if is_port_available(current_port): + available_ports.append(current_port) + current_port += random.randint(100, 1000) + + return available_ports + + +def cleanup_processes(processes: List[mp.Process]): + for process in processes: + logger.info(f"Terminating process group {process.pid}") + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + # Process group may already be terminated + pass + + # Wait for processes to terminate + for process in processes: + process.join(timeout=5) + if process.is_alive(): + logger.warning( + f"Process {process.pid} did not terminate gracefully, forcing kill" + ) + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + logger.info("All process groups terminated") + + +def main(): + # CUDA runtime isn't fork-safe, which can lead to subtle bugs or crashes + mp.set_start_method("spawn") + + parser = argparse.ArgumentParser( + description="Launch SGLang router and server processes" + ) + + ServerArgs.add_cli_args(parser) + RouterArgs.add_cli_args(parser, use_router_prefix=True, exclude_host_port=True) + parser.add_argument( + "--router-dp-worker-base-port", + type=int, + default=31000, + help="Base port number for data parallel workers", + ) + + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Find available ports for workers + worker_ports = find_available_ports( + args.router_dp_worker_base_port, server_args.dp_size + ) + + # Start server processes + server_processes = [] + + for i, worker_port in enumerate(worker_ports): + logger.info(f"Launching DP server process {i} on port {worker_port}") + proc = launch_server_process(server_args, worker_port, i) + server_processes.append(proc) + + signal.signal(signal.SIGINT, lambda sig, frame: cleanup_processes(server_processes)) + signal.signal( + signal.SIGTERM, lambda sig, frame: cleanup_processes(server_processes) + ) + signal.signal( + signal.SIGQUIT, lambda sig, frame: cleanup_processes(server_processes) + ) + + # Update router args with worker URLs + router_args.worker_urls = [ + f"http://{server_args.host}:{port}" for port in worker_ports + ] + + # Start the router + try: + launch_router(router_args) + except Exception as e: + logger.error(f"Failed to start router: {e}") + cleanup_processes(server_processes) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/sgl-router/py_src/sglang_router/router.py b/sgl-router/py_src/sglang_router/router.py new file mode 100644 index 0000000..b875716 --- /dev/null +++ b/sgl-router/py_src/sglang_router/router.py @@ -0,0 +1,73 @@ +from typing import List, Optional + +from sglang_router_rs import PolicyType +from sglang_router_rs import Router as _Router + + +class Router: + """ + A high-performance router for distributing requests across worker nodes. + + Args: + worker_urls: List of URLs for worker nodes that will handle requests. Each URL should include + the protocol, host, and port (e.g., ['http://worker1:8000', 'http://worker2:8000']) + policy: Load balancing policy to use. Options: + - PolicyType.Random: Randomly select workers + - PolicyType.RoundRobin: Distribute requests in round-robin fashion + - PolicyType.CacheAware: Distribute requests based on cache state and load balance + host: Host address to bind the router server. Default: '127.0.0.1' + port: Port number to bind the router server. Default: 3001 + worker_startup_timeout_secs: Timeout in seconds for worker startup. Default: 300 + worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10 + cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker + if the match rate exceeds threshold, otherwise routes to the worker with the smallest + tree. Default: 0.5 + balance_abs_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold + AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 32 + balance_rel_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold + AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 1.0001 + eviction_interval_secs: Interval in seconds between cache eviction operations in cache-aware + routing. Default: 60 + max_payload_size: Maximum payload size in bytes. Default: 4MB + max_tree_size: Maximum size of the approximation tree for cache-aware routing. Default: 2^24 + verbose: Enable verbose logging. Default: False + """ + + def __init__( + self, + worker_urls: List[str], + policy: PolicyType = PolicyType.RoundRobin, + host: str = "127.0.0.1", + port: int = 3001, + worker_startup_timeout_secs: int = 300, + worker_startup_check_interval: int = 10, + cache_threshold: float = 0.50, + balance_abs_threshold: int = 32, + balance_rel_threshold: float = 1.0001, + eviction_interval_secs: int = 60, + max_tree_size: int = 2**24, + max_payload_size: int = 4 * 1024 * 1024, # 4MB + verbose: bool = False, + ): + self._router = _Router( + worker_urls=worker_urls, + policy=policy, + host=host, + port=port, + worker_startup_timeout_secs=worker_startup_timeout_secs, + worker_startup_check_interval=worker_startup_check_interval, + cache_threshold=cache_threshold, + balance_abs_threshold=balance_abs_threshold, + balance_rel_threshold=balance_rel_threshold, + eviction_interval_secs=eviction_interval_secs, + max_tree_size=max_tree_size, + max_payload_size=max_payload_size, + verbose=verbose, + ) + + def start(self) -> None: + """Start the router server. + + This method blocks until the server is shut down. + """ + self._router.start() diff --git a/sgl-router/py_src/sglang_router/version.py b/sgl-router/py_src/sglang_router/version.py new file mode 100644 index 0000000..bbab024 --- /dev/null +++ b/sgl-router/py_src/sglang_router/version.py @@ -0,0 +1 @@ +__version__ = "0.1.4" diff --git a/sgl-router/py_test/run_suite.py b/sgl-router/py_test/run_suite.py new file mode 100644 index 0000000..e1434b0 --- /dev/null +++ b/sgl-router/py_test/run_suite.py @@ -0,0 +1,20 @@ +import argparse +import glob + +from sglang.test.test_utils import TestFile, run_unittest_files + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--timeout-per-file", + type=int, + default=1000, + help="The time limit for running one file in seconds.", + ) + args = arg_parser.parse_args() + + files = glob.glob("**/test_*.py", recursive=True) + + test_files = [TestFile(name=file) for file in files] + exit_code = run_unittest_files(test_files, args.timeout_per_file) + exit(exit_code) diff --git a/sgl-router/py_test/test_launch_router.py b/sgl-router/py_test/test_launch_router.py new file mode 100644 index 0000000..27ed64d --- /dev/null +++ b/sgl-router/py_test/test_launch_router.py @@ -0,0 +1,83 @@ +import multiprocessing +import time +import unittest +from types import SimpleNamespace + + +def terminate_process(process: multiprocessing.Process, timeout: float = 1.0) -> None: + """Terminate a process gracefully, with forced kill as fallback. + + Args: + process: The process to terminate + timeout: Seconds to wait for graceful termination before forcing kill + """ + if not process.is_alive(): + return + + process.terminate() + process.join(timeout=timeout) + if process.is_alive(): + process.kill() # Force kill if terminate didn't work + process.join() + + +class TestLaunchRouter(unittest.TestCase): + def setUp(self): + """Set up default arguments for router tests.""" + self.default_args = SimpleNamespace( + host="127.0.0.1", + port=30000, + policy="cache_aware", + worker_startup_timeout_secs=600, + worker_startup_check_interval=10, + cache_threshold=0.5, + balance_abs_threshold=32, + balance_rel_threshold=1.0001, + eviction_interval=60, + max_tree_size=2**24, + max_payload_size=4 * 1024 * 1024, # 4MB + verbose=False, + ) + + def create_router_args(self, **kwargs): + """Create router arguments by updating default args with provided kwargs.""" + args_dict = vars(self.default_args).copy() + args_dict.update(kwargs) + return SimpleNamespace(**args_dict) + + def run_router_process(self, args): + """Run router in a separate process and verify it starts successfully.""" + + def run_router(): + try: + from sglang_router.launch_router import launch_router + + router = launch_router(args) + if router is None: + return 1 + return 0 + except Exception as e: + print(e) + return 1 + + process = multiprocessing.Process(target=run_router) + try: + process.start() + # Wait 3 seconds + time.sleep(3) + # Process is still running means router started successfully + self.assertTrue(process.is_alive()) + finally: + terminate_process(process) + + def test_launch_router_common(self): + args = self.create_router_args(worker_urls=["http://localhost:8000"]) + self.run_router_process(args) + + def test_launch_router_with_empty_worker_urls(self): + args = self.create_router_args(worker_urls=[]) + self.run_router_process(args) + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-router/py_test/test_launch_server.py b/sgl-router/py_test/test_launch_server.py new file mode 100644 index 0000000..80659fc --- /dev/null +++ b/sgl-router/py_test/test_launch_server.py @@ -0,0 +1,394 @@ +import socket +import subprocess +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, +) + + +def popen_launch_router( + model: str, + base_url: str, + dp_size: int, + timeout: float, + policy: str = "cache_aware", + max_payload_size: int = None, + api_key: str = None, +): + """ + Launch the router server process. + + Args: + model: Model path/name + base_url: Server base URL + dp_size: Data parallel size + timeout: Server launch timeout + policy: Router policy, one of "cache_aware", "round_robin", "random" + max_payload_size: Maximum payload size in bytes + api_key: API key for the router + """ + _, host, port = base_url.split(":") + host = host[2:] + + command = [ + "python3", + "-m", + "sglang_router.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + "--dp", + str(dp_size), + "--router-eviction-interval", + "5", + "--router-policy", + policy, + ] + + if api_key is not None: + command.extend(["--api-key", api_key]) + + if max_payload_size is not None: + command.extend(["--router-max-payload-size", str(max_payload_size)]) + + process = subprocess.Popen(command, stdout=None, stderr=None) + + start_time = time.time() + with requests.Session() as session: + while time.time() - start_time < timeout: + try: + response = session.get(f"{base_url}/health") + if response.status_code == 200: + print(f"Router {base_url} is healthy") + return process + except requests.RequestException: + pass + time.sleep(10) + + raise TimeoutError("Router failed to start within the timeout period.") + + +def find_available_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def popen_launch_server( + model: str, + base_url: str, + timeout: float, +): + _, host, port = base_url.split(":") + host = host[2:] + + command = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + "--base-gpu-id", + "1", + ] + + process = subprocess.Popen(command, stdout=None, stderr=None) + + # intentionally don't wait and defer the job to the router health check + return process + + +def terminate_and_wait(process, timeout=300): + """Terminate a process and wait until it is terminated. + + Args: + process: subprocess.Popen object + timeout: maximum time to wait in seconds + + Raises: + TimeoutError: if process does not terminate within timeout + """ + if process is None: + return + + process.terminate() + start_time = time.time() + + while process.poll() is None: + print(f"Terminating process {process.pid}") + if time.time() - start_time > timeout: + raise TimeoutError( + f"Process {process.pid} failed to terminate within {timeout}s" + ) + time.sleep(1) + + print(f"Process {process.pid} is successfully terminated") + + +class TestLaunchServer(unittest.TestCase): + def setUp(self): + self.model = DEFAULT_MODEL_NAME_FOR_TEST + self.base_url = DEFAULT_URL_FOR_TEST + self.process = None + self.other_process = [] + + def tearDown(self): + print("Running tearDown...") + if self.process: + terminate_and_wait(self.process) + for process in self.other_process: + terminate_and_wait(process) + print("tearDown done") + + def test_1_mmlu(self): + print("Running test_1_mmlu...") + # DP size = 2 + self.process = popen_launch_router( + self.model, + self.base_url, + dp_size=2, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + policy="cache_aware", + ) + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + score = metrics["score"] + THRESHOLD = 0.65 + passed = score >= THRESHOLD + msg = f"MMLU test {'passed' if passed else 'failed'} with score {score:.3f} (threshold: {THRESHOLD})" + self.assertGreaterEqual(score, THRESHOLD, msg) + + def test_2_add_and_remove_worker(self): + print("Running test_2_add_and_remove_worker...") + # DP size = 1 + self.process = popen_launch_router( + self.model, + self.base_url, + dp_size=1, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + policy="round_robin", # use round robin to make sure every worker processes requests + ) + # 1. start a worker + port = find_available_port() + worker_url = f"http://127.0.0.1:{port}" + worker_process = popen_launch_server( + self.model, worker_url, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + self.other_process.append(worker_process) + + # 2. use /add_worker api to add it the the router. It will be used by router after it is healthy + with requests.Session() as session: + response = session.post(f"{self.base_url}/add_worker?url={worker_url}") + print(f"status code: {response.status_code}, response: {response.text}") + self.assertEqual(response.status_code, 200) + + # 3. run mmlu + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + score = metrics["score"] + THRESHOLD = 0.65 + passed = score >= THRESHOLD + msg = f"MMLU test {'passed' if passed else 'failed'} with score {score:.3f} (threshold: {THRESHOLD})" + self.assertGreaterEqual(score, THRESHOLD, msg) + + # 4. use /remove_worker api to remove it from the router + with requests.Session() as session: + response = session.post(f"{self.base_url}/remove_worker?url={worker_url}") + print(f"status code: {response.status_code}, response: {response.text}") + self.assertEqual(response.status_code, 200) + + # 5. run mmlu again + metrics = run_eval(args) + score = metrics["score"] + THRESHOLD = 0.65 + passed = score >= THRESHOLD + msg = f"MMLU test {'passed' if passed else 'failed'} with score {score:.3f} (threshold: {THRESHOLD})" + self.assertGreaterEqual(score, THRESHOLD, msg) + + def test_3_lazy_fault_tolerance(self): + print("Running test_3_lazy_fault_tolerance...") + # DP size = 1 + self.process = popen_launch_router( + self.model, + self.base_url, + dp_size=1, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + policy="round_robin", + ) + + # 1. start a worker + port = find_available_port() + worker_url = f"http://127.0.0.1:{port}" + worker_process = popen_launch_server( + self.model, worker_url, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + self.other_process.append(worker_process) + + # 2. use /add_worker api to add it the the router. It will be used by router after it is healthy + with requests.Session() as session: + response = session.post(f"{self.base_url}/add_worker?url={worker_url}") + print(f"status code: {response.status_code}, response: {response.text}") + self.assertEqual(response.status_code, 200) + + # Start a thread to kill the worker after 10 seconds to mimic abrupt worker failure + def kill_worker(): + time.sleep(10) + kill_process_tree(worker_process.pid) + print("Worker process killed") + + import threading + + kill_thread = threading.Thread(target=kill_worker) + kill_thread.daemon = True + kill_thread.start() + + # 3. run mmlu + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=256, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + score = metrics["score"] + THRESHOLD = 0.65 + passed = score >= THRESHOLD + msg = f"MMLU test {'passed' if passed else 'failed'} with score {score:.3f} (threshold: {THRESHOLD})" + self.assertGreaterEqual(score, THRESHOLD, msg) + + def test_4_payload_size(self): + print("Running test_4_payload_size...") + # Start router with 3MB limit + self.process = popen_launch_router( + self.model, + self.base_url, + dp_size=1, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + policy="round_robin", + max_payload_size=1 * 1024 * 1024, # 1MB limit + ) + + # Test case 1: Payload just under 1MB should succeed + payload_0_5_mb = { + "text": "x" * int(0.5 * 1024 * 1024), # 0.5MB of text + "temperature": 0.0, + } + + with requests.Session() as session: + response = session.post( + f"{self.base_url}/generate", + json=payload_0_5_mb, + headers={"Content-Type": "application/json"}, + ) + self.assertEqual( + response.status_code, + 200, + f"0.5MB payload should succeed but got status {response.status_code}", + ) + + # Test case 2: Payload over 1MB should fail + payload_1_plus_mb = { + "text": "x" * int((1.2 * 1024 * 1024)), # 1.2MB of text + "temperature": 0.0, + } + + with requests.Session() as session: + response = session.post( + f"{self.base_url}/generate", + json=payload_1_plus_mb, + headers={"Content-Type": "application/json"}, + ) + self.assertEqual( + response.status_code, + 413, # Payload Too Large + f"1.2MB payload should fail with 413 but got status {response.status_code}", + ) + + def test_5_api_key(self): + print("Running test_5_api_key...") + + self.process = popen_launch_router( + self.model, + self.base_url, + dp_size=1, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + policy="round_robin", + api_key="correct_api_key", + ) + + # # Test case 1: request without api key should fail + with requests.Session() as session: + response = session.post( + f"{self.base_url}/generate", + json={"text": "Kanye west is, ", "temperature": 0}, + ) + print(f"status code: {response.status_code}, response: {response.text}") + self.assertEqual( + response.status_code, + 401, + "Request without api key should fail with 401", + ) + + # Test case 2: request with invalid api key should fail + with requests.Session() as session: + response = requests.post( + f"{self.base_url}/generate", + json={"text": "Kanye west is, ", "temperature": 0}, + headers={"Authorization": "Bearer 123"}, + ) + print(f"status code: {response.status_code}, response: {response.text}") + self.assertEqual( + response.status_code, + 401, + "Request with invalid api key should fail with 401", + ) + + # Test case 3: request with correct api key should succeed + with requests.Session() as session: + response = session.post( + f"{self.base_url}/generate", + json={"text": "Kanye west is ", "temperature": 0}, + headers={"Authorization": "Bearer correct_api_key"}, + ) + print(f"status code: {response.status_code}, response: {response.text}") + self.assertEqual( + response.status_code, 200, "Request with correct api key should succeed" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-router/pyproject.toml b/sgl-router/pyproject.toml new file mode 100644 index 0000000..b55067d --- /dev/null +++ b/sgl-router/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools-rust>=1.5.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang-router" +version = "0.1.4" +description = "SGLang router is a standalone module implemented in Rust to achieve data parallelism across SGLang instances." +authors = [{name = "Byron Hsu", email = "byronhsu1230@gmail.com"}] +requires-python = ">=3.8" +readme = "README.md" +license = { text = "Apache-2.0" } +classifiers = [ + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Rust", + "Programming Language :: Python :: 3", +] + +# https://github.com/PyO3/setuptools-rust?tab=readme-ov-file +[tool.setuptools.packages] +find = { where = ["py_src"] } + +# workaround for https://github.com/pypa/twine/issues/1216 +[tool.setuptools] +license-files = [] + +[[tool.setuptools-rust.ext-modules]] +target = "sglang_router_rs" +path = "Cargo.toml" +binding = "PyO3" diff --git a/sgl-router/src/lib.rs b/sgl-router/src/lib.rs new file mode 100644 index 0000000..ba9aeac --- /dev/null +++ b/sgl-router/src/lib.rs @@ -0,0 +1,123 @@ +use pyo3::prelude::*; +pub mod router; +pub mod server; +pub mod tree; + +#[pyclass(eq)] +#[derive(Clone, PartialEq)] +pub enum PolicyType { + Random, + RoundRobin, + CacheAware, +} + +#[pyclass] +struct Router { + host: String, + port: u16, + worker_urls: Vec, + policy: PolicyType, + worker_startup_timeout_secs: u64, + worker_startup_check_interval: u64, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + max_payload_size: usize, + verbose: bool, +} + +#[pymethods] +impl Router { + #[new] + #[pyo3(signature = ( + worker_urls, + policy = PolicyType::RoundRobin, + host = String::from("127.0.0.1"), + port = 3001, + worker_startup_timeout_secs = 300, + worker_startup_check_interval = 10, + cache_threshold = 0.50, + balance_abs_threshold = 32, + balance_rel_threshold = 1.0001, + eviction_interval_secs = 60, + max_tree_size = 2usize.pow(24), + max_payload_size = 4 * 1024 * 1024, + verbose = false + ))] + fn new( + worker_urls: Vec, + policy: PolicyType, + host: String, + port: u16, + worker_startup_timeout_secs: u64, + worker_startup_check_interval: u64, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + max_payload_size: usize, + verbose: bool, + ) -> PyResult { + Ok(Router { + host, + port, + worker_urls, + policy, + worker_startup_timeout_secs, + worker_startup_check_interval, + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + max_payload_size, + verbose, + }) + } + + fn start(&self) -> PyResult<()> { + let policy_config = match &self.policy { + PolicyType::Random => router::PolicyConfig::RandomConfig { + timeout_secs: self.worker_startup_timeout_secs, + interval_secs: self.worker_startup_check_interval, + }, + PolicyType::RoundRobin => router::PolicyConfig::RoundRobinConfig { + timeout_secs: self.worker_startup_timeout_secs, + interval_secs: self.worker_startup_check_interval, + }, + PolicyType::CacheAware => router::PolicyConfig::CacheAwareConfig { + timeout_secs: self.worker_startup_timeout_secs, + interval_secs: self.worker_startup_check_interval, + cache_threshold: self.cache_threshold, + balance_abs_threshold: self.balance_abs_threshold, + balance_rel_threshold: self.balance_rel_threshold, + eviction_interval_secs: self.eviction_interval_secs, + max_tree_size: self.max_tree_size, + }, + }; + + actix_web::rt::System::new().block_on(async move { + server::startup(server::ServerConfig { + host: self.host.clone(), + port: self.port, + worker_urls: self.worker_urls.clone(), + policy_config, + verbose: self.verbose, + max_payload_size: self.max_payload_size, + }) + .await + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + Ok(()) + }) + } +} + +#[pymodule] +fn sglang_router_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/sgl-router/src/router.rs b/sgl-router/src/router.rs new file mode 100644 index 0000000..e722732 --- /dev/null +++ b/sgl-router/src/router.rs @@ -0,0 +1,828 @@ +use crate::tree::Tree; +use actix_web::http::header::{HeaderValue, CONTENT_TYPE}; +use actix_web::{HttpRequest, HttpResponse}; +use bytes::Bytes; +use futures_util::{StreamExt, TryStreamExt}; +use log::{debug, error, info, warn}; +use serde_json::Value; +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::atomic::AtomicUsize; +use std::sync::{Arc, Mutex, RwLock}; +use std::thread; +use std::time::Duration; +use tokio; + +fn copy_request_headers(req: &HttpRequest) -> Vec<(String, String)> { + req.headers() + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|v| (name.to_string(), v.to_string())) + }) + .collect() +} + +#[derive(Debug)] +pub enum Router { + RoundRobin { + worker_urls: Arc>>, + current_index: AtomicUsize, + timeout_secs: u64, + interval_secs: u64, + }, + Random { + worker_urls: Arc>>, + timeout_secs: u64, + interval_secs: u64, + }, + CacheAware { + /* + Cache-Aware Load Balancing Router + + This router combines two strategies to optimize both cache utilization and request distribution: + + 1. Cache-Aware Routing (Approximate Tree) + 2. Load Balancing (Shortest Queue with Balance Thresholds) + + The router dynamically switches between these strategies based on load conditions: + - Uses load balancing when the system is imbalanced + - Uses cache-aware routing when the system is balanced + + A system is considered imbalanced if both conditions are met: + 1. (max - min) > abs_threshold + 2. max > rel_threshold * min + + Strategy Details: + + 1. Cache-Aware Routing (Approximate Tree) + ------------------------------------------- + This strategy maintains an approximate radix tree for each worker based on request history, + eliminating the need for direct cache state queries. The tree stores raw text characters + instead of token IDs to avoid tokenization overhead. + + Process: + a. For each request, find the worker with the highest prefix match + b. If match rate > cache_threshold: + Route to the worker with highest match (likely has relevant data cached) + c. If match rate ≤ cache_threshold: + Route to the worker with smallest tree size (most available cache capacity) + d. Background maintenance: + Periodically evict least recently used leaf nodes to prevent memory overflow + + 2. Load Balancing (Shortest Queue) + ------------------------------------------- + This strategy tracks pending request counts per worker and routes new requests + to the least busy worker when the system is detected to be imbalanced. + + Configuration Parameters: + ------------------------ + 1. cache_threshold: (float, 0.0 to 1.0) + Minimum prefix match ratio to use highest-match routing. + Below this threshold, routes to worker with most available cache space. + + 2. balance_abs_threshold: (integer) + Absolute difference threshold for load imbalance detection. + System is potentially imbalanced if (max_load - min_load) > abs_threshold + + 3. balance_rel_threshold: (float) + Relative ratio threshold for load imbalance detection. + System is potentially imbalanced if max_load > min_load * rel_threshold + Used in conjunction with abs_threshold to determine final imbalance state. + + 4. eviction_interval_secs: (integer) + Interval between LRU eviction cycles for the approximate trees. + + 5. max_tree_size: (integer) + Maximum nodes per tree. When exceeded, LRU leaf nodes are evicted + during the next eviction cycle. + */ + worker_urls: Arc>>, + tree: Arc>, + running_queue: Arc>>, + processed_queue: Arc>>, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + timeout_secs: u64, + interval_secs: u64, + _eviction_thread: Option>, + }, +} + +#[derive(Debug, Clone)] +pub enum PolicyConfig { + RandomConfig { + timeout_secs: u64, + interval_secs: u64, + }, + RoundRobinConfig { + timeout_secs: u64, + interval_secs: u64, + }, + CacheAwareConfig { + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + timeout_secs: u64, + interval_secs: u64, + }, +} + +impl Router { + pub fn new(worker_urls: Vec, policy_config: PolicyConfig) -> Result { + // Get timeout and interval from policy config + let (timeout_secs, interval_secs) = match &policy_config { + PolicyConfig::RandomConfig { + timeout_secs, + interval_secs, + } => (*timeout_secs, *interval_secs), + PolicyConfig::RoundRobinConfig { + timeout_secs, + interval_secs, + } => (*timeout_secs, *interval_secs), + PolicyConfig::CacheAwareConfig { + timeout_secs, + interval_secs, + .. + } => (*timeout_secs, *interval_secs), + }; + + // Wait until all workers are healthy + Self::wait_for_healthy_workers(&worker_urls, timeout_secs, interval_secs)?; + + // Create router based on policy... + Ok(match policy_config { + PolicyConfig::RandomConfig { + timeout_secs, + interval_secs, + } => Router::Random { + worker_urls: Arc::new(RwLock::new(worker_urls)), + timeout_secs, + interval_secs, + }, + PolicyConfig::RoundRobinConfig { + timeout_secs, + interval_secs, + } => Router::RoundRobin { + worker_urls: Arc::new(RwLock::new(worker_urls)), + current_index: std::sync::atomic::AtomicUsize::new(0), + timeout_secs, + interval_secs, + }, + PolicyConfig::CacheAwareConfig { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + timeout_secs, + interval_secs, + } => { + let mut running_queue = HashMap::new(); + for url in &worker_urls { + running_queue.insert(url.clone(), 0); + } + + let mut processed_queue = HashMap::new(); + for url in &worker_urls { + processed_queue.insert(url.clone(), 0); + } + + let tree = Arc::new(Mutex::new(Tree::new())); + let running_queue = Arc::new(Mutex::new(running_queue)); + let processed_queue = Arc::new(Mutex::new(processed_queue)); + + // Create background eviction thread + let tree_clone = Arc::clone(&tree); + let processed_queue_clone = Arc::clone(&processed_queue); + let running_queue_clone = Arc::clone(&running_queue); + let eviction_thread = thread::spawn(move || { + loop { + // Sleep for the specified interval + thread::sleep(Duration::from_secs(eviction_interval_secs)); + + let locked_tree_clone = tree_clone.lock().unwrap(); + // Run eviction + locked_tree_clone.evict_tenant_by_size(max_tree_size); + + // Print the process queue + let locked_processed_queue = processed_queue_clone.lock().unwrap(); + info!("Processed Queue: {:?}", locked_processed_queue); + + // Print the running queue + let locked_running_queue = running_queue_clone.lock().unwrap(); + info!("Running Queue: {:?}", locked_running_queue); + } + }); + + for url in &worker_urls { + tree.lock().unwrap().insert(&"".to_string(), url); + } + + Router::CacheAware { + worker_urls: Arc::new(RwLock::new(worker_urls)), + tree, + running_queue, + processed_queue, + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + timeout_secs, + interval_secs, + _eviction_thread: Some(eviction_thread), + } + } + }) + } + + fn wait_for_healthy_workers( + worker_urls: &[String], + timeout_secs: u64, + interval_secs: u64, + ) -> Result<(), String> { + let start_time = std::time::Instant::now(); + let sync_client = reqwest::blocking::Client::new(); + + loop { + if start_time.elapsed() > Duration::from_secs(timeout_secs) { + error!( + "Timeout {}s waiting for workers {:?} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + timeout_secs, worker_urls + ); + return Err(format!( + "Timeout {}s waiting for workers {:?} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + timeout_secs, worker_urls + )); + } + + let mut all_healthy = true; + let mut unhealthy_workers = Vec::new(); + + for url in worker_urls { + match sync_client.get(&format!("{}/health", url)).send() { + Ok(res) => { + if !res.status().is_success() { + let msg = format!( + "Worker heatlh check is pending with status {}", + res.status() + ); + info!("{}", msg); + all_healthy = false; + unhealthy_workers.push((url, msg)); + } + } + Err(_) => { + let msg = format!("Worker is not ready yet"); + info!("{}", msg); + all_healthy = false; + unhealthy_workers.push((url, msg)); + } + } + } + + if all_healthy { + info!("All workers are healthy"); + return Ok(()); + } else { + info!("Initializing workers:"); + for (url, reason) in &unhealthy_workers { + info!(" {} - {}", url, reason); + } + thread::sleep(Duration::from_secs(interval_secs)); + } + } + } + + fn select_first_worker(&self) -> Result { + match self { + Router::RoundRobin { worker_urls, .. } + | Router::Random { worker_urls, .. } + | Router::CacheAware { worker_urls, .. } => { + if worker_urls.read().unwrap().is_empty() { + Err("No workers are available".to_string()) + } else { + Ok(worker_urls.read().unwrap()[0].clone()) + } + } + } + } + + async fn send_request( + &self, + client: &reqwest::Client, + worker_url: &str, + route: &str, + req: &HttpRequest, + ) -> HttpResponse { + let mut request_builder = client.get(format!("{}{}", worker_url, route)); + + // Copy all headers from original request except for /health because it does not need authorization + if route != "/health" { + for (name, value) in copy_request_headers(req) { + request_builder = request_builder.header(name, value); + } + } + + match request_builder.send().await { + Ok(res) => { + let status = actix_web::http::StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR); + + match res.bytes().await { + Ok(body) => HttpResponse::build(status).body(body.to_vec()), + Err(e) => HttpResponse::InternalServerError() + .body(format!("Failed to read response body: {}", e)), + } + } + Err(e) => HttpResponse::InternalServerError().body(format!( + "Failed to send request to worker {}: {}", + worker_url, e + )), + } + } + + pub async fn route_to_first( + &self, + client: &reqwest::Client, + route: &str, + req: &HttpRequest, + ) -> HttpResponse { + const MAX_REQUEST_RETRIES: u32 = 3; + const MAX_TOTAL_RETRIES: u32 = 6; + let mut total_retries = 0; + + while total_retries < MAX_TOTAL_RETRIES { + match self.select_first_worker() { + Ok(worker_url) => { + let mut request_retries = 0; + + // Try the same worker multiple times + while request_retries < MAX_REQUEST_RETRIES { + if total_retries >= 1 { + info!("Retrying request after {} failed attempts", total_retries); + } + + let response = self.send_request(client, &worker_url, route, req).await; + + if response.status().is_success() { + return response; + } else { + // if the worker is healthy, it means the request is bad, so return the error response + let health_response = + self.send_request(client, &worker_url, "/health", req).await; + if health_response.status().is_success() { + return response; + } + } + + warn!( + "Request to {} failed (attempt {}/{})", + worker_url, + request_retries + 1, + MAX_REQUEST_RETRIES + ); + + request_retries += 1; + total_retries += 1; + + if request_retries == MAX_REQUEST_RETRIES { + warn!("Removing failed worker: {}", worker_url); + self.remove_worker(&worker_url); + break; + } + } + } + Err(e) => return HttpResponse::InternalServerError().body(e), + } + } + + HttpResponse::InternalServerError().body("All retry attempts failed") + } + + fn get_text_from_request(&self, body: &Bytes, route: &str) -> String { + // Convert body to JSON + let json: Value = match serde_json::from_slice(body) { + Ok(j) => j, + Err(_) => { + warn!("Failed to parse JSON from request body."); + return String::new(); + } + }; + + match route { + "/generate" => { + // For /generate, always use the "text" field. + match json.get("text").and_then(Value::as_str) { + Some(text) => text.to_string(), + None => { + warn!("No 'text' field found in request body for route /generate."); + String::new() + } + } + } + "/v1/chat/completions" | "/v1/completions" => { + // For these routes, try "messages", then "prompt", then "text". + if let Some(messages) = json.get("messages") { + serde_json::to_string(messages).unwrap_or_default() + } else if let Some(prompt) = json.get("prompt").and_then(Value::as_str) { + prompt.to_string() + } else { + warn!("Failed to find 'messages', 'prompt' in request body."); + String::new() + } + } + _ => { + warn!("Unknown route: {} - defaulting to fallback string", route); + String::new() + } + } + } + + // TODO: return Result instead of panicking + fn select_generate_worker(&self, body: &Bytes, route: &str) -> String { + let text = self.get_text_from_request(&body, route); + + let worker_url = match self { + Router::RoundRobin { + worker_urls, + current_index, + .. + } => { + let idx = current_index + .fetch_update( + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + |x| Some((x + 1) % worker_urls.read().unwrap().len()), + ) + .unwrap(); + worker_urls.read().unwrap()[idx].clone() + } + + Router::Random { worker_urls, .. } => worker_urls.read().unwrap() + [rand::random::() % worker_urls.read().unwrap().len()] + .clone(), + + Router::CacheAware { + worker_urls, + tree, + running_queue, + processed_queue, + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + .. + } => { + // TODO: delay scheduling if cache hit rate is high because it may cause imbalance. prioritize low hit rate ones + + let tree = tree.lock().unwrap(); + let mut running_queue = running_queue.lock().unwrap(); + + // Get current load statistics + let max_load = *running_queue.values().max().unwrap_or(&0); + let min_load = *running_queue.values().min().unwrap_or(&0); + + // Load is considered imbalanced if: + // 1. (max - min) > abs_threshold AND + // 2. max > rel_threshold * min + let is_imbalanced = max_load.saturating_sub(min_load) > *balance_abs_threshold + && (max_load as f32) > (min_load as f32 * balance_rel_threshold); + + let selected_url = if is_imbalanced { + // Log load balancing trigger and current queue state + info!( + "Load balancing triggered due to workload imbalance:\n\ + Max load: {}, Min load: {}\n\ + Current running queue: {:?}", + max_load, min_load, running_queue + ); + + // Use shortest queue routing when load is imbalanced + running_queue + .iter() + .min_by_key(|(_url, &count)| count) + .map(|(url, _)| url.clone()) + .unwrap_or_else(|| worker_urls.read().unwrap()[0].clone()) + } else { + // Use cache-aware routing when load is balanced + let (matched_text, matched_worker) = tree.prefix_match(&text); + let matched_rate = + matched_text.chars().count() as f32 / text.chars().count() as f32; + + if matched_rate > *cache_threshold { + matched_worker.to_string() + } else { + tree.get_smallest_tenant() + } + }; + + // Update queues and tree + *running_queue.get_mut(&selected_url).unwrap() += 1; + + *processed_queue + .lock() + .unwrap() + .get_mut(&selected_url) + .unwrap() += 1; + tree.insert(&text, &selected_url); + + selected_url + } + }; + + worker_url + } + + async fn send_generate_request( + &self, + client: &reqwest::Client, + req: &HttpRequest, + body: &Bytes, + route: &str, + worker_url: &str, + ) -> HttpResponse { + let is_stream = serde_json::from_slice::(&body) + .map(|v| v.get("stream").and_then(|s| s.as_bool()).unwrap_or(false)) + .unwrap_or(false); + + let mut request_builder = client + .post(format!("{}{}", worker_url, route)) + .body(body.to_vec()); + + // Copy all headers from original request + for (name, value) in copy_request_headers(req) { + request_builder = request_builder.header(name, value); + } + + let res = match request_builder.send().await { + Ok(res) => res, + Err(_) => return HttpResponse::InternalServerError().finish(), + }; + + let status = actix_web::http::StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(actix_web::http::StatusCode::INTERNAL_SERVER_ERROR); + + if !is_stream { + // For non-streaming requests, get response first + let response = match res.bytes().await { + Ok(body) => HttpResponse::build(status).body(body.to_vec()), + Err(e) => { + let error_msg = format!("Failed to get response body: {}", e); + HttpResponse::InternalServerError().body(error_msg) + } + }; + + // Then decrement running queue counter if using CacheAware + if let Router::CacheAware { running_queue, .. } = self { + if let Ok(mut queue) = running_queue.lock() { + if let Some(count) = queue.get_mut(worker_url) { + *count = count.saturating_sub(1); + } + } + } + + response + } else if let Router::CacheAware { running_queue, .. } = self { + let running_queue = Arc::clone(running_queue); + let worker_url = worker_url.to_string(); + + HttpResponse::build(status) + .insert_header((CONTENT_TYPE, HeaderValue::from_static("text/event-stream"))) + .streaming( + res.bytes_stream() + .map_err(|_| { + actix_web::error::ErrorInternalServerError("Failed to read stream") + }) + .inspect(move |bytes| { + let bytes = bytes.as_ref().unwrap(); + if bytes + .as_ref() + .windows(12) + .any(|window| window == b"data: [DONE]") + { + let mut locked_queue = running_queue.lock().unwrap(); + let count = locked_queue.get_mut(&worker_url).unwrap(); + *count = count.saturating_sub(1); + debug!("Streaming is done!!") + } + }), + ) + } else { + HttpResponse::build(status) + .insert_header((CONTENT_TYPE, HeaderValue::from_static("text/event-stream"))) + .streaming(res.bytes_stream().map_err(|_| { + actix_web::error::ErrorInternalServerError("Failed to read stream") + })) + } + } + + pub async fn route_generate_request( + &self, + client: &reqwest::Client, + req: &HttpRequest, + body: &Bytes, + route: &str, + ) -> HttpResponse { + const MAX_REQUEST_RETRIES: u32 = 3; + const MAX_TOTAL_RETRIES: u32 = 6; + let mut total_retries = 0; + + while total_retries < MAX_TOTAL_RETRIES { + let worker_url = self.select_generate_worker(body, route); + let mut request_retries = 0; + + // Try the same worker multiple times + while request_retries < MAX_REQUEST_RETRIES { + if total_retries >= 1 { + info!("Retrying request after {} failed attempts", total_retries); + } + let response = self + .send_generate_request(client, req, body, route, &worker_url) + .await; + + if response.status().is_success() { + return response; + } else { + // if the worker is healthy, it means the request is bad, so return the error response + let health_response = + self.send_request(client, &worker_url, "/health", req).await; + if health_response.status().is_success() { + return response; + } + } + + warn!( + "Generate request to {} failed (attempt {}/{})", + worker_url, + request_retries + 1, + MAX_REQUEST_RETRIES + ); + + request_retries += 1; + total_retries += 1; + + if request_retries == MAX_REQUEST_RETRIES { + warn!("Removing failed worker: {}", worker_url); + self.remove_worker(&worker_url); + break; + } + } + } + + HttpResponse::InternalServerError().body("All retry attempts failed") + } + + pub async fn add_worker(&self, worker_url: &str) -> Result { + let (timeout_secs, interval_secs) = match self { + Router::Random { + timeout_secs, + interval_secs, + .. + } => (*timeout_secs, *interval_secs), + Router::RoundRobin { + timeout_secs, + interval_secs, + .. + } => (*timeout_secs, *interval_secs), + Router::CacheAware { + timeout_secs, + interval_secs, + .. + } => (*timeout_secs, *interval_secs), + }; + + let start_time = std::time::Instant::now(); + let client = reqwest::Client::new(); + + loop { + if start_time.elapsed() > Duration::from_secs(timeout_secs) { + error!( + "Timeout {}s waiting for worker {} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + timeout_secs, worker_url + ); + return Err(format!( + "Timeout {}s waiting for worker {} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + timeout_secs, worker_url + )); + } + + match client.get(&format!("{}/health", worker_url)).send().await { + Ok(res) => { + if res.status().is_success() { + match self { + Router::RoundRobin { worker_urls, .. } + | Router::Random { worker_urls, .. } + | Router::CacheAware { worker_urls, .. } => { + info!("Worker {} health check passed", worker_url); + let mut urls = worker_urls.write().unwrap(); + if urls.contains(&worker_url.to_string()) { + return Err(format!("Worker {} already exists", worker_url)); + } + info!("Added worker: {}", worker_url); + urls.push(worker_url.to_string()); + } + } + + // If cache aware, initialize the queues for the new worker + if let Router::CacheAware { + running_queue, + processed_queue, + tree, + .. + } = self + { + // Add worker to running queue with initial count of 0 + running_queue + .lock() + .unwrap() + .insert(worker_url.to_string(), 0); + + // Add worker to processed queue with initial count of 0 + processed_queue + .lock() + .unwrap() + .insert(worker_url.to_string(), 0); + + // Add worker to tree + tree.lock().unwrap().insert(&"".to_string(), &worker_url); + } + + return Ok(format!("Successfully added worker: {}", worker_url)); + } else { + info!( + "Worker {} health check is pending with status: {}.", + worker_url, + res.status() + ); + // if the url does not have http or https prefix, warn users + if !worker_url.starts_with("http://") && !worker_url.starts_with("https://") + { + warn!("The worker url {} does not have http or https prefix. Please add the prefix to the url.", worker_url); + } + + tokio::time::sleep(Duration::from_secs(interval_secs)).await; + continue; + } + } + Err(e) => { + info!( + "Worker {} health check is pending with error: {}", + worker_url, e + ); + + // if the url does not have http or https prefix, warn users + if !worker_url.starts_with("http://") && !worker_url.starts_with("https://") { + warn!("The worker url {} does not have http or https prefix. Please add the prefix to the url.", worker_url); + } + + tokio::time::sleep(Duration::from_secs(interval_secs)).await; + continue; + } + } + } + } + + pub fn remove_worker(&self, worker_url: &str) { + match self { + Router::RoundRobin { worker_urls, .. } + | Router::Random { worker_urls, .. } + | Router::CacheAware { worker_urls, .. } => { + let mut urls = worker_urls.write().unwrap(); + if let Some(index) = urls.iter().position(|url| url == &worker_url) { + urls.remove(index); + info!("Removed worker: {}", worker_url); + } else { + warn!("Worker {} not found, skipping removal", worker_url); + return; + } + } + } + + // if cache aware, remove the worker from the tree + if let Router::CacheAware { + tree, + running_queue, + processed_queue, + .. + } = self + { + tree.lock().unwrap().remove_tenant(&worker_url); + running_queue + .lock() + .unwrap() + .remove(&worker_url.to_string()); + processed_queue + .lock() + .unwrap() + .remove(&worker_url.to_string()); + info!( + "Removed worker from tree and cleaned up queues: {}", + worker_url + ); + } + } +} diff --git a/sgl-router/src/server.rs b/sgl-router/src/server.rs new file mode 100644 index 0000000..a381cdc --- /dev/null +++ b/sgl-router/src/server.rs @@ -0,0 +1,226 @@ +use crate::router::PolicyConfig; +use crate::router::Router; +use actix_web::{ + error, get, post, web, App, Error, HttpRequest, HttpResponse, HttpServer, Responder, +}; +use bytes::Bytes; +use env_logger::Builder; +use futures_util::StreamExt; +use log::{info, LevelFilter}; +use std::collections::HashMap; +use std::io::Write; +use std::time::Duration; + +#[derive(Debug)] +pub struct AppState { + router: Router, + client: reqwest::Client, +} + +impl AppState { + pub fn new( + worker_urls: Vec, + client: reqwest::Client, + policy_config: PolicyConfig, + ) -> Result { + // Create router based on policy + let router = Router::new(worker_urls, policy_config)?; + Ok(Self { router, client }) + } +} + +async fn sink_handler(_req: HttpRequest, mut payload: web::Payload) -> Result { + // Drain the payload + while let Some(chunk) = payload.next().await { + if let Err(err) = chunk { + println!("Error while draining payload: {:?}", err); + break; + } + } + Ok(HttpResponse::NotFound().finish()) +} + +// Custom error handler for JSON payload errors. +fn json_error_handler(_err: error::JsonPayloadError, _req: &HttpRequest) -> Error { + error::ErrorPayloadTooLarge("Payload too large") +} + +#[get("/health")] +async fn health(req: HttpRequest, data: web::Data) -> impl Responder { + data.router + .route_to_first(&data.client, "/health", &req) + .await +} + +#[get("/health_generate")] +async fn health_generate(req: HttpRequest, data: web::Data) -> impl Responder { + data.router + .route_to_first(&data.client, "/health_generate", &req) + .await +} + +#[get("/get_server_info")] +async fn get_server_info(req: HttpRequest, data: web::Data) -> impl Responder { + data.router + .route_to_first(&data.client, "/get_server_info", &req) + .await +} + +#[get("/v1/models")] +async fn v1_models(req: HttpRequest, data: web::Data) -> impl Responder { + data.router + .route_to_first(&data.client, "/v1/models", &req) + .await +} + +#[get("/get_model_info")] +async fn get_model_info(req: HttpRequest, data: web::Data) -> impl Responder { + data.router + .route_to_first(&data.client, "/get_model_info", &req) + .await +} + +#[post("/generate")] +async fn generate(req: HttpRequest, body: Bytes, data: web::Data) -> impl Responder { + data.router + .route_generate_request(&data.client, &req, &body, "/generate") + .await +} + +#[post("/v1/chat/completions")] +async fn v1_chat_completions( + req: HttpRequest, + body: Bytes, + data: web::Data, +) -> impl Responder { + data.router + .route_generate_request(&data.client, &req, &body, "/v1/chat/completions") + .await +} + +#[post("/v1/completions")] +async fn v1_completions( + req: HttpRequest, + body: Bytes, + data: web::Data, +) -> impl Responder { + data.router + .route_generate_request(&data.client, &req, &body, "/v1/completions") + .await +} + +#[post("/add_worker")] +async fn add_worker( + query: web::Query>, + data: web::Data, +) -> impl Responder { + let worker_url = match query.get("url") { + Some(url) => url.to_string(), + None => { + return HttpResponse::BadRequest() + .body("Worker URL required. Provide 'url' query parameter") + } + }; + + match data.router.add_worker(&worker_url).await { + Ok(message) => HttpResponse::Ok().body(message), + Err(error) => HttpResponse::BadRequest().body(error), + } +} + +#[post("/remove_worker")] +async fn remove_worker( + query: web::Query>, + data: web::Data, +) -> impl Responder { + let worker_url = match query.get("url") { + Some(url) => url.to_string(), + None => return HttpResponse::BadRequest().finish(), + }; + data.router.remove_worker(&worker_url); + HttpResponse::Ok().body(format!("Successfully removed worker: {}", worker_url)) +} + +pub struct ServerConfig { + pub host: String, + pub port: u16, + pub worker_urls: Vec, + pub policy_config: PolicyConfig, + pub verbose: bool, + pub max_payload_size: usize, +} + +pub async fn startup(config: ServerConfig) -> std::io::Result<()> { + // Initialize logger + Builder::new() + .format(|buf, record| { + use chrono::Local; + writeln!( + buf, + "[Router (Rust)] {} - {} - {}", + Local::now().format("%Y-%m-%d %H:%M:%S"), + record.level(), + record.args() + ) + }) + .filter( + None, + if config.verbose { + LevelFilter::Debug + } else { + LevelFilter::Info + }, + ) + .init(); + + info!("🚧 Initializing router on {}:{}", config.host, config.port); + info!("🚧 Initializing workers on {:?}", config.worker_urls); + info!("🚧 Policy Config: {:?}", config.policy_config); + info!( + "🚧 Max payload size: {} MB", + config.max_payload_size / (1024 * 1024) + ); + + let client = reqwest::Client::builder() + .pool_idle_timeout(Some(Duration::from_secs(50))) + .build() + .expect("Failed to create HTTP client"); + + let app_state = web::Data::new( + AppState::new( + config.worker_urls.clone(), + client, + config.policy_config.clone(), + ) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?, + ); + + info!("✅ Serving router on {}:{}", config.host, config.port); + info!("✅ Serving workers on {:?}", config.worker_urls); + + HttpServer::new(move || { + App::new() + .app_data(app_state.clone()) + .app_data( + web::JsonConfig::default() + .limit(config.max_payload_size) + .error_handler(json_error_handler), + ) + .app_data(web::PayloadConfig::default().limit(config.max_payload_size)) + .service(generate) + .service(v1_chat_completions) + .service(v1_completions) + .service(v1_models) + .service(get_model_info) + .service(health) + .service(health_generate) + .service(get_server_info) + .service(add_worker) + .service(remove_worker) + // Default handler for unmatched routes. + .default_service(web::route().to(sink_handler)) + }) + .bind((config.host, config.port))? + .run() + .await +} diff --git a/sgl-router/src/tree.rs b/sgl-router/src/tree.rs new file mode 100644 index 0000000..e8dc8b7 --- /dev/null +++ b/sgl-router/src/tree.rs @@ -0,0 +1,1483 @@ +use dashmap::mapref::entry::Entry; +use dashmap::DashMap; +use log::info; + +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::collections::HashMap; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::RwLock; + +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; + +type NodeRef = Arc; + +#[derive(Debug)] +struct Node { + children: DashMap, + text: RwLock, + tenant_last_access_time: DashMap, + parent: RwLock>, +} + +#[derive(Debug)] +pub struct Tree { + root: NodeRef, + pub tenant_char_count: DashMap, +} + +// For the heap + +struct EvictionEntry { + timestamp: u128, + tenant: String, + node: NodeRef, +} + +impl Eq for EvictionEntry {} + +impl PartialOrd for EvictionEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.timestamp.cmp(&other.timestamp)) + } +} + +impl Ord for EvictionEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.timestamp.cmp(&other.timestamp) + } +} + +impl PartialEq for EvictionEntry { + fn eq(&self, other: &Self) -> bool { + self.timestamp == other.timestamp + } +} + +// For char operations +// Note that in rust, `.len()` or slice is operated on the "byte" level. It causes issues for UTF-8 characters because one character might use multiple bytes. +// https://en.wikipedia.org/wiki/UTF-8 + +fn shared_prefix_count(a: &str, b: &str) -> usize { + let mut i = 0; + let mut a_iter = a.chars(); + let mut b_iter = b.chars(); + + loop { + match (a_iter.next(), b_iter.next()) { + (Some(a_char), Some(b_char)) if a_char == b_char => { + i += 1; + } + _ => break, + } + } + + return i; +} + +fn slice_by_chars(s: &str, start: usize, end: usize) -> String { + s.chars().skip(start).take(end - start).collect() +} + +impl Tree { + /* + Thread-safe multi tenant radix tree + + 1. Storing data for multiple tenants (the overlap of multiple radix tree) + 2. Node-level lock to enable concurrent acesss on nodes + 3. Leaf LRU eviction based on tenant access time + */ + + pub fn new() -> Self { + Tree { + root: Arc::new(Node { + children: DashMap::new(), + text: RwLock::new("".to_string()), + tenant_last_access_time: DashMap::new(), + parent: RwLock::new(None), + }), + tenant_char_count: DashMap::new(), + } + } + + pub fn insert(&self, text: &str, tenant: &str) { + // Insert text into tree with given tenant + + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + curr.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + self.tenant_char_count + .entry(tenant.to_string()) + .or_insert(0); + + let mut prev = Arc::clone(&self.root); + + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + + curr = prev; + + // dashmap.entry locks the entry until the op is done + // if using contains_key + insert, there will be an issue that + // 1. "apple" and "app" entered at the same time + // 2. and get inserted to the dashmap concurrently, so only one is inserted + + match curr.children.entry(first_char) { + Entry::Vacant(entry) => { + /* + no matched + [curr] + becomes + [curr] => [new node] + */ + + let curr_text = slice_by_chars(text, curr_idx, text_count); + let curr_text_count = curr_text.chars().count(); + let new_node = Arc::new(Node { + children: DashMap::new(), + text: RwLock::new(curr_text), + tenant_last_access_time: DashMap::new(), + parent: RwLock::new(Some(Arc::clone(&curr))), + }); + + // Increment char count when creating new node with tenant + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += curr_text_count) + .or_insert(curr_text_count); + + new_node + .tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + entry.insert(Arc::clone(&new_node)); + + prev = Arc::clone(&new_node); + curr_idx = text_count; + } + + Entry::Occupied(mut entry) => { + // matched + let matched_node = entry.get().clone(); + + let matched_node_text = matched_node.text.read().unwrap().to_owned(); + let matched_node_text_count = matched_node_text.chars().count(); + + let curr_text = slice_by_chars(text, curr_idx, text_count); + let shared_count = shared_prefix_count(&matched_node_text, &curr_text); + + if shared_count < matched_node_text_count { + /* + split the matched node + [curr] -> [matched_node] => + becomes + [curr] -> [new_node] -> [contracted_matched_node] + */ + + let matched_text = slice_by_chars(&matched_node_text, 0, shared_count); + let contracted_text = slice_by_chars( + &matched_node_text, + shared_count, + matched_node_text_count, + ); + let matched_text_count = matched_text.chars().count(); + + let new_node = Arc::new(Node { + text: RwLock::new(matched_text), + children: DashMap::new(), + parent: RwLock::new(Some(Arc::clone(&curr))), + tenant_last_access_time: matched_node.tenant_last_access_time.clone(), + }); + + let first_new_char = contracted_text.chars().nth(0).unwrap(); + new_node + .children + .insert(first_new_char, Arc::clone(&matched_node)); + + entry.insert(Arc::clone(&new_node)); + + *matched_node.text.write().unwrap() = contracted_text; + *matched_node.parent.write().unwrap() = Some(Arc::clone(&new_node)); + + prev = Arc::clone(&new_node); + + // Increment char count for the tenant in the new split node + if !prev.tenant_last_access_time.contains_key(tenant) { + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += matched_text_count) + .or_insert(matched_text_count); + } + + prev.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + curr_idx += shared_count; + } else { + // move to next node + prev = Arc::clone(&matched_node); + + // Increment char count when adding tenant to existing node + if !prev.tenant_last_access_time.contains_key(tenant) { + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += matched_node_text_count) + .or_insert(matched_node_text_count); + } + + prev.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + curr_idx += shared_count; + } + } + } + } + } + + #[allow(unused_assignments)] + pub fn prefix_match(&self, text: &str) -> (String, String) { + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let mut prev = Arc::clone(&self.root); + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + let curr_text = slice_by_chars(text, curr_idx, text_count); + + curr = prev.clone(); + + match curr.children.entry(first_char) { + Entry::Occupied(entry) => { + let matched_node = entry.get().clone(); + let shared_count = + shared_prefix_count(&matched_node.text.read().unwrap(), &curr_text); + + let matched_node_text_count = matched_node.text.read().unwrap().chars().count(); + + if shared_count == matched_node_text_count { + // Full match with current node's text, continue to next node + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + } else { + // Partial match, stop here + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + break; + } + } + Entry::Vacant(_) => { + // No match found, stop here + break; + } + } + } + + curr = prev.clone(); + + // Select the first tenant (key in the map) + let tenant = curr + .tenant_last_access_time + .iter() + .next() + .map(|kv| kv.key().to_owned()) + .unwrap_or("empty".to_string()); + + // Traverse from the curr node to the root and update the timestamp + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + if !tenant.eq("empty") { + let mut current_node = Some(curr); + while let Some(node) = current_node { + node.tenant_last_access_time + .insert(tenant.clone(), timestamp_ms); + current_node = node.parent.read().unwrap().clone(); + } + } + + let ret_text = slice_by_chars(text, 0, curr_idx); + (ret_text, tenant) + } + + #[allow(unused_assignments)] + pub fn prefix_match_tenant(&self, text: &str, tenant: &str) -> String { + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let mut prev = Arc::clone(&self.root); + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + let curr_text = slice_by_chars(text, curr_idx, text_count); + + curr = prev.clone(); + + match curr.children.entry(first_char) { + Entry::Occupied(entry) => { + let matched_node = entry.get().clone(); + + // Only continue matching if this node belongs to the specified tenant + if !matched_node.tenant_last_access_time.contains_key(tenant) { + break; + } + + let shared_count = + shared_prefix_count(&matched_node.text.read().unwrap(), &curr_text); + + let matched_node_text_count = matched_node.text.read().unwrap().chars().count(); + + if shared_count == matched_node_text_count { + // Full match with current node's text, continue to next node + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + } else { + // Partial match, stop here + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + break; + } + } + Entry::Vacant(_) => { + // No match found, stop here + break; + } + } + } + + curr = prev.clone(); + + // Only update timestamp if we found a match for the specified tenant + if curr.tenant_last_access_time.contains_key(tenant) { + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + let mut current_node = Some(curr); + while let Some(node) = current_node { + node.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + current_node = node.parent.read().unwrap().clone(); + } + } + + slice_by_chars(text, 0, curr_idx) + } + + fn leaf_of(node: &NodeRef) -> Vec { + /* + Return the list of tenants if it's a leaf for the tenant + */ + let mut candidates: HashMap = node + .tenant_last_access_time + .iter() + .map(|entry| (entry.key().clone(), true)) + .collect(); + + for child in node.children.iter() { + for tenant in child.value().tenant_last_access_time.iter() { + candidates.insert(tenant.key().clone(), false); + } + } + + candidates + .into_iter() + .filter(|(_, is_leaf)| *is_leaf) + .map(|(tenant, _)| tenant) + .collect() + } + + pub fn evict_tenant_by_size(&self, max_size: usize) { + // Calculate used size and collect leaves + let mut stack = vec![Arc::clone(&self.root)]; + let mut pq = BinaryHeap::new(); + + while let Some(curr) = stack.pop() { + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + + // Add leaves to priority queue + for tenant in Tree::leaf_of(&curr) { + if let Some(timestamp) = curr.tenant_last_access_time.get(&tenant) { + pq.push(Reverse(EvictionEntry { + timestamp: *timestamp, + tenant: tenant.clone(), + node: Arc::clone(&curr), + })); + } + } + } + + info!("Before eviction - Used size per tenant:"); + for entry in self.tenant_char_count.iter() { + info!("Tenant: {}, Size: {}", entry.key(), entry.value()); + } + + // Process eviction + while let Some(Reverse(entry)) = pq.pop() { + let EvictionEntry { tenant, node, .. } = entry; + + if let Some(used_size) = self.tenant_char_count.get(&tenant) { + if *used_size <= max_size { + continue; + } + } + + // Decrement when removing tenant from node + if node.tenant_last_access_time.contains_key(&tenant) { + self.tenant_char_count + .entry(tenant.clone()) + .and_modify(|count| { + if *count > 0 { + *count -= node.text.read().unwrap().chars().count(); + } + }); + } + + // Remove tenant from node + node.tenant_last_access_time.remove(&tenant); + + // Remove empty nodes + if node.children.is_empty() && node.tenant_last_access_time.is_empty() { + if let Some(parent) = node.parent.write().unwrap().as_ref() { + let first_char = node.text.read().unwrap().chars().next().unwrap(); + parent.children.remove(&first_char); + } + } + + // Add parent to queue if it becomes a leaf + if let Some(parent) = node.parent.read().unwrap().as_ref() { + if Tree::leaf_of(parent).contains(&tenant) { + if let Some(timestamp) = parent.tenant_last_access_time.get(&tenant) { + pq.push(Reverse(EvictionEntry { + timestamp: *timestamp, + tenant: tenant.clone(), + node: Arc::clone(parent), + })); + } + } + }; + } + + info!("After eviction - Used size per tenant:"); + for entry in self.tenant_char_count.iter() { + info!("Tenant: {}, Size: {}", entry.key(), entry.value()); + } + } + + pub fn remove_tenant(&self, tenant: &str) { + // 1. Find all the leaves for the tenant + let mut stack = vec![Arc::clone(&self.root)]; + let mut queue = VecDeque::new(); + + while let Some(curr) = stack.pop() { + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + + if Tree::leaf_of(&curr).contains(&tenant.to_string()) { + queue.push_back(Arc::clone(&curr)); + } + } + + // 2. Start from the leaves and traverse up to the root, removing the tenant from each node + while let Some(curr) = queue.pop_front() { + // remove tenant from node + curr.tenant_last_access_time.remove(&tenant.to_string()); + + // remove empty nodes + if curr.children.is_empty() && curr.tenant_last_access_time.is_empty() { + if let Some(parent) = curr.parent.read().unwrap().as_ref() { + let first_char = curr.text.read().unwrap().chars().next().unwrap(); + parent.children.remove(&first_char); + } + } + + // add parent to queue if it becomes a leaf + if let Some(parent) = curr.parent.read().unwrap().as_ref() { + if Tree::leaf_of(parent).contains(&tenant.to_string()) { + queue.push_back(Arc::clone(&parent)); + } + } + } + + // 3. Remove the tenant from the tenant_char_count map + self.tenant_char_count.remove(&tenant.to_string()); + } + + pub fn get_tenant_char_count(&self) -> HashMap { + self.tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect() + } + + pub fn get_smallest_tenant(&self) -> String { + // Return a placeholder if there are no tenants + if self.tenant_char_count.is_empty() { + return "empty".to_string(); + } + + // Find the tenant with minimum char count + let mut min_tenant = None; + let mut min_count = usize::MAX; + + for entry in self.tenant_char_count.iter() { + let tenant = entry.key(); + let count = *entry.value(); + + if count < min_count { + min_count = count; + min_tenant = Some(tenant.clone()); + } + } + + // Return the found tenant or "empty" if somehow none was found + min_tenant.unwrap_or_else(|| "empty".to_string()) + } + + pub fn get_used_size_per_tenant(&self) -> HashMap { + // perform a DFS to traverse all nodes and calculate the total size used by each tenant + + let mut used_size_per_tenant: HashMap = HashMap::new(); + let mut stack = vec![Arc::clone(&self.root)]; + + while let Some(curr) = stack.pop() { + let text_count = curr.text.read().unwrap().chars().count(); + + for tenant in curr.tenant_last_access_time.iter() { + let size = used_size_per_tenant + .entry(tenant.key().clone()) + .or_insert(0); + *size += text_count; + } + + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + } + + used_size_per_tenant + } + + fn node_to_string(node: &NodeRef, prefix: &str, is_last: bool) -> String { + let mut result = String::new(); + + // Add prefix and branch character + result.push_str(prefix); + result.push_str(if is_last { "└── " } else { "├── " }); + + // Add node text + let node_text = node.text.read().unwrap(); + result.push_str(&format!("'{}' [", node_text)); + + // Add tenant information with timestamps + let mut tenant_info = Vec::new(); + for entry in node.tenant_last_access_time.iter() { + let tenant_id = entry.key(); + let timestamp_ms = entry.value(); + + // Convert milliseconds to seconds and remaining milliseconds + let seconds = (timestamp_ms / 1000) as u64; + let millis = (timestamp_ms % 1000) as u32; + + // Create SystemTime from Unix timestamp + let system_time = UNIX_EPOCH + Duration::from_secs(seconds); + + // Format time as HH:MM:SS.mmm + let datetime = system_time.duration_since(UNIX_EPOCH).unwrap(); + let hours = (datetime.as_secs() % 86400) / 3600; + let minutes = (datetime.as_secs() % 3600) / 60; + let seconds = datetime.as_secs() % 60; + + tenant_info.push(format!( + "{} | {:02}:{:02}:{:02}.{:03}", + tenant_id, hours, minutes, seconds, millis + )); + } + + result.push_str(&tenant_info.join(", ")); + result.push_str("]\n"); + + // Process children + let children: Vec<_> = node.children.iter().collect(); + let child_count = children.len(); + + for (i, entry) in children.iter().enumerate() { + let is_last_child = i == child_count - 1; + let new_prefix = format!("{}{}", prefix, if is_last { " " } else { "│ " }); + + result.push_str(&Tree::node_to_string( + entry.value(), + &new_prefix, + is_last_child, + )); + } + + result + } + + pub fn pretty_print(&self) { + if self.root.children.is_empty() { + return; + } + + let mut result = String::new(); + let children: Vec<_> = self.root.children.iter().collect(); + let child_count = children.len(); + + for (i, entry) in children.iter().enumerate() { + let is_last = i == child_count - 1; + result.push_str(&Tree::node_to_string(entry.value(), "", is_last)); + } + + println!("{result}"); + + return; + } +} + +// Unit tests +#[cfg(test)] +mod tests { + use rand::distributions::Alphanumeric; + use rand::distributions::DistString; + use rand::thread_rng; + use rand::Rng; + use std::thread; + use std::time::Instant; + + use super::*; + + #[test] + fn test_get_smallest_tenant() { + let tree = Tree::new(); + + // Test empty tree + assert_eq!(tree.get_smallest_tenant(), "empty"); + + // Insert data for tenant1 - "ap" + "icot" = 6 chars + tree.insert("ap", "tenant1"); + tree.insert("icot", "tenant1"); + + // Insert data for tenant2 - "cat" = 3 chars + tree.insert("cat", "tenant2"); + + // Test - tenant2 should be smallest with 3 chars vs 6 chars + assert_eq!( + tree.get_smallest_tenant(), + "tenant2", + "Expected tenant2 to be smallest with 3 characters." + ); + + // Insert overlapping data for tenant3 and tenant4 to test equal counts + // tenant3: "do" = 2 chars + // tenant4: "hi" = 2 chars + tree.insert("do", "tenant3"); + tree.insert("hi", "tenant4"); + + // Test - should return either tenant3 or tenant4 (both have 2 chars) + let smallest = tree.get_smallest_tenant(); + assert!( + smallest == "tenant3" || smallest == "tenant4", + "Expected either tenant3 or tenant4 (both have 2 characters), got {}", + smallest + ); + + // Add more text to tenant4 to make it larger + tree.insert("hello", "tenant4"); // Now tenant4 has "hi" + "hello" = 6 chars + + // Now tenant3 should be smallest (2 chars vs 6 chars for tenant4) + assert_eq!( + tree.get_smallest_tenant(), + "tenant3", + "Expected tenant3 to be smallest with 2 characters" + ); + + // Test eviction + tree.evict_tenant_by_size(3); // This should evict tenants with more than 3 chars + + let post_eviction_smallest = tree.get_smallest_tenant(); + println!("Smallest tenant after eviction: {}", post_eviction_smallest); + } + + #[test] + fn test_tenant_char_count() { + let tree = Tree::new(); + + // Phase 1: Initial insertions + tree.insert("apple", "tenant1"); + tree.insert("apricot", "tenant1"); + tree.insert("banana", "tenant1"); + tree.insert("amplify", "tenant2"); + tree.insert("application", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 1 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 1: Initial insertions" + ); + + // Phase 2: Additional insertions + tree.insert("apartment", "tenant1"); + tree.insert("appetite", "tenant2"); + tree.insert("ball", "tenant1"); + tree.insert("box", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 2 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 2: Additional insertions" + ); + + // Phase 3: Overlapping insertions + tree.insert("zebra", "tenant1"); + tree.insert("zebra", "tenant2"); + tree.insert("zero", "tenant1"); + tree.insert("zero", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 3 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 3: Overlapping insertions" + ); + + // Phase 4: Eviction test + tree.evict_tenant_by_size(10); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 4 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!(maintained_counts, computed_sizes, "Phase 4: After eviction"); + } + + fn random_string(len: usize) -> String { + Alphanumeric.sample_string(&mut thread_rng(), len) + } + + #[test] + fn test_cold_start() { + let tree = Tree::new(); + + let (matched_text, tenant) = tree.prefix_match("hello"); + + assert_eq!(matched_text, ""); + assert_eq!(tenant, "empty"); + } + + #[test] + fn test_exact_match_seq() { + let tree = Tree::new(); + tree.insert("hello", "tenant1"); + tree.pretty_print(); + tree.insert("apple", "tenant2"); + tree.pretty_print(); + tree.insert("banana", "tenant3"); + tree.pretty_print(); + + let (matched_text, tenant) = tree.prefix_match("hello"); + assert_eq!(matched_text, "hello"); + assert_eq!(tenant, "tenant1"); + + let (matched_text, tenant) = tree.prefix_match("apple"); + assert_eq!(matched_text, "apple"); + assert_eq!(tenant, "tenant2"); + + let (matched_text, tenant) = tree.prefix_match("banana"); + assert_eq!(matched_text, "banana"); + assert_eq!(tenant, "tenant3"); + } + + #[test] + fn test_exact_match_concurrent() { + let tree = Arc::new(Tree::new()); + + // spawn 3 threads for insert + let tree_clone = Arc::clone(&tree); + + let texts = vec!["hello", "apple", "banana"]; + let tenants = vec!["tenant1", "tenant2", "tenant3"]; + + let mut handles = vec![]; + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = tenants[i]; + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + // spawn 3 threads for match + let mut handles = vec![]; + + let tree_clone = Arc::clone(&tree); + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = tenants[i]; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_partial_match_concurrent() { + let tree = Arc::new(Tree::new()); + + // spawn 3 threads for insert + let tree_clone = Arc::clone(&tree); + + let texts = vec!["apple", "apabc", "acbdeds"]; + + let mut handles = vec![]; + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = "tenant0"; + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + // spawn 3 threads for match + let mut handles = vec![]; + + let tree_clone = Arc::clone(&tree); + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = "tenant0"; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_group_prefix_insert_match_concurrent() { + let prefix = vec![ + "Clock strikes midnight, I'm still wide awake", + "Got dreams bigger than these city lights", + "Time waits for no one, gotta make my move", + "Started from the bottom, that's no metaphor", + ]; + let suffix = vec![ + "Got too much to prove, ain't got time to lose", + "History in the making, yeah, you can't erase this", + ]; + let tree = Arc::new(Tree::new()); + + let mut handles = vec![]; + + for i in 0..prefix.len() { + for j in 0..suffix.len() { + let tree_clone = Arc::clone(&tree); + let text = format!("{} {}", prefix[i], suffix[j]); + let tenant = format!("tenant{}", i); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + tree.pretty_print(); + + // check matching using multi threads + + let mut handles = vec![]; + + for i in 0..prefix.len() { + let tree_clone = Arc::clone(&tree); + let text = prefix[i]; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + let tenant = format!("tenant{}", i); + assert_eq!(matched_text, text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_mixed_concurrent_insert_match() { + // ensure it does not deadlock instead of doing correctness check + + let prefix = vec![ + "Clock strikes midnight, I'm still wide awake", + "Got dreams bigger than these city lights", + "Time waits for no one, gotta make my move", + "Started from the bottom, that's no metaphor", + ]; + let suffix = vec![ + "Got too much to prove, ain't got time to lose", + "History in the making, yeah, you can't erase this", + ]; + let tree = Arc::new(Tree::new()); + + let mut handles = vec![]; + + for i in 0..prefix.len() { + for j in 0..suffix.len() { + let tree_clone = Arc::clone(&tree); + let text = format!("{} {}", prefix[i], suffix[j]); + let tenant = format!("tenant{}", i); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + } + + // check matching using multi threads + + for i in 0..prefix.len() { + let tree_clone = Arc::clone(&tree); + let text = prefix[i]; + + let handle = thread::spawn(move || { + let (_matched_text, _matched_tenant) = tree_clone.prefix_match(text); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_utf8_split_seq() { + // The string should be indexed and splitted by a utf-8 value basis instead of byte basis + // use .chars() to get the iterator of the utf-8 value + let tree = Arc::new(Tree::new()); + + let test_pairs = vec![ + ("你好嗎", "tenant1"), + ("你好喔", "tenant2"), + ("你心情好嗎", "tenant3"), + ]; + + // Insert sequentially + for i in 0..test_pairs.len() { + let text = test_pairs[i].0; + let tenant = test_pairs[i].1; + tree.insert(text, tenant); + } + + tree.pretty_print(); + + // Test sequentially + + for i in 0..test_pairs.len() { + let (matched_text, matched_tenant) = tree.prefix_match(test_pairs[i].0); + assert_eq!(matched_text, test_pairs[i].0); + assert_eq!(matched_tenant, test_pairs[i].1); + } + } + + #[test] + fn test_utf8_split_concurrent() { + let tree = Arc::new(Tree::new()); + + let test_pairs = vec![ + ("你好嗎", "tenant1"), + ("你好喔", "tenant2"), + ("你心情好嗎", "tenant3"), + ]; + + // Create multiple threads for insertion + let mut handles = vec![]; + + for i in 0..test_pairs.len() { + let tree_clone = Arc::clone(&tree); + let text = test_pairs[i].0.to_string(); + let tenant = test_pairs[i].1.to_string(); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + + // Wait for all insertions to complete + for handle in handles { + handle.join().unwrap(); + } + + tree.pretty_print(); + + // Create multiple threads for matching + let mut handles = vec![]; + + for i in 0..test_pairs.len() { + let tree_clone = Arc::clone(&tree); + let text = test_pairs[i].0.to_string(); + let tenant = test_pairs[i].1.to_string(); + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(&text); + assert_eq!(matched_text, text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // Wait for all matches to complete + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_simple_eviction() { + let tree = Tree::new(); + let max_size = 5; + + // Insert strings for both tenants + tree.insert("hello", "tenant1"); // size 5 + + tree.insert("hello", "tenant2"); // size 5 + thread::sleep(Duration::from_millis(10)); + tree.insert("world", "tenant2"); // size 5, total for tenant2 = 10 + + tree.pretty_print(); + + // Verify initial sizes + let sizes_before = tree.get_used_size_per_tenant(); + assert_eq!(sizes_before.get("tenant1").unwrap(), &5); // "hello" = 5 + assert_eq!(sizes_before.get("tenant2").unwrap(), &10); // "hello" + "world" = 10 + + // Evict - should remove "hello" from tenant2 as it's the oldest + tree.evict_tenant_by_size(max_size); + + tree.pretty_print(); + + // Verify sizes after eviction + let sizes_after = tree.get_used_size_per_tenant(); + assert_eq!(sizes_after.get("tenant1").unwrap(), &5); // Should be unchanged + assert_eq!(sizes_after.get("tenant2").unwrap(), &5); // Only "world" remains + + // Verify "world" remains for tenant2 + let (matched, tenant) = tree.prefix_match("world"); + assert_eq!(matched, "world"); + assert_eq!(tenant, "tenant2"); + } + + #[test] + fn test_advanced_eviction() { + let tree = Tree::new(); + + // Set limits for each tenant + let max_size: usize = 100; + + // Define prefixes + let prefixes = vec!["aqwefcisdf", "iajsdfkmade", "kjnzxcvewqe", "iejksduqasd"]; + + // Insert strings with shared prefixes + for _i in 0..100 { + for (j, prefix) in prefixes.iter().enumerate() { + let random_suffix = random_string(10); + let text = format!("{}{}", prefix, random_suffix); + let tenant = format!("tenant{}", j + 1); + tree.insert(&text, &tenant); + } + } + + // Perform eviction + tree.evict_tenant_by_size(max_size); + + // Check sizes after eviction + let sizes_after = tree.get_used_size_per_tenant(); + // Verify all tenants are under their size limits + for (tenant, &size) in sizes_after.iter() { + assert!( + size <= max_size, + "Tenant {} exceeds size limit. Current size: {}, Limit: {}", + tenant, + size, + max_size + ); + } + } + + #[test] + fn test_concurrent_operations_with_eviction() { + // Ensure eviction works fine with concurrent insert and match operations for a given period + + let tree = Arc::new(Tree::new()); + let mut handles = vec![]; + let test_duration = Duration::from_secs(10); + let start_time = Instant::now(); + let max_size = 100; // Single max size for all tenants + + // Spawn eviction thread + { + let tree = Arc::clone(&tree); + let handle = thread::spawn(move || { + while start_time.elapsed() < test_duration { + // Run eviction + tree.evict_tenant_by_size(max_size); + + // Sleep for 5 seconds + thread::sleep(Duration::from_secs(5)); + } + }); + handles.push(handle); + } + + // Spawn 4 worker threads + for thread_id in 0..4 { + let tree = Arc::clone(&tree); + let handle = thread::spawn(move || { + let mut rng = rand::thread_rng(); + let tenant = format!("tenant{}", thread_id + 1); + let prefix = format!("prefix{}", thread_id); + + while start_time.elapsed() < test_duration { + // Random decision: match or insert (70% match, 30% insert) + if rng.gen_bool(0.7) { + // Perform match operation + let random_len = rng.gen_range(3..10); + let search_str = format!("{}{}", prefix, random_string(random_len)); + let (_matched, _) = tree.prefix_match(&search_str); + } else { + // Perform insert operation + let random_len = rng.gen_range(5..15); + let insert_str = format!("{}{}", prefix, random_string(random_len)); + tree.insert(&insert_str, &tenant); + // println!("Thread {} inserted: {}", thread_id, insert_str); + } + + // Small random sleep to vary timing + thread::sleep(Duration::from_millis(rng.gen_range(10..100))); + } + }); + handles.push(handle); + } + + // Wait for all threads to complete + for handle in handles { + handle.join().unwrap(); + } + + // final eviction + tree.evict_tenant_by_size(max_size); + + // Final size check + let final_sizes = tree.get_used_size_per_tenant(); + println!("Final sizes after test completion: {:?}", final_sizes); + + // Verify all tenants are under limit + for (_, &size) in final_sizes.iter() { + assert!( + size <= max_size, + "Tenant exceeds size limit. Final size: {}, Limit: {}", + size, + max_size + ); + } + } + + #[test] + fn test_leaf_of() { + let tree = Tree::new(); + + // Single node + tree.insert("hello", "tenant1"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert_eq!(leaves, vec!["tenant1"]); + + // Node with multiple tenants + tree.insert("hello", "tenant2"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert_eq!(leaves.len(), 2); + assert!(leaves.contains(&"tenant1".to_string())); + assert!(leaves.contains(&"tenant2".to_string())); + + // Non-leaf node + tree.insert("hi", "tenant1"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert!(leaves.is_empty()); + } + + #[test] + fn test_get_used_size_per_tenant() { + let tree = Tree::new(); + + // Single tenant + tree.insert("hello", "tenant1"); + tree.insert("world", "tenant1"); + let sizes = tree.get_used_size_per_tenant(); + + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant1").unwrap(), &10); // "hello" + "world" + + // Multiple tenants sharing nodes + tree.insert("hello", "tenant2"); + tree.insert("help", "tenant2"); + let sizes = tree.get_used_size_per_tenant(); + + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant1").unwrap(), &10); + assert_eq!(sizes.get("tenant2").unwrap(), &6); // "hello" + "p" + + // UTF-8 characters + tree.insert("你好", "tenant3"); + let sizes = tree.get_used_size_per_tenant(); + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant3").unwrap(), &2); // 2 Chinese characters + + tree.pretty_print(); + } + + #[test] + fn test_prefix_match_tenant() { + let tree = Tree::new(); + + // Insert overlapping prefixes for different tenants + tree.insert("hello", "tenant1"); // tenant1: hello + tree.insert("hello", "tenant2"); // tenant2: hello + tree.insert("hello world", "tenant2"); // tenant2: hello -> world + tree.insert("help", "tenant1"); // tenant1: hel -> p + tree.insert("helicopter", "tenant2"); // tenant2: hel -> icopter + + // Test tenant1's data + assert_eq!(tree.prefix_match_tenant("hello", "tenant1"), "hello"); // Full match for tenant1 + assert_eq!(tree.prefix_match_tenant("help", "tenant1"), "help"); // Exclusive to tenant1 + assert_eq!(tree.prefix_match_tenant("hel", "tenant1"), "hel"); // Shared prefix + assert_eq!(tree.prefix_match_tenant("hello world", "tenant1"), "hello"); // Should stop at tenant1's boundary + assert_eq!(tree.prefix_match_tenant("helicopter", "tenant1"), "hel"); // Should stop at tenant1's boundary + + // Test tenant2's data + assert_eq!(tree.prefix_match_tenant("hello", "tenant2"), "hello"); // Full match for tenant2 + assert_eq!( + tree.prefix_match_tenant("hello world", "tenant2"), + "hello world" + ); // Exclusive to tenant2 + assert_eq!( + tree.prefix_match_tenant("helicopter", "tenant2"), + "helicopter" + ); // Exclusive to tenant2 + assert_eq!(tree.prefix_match_tenant("hel", "tenant2"), "hel"); // Shared prefix + assert_eq!(tree.prefix_match_tenant("help", "tenant2"), "hel"); // Should stop at tenant2's boundary + + // Test non-existent tenant + assert_eq!(tree.prefix_match_tenant("hello", "tenant3"), ""); // Non-existent tenant + assert_eq!(tree.prefix_match_tenant("help", "tenant3"), ""); // Non-existent tenant + } + + #[test] + fn test_simple_tenant_eviction() { + let tree = Tree::new(); + + // Insert data for multiple tenants + tree.insert("hello", "tenant1"); + tree.insert("world", "tenant1"); + tree.insert("hello", "tenant2"); + tree.insert("help", "tenant2"); + + // Verify initial state + let initial_sizes = tree.get_used_size_per_tenant(); + assert_eq!(initial_sizes.get("tenant1").unwrap(), &10); // "hello" + "world" + assert_eq!(initial_sizes.get("tenant2").unwrap(), &6); // "hello" + "p" + + // Evict tenant1 + tree.remove_tenant("tenant1"); + + // Verify after eviction + let final_sizes = tree.get_used_size_per_tenant(); + assert!( + !final_sizes.contains_key("tenant1"), + "tenant1 should be completely removed" + ); + assert_eq!( + final_sizes.get("tenant2").unwrap(), + &6, + "tenant2 should be unaffected" + ); + + // Verify tenant1's data is inaccessible + assert_eq!(tree.prefix_match_tenant("hello", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("world", "tenant1"), ""); + + // Verify tenant2's data is still accessible + assert_eq!(tree.prefix_match_tenant("hello", "tenant2"), "hello"); + assert_eq!(tree.prefix_match_tenant("help", "tenant2"), "help"); + } + + #[test] + fn test_complex_tenant_eviction() { + let tree = Tree::new(); + + // Create a more complex tree structure with shared prefixes + tree.insert("apple", "tenant1"); + tree.insert("application", "tenant1"); + tree.insert("apple", "tenant2"); + tree.insert("appetite", "tenant2"); + tree.insert("banana", "tenant1"); + tree.insert("banana", "tenant2"); + tree.insert("ball", "tenant2"); + + // Verify initial state + let initial_sizes = tree.get_used_size_per_tenant(); + println!("Initial sizes: {:?}", initial_sizes); + tree.pretty_print(); + + // Evict tenant1 + tree.remove_tenant("tenant1"); + + // Verify final state + let final_sizes = tree.get_used_size_per_tenant(); + println!("Final sizes: {:?}", final_sizes); + tree.pretty_print(); + + // Verify tenant1 is completely removed + assert!( + !final_sizes.contains_key("tenant1"), + "tenant1 should be completely removed" + ); + + // Verify all tenant1's data is inaccessible + assert_eq!(tree.prefix_match_tenant("apple", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("application", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("banana", "tenant1"), ""); + + // Verify tenant2's data is intact + assert_eq!(tree.prefix_match_tenant("apple", "tenant2"), "apple"); + assert_eq!(tree.prefix_match_tenant("appetite", "tenant2"), "appetite"); + assert_eq!(tree.prefix_match_tenant("banana", "tenant2"), "banana"); + assert_eq!(tree.prefix_match_tenant("ball", "tenant2"), "ball"); + + // Verify the tree structure is still valid for tenant2 + let tenant2_size = final_sizes.get("tenant2").unwrap(); + assert_eq!(tenant2_size, &(5 + 5 + 6 + 2)); // "apple" + "etite" + "banana" + "ll" + } +} diff --git a/sgl-router/v0.1.0.md b/sgl-router/v0.1.0.md new file mode 100644 index 0000000..747731a --- /dev/null +++ b/sgl-router/v0.1.0.md @@ -0,0 +1,63 @@ +# SGLang Router v0.1.0: Dynamic Scaling and Fault Tolerance + +We have released `sglang-router` v0.1.0 equipped with dynamic scaling and fault tolerance! It is essential for the router to be able to dynamically scale the number of workers and handle worker failures. To achieve this, we have implemented the following features: + +## 1. Dynamic scaling: The router can dynamically scale the number of workers based on the request load. + +We offer `/add_worker` and `/remove_worker` APIs to dynamically add or remove workers from the router. + +- `/add_worker` + +Usage: + +```bash +$ curl -X POST http://localhost:30000/add_worker?url=http://worker_url_1 +``` + +Example: + +```bash +$ python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30001 +$ curl -X POST http://localhost:30000/add_worker?url=http://127.0.0.1:30001 +Successfully added worker: http://127.0.0.1:30001 +``` + +- `/remove_worker` + +Usage: + +```bash +$ curl -X POST http://localhost:30000/remove_worker?url=http://worker_url_1 +``` + +Example: + +```bash +$ curl -X POST http://localhost:30000/remove_worker?url=http://127.0.0.1:30001 +Successfully removed worker: http://127.0.0.1:30001 +``` + +Note: + +- For cache-aware router, the worker will be removed from the tree and the queues. + +## 2. Fault tolerance: The router can handle worker failures and automatically remove the failed worker from the router. + +We provide retries based for failure tolerance. + +1. If the request to a worker fails for `max_worker_retries` times, the router will remove the worker from the router and move on to the next worker. +2. If the total number of retries exceeds `max_total_retries`, the router will return an error. + +Note: + +- `max_worker_retries` is 3 and `max_total_retries` is 6 by default. + +## Closing remarks: + +1. Please read the full usage at https://docs.sglang.ai/router/router.html +2. The feature is still under active improvement, so please don't hesitate to raise issues or submit PRs if you have any suggestions or feedback. + + +# Release Instructions + +Update the version in `rust/pyproject.toml` and `py_src/sglang_router/version.py`. diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..868061b --- /dev/null +++ b/test/README.md @@ -0,0 +1,47 @@ +# Run Unit Tests + +SGLang uses the built-in library [unittest](https://docs.python.org/3/library/unittest.html) as the testing framework. + +## Test Backend Runtime +```bash +cd sglang/test/srt + +# Run a single file +python3 test_srt_endpoint.py + +# Run a single test +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_simple_decode + +# Run a suite with multiple files +python3 run_suite.py --suite per-commit +``` + +## Test Frontend Language +```bash +cd sglang/test/lang +export OPENAI_API_KEY=sk-***** + +# Run a single file +python3 test_openai_backend.py + +# Run a single test +python3 -m unittest test_openai_backend.TestOpenAIServer.test_few_shot_qa + +# Run a suite with multiple files +python3 run_suite.py --suite per-commit +``` + +## Adding or Updating Tests in CI + +- Create new test files under `test/srt` or `test/lang` depending on the type of test. +- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py` or `test/lang/run_suite.py`) so they’re picked up in CI. For most small test cases, they can be added to the `per-commit` suite. +- The CI will run the `per-commit` and `nightly` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows). + + +## Writing Elegant Test Cases + +- Examine existing tests in [sglang/test](https://github.com/sgl-project/sglang/tree/main/test) for practical examples. +- Keep each test function focused on a single scenario or piece of functionality. +- Give tests descriptive names reflecting their purpose. +- Use robust assertions (e.g., assert, unittest methods) to validate outcomes. +- Clean up resources to avoid side effects and preserve test independence. diff --git a/test/lang/run_suite.py b/test/lang/run_suite.py new file mode 100644 index 0000000..04efba5 --- /dev/null +++ b/test/lang/run_suite.py @@ -0,0 +1,38 @@ +import argparse +import glob + +from sglang.test.test_utils import TestFile, run_unittest_files + +suites = { + "per-commit": [ + TestFile("test_srt_backend.py"), + # Skip this due to some OPENAI_API_KEY issues + # "test_openai_backend.py", + ], +} + + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--timeout-per-file", + type=int, + default=1000, + help="The time limit for running one file in seconds.", + ) + arg_parser.add_argument( + "--suite", + type=str, + default=list(suites.keys())[0], + choices=list(suites.keys()) + ["all"], + help="The suite to run", + ) + args = arg_parser.parse_args() + + if args.suite == "all": + files = glob.glob("**/test_*.py", recursive=True) + else: + files = suites[args.suite] + + exit_code = run_unittest_files(files, args.timeout_per_file) + exit(exit_code) diff --git a/test/lang/test_anthropic_backend.py b/test/lang/test_anthropic_backend.py new file mode 100644 index 0000000..dc8f0b1 --- /dev/null +++ b/test/lang/test_anthropic_backend.py @@ -0,0 +1,25 @@ +import json +import unittest + +from sglang import Anthropic, set_default_backend +from sglang.test.test_programs import test_mt_bench, test_stream +from sglang.test.test_utils import CustomTestCase + + +class TestAnthropicBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = Anthropic("claude-3-haiku-20240307") + set_default_backend(cls.backend) + + def test_mt_bench(self): + test_mt_bench() + + def test_stream(self): + test_stream() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_bind_cache.py b/test/lang/test_bind_cache.py new file mode 100644 index 0000000..d5beefa --- /dev/null +++ b/test/lang/test_bind_cache.py @@ -0,0 +1,51 @@ +import unittest + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestBind(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = sgl.Runtime(model_path=DEFAULT_MODEL_NAME_FOR_TEST) + sgl.set_default_backend(cls.backend) + + @classmethod + def tearDownClass(cls): + cls.backend.shutdown() + + def test_bind(self): + @sgl.function + def few_shot_qa(s, prompt, question): + s += prompt + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + few_shot_qa_2 = few_shot_qa.bind( + prompt="The following are questions with answers.\n\n" + ) + + tracer = few_shot_qa_2.trace() + print(tracer.last_node.print_graph_dfs() + "\n") + + def test_cache(self): + @sgl.function + def few_shot_qa(s, prompt, question): + s += prompt + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + few_shot_qa_2 = few_shot_qa.bind( + prompt="Answer the following questions as if you were a 5-year-old kid.\n\n" + ) + few_shot_qa_2.cache(self.backend) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_choices.py b/test/lang/test_choices.py new file mode 100644 index 0000000..89e7ca1 --- /dev/null +++ b/test/lang/test_choices.py @@ -0,0 +1,91 @@ +import unittest + +import numpy as np + +from sglang.lang.choices import ( + greedy_token_selection, + token_length_normalized, + unconditional_likelihood_normalized, +) +from sglang.test.test_utils import CustomTestCase + +MOCK_CHOICES_INPUT_DATA = { + "choices": [ + "organ", # ["organ"] + "organism", # ["organ", "ism"] + "antidisestablishmentarianism", # ["ant", "id", "is", "est", "ablish", "ment", "arian", "ism"] + ], + "normalized_prompt_logprobs": [-0.1, -0.2, -0.05], + "input_token_logprobs": [ + [[-0.1, 1, None]], + [[-0.1, 1, None], [-0.3, 2, None]], + [ + [-0.4, 3, None], + [-0.25, 4, None], + [-0.1, 5, None], + [-0.01, 6, None], + [-0.01, 7, None], + [-0.01, 8, None], + [-0.01, 9, None], + [-0.01, 2, None], + ], + ], + "output_token_logprobs": [ + [[-0.1, 10, None]], + [[-0.1, 10, None]], + [[-0.1, 10, None]], + ], + "unconditional_token_logprobs": [ + [[None, 1, None]], + [[None, 1, None], [-1.4, 2, None]], + [ + [None, 3, None], + [-0.25, 4, None], + [-0.1, 5, None], + [-0.01, 6, None], + [-0.01, 7, None], + [-0.01, 8, None], + [-0.01, 9, None], + [-0.01, 2, None], + ], + ], +} + + +class TestChoices(CustomTestCase): + + def test_token_length_normalized(self): + """Confirm 'antidisestablishmentarianism' is selected due to high confidences for + its later tokens resulting in highest token length normalized prompt logprob.""" + decision = token_length_normalized(**MOCK_CHOICES_INPUT_DATA) + assert decision.decision == "antidisestablishmentarianism" + + def test_greedy_token_selection(self): + """Confirm 'organ' is selected due it having the joint highest initial token + logprob, and a higher average logprob than organism's second token.""" + decision = greedy_token_selection(**MOCK_CHOICES_INPUT_DATA) + assert decision.decision == "organ" + assert np.allclose( + decision.meta_info["greedy_logprob_matrix"], + [ + [-0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1], + [-0.1, -0.3, -0.2, -0.2, -0.2, -0.2, -0.2, -0.2], + [-0.4, -0.25, -0.1, -0.01, -0.01, -0.01, -0.01, -0.01], + ], + atol=0.01, + ) + + def test_unconditional_likelihood_normalized(self): + """Confirm 'organism' is selected due to it having the highest average token logprob + once normalized by the unconditional token logprobs.""" + decision = unconditional_likelihood_normalized(**MOCK_CHOICES_INPUT_DATA) + assert decision.decision == "organism" + assert np.allclose( + decision.meta_info["normalized_unconditional_prompt_logprobs"], + [-0.1, 0.5, -0.05], + atol=0.01, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_litellm_backend.py b/test/lang/test_litellm_backend.py new file mode 100644 index 0000000..74c3a18 --- /dev/null +++ b/test/lang/test_litellm_backend.py @@ -0,0 +1,25 @@ +import json +import unittest + +from sglang import LiteLLM, set_default_backend +from sglang.test.test_programs import test_mt_bench, test_stream +from sglang.test.test_utils import CustomTestCase + + +class TestAnthropicBackend(CustomTestCase): + chat_backend = None + + @classmethod + def setUpClass(cls): + cls.chat_backend = LiteLLM("gpt-3.5-turbo") + set_default_backend(cls.chat_backend) + + def test_mt_bench(self): + test_mt_bench() + + def test_stream(self): + test_stream() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_openai_backend.py b/test/lang/test_openai_backend.py new file mode 100644 index 0000000..dbef7d4 --- /dev/null +++ b/test/lang/test_openai_backend.py @@ -0,0 +1,92 @@ +import unittest + +from sglang import OpenAI, set_default_backend +from sglang.test.test_programs import ( + test_chat_completion_speculative, + test_completion_speculative, + test_decode_int, + test_decode_json, + test_expert_answer, + test_few_shot_qa, + test_image_qa, + test_mt_bench, + test_parallel_decoding, + test_parallel_encoding, + test_react, + test_select, + test_stream, + test_tool_use, +) +from sglang.test.test_utils import CustomTestCase + + +class TestOpenAIBackend(CustomTestCase): + instruct_backend = None + chat_backend = None + chat_vision_backend = None + + @classmethod + def setUpClass(cls): + cls.instruct_backend = OpenAI("gpt-3.5-turbo-instruct") + cls.chat_backend = OpenAI("gpt-3.5-turbo") + cls.chat_vision_backend = OpenAI("gpt-4-turbo") + + def test_few_shot_qa(self): + set_default_backend(self.instruct_backend) + test_few_shot_qa() + + def test_mt_bench(self): + set_default_backend(self.chat_backend) + test_mt_bench() + + def test_select(self): + set_default_backend(self.instruct_backend) + test_select(check_answer=True) + + def test_decode_int(self): + set_default_backend(self.instruct_backend) + test_decode_int() + + def test_decode_json(self): + set_default_backend(self.instruct_backend) + test_decode_json() + + def test_expert_answer(self): + set_default_backend(self.instruct_backend) + test_expert_answer() + + def test_tool_use(self): + set_default_backend(self.instruct_backend) + test_tool_use() + + def test_react(self): + set_default_backend(self.instruct_backend) + test_react() + + def test_parallel_decoding(self): + set_default_backend(self.instruct_backend) + test_parallel_decoding() + + def test_parallel_encoding(self): + set_default_backend(self.instruct_backend) + test_parallel_encoding() + + def test_image_qa(self): + set_default_backend(self.chat_vision_backend) + test_image_qa() + + def test_stream(self): + set_default_backend(self.instruct_backend) + test_stream() + + def test_completion_speculative(self): + set_default_backend(self.instruct_backend) + test_completion_speculative() + + def test_chat_completion_speculative(self): + set_default_backend(self.chat_backend) + test_chat_completion_speculative() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_srt_backend.py b/test/lang/test_srt_backend.py new file mode 100644 index 0000000..0e05eb9 --- /dev/null +++ b/test/lang/test_srt_backend.py @@ -0,0 +1,86 @@ +""" +Usage: +python3 -m unittest test_srt_backend.TestSRTBackend.test_gen_min_new_tokens +python3 -m unittest test_srt_backend.TestSRTBackend.test_hellaswag_select +""" + +import unittest + +import sglang as sgl +from sglang.test.test_programs import ( + test_decode_int, + test_decode_json_regex, + test_dtype_gen, + test_expert_answer, + test_few_shot_qa, + test_gen_min_new_tokens, + test_hellaswag_select, + test_mt_bench, + test_parallel_decoding, + test_regex, + test_select, + test_stream, + test_tool_use, +) +from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestSRTBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = sgl.Runtime( + model_path=DEFAULT_MODEL_NAME_FOR_TEST, cuda_graph_max_bs=4 + ) + sgl.set_default_backend(cls.backend) + + @classmethod + def tearDownClass(cls): + cls.backend.shutdown() + + def test_few_shot_qa(self): + test_few_shot_qa() + + def test_mt_bench(self): + test_mt_bench() + + def test_select(self): + test_select(check_answer=False) + + def test_decode_int(self): + test_decode_int() + + def test_decode_json_regex(self): + test_decode_json_regex() + + def test_expert_answer(self): + test_expert_answer() + + def test_tool_use(self): + test_tool_use() + + def test_parallel_decoding(self): + test_parallel_decoding() + + def test_stream(self): + test_stream() + + def test_regex(self): + test_regex() + + def test_dtype_gen(self): + test_dtype_gen() + + def test_hellaswag_select(self): + # Run twice to capture more bugs + for _ in range(2): + accuracy, latency = test_hellaswag_select() + self.assertGreater(accuracy, 0.60) + + def test_gen_min_new_tokens(self): + test_gen_min_new_tokens() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_tracing.py b/test/lang/test_tracing.py new file mode 100644 index 0000000..3f02ac5 --- /dev/null +++ b/test/lang/test_tracing.py @@ -0,0 +1,129 @@ +import unittest + +import sglang as sgl +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.test.test_utils import CustomTestCase + + +class TestTracing(CustomTestCase): + def test_few_shot_qa(self): + @sgl.function + def few_shot_qa(s, question): + s += "The following are questions with answers.\n\n" + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + tracer = few_shot_qa.trace() + # print(tracer.last_node.print_graph_dfs() + "\n") + + def test_select(self): + @sgl.function + def capital(s): + s += "The capital of France is" + s += sgl.select("capital", ["Paris. ", "London. "]) + s += "It is a city" + sgl.gen("description", stop=".") + + tracer = capital.trace() + # print(tracer.last_node.print_graph_dfs() + "\n") + + def test_raise_warning(self): + @sgl.function + def wrong(s, question): + s += f"I want to ask {question}" + + try: + tracer = wrong.trace() + raised = False + except TypeError: + raised = True + + assert raised + + def test_multi_function(self): + @sgl.function + def expand(s, tip): + s += ( + "Please expand the following tip into a detailed paragraph:" + + tip + + "\n" + ) + s += sgl.gen("detailed_tip") + + @sgl.function + def tip_suggestion(s, topic): + s += "Here are 2 tips for " + topic + ".\n" + + s += "1." + sgl.gen("tip_1", stop=["\n", ":", "."]) + "\n" + s += "2." + sgl.gen("tip_2", stop=["\n", ":", "."]) + "\n" + + branch1 = expand(tip=s["tip_1"]) + branch2 = expand(tip=s["tip_2"]) + + s += "Tip 1: " + branch1["detailed_tip"] + "\n" + s += "Tip 2: " + branch2["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + compiled = tip_suggestion.compile() + # compiled.print_graph() + + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + state = compiled.run(topic="staying healthy") + # print(state.text() + "\n") + + states = compiled.run_batch( + [ + {"topic": "staying healthy"}, + {"topic": "staying happy"}, + {"topic": "earning money"}, + ], + temperature=0, + ) + # for s in states: + # print(s.text() + "\n") + + def test_role(self): + @sgl.function + def multi_turn_chat(s): + s += sgl.user("Who are you?") + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user("Who created you?") + s += sgl.assistant(sgl.gen("answer_2")) + + backend = BaseBackend() + backend.chat_template = get_chat_template("llama-2-chat") + + compiled = multi_turn_chat.compile(backend=backend) + # compiled.print_graph() + + def test_fork(self): + @sgl.function + def tip_suggestion(s): + s += ( + "Here are three tips for staying healthy: " + "1. Balanced Diet; " + "2. Regular Exercise; " + "3. Adequate Sleep\n" + ) + + forks = s.fork(3) + for i in range(3): + forks[i] += f"Now, expand tip {i+1} into a paragraph:\n" + forks[i] += sgl.gen(f"detailed_tip") + + s += "Tip 1:" + forks[0]["detailed_tip"] + "\n" + s += "Tip 2:" + forks[1]["detailed_tip"] + "\n" + s += "Tip 3:" + forks[2]["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + tracer = tip_suggestion.trace() + # print(tracer.last_node.print_graph_dfs()) + + a = tip_suggestion.run(backend=sgl.OpenAI("gpt-3.5-turbo-instruct")) + # print(a.text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_vertexai_backend.py b/test/lang/test_vertexai_backend.py new file mode 100644 index 0000000..83ce7fc --- /dev/null +++ b/test/lang/test_vertexai_backend.py @@ -0,0 +1,53 @@ +import unittest + +from sglang import VertexAI, set_default_backend +from sglang.test.test_programs import ( + test_expert_answer, + test_few_shot_qa, + test_image_qa, + test_mt_bench, + test_parallel_decoding, + test_parallel_encoding, + test_stream, +) +from sglang.test.test_utils import CustomTestCase + + +class TestVertexAIBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = VertexAI("gemini-1.5-pro-001") + + def test_few_shot_qa(self): + set_default_backend(self.backend) + test_few_shot_qa() + + def test_mt_bench(self): + set_default_backend(self.backend) + test_mt_bench() + + def test_expert_answer(self): + set_default_backend(self.backend) + test_expert_answer(check_answer=False) + + def test_parallel_decoding(self): + set_default_backend(self.backend) + test_parallel_decoding() + + def test_parallel_encoding(self): + set_default_backend(self.backend) + test_parallel_encoding() + + def test_image_qa(self): + set_default_backend(self.backend) + test_image_qa() + + def test_stream(self): + set_default_backend(self.backend) + test_stream() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/configs/random_config.yaml b/test/srt/configs/random_config.yaml new file mode 100644 index 0000000..eae8c27 --- /dev/null +++ b/test/srt/configs/random_config.yaml @@ -0,0 +1,25 @@ +tasks: + - name: sglang-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: vllm-128-4 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: vllm-2000-100 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: vllm-4000-200 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 + - name: vllm-32000-100 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 diff --git a/test/srt/configs/random_flashinfer_vs_triton_config.yaml b/test/srt/configs/random_flashinfer_vs_triton_config.yaml new file mode 100644 index 0000000..7f4a386 --- /dev/null +++ b/test/srt/configs/random_flashinfer_vs_triton_config.yaml @@ -0,0 +1,25 @@ +tasks: + - name: sglang-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-triton-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-triton-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-triton-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 + - name: sglang-triton-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 diff --git a/test/srt/configs/sharegpt_config.yaml b/test/srt/configs/sharegpt_config.yaml new file mode 100644 index 0000000..a80b96c --- /dev/null +++ b/test/srt/configs/sharegpt_config.yaml @@ -0,0 +1,7 @@ +tasks: + - name: sglang-benchmark + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --request-rate 16 + - name: vllm-benchmark + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --request-rate 16 diff --git a/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json b/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json new file mode 100644 index 0000000..f1652ec --- /dev/null +++ b/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json @@ -0,0 +1 @@ +{"model.layers.0.self_attn.q_proj": [[39, 106, 104, 102, 33, 95, 13, 44, 11, 29, 12, 10, 53, 126, 27, 114, 121, 8, 124, 113, 112, 15, 23, 89, 69, 111, 54, 80, 4, 1, 20, 24, 83, 63, 115, 122, 66, 42, 22, 110, 3, 73, 21, 61, 97, 19, 25, 88, 117, 119, 116, 85, 70, 5, 56, 118, 68, 123, 2, 86, 71, 127, 93, 49, 109, 50, 67, 52, 91, 40, 17, 108, 60, 55, 78, 62, 65, 47, 6, 87, 51, 84, 58, 82, 94, 79, 57, 103, 76, 48, 0, 18, 81, 96, 9, 31, 16, 92, 26, 43, 30, 37, 74, 100, 46, 38, 32, 14, 90, 36, 101, 75, 35, 125, 45, 7, 72, 41, 77, 105, 28, 59, 34, 99, 98, 107, 120, 64], [39, 104, 102, 106, 33, 44, 95, 53, 126, 111, 114, 110, 29, 14, 113, 54, 121, 112, 109, 80, 119, 12, 63, 25, 78, 123, 42, 10, 24, 127, 108, 115, 122, 60, 116, 97, 9, 93, 61, 17, 117, 56, 82, 91, 21, 124, 62, 0, 43, 19, 31, 26, 49, 52, 57, 59, 1, 50, 41, 15, 46, 86, 88, 81, 118, 125, 2, 3, 55, 105, 84, 75, 120, 70, 47, 85, 32, 51, 37, 16, 22, 23, 89, 87, 67, 30, 68, 69, 77, 48, 66, 27, 36, 107, 20, 76, 94, 79, 96, 83, 99, 38, 100, 18, 8, 35, 28, 101, 13, 72, 92, 7, 4, 98, 90, 103, 6, 34, 74, 11, 65, 40, 71, 45, 58, 64, 73, 5], [106, 104, 39, 102, 2, 44, 95, 53, 121, 3, 4, 33, 1, 0, 122, 111, 126, 115, 63, 71, 119, 112, 113, 70, 109, 29, 61, 52, 114, 54, 127, 21, 10, 72, 5, 110, 116, 49, 123, 11, 12, 55, 24, 45, 15, 19, 117, 73, 80, 50, 6, 23, 64, 124, 75, 69, 25, 83, 14, 77, 97, 42, 78, 13, 79, 85, 27, 56, 87, 86, 57, 17, 118, 91, 81, 74, 65, 76, 9, 41, 88, 18, 51, 22, 62, 93, 89, 67, 105, 84, 82, 38, 48, 120, 60, 47, 26, 20, 43, 68, 28, 66, 108, 59, 58, 8, 125, 98, 46, 31, 90, 36, 94, 92, 34, 107, 35, 7, 99, 100, 30, 32, 37, 101, 16, 96, 40, 103], [39, 104, 106, 102, 33, 44, 53, 111, 113, 95, 54, 109, 41, 116, 124, 63, 121, 122, 61, 56, 114, 110, 60, 108, 127, 112, 125, 43, 52, 126, 12, 80, 119, 59, 62, 82, 123, 97, 117, 25, 37, 0, 88, 29, 57, 120, 26, 91, 115, 78, 24, 21, 118, 49, 93, 1, 42, 14, 58, 107, 10, 84, 67, 7, 9, 86, 87, 3, 50, 51, 64, 98, 18, 20, 72, 66, 100, 17, 32, 81, 69, 79, 2, 101, 105, 85, 15, 77, 31, 22, 75, 70, 68, 90, 55, 19, 89, 11, 74, 76, 8, 38, 30, 4, 65, 5, 45, 92, 23, 46, 73, 13, 94, 47, 35, 36, 6, 48, 34, 28, 99, 27, 96, 103, 71, 83, 16, 40], [41, 103, 38, 33, 42, 93, 109, 95, 88, 44, 107, 17, 25, 20, 110, 21, 47, 40, 78, 116, 115, 126, 59, 61, 64, 57, 48, 83, 119, 12, 45, 125, 11, 18, 124, 118, 53, 56, 89, 70, 22, 77, 86, 50, 92, 4, 87, 24, 97, 65, 49, 73, 26, 62, 3, 16, 108, 66, 72, 104, 67, 80, 58, 71, 79, 23, 91, 122, 19, 43, 113, 39, 14, 102, 111, 27, 9, 121, 10, 117, 29, 60, 69, 90, 76, 120, 0, 84, 15, 114, 51, 74, 68, 98, 6, 81, 85, 2, 1, 82, 100, 112, 99, 13, 52, 123, 94, 8, 35, 28, 55, 96, 63, 54, 7, 105, 5, 36, 127, 37, 32, 75, 34, 106, 31, 30, 101, 46], [38, 103, 44, 41, 107, 33, 42, 88, 95, 40, 93, 87, 25, 78, 110, 47, 48, 26, 20, 18, 21, 17, 64, 59, 45, 22, 109, 89, 118, 62, 91, 115, 116, 53, 49, 61, 126, 90, 16, 11, 119, 92, 111, 65, 125, 124, 43, 85, 83, 60, 24, 4, 77, 73, 104, 117, 57, 120, 86, 3, 76, 97, 29, 23, 102, 39, 51, 67, 70, 12, 122, 71, 9, 15, 30, 113, 56, 66, 0, 106, 7, 50, 108, 100, 114, 28, 46, 121, 80, 94, 72, 10, 6, 79, 69, 55, 19, 58, 74, 68, 123, 81, 27, 13, 2, 82, 98, 35, 8, 34, 36, 1, 101, 5, 105, 54, 63, 96, 37, 84, 127, 52, 32, 31, 14, 75, 112, 99], [103, 41, 107, 33, 38, 93, 110, 116, 64, 42, 48, 20, 123, 61, 124, 17, 126, 115, 88, 49, 25, 53, 47, 78, 57, 122, 21, 59, 95, 18, 44, 89, 77, 97, 16, 11, 125, 66, 86, 120, 119, 109, 4, 65, 12, 118, 71, 62, 22, 24, 80, 14, 50, 74, 73, 51, 70, 3, 7, 121, 67, 83, 27, 111, 72, 5, 69, 76, 23, 45, 84, 29, 60, 9, 31, 10, 81, 26, 8, 87, 19, 13, 0, 40, 2, 56, 113, 85, 82, 127, 91, 43, 34, 108, 117, 114, 6, 15, 96, 106, 79, 75, 94, 102, 100, 101, 68, 98, 63, 58, 99, 105, 55, 30, 90, 36, 1, 35, 32, 46, 112, 28, 104, 39, 37, 92, 52, 54], [103, 41, 33, 38, 42, 93, 95, 107, 110, 25, 59, 17, 48, 116, 88, 109, 44, 57, 11, 83, 20, 125, 78, 21, 62, 86, 115, 70, 97, 77, 4, 118, 18, 47, 43, 64, 12, 24, 50, 126, 22, 124, 45, 3, 61, 72, 65, 39, 89, 84, 113, 53, 73, 29, 49, 120, 66, 80, 79, 27, 71, 122, 10, 117, 40, 9, 92, 16, 67, 81, 19, 23, 119, 123, 51, 105, 36, 8, 15, 55, 108, 111, 60, 90, 14, 26, 87, 104, 74, 56, 91, 75, 0, 28, 69, 68, 100, 2, 102, 99, 58, 52, 85, 6, 63, 34, 54, 94, 127, 96, 76, 32, 121, 30, 1, 31, 7, 114, 82, 46, 37, 13, 112, 35, 5, 101, 98, 106], [102, 41, 112, 101, 108, 103, 33, 104, 29, 27, 115, 30, 42, 38, 16, 45, 114, 20, 32, 100, 121, 89, 98, 111, 87, 62, 59, 126, 13, 63, 107, 11, 43, 88, 117, 75, 118, 25, 84, 34, 31, 50, 51, 55, 23, 95, 56, 58, 53, 78, 94, 97, 82, 92, 8, 18, 127, 113, 110, 19, 40, 21, 35, 39, 85, 119, 37, 125, 96, 54, 6, 72, 90, 91, 14, 26, 124, 105, 9, 44, 28, 17, 116, 120, 60, 48, 80, 57, 1, 123, 79, 77, 86, 24, 69, 73, 15, 122, 68, 99, 81, 74, 4, 52, 71, 67, 7, 65, 76, 66, 61, 109, 106, 46, 93, 2, 64, 12, 10, 49, 3, 47, 83, 70, 36, 22, 0, 5], [107, 104, 102, 33, 108, 45, 95, 92, 29, 42, 103, 21, 99, 43, 85, 101, 23, 50, 27, 28, 113, 98, 19, 15, 120, 83, 26, 46, 74, 97, 34, 12, 86, 123, 52, 127, 90, 79, 39, 7, 17, 41, 112, 49, 126, 77, 48, 38, 125, 89, 44, 61, 111, 18, 100, 115, 32, 25, 35, 119, 93, 62, 30, 60, 84, 96, 31, 80, 122, 87, 114, 53, 55, 47, 56, 110, 13, 57, 63, 37, 51, 54, 116, 117, 68, 124, 105, 78, 40, 81, 118, 88, 9, 91, 11, 109, 76, 24, 59, 71, 121, 82, 6, 20, 1, 72, 106, 16, 58, 73, 5, 94, 67, 10, 69, 14, 4, 22, 75, 2, 36, 70, 8, 66, 3, 65, 64, 0], [102, 103, 33, 29, 41, 42, 104, 45, 64, 1, 107, 108, 66, 27, 16, 112, 88, 115, 3, 50, 101, 20, 127, 67, 118, 75, 58, 116, 53, 72, 13, 84, 68, 19, 86, 25, 89, 70, 11, 30, 21, 77, 117, 4, 6, 14, 56, 59, 69, 82, 80, 100, 126, 95, 32, 92, 97, 39, 85, 124, 79, 90, 121, 52, 54, 26, 5, 113, 71, 7, 123, 73, 51, 8, 0, 63, 111, 38, 23, 57, 15, 119, 10, 43, 60, 78, 22, 83, 87, 9, 74, 37, 99, 98, 76, 65, 61, 55, 17, 81, 18, 31, 24, 94, 28, 2, 47, 44, 40, 110, 114, 96, 105, 46, 109, 120, 34, 48, 12, 35, 62, 93, 106, 49, 91, 122, 125, 36], [103, 45, 102, 33, 26, 42, 95, 87, 79, 123, 104, 19, 113, 112, 41, 29, 21, 76, 108, 120, 30, 71, 10, 97, 81, 27, 101, 74, 44, 32, 115, 122, 89, 17, 12, 23, 7, 92, 25, 53, 15, 100, 90, 117, 13, 39, 105, 99, 83, 51, 85, 84, 50, 4, 20, 109, 61, 88, 35, 16, 98, 54, 94, 110, 125, 55, 66, 127, 70, 47, 65, 124, 116, 46, 80, 67, 114, 49, 57, 56, 78, 38, 86, 18, 60, 82, 11, 5, 121, 119, 126, 62, 48, 58, 59, 28, 111, 93, 37, 63, 2, 73, 52, 6, 8, 24, 3, 96, 106, 43, 40, 107, 1, 34, 118, 22, 0, 68, 69, 77, 91, 64, 72, 36, 75, 31, 14, 9], [104, 103, 33, 43, 44, 111, 102, 90, 49, 89, 8, 41, 93, 37, 116, 113, 95, 122, 123, 23, 74, 42, 14, 17, 83, 68, 85, 106, 91, 92, 13, 78, 63, 127, 115, 81, 112, 87, 121, 3, 70, 48, 72, 120, 28, 124, 53, 71, 30, 19, 47, 86, 59, 51, 96, 56, 94, 114, 108, 118, 76, 79, 55, 38, 29, 110, 5, 82, 15, 21, 11, 20, 26, 66, 6, 84, 22, 50, 9, 27, 100, 65, 61, 64, 57, 25, 35, 7, 16, 101, 80, 45, 52, 36, 46, 109, 107, 24, 126, 2, 4, 12, 67, 125, 75, 10, 32, 117, 97, 1, 69, 60, 77, 88, 98, 99, 73, 105, 58, 18, 62, 54, 34, 31, 0, 119, 40, 39], [103, 104, 102, 33, 43, 44, 111, 90, 115, 106, 113, 89, 42, 74, 56, 93, 48, 95, 85, 127, 41, 72, 92, 37, 107, 71, 17, 87, 123, 112, 122, 50, 83, 22, 55, 57, 116, 14, 53, 124, 49, 114, 26, 47, 110, 23, 63, 70, 98, 100, 51, 108, 121, 59, 13, 35, 28, 81, 94, 79, 32, 120, 29, 76, 101, 105, 126, 38, 7, 82, 9, 6, 118, 24, 125, 91, 20, 88, 15, 99, 36, 109, 18, 62, 34, 46, 84, 19, 3, 61, 2, 30, 11, 27, 86, 52, 58, 75, 25, 68, 12, 31, 117, 69, 54, 96, 119, 65, 78, 80, 8, 45, 16, 77, 40, 1, 21, 73, 10, 4, 66, 64, 60, 67, 0, 39, 97, 5], [104, 44, 103, 43, 33, 49, 111, 102, 95, 41, 37, 123, 27, 91, 9, 84, 42, 89, 61, 122, 24, 76, 124, 116, 30, 85, 79, 96, 51, 48, 118, 28, 87, 92, 98, 127, 59, 23, 109, 121, 20, 15, 88, 29, 100, 106, 53, 34, 35, 101, 126, 16, 120, 54, 108, 115, 47, 93, 83, 17, 56, 18, 90, 113, 55, 38, 22, 63, 8, 81, 13, 94, 72, 117, 60, 50, 114, 36, 12, 71, 45, 57, 105, 32, 110, 119, 62, 74, 58, 97, 31, 112, 125, 52, 19, 46, 86, 5, 80, 68, 73, 3, 78, 99, 107, 6, 70, 75, 40, 14, 65, 21, 39, 7, 25, 77, 64, 2, 11, 82, 26, 10, 69, 4, 0, 66, 1, 67], [102, 33, 44, 104, 43, 103, 79, 28, 27, 87, 95, 9, 12, 124, 49, 21, 23, 99, 76, 90, 85, 35, 32, 31, 29, 80, 94, 61, 98, 114, 84, 91, 18, 15, 73, 89, 81, 106, 112, 62, 51, 41, 54, 56, 58, 59, 117, 45, 119, 88, 26, 110, 105, 52, 115, 42, 111, 24, 30, 92, 109, 38, 123, 37, 108, 113, 57, 118, 60, 125, 82, 25, 86, 75, 96, 19, 16, 14, 40, 46, 77, 20, 101, 93, 122, 126, 36, 53, 100, 55, 47, 71, 39, 11, 72, 34, 50, 63, 116, 48, 107, 70, 121, 13, 120, 17, 127, 78, 83, 3, 22, 7, 6, 5, 8, 66, 74, 97, 69, 10, 68, 67, 64, 4, 65, 0, 1, 2], [39, 41, 101, 97, 29, 110, 24, 43, 32, 49, 121, 84, 120, 63, 3, 56, 44, 8, 59, 21, 38, 98, 40, 6, 60, 127, 53, 71, 2, 25, 52, 116, 99, 13, 62, 4, 125, 72, 46, 18, 23, 7, 31, 68, 85, 15, 92, 42, 79, 76, 115, 123, 87, 75, 109, 57, 108, 95, 102, 12, 54, 0, 36, 66, 61, 19, 50, 124, 94, 65, 111, 82, 69, 126, 77, 80, 83, 106, 113, 51, 33, 100, 114, 91, 14, 20, 78, 10, 47, 27, 117, 86, 48, 118, 67, 89, 11, 112, 58, 16, 35, 88, 30, 45, 28, 5, 17, 70, 93, 73, 90, 34, 105, 22, 26, 55, 64, 119, 107, 9, 122, 104, 103, 96, 81, 1, 74, 37], [39, 41, 97, 32, 101, 43, 29, 26, 116, 31, 54, 80, 99, 55, 125, 18, 21, 59, 24, 78, 73, 111, 23, 76, 71, 16, 87, 82, 119, 3, 25, 28, 11, 121, 112, 12, 123, 63, 98, 120, 85, 83, 47, 91, 94, 56, 13, 103, 7, 33, 17, 88, 51, 127, 4, 95, 104, 19, 38, 108, 60, 86, 100, 79, 124, 77, 30, 27, 109, 81, 52, 50, 84, 61, 67, 37, 93, 49, 92, 5, 10, 118, 46, 58, 69, 74, 110, 14, 53, 45, 122, 126, 57, 113, 114, 15, 42, 107, 68, 22, 72, 9, 62, 90, 89, 70, 115, 48, 102, 8, 75, 36, 40, 6, 117, 105, 2, 0, 96, 20, 106, 44, 35, 65, 66, 1, 34, 64], [6, 39, 41, 70, 101, 72, 8, 43, 74, 60, 108, 53, 92, 110, 98, 113, 50, 32, 44, 47, 61, 87, 62, 95, 28, 111, 26, 38, 58, 57, 67, 124, 18, 52, 56, 51, 99, 97, 112, 123, 118, 125, 116, 59, 109, 33, 40, 89, 120, 106, 19, 121, 30, 46, 14, 31, 68, 126, 88, 11, 4, 104, 24, 45, 2, 63, 127, 10, 119, 105, 80, 90, 107, 83, 34, 114, 36, 78, 3, 27, 69, 42, 55, 9, 20, 73, 122, 5, 15, 86, 25, 49, 48, 29, 21, 94, 66, 100, 22, 0, 54, 65, 75, 84, 17, 96, 93, 91, 35, 77, 12, 71, 115, 102, 103, 79, 7, 23, 81, 117, 37, 85, 16, 13, 64, 76, 1, 82], [41, 39, 70, 101, 10, 32, 19, 43, 87, 97, 11, 83, 46, 14, 60, 4, 123, 15, 125, 38, 44, 108, 56, 86, 16, 9, 118, 53, 18, 5, 109, 2, 92, 76, 78, 62, 61, 23, 120, 127, 24, 68, 49, 59, 42, 66, 84, 75, 21, 110, 8, 27, 106, 72, 63, 111, 47, 52, 121, 50, 73, 126, 28, 40, 17, 99, 67, 107, 79, 122, 89, 82, 33, 124, 105, 77, 58, 26, 1, 85, 25, 114, 119, 51, 22, 57, 98, 48, 31, 71, 94, 103, 115, 116, 13, 7, 102, 74, 113, 3, 20, 96, 6, 0, 81, 90, 69, 45, 104, 29, 34, 100, 12, 36, 91, 88, 80, 64, 54, 95, 30, 55, 35, 65, 93, 37, 112, 117], [40, 102, 31, 107, 22, 39, 78, 98, 9, 6, 73, 88, 12, 81, 17, 67, 76, 106, 25, 70, 48, 85, 86, 110, 91, 92, 114, 74, 82, 2, 58, 15, 83, 59, 14, 27, 108, 117, 127, 66, 79, 111, 47, 30, 51, 50, 5, 21, 57, 52, 109, 36, 89, 23, 35, 49, 101, 63, 72, 90, 122, 54, 100, 26, 84, 56, 124, 33, 18, 55, 62, 126, 29, 123, 11, 95, 119, 20, 96, 28, 77, 19, 24, 4, 87, 32, 115, 37, 99, 41, 53, 97, 125, 112, 8, 104, 71, 43, 93, 3, 116, 65, 16, 10, 64, 75, 94, 113, 120, 105, 68, 60, 46, 103, 7, 61, 69, 34, 13, 44, 0, 118, 80, 45, 121, 1, 42, 38], [40, 73, 70, 14, 39, 17, 110, 102, 114, 85, 82, 91, 51, 106, 99, 49, 48, 41, 98, 83, 36, 75, 115, 46, 88, 119, 92, 30, 69, 57, 28, 32, 15, 109, 126, 63, 86, 31, 123, 84, 6, 37, 72, 107, 90, 76, 77, 53, 71, 60, 22, 23, 124, 55, 93, 100, 21, 33, 58, 111, 81, 8, 24, 18, 125, 19, 44, 112, 79, 113, 59, 2, 74, 65, 62, 97, 35, 0, 61, 45, 78, 89, 101, 5, 4, 66, 67, 50, 42, 122, 11, 117, 26, 16, 108, 56, 3, 10, 9, 121, 25, 116, 94, 1, 105, 118, 12, 47, 120, 20, 96, 43, 29, 87, 64, 52, 13, 104, 80, 127, 7, 38, 54, 95, 27, 34, 68, 103], [40, 99, 41, 107, 9, 66, 96, 37, 91, 36, 73, 24, 58, 111, 94, 31, 47, 86, 114, 50, 92, 89, 10, 52, 29, 59, 34, 6, 39, 18, 127, 44, 100, 22, 76, 21, 56, 122, 121, 57, 102, 79, 70, 12, 46, 84, 81, 85, 63, 116, 72, 88, 43, 110, 5, 83, 87, 45, 69, 27, 3, 2, 8, 78, 51, 115, 106, 67, 74, 7, 54, 118, 19, 42, 104, 124, 11, 97, 4, 75, 105, 35, 33, 62, 55, 28, 30, 0, 82, 77, 95, 93, 120, 98, 20, 25, 117, 32, 53, 17, 90, 26, 71, 108, 64, 14, 23, 101, 48, 49, 65, 15, 119, 125, 103, 80, 68, 112, 1, 126, 13, 16, 113, 123, 61, 109, 38, 60], [40, 41, 102, 37, 92, 32, 90, 88, 96, 107, 23, 31, 74, 39, 84, 97, 25, 75, 82, 35, 79, 10, 91, 15, 7, 4, 18, 116, 20, 98, 87, 99, 69, 33, 86, 111, 100, 122, 121, 127, 65, 58, 13, 8, 56, 47, 95, 9, 50, 114, 81, 110, 59, 118, 72, 48, 77, 17, 29, 24, 21, 52, 104, 22, 54, 28, 94, 83, 85, 51, 57, 14, 73, 36, 16, 19, 66, 76, 71, 11, 93, 78, 120, 27, 6, 108, 5, 117, 43, 49, 63, 3, 67, 2, 61, 26, 64, 70, 68, 0, 101, 105, 124, 62, 55, 30, 80, 89, 103, 34, 44, 113, 45, 60, 1, 38, 106, 42, 115, 109, 12, 125, 119, 126, 53, 112, 123, 46], [103, 105, 96, 47, 59, 45, 42, 30, 19, 93, 14, 112, 113, 32, 86, 0, 127, 104, 38, 21, 84, 57, 48, 89, 72, 124, 87, 81, 51, 80, 55, 44, 25, 63, 126, 94, 118, 123, 34, 52, 68, 26, 40, 7, 115, 27, 3, 33, 88, 11, 120, 8, 125, 1, 23, 66, 70, 110, 77, 5, 107, 39, 50, 73, 58, 46, 13, 119, 6, 91, 9, 69, 15, 17, 76, 74, 43, 54, 24, 79, 95, 75, 22, 82, 78, 90, 12, 121, 92, 16, 71, 31, 114, 4, 117, 116, 18, 20, 67, 61, 53, 97, 37, 28, 41, 49, 35, 60, 36, 122, 99, 10, 85, 2, 109, 100, 101, 108, 83, 102, 56, 111, 65, 62, 98, 29, 64, 106], [104, 42, 103, 96, 45, 34, 47, 93, 32, 89, 27, 30, 84, 94, 26, 21, 81, 112, 116, 87, 105, 19, 51, 102, 86, 25, 38, 88, 80, 15, 41, 23, 119, 33, 20, 118, 52, 111, 114, 113, 44, 110, 85, 57, 98, 125, 124, 13, 48, 54, 11, 107, 43, 73, 55, 37, 106, 12, 100, 59, 10, 76, 24, 127, 77, 82, 115, 60, 14, 72, 101, 68, 90, 121, 29, 91, 0, 63, 97, 58, 75, 9, 61, 46, 126, 108, 4, 39, 56, 18, 22, 28, 70, 117, 120, 53, 109, 66, 6, 123, 3, 1, 95, 92, 49, 31, 50, 122, 99, 40, 69, 5, 62, 67, 83, 79, 16, 36, 64, 78, 2, 8, 74, 7, 35, 65, 71, 17], [103, 104, 96, 42, 45, 105, 30, 19, 93, 59, 47, 89, 113, 14, 86, 112, 84, 21, 32, 124, 80, 87, 57, 81, 44, 0, 72, 127, 88, 115, 25, 26, 48, 63, 119, 123, 51, 68, 118, 34, 52, 15, 55, 98, 126, 61, 43, 73, 24, 110, 102, 46, 107, 125, 121, 23, 116, 114, 41, 56, 54, 66, 27, 8, 3, 70, 91, 76, 6, 11, 39, 13, 1, 33, 97, 74, 4, 94, 22, 77, 58, 7, 37, 95, 82, 50, 36, 17, 117, 120, 53, 122, 75, 16, 90, 79, 67, 111, 40, 5, 64, 9, 2, 78, 18, 38, 31, 71, 60, 65, 12, 100, 108, 92, 85, 69, 29, 49, 99, 101, 62, 20, 109, 28, 35, 10, 106, 83], [42, 104, 103, 96, 45, 30, 89, 112, 34, 27, 113, 32, 102, 47, 93, 26, 105, 84, 94, 21, 81, 86, 88, 87, 124, 25, 59, 80, 44, 15, 19, 38, 114, 20, 57, 85, 52, 51, 23, 118, 127, 125, 119, 43, 100, 29, 33, 41, 46, 116, 82, 13, 126, 48, 11, 107, 77, 73, 98, 72, 122, 106, 54, 37, 95, 55, 24, 76, 12, 115, 10, 14, 97, 39, 110, 68, 101, 123, 56, 63, 92, 4, 0, 6, 91, 53, 90, 121, 50, 40, 61, 111, 22, 75, 9, 18, 16, 70, 117, 31, 60, 3, 58, 99, 66, 49, 74, 67, 1, 108, 109, 79, 2, 120, 35, 62, 64, 36, 28, 7, 83, 8, 69, 65, 5, 78, 71, 17], [100, 39, 31, 50, 91, 25, 47, 97, 96, 42, 116, 40, 90, 60, 41, 46, 19, 70, 44, 89, 22, 109, 48, 33, 95, 77, 27, 107, 2, 43, 101, 0, 18, 17, 108, 80, 49, 13, 53, 7, 10, 103, 64, 45, 21, 94, 12, 74, 79, 76, 118, 115, 124, 5, 73, 120, 35, 56, 99, 28, 55, 81, 54, 110, 14, 114, 126, 6, 68, 125, 127, 123, 93, 112, 113, 38, 58, 57, 61, 122, 32, 121, 75, 62, 84, 88, 119, 111, 59, 106, 3, 63, 72, 52, 30, 36, 26, 51, 86, 34, 98, 102, 29, 117, 16, 9, 8, 67, 83, 105, 11, 71, 65, 15, 4, 69, 1, 24, 104, 87, 66, 20, 82, 37, 92, 78, 23, 85], [40, 25, 64, 101, 42, 41, 45, 110, 72, 113, 56, 112, 108, 68, 107, 111, 114, 39, 2, 61, 125, 55, 119, 51, 97, 63, 31, 121, 22, 57, 54, 126, 59, 74, 123, 80, 87, 96, 52, 62, 18, 120, 7, 48, 0, 84, 58, 28, 91, 70, 115, 93, 11, 5, 14, 49, 60, 17, 30, 76, 66, 44, 19, 24, 88, 77, 3, 100, 13, 50, 1, 79, 109, 69, 90, 95, 99, 102, 104, 33, 53, 71, 21, 73, 8, 26, 106, 65, 75, 116, 117, 46, 98, 105, 89, 92, 67, 47, 35, 34, 81, 12, 4, 6, 122, 36, 78, 85, 10, 94, 43, 82, 83, 9, 118, 86, 103, 15, 127, 37, 16, 38, 27, 124, 23, 20, 32, 29], [0, 39, 22, 42, 96, 41, 91, 18, 10, 107, 79, 40, 14, 21, 19, 50, 77, 71, 80, 88, 70, 84, 48, 5, 45, 85, 44, 17, 11, 81, 100, 46, 28, 60, 76, 83, 72, 110, 66, 63, 57, 36, 47, 109, 2, 108, 56, 125, 120, 35, 111, 93, 61, 73, 114, 24, 112, 49, 55, 113, 87, 90, 121, 30, 64, 4, 99, 126, 116, 15, 51, 103, 52, 54, 6, 82, 97, 43, 115, 86, 119, 123, 58, 7, 101, 117, 69, 53, 13, 33, 59, 9, 62, 32, 68, 75, 31, 8, 3, 1, 12, 92, 29, 20, 78, 65, 104, 37, 27, 106, 122, 74, 16, 23, 34, 95, 25, 118, 26, 89, 94, 67, 98, 38, 105, 102, 127, 124], [108, 54, 112, 119, 59, 62, 113, 110, 123, 111, 26, 51, 61, 107, 55, 125, 89, 115, 114, 45, 121, 98, 58, 57, 63, 48, 52, 49, 34, 109, 122, 41, 126, 43, 127, 38, 32, 95, 56, 94, 100, 33, 120, 53, 50, 24, 124, 29, 40, 101, 44, 90, 25, 85, 8, 102, 60, 97, 88, 86, 83, 27, 42, 118, 46, 99, 66, 105, 4, 12, 19, 10, 47, 116, 96, 13, 82, 22, 16, 92, 30, 64, 31, 37, 117, 72, 28, 91, 87, 106, 69, 2, 103, 17, 9, 36, 70, 5, 84, 65, 76, 104, 75, 11, 39, 0, 78, 73, 20, 77, 23, 3, 6, 81, 68, 15, 74, 79, 80, 21, 35, 7, 93, 18, 71, 14, 67, 1]], "model.layers.0.self_attn.k_proj": [[0, 97, 42, 47, 103, 86, 93, 1, 25, 38, 88, 49, 40, 108, 65, 91, 117, 48, 57, 66, 78, 46, 68, 9, 64, 50, 52, 19, 45, 31, 118, 2, 55, 51, 70, 67, 85, 61, 77, 127, 82, 17, 63, 120, 80, 126, 87, 6, 75, 124, 72, 123, 3, 54, 62, 122, 14, 121, 7, 69, 53, 5, 56, 12, 10, 84, 41, 119, 115, 81, 34, 4, 76, 101, 107, 105, 60, 26, 74, 23, 79, 89, 15, 73, 24, 83, 22, 21, 18, 58, 43, 28, 33, 116, 29, 35, 27, 11, 112, 32, 30, 92, 16, 37, 104, 98, 59, 99, 94, 39, 90, 114, 13, 96, 36, 113, 100, 109, 95, 125, 111, 20, 106, 71, 8, 110, 44, 102], [39, 97, 105, 102, 112, 46, 65, 43, 29, 113, 64, 51, 77, 68, 111, 25, 86, 125, 18, 67, 106, 70, 9, 80, 17, 4, 59, 11, 116, 31, 124, 24, 126, 12, 88, 3, 108, 78, 52, 71, 73, 16, 45, 47, 90, 66, 42, 22, 53, 57, 21, 61, 122, 118, 20, 1, 6, 121, 114, 93, 109, 54, 89, 62, 76, 120, 74, 48, 69, 123, 33, 87, 50, 13, 55, 7, 10, 127, 84, 95, 83, 115, 41, 23, 26, 119, 2, 117, 60, 103, 56, 5, 30, 37, 72, 81, 40, 100, 101, 91, 14, 104, 94, 15, 92, 19, 38, 85, 8, 0, 49, 79, 75, 35, 36, 28, 63, 98, 82, 32, 34, 99, 110, 44, 27, 58, 96, 107], [38, 39, 97, 109, 105, 89, 25, 49, 106, 78, 44, 40, 20, 48, 36, 126, 33, 64, 62, 31, 104, 108, 82, 27, 122, 29, 120, 94, 34, 96, 113, 117, 51, 42, 107, 103, 11, 8, 115, 16, 18, 84, 123, 14, 30, 37, 124, 91, 125, 50, 88, 56, 61, 41, 17, 87, 114, 66, 21, 47, 12, 53, 54, 19, 60, 26, 52, 73, 28, 24, 23, 118, 90, 32, 81, 83, 59, 102, 85, 15, 1, 74, 93, 57, 121, 43, 75, 58, 3, 112, 70, 92, 9, 69, 13, 127, 111, 79, 5, 6, 80, 119, 22, 63, 55, 76, 99, 7, 116, 35, 110, 77, 68, 4, 101, 72, 86, 46, 67, 10, 71, 45, 0, 2, 95, 98, 65, 100], [107, 39, 108, 64, 40, 47, 105, 97, 113, 29, 65, 30, 3, 66, 36, 83, 48, 89, 116, 122, 101, 90, 2, 95, 17, 13, 16, 33, 74, 38, 85, 23, 19, 63, 87, 27, 109, 5, 71, 114, 24, 81, 26, 84, 104, 53, 50, 92, 82, 93, 4, 34, 22, 121, 106, 76, 1, 14, 78, 68, 35, 79, 115, 103, 120, 123, 9, 42, 55, 41, 69, 7, 126, 118, 88, 18, 127, 28, 112, 51, 59, 80, 54, 124, 111, 117, 62, 91, 15, 43, 77, 96, 125, 110, 98, 6, 45, 49, 119, 20, 100, 94, 60, 25, 75, 58, 86, 56, 52, 44, 0, 57, 21, 61, 12, 10, 46, 31, 102, 11, 32, 70, 8, 72, 99, 67, 37, 73], [103, 105, 107, 33, 0, 37, 65, 83, 24, 28, 95, 87, 78, 19, 84, 23, 25, 77, 34, 96, 125, 54, 94, 14, 92, 35, 21, 89, 79, 5, 116, 123, 22, 59, 13, 18, 29, 55, 73, 81, 36, 12, 106, 52, 127, 111, 102, 56, 121, 15, 97, 42, 63, 46, 48, 120, 11, 45, 91, 51, 71, 76, 126, 93, 85, 66, 119, 122, 90, 8, 68, 60, 26, 98, 2, 118, 40, 39, 109, 16, 113, 57, 32, 58, 3, 27, 112, 62, 61, 44, 30, 7, 80, 115, 114, 124, 41, 74, 117, 50, 17, 110, 49, 31, 69, 53, 47, 108, 4, 64, 82, 9, 104, 67, 100, 75, 43, 1, 88, 72, 38, 101, 10, 20, 70, 86, 99, 6], [104, 38, 43, 103, 95, 42, 45, 46, 105, 112, 84, 113, 53, 50, 59, 125, 96, 90, 115, 88, 60, 23, 13, 62, 34, 61, 91, 82, 58, 124, 0, 57, 92, 63, 111, 21, 123, 120, 119, 20, 79, 19, 55, 54, 17, 15, 126, 22, 81, 87, 27, 127, 44, 72, 49, 85, 51, 86, 56, 52, 117, 80, 48, 75, 114, 31, 89, 18, 94, 116, 29, 121, 122, 1, 33, 12, 16, 118, 32, 7, 35, 101, 64, 40, 78, 47, 24, 109, 65, 69, 71, 14, 76, 73, 100, 11, 97, 110, 41, 8, 26, 4, 108, 67, 39, 93, 25, 36, 70, 28, 10, 66, 74, 6, 77, 37, 107, 99, 5, 68, 83, 9, 98, 3, 102, 106, 30, 2], [39, 111, 32, 1, 0, 109, 2, 6, 48, 4, 40, 41, 29, 108, 67, 84, 66, 15, 107, 11, 65, 81, 115, 70, 89, 73, 52, 21, 30, 57, 104, 119, 80, 42, 118, 87, 124, 123, 13, 106, 72, 3, 49, 86, 58, 98, 63, 27, 114, 110, 127, 113, 126, 76, 53, 26, 25, 14, 83, 59, 96, 55, 77, 88, 19, 94, 121, 93, 23, 125, 7, 78, 51, 120, 46, 82, 116, 71, 68, 105, 85, 24, 97, 8, 61, 74, 103, 10, 54, 47, 117, 122, 34, 91, 18, 92, 20, 62, 100, 12, 31, 75, 56, 5, 90, 9, 79, 28, 33, 101, 69, 99, 22, 36, 95, 35, 43, 50, 60, 16, 112, 37, 44, 102, 17, 45, 64, 38], [64, 66, 4, 106, 6, 67, 1, 71, 10, 69, 105, 46, 43, 104, 37, 44, 92, 47, 109, 103, 94, 15, 8, 83, 85, 7, 56, 53, 73, 86, 126, 116, 32, 78, 22, 82, 75, 24, 77, 49, 9, 120, 76, 79, 50, 91, 114, 52, 127, 48, 17, 16, 35, 19, 124, 18, 102, 12, 20, 68, 58, 57, 63, 60, 115, 13, 113, 121, 123, 14, 81, 117, 45, 62, 59, 55, 51, 111, 122, 125, 23, 61, 119, 118, 110, 21, 3, 100, 112, 33, 54, 38, 72, 34, 2, 11, 80, 65, 29, 70, 88, 27, 0, 36, 98, 84, 89, 97, 74, 93, 30, 5, 107, 90, 96, 26, 87, 95, 28, 39, 108, 41, 42, 99, 101, 31, 25, 40]], "model.layers.0.self_attn.qk_proj": [[104, 39, 33, 42, 103, 41, 97, 64, 0, 93, 108, 25, 107, 89, 105, 113, 126, 29, 102, 121, 111, 48, 57, 95, 59, 115, 47, 123, 70, 88, 53, 63, 96, 30, 24, 20, 43, 87, 116, 23, 38, 109, 78, 124, 49, 19, 61, 112, 52, 51, 50, 120, 27, 83, 119, 22, 32, 114, 45, 21, 106, 125, 122, 40, 44, 127, 86, 54, 17, 118, 56, 85, 55, 14, 84, 46, 65, 81, 68, 1, 91, 3, 62, 4, 66, 18, 82, 77, 9, 79, 117, 2, 90, 92, 110, 8, 80, 67, 13, 76, 15, 75, 60, 12, 26, 16, 31, 11, 73, 10, 58, 71, 74, 7, 5, 69, 28, 101, 6, 34, 98, 94, 72, 37, 100, 35, 36, 99], [39, 104, 33, 42, 103, 41, 64, 97, 108, 0, 93, 107, 105, 25, 113, 126, 29, 89, 95, 102, 111, 59, 47, 38, 123, 121, 124, 88, 115, 50, 53, 106, 116, 48, 85, 119, 30, 70, 43, 63, 40, 87, 96, 24, 21, 57, 49, 23, 122, 27, 20, 22, 44, 114, 109, 52, 19, 120, 54, 61, 45, 67, 127, 17, 78, 32, 81, 83, 84, 62, 56, 125, 86, 51, 46, 14, 1, 118, 92, 112, 55, 79, 26, 18, 12, 2, 82, 15, 76, 65, 90, 3, 77, 91, 66, 117, 110, 73, 74, 4, 31, 9, 16, 13, 68, 75, 10, 80, 60, 8, 7, 58, 11, 71, 28, 98, 94, 6, 5, 72, 69, 34, 101, 100, 35, 36, 37, 99], [104, 39, 33, 42, 103, 41, 97, 108, 93, 0, 64, 25, 89, 29, 105, 107, 113, 95, 126, 102, 59, 123, 111, 115, 121, 47, 57, 88, 63, 24, 48, 124, 96, 38, 53, 116, 30, 87, 19, 52, 119, 43, 23, 20, 21, 40, 61, 127, 27, 109, 51, 84, 50, 22, 112, 106, 49, 32, 78, 86, 125, 14, 85, 54, 56, 17, 45, 120, 114, 70, 122, 44, 3, 118, 81, 46, 83, 65, 62, 82, 1, 91, 92, 15, 55, 76, 2, 117, 110, 12, 90, 79, 80, 26, 67, 16, 18, 77, 68, 11, 66, 13, 73, 9, 4, 31, 75, 6, 58, 74, 60, 72, 71, 10, 8, 28, 69, 7, 34, 98, 5, 94, 101, 100, 37, 35, 36, 99], [39, 104, 42, 33, 103, 41, 97, 108, 64, 0, 105, 93, 107, 113, 126, 89, 25, 29, 59, 124, 123, 111, 119, 63, 102, 115, 121, 95, 53, 47, 48, 50, 57, 87, 106, 96, 116, 43, 88, 38, 49, 30, 61, 23, 24, 22, 54, 20, 51, 6, 122, 44, 114, 27, 40, 109, 52, 78, 85, 125, 21, 83, 45, 17, 19, 118, 86, 84, 112, 62, 65, 81, 56, 55, 14, 120, 79, 127, 32, 68, 66, 67, 1, 46, 3, 9, 91, 110, 18, 4, 12, 73, 82, 26, 2, 72, 70, 92, 77, 15, 117, 76, 13, 16, 90, 58, 80, 75, 10, 31, 60, 74, 11, 7, 71, 69, 28, 5, 98, 94, 101, 34, 8, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 41, 97, 64, 108, 0, 93, 105, 25, 89, 107, 113, 126, 115, 102, 59, 29, 123, 47, 95, 119, 121, 63, 53, 111, 124, 57, 48, 116, 43, 6, 20, 30, 23, 88, 96, 24, 106, 109, 38, 87, 50, 52, 22, 21, 49, 27, 17, 40, 61, 19, 112, 122, 78, 3, 54, 14, 44, 45, 114, 51, 91, 84, 86, 46, 65, 85, 125, 1, 127, 118, 120, 32, 81, 56, 62, 82, 83, 26, 73, 72, 55, 15, 9, 2, 110, 4, 79, 66, 76, 92, 117, 12, 67, 18, 16, 77, 11, 13, 68, 31, 80, 90, 75, 60, 58, 10, 74, 71, 70, 7, 98, 34, 28, 69, 5, 94, 101, 8, 100, 37, 36, 35, 99], [39, 104, 42, 33, 103, 41, 97, 108, 0, 64, 25, 93, 89, 105, 107, 126, 95, 113, 29, 102, 115, 123, 59, 48, 121, 53, 124, 6, 30, 88, 111, 47, 87, 63, 96, 106, 116, 23, 119, 38, 27, 24, 43, 57, 20, 49, 40, 22, 61, 44, 50, 54, 52, 122, 21, 109, 1, 51, 86, 81, 85, 19, 78, 114, 3, 84, 45, 125, 112, 32, 127, 14, 83, 118, 120, 17, 2, 46, 56, 55, 76, 62, 82, 91, 92, 18, 72, 77, 65, 79, 9, 117, 15, 12, 66, 90, 110, 67, 68, 4, 26, 16, 31, 13, 80, 73, 75, 10, 74, 60, 7, 11, 71, 58, 98, 5, 94, 28, 34, 70, 69, 101, 8, 100, 35, 37, 99, 36], [39, 104, 33, 42, 103, 97, 41, 0, 64, 93, 108, 113, 105, 107, 25, 89, 126, 47, 29, 95, 102, 30, 121, 59, 123, 115, 48, 111, 53, 6, 124, 88, 24, 50, 119, 49, 116, 27, 38, 106, 63, 43, 57, 96, 87, 23, 21, 32, 20, 125, 52, 61, 22, 109, 112, 44, 85, 45, 51, 19, 83, 122, 114, 40, 14, 54, 55, 78, 17, 120, 2, 84, 86, 62, 1, 118, 127, 66, 91, 3, 56, 72, 110, 4, 81, 18, 82, 77, 46, 68, 26, 79, 65, 16, 13, 9, 15, 76, 67, 12, 80, 92, 75, 117, 31, 90, 73, 60, 74, 58, 10, 11, 71, 7, 28, 70, 5, 69, 98, 34, 94, 101, 8, 100, 37, 35, 36, 99], [39, 33, 104, 42, 103, 97, 41, 93, 64, 108, 0, 25, 105, 113, 89, 107, 126, 29, 95, 47, 59, 102, 48, 88, 53, 115, 116, 123, 124, 111, 24, 57, 30, 38, 96, 63, 87, 52, 121, 19, 43, 23, 27, 20, 50, 119, 51, 85, 32, 6, 40, 21, 106, 49, 109, 61, 127, 78, 125, 86, 122, 84, 22, 54, 81, 120, 45, 112, 14, 114, 91, 17, 83, 56, 46, 44, 118, 2, 67, 16, 90, 110, 66, 62, 1, 117, 4, 79, 65, 76, 92, 72, 68, 82, 15, 18, 75, 13, 26, 55, 12, 9, 3, 80, 31, 77, 11, 73, 10, 58, 74, 60, 70, 28, 7, 98, 71, 34, 94, 101, 69, 5, 100, 37, 8, 35, 36, 99], [39, 104, 33, 42, 103, 41, 97, 64, 0, 108, 93, 105, 107, 113, 25, 95, 89, 29, 126, 48, 59, 124, 102, 88, 47, 38, 115, 123, 111, 53, 50, 106, 21, 27, 44, 30, 96, 43, 87, 116, 121, 109, 52, 23, 24, 119, 40, 63, 49, 57, 85, 19, 22, 114, 86, 122, 20, 127, 78, 54, 83, 56, 51, 17, 32, 84, 61, 14, 81, 90, 125, 120, 112, 45, 46, 118, 66, 1, 70, 82, 15, 62, 12, 79, 91, 55, 2, 76, 26, 65, 67, 18, 92, 6, 110, 9, 117, 73, 3, 72, 77, 16, 31, 68, 10, 13, 80, 74, 75, 4, 60, 11, 28, 58, 7, 98, 94, 34, 71, 5, 69, 8, 101, 100, 37, 35, 36, 99], [33, 104, 39, 42, 103, 41, 97, 89, 25, 93, 108, 29, 64, 95, 0, 113, 102, 107, 105, 126, 38, 87, 59, 88, 96, 48, 27, 24, 53, 47, 32, 116, 30, 115, 124, 85, 123, 19, 21, 40, 106, 23, 57, 43, 109, 121, 111, 22, 44, 20, 83, 63, 122, 78, 52, 50, 84, 86, 81, 49, 127, 2, 56, 45, 17, 26, 119, 66, 14, 114, 54, 61, 51, 125, 70, 91, 82, 4, 120, 112, 76, 12, 46, 118, 92, 18, 1, 67, 15, 55, 117, 68, 79, 65, 62, 80, 16, 90, 13, 77, 73, 74, 72, 31, 75, 9, 11, 10, 3, 110, 71, 7, 58, 60, 6, 28, 8, 34, 94, 98, 101, 69, 5, 100, 37, 36, 35, 99], [39, 104, 33, 42, 103, 0, 41, 64, 97, 108, 25, 93, 113, 105, 107, 126, 89, 59, 29, 115, 111, 47, 57, 121, 102, 63, 123, 48, 95, 124, 70, 52, 88, 96, 116, 53, 119, 43, 87, 61, 49, 23, 24, 30, 38, 50, 19, 125, 106, 20, 51, 21, 32, 109, 56, 22, 54, 14, 78, 1, 112, 120, 83, 67, 45, 40, 114, 44, 122, 118, 84, 127, 27, 46, 86, 81, 68, 65, 3, 17, 62, 85, 55, 91, 110, 2, 79, 18, 66, 77, 82, 4, 76, 9, 13, 117, 92, 15, 90, 26, 12, 73, 60, 11, 16, 31, 74, 58, 75, 80, 10, 8, 71, 7, 72, 28, 5, 69, 6, 98, 101, 94, 34, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 0, 41, 64, 108, 97, 107, 105, 113, 93, 126, 25, 89, 59, 121, 63, 111, 115, 70, 123, 47, 29, 95, 102, 119, 48, 124, 96, 53, 116, 49, 57, 51, 88, 52, 87, 50, 61, 27, 30, 65, 109, 43, 106, 22, 120, 24, 23, 21, 114, 38, 56, 19, 122, 40, 54, 14, 45, 125, 118, 83, 84, 112, 20, 46, 44, 127, 55, 32, 86, 81, 78, 62, 17, 67, 82, 85, 3, 91, 2, 90, 117, 79, 66, 77, 92, 68, 4, 110, 76, 15, 13, 60, 73, 26, 31, 1, 9, 12, 75, 18, 80, 8, 58, 74, 11, 16, 71, 10, 28, 7, 69, 5, 98, 72, 34, 94, 6, 101, 100, 37, 36, 35, 99], [39, 33, 104, 42, 103, 97, 41, 108, 89, 93, 95, 105, 0, 113, 25, 64, 29, 126, 107, 30, 102, 27, 124, 48, 38, 47, 115, 53, 106, 88, 24, 59, 23, 116, 123, 21, 96, 40, 32, 87, 86, 111, 50, 121, 44, 85, 43, 22, 122, 20, 109, 63, 49, 70, 57, 119, 52, 83, 81, 19, 51, 14, 91, 17, 114, 118, 78, 54, 127, 125, 45, 112, 61, 26, 92, 84, 2, 56, 76, 66, 18, 15, 82, 12, 65, 62, 79, 46, 1, 120, 117, 80, 55, 73, 31, 68, 90, 110, 16, 67, 77, 3, 8, 13, 75, 74, 4, 9, 6, 10, 11, 60, 34, 58, 71, 28, 98, 94, 7, 72, 69, 101, 5, 100, 36, 37, 35, 99], [33, 104, 39, 42, 103, 41, 97, 93, 108, 89, 25, 0, 64, 95, 105, 113, 29, 107, 126, 102, 121, 115, 53, 30, 88, 38, 48, 47, 96, 59, 124, 123, 57, 43, 87, 21, 111, 116, 22, 23, 27, 63, 32, 40, 51, 52, 50, 106, 20, 109, 86, 44, 83, 122, 24, 14, 49, 78, 119, 127, 19, 112, 84, 81, 61, 54, 17, 114, 85, 55, 120, 91, 45, 46, 70, 18, 82, 62, 79, 65, 12, 2, 56, 92, 3, 66, 118, 4, 125, 90, 73, 26, 117, 13, 1, 68, 8, 80, 15, 76, 6, 31, 67, 77, 110, 16, 11, 9, 75, 10, 60, 74, 28, 7, 71, 5, 58, 98, 34, 101, 94, 69, 100, 72, 37, 35, 36, 99], [39, 104, 33, 42, 103, 0, 97, 41, 108, 93, 64, 113, 25, 107, 126, 47, 105, 89, 59, 29, 115, 124, 48, 63, 121, 123, 53, 95, 57, 111, 116, 102, 52, 30, 96, 24, 49, 88, 119, 51, 87, 23, 38, 20, 106, 78, 32, 61, 83, 44, 43, 27, 21, 22, 50, 127, 109, 122, 86, 54, 85, 19, 114, 6, 17, 125, 14, 3, 40, 112, 4, 81, 118, 46, 45, 56, 65, 62, 84, 2, 55, 120, 82, 66, 8, 91, 18, 76, 68, 79, 26, 117, 1, 12, 9, 11, 67, 73, 90, 70, 13, 15, 16, 110, 92, 77, 31, 60, 75, 80, 71, 10, 7, 58, 74, 28, 5, 98, 34, 69, 94, 72, 101, 37, 100, 35, 36, 99], [39, 104, 33, 42, 103, 41, 64, 0, 97, 108, 93, 113, 25, 105, 107, 89, 126, 115, 29, 102, 59, 123, 95, 111, 47, 6, 48, 121, 53, 88, 63, 57, 119, 124, 43, 49, 116, 30, 52, 109, 24, 38, 106, 87, 50, 96, 61, 20, 23, 21, 19, 44, 27, 114, 14, 122, 125, 51, 54, 85, 22, 112, 56, 40, 120, 84, 83, 118, 127, 17, 65, 62, 78, 55, 45, 46, 32, 66, 81, 91, 3, 86, 1, 67, 8, 76, 18, 79, 117, 68, 2, 73, 110, 26, 90, 82, 77, 9, 4, 13, 92, 12, 16, 80, 74, 15, 60, 31, 11, 75, 10, 7, 58, 28, 71, 70, 94, 69, 5, 98, 34, 101, 72, 37, 100, 35, 99, 36], [39, 33, 104, 42, 103, 41, 97, 108, 0, 93, 64, 105, 107, 25, 89, 95, 113, 126, 29, 102, 59, 30, 111, 121, 47, 123, 88, 38, 124, 115, 87, 6, 24, 43, 106, 50, 53, 48, 116, 119, 40, 27, 21, 23, 96, 86, 109, 85, 63, 20, 44, 83, 32, 122, 22, 84, 49, 52, 57, 17, 14, 91, 114, 81, 51, 19, 61, 54, 127, 45, 56, 65, 120, 92, 125, 90, 79, 66, 2, 76, 78, 112, 82, 46, 118, 26, 15, 18, 62, 55, 80, 3, 13, 67, 8, 68, 9, 31, 12, 117, 1, 74, 73, 16, 110, 4, 77, 11, 75, 10, 60, 58, 71, 7, 28, 34, 94, 98, 5, 72, 69, 100, 70, 101, 35, 36, 37, 99], [39, 104, 33, 42, 103, 41, 97, 108, 93, 64, 0, 89, 29, 105, 113, 25, 126, 107, 95, 115, 47, 124, 111, 116, 88, 59, 48, 121, 123, 53, 57, 87, 102, 24, 38, 63, 30, 20, 23, 96, 43, 21, 6, 40, 127, 22, 106, 119, 52, 50, 27, 51, 109, 49, 61, 32, 54, 17, 78, 84, 44, 112, 114, 19, 83, 118, 85, 82, 56, 120, 45, 122, 14, 125, 81, 1, 46, 117, 86, 65, 91, 67, 62, 68, 26, 2, 92, 90, 55, 4, 79, 12, 77, 66, 15, 18, 13, 76, 31, 3, 9, 110, 16, 73, 75, 80, 11, 74, 10, 60, 58, 8, 7, 98, 71, 70, 5, 72, 69, 28, 34, 94, 101, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 64, 97, 41, 0, 108, 93, 107, 113, 89, 105, 126, 25, 29, 59, 121, 115, 102, 95, 47, 111, 123, 124, 48, 63, 53, 57, 88, 119, 43, 30, 38, 24, 50, 116, 96, 49, 61, 51, 106, 87, 20, 52, 54, 19, 23, 21, 44, 27, 22, 81, 6, 14, 125, 122, 85, 32, 112, 114, 109, 127, 120, 55, 78, 62, 40, 84, 83, 56, 2, 45, 17, 86, 118, 46, 1, 67, 82, 91, 65, 3, 110, 68, 79, 4, 66, 12, 26, 77, 18, 73, 15, 90, 117, 92, 76, 9, 70, 13, 80, 11, 16, 10, 75, 31, 74, 58, 60, 7, 71, 69, 72, 8, 28, 5, 98, 94, 34, 101, 37, 100, 35, 99, 36], [39, 33, 104, 42, 103, 41, 108, 97, 0, 64, 113, 25, 93, 89, 107, 105, 29, 95, 126, 102, 115, 59, 48, 124, 123, 121, 111, 38, 88, 53, 63, 21, 24, 47, 106, 116, 30, 87, 23, 49, 22, 27, 96, 119, 85, 43, 20, 57, 122, 40, 52, 109, 44, 65, 32, 14, 83, 17, 81, 51, 19, 61, 50, 54, 114, 125, 78, 91, 127, 3, 84, 112, 70, 79, 56, 86, 67, 120, 46, 45, 12, 62, 18, 2, 118, 26, 76, 66, 73, 55, 82, 92, 4, 16, 13, 6, 90, 1, 15, 9, 117, 110, 31, 77, 80, 74, 10, 68, 75, 72, 7, 71, 11, 28, 58, 60, 94, 98, 34, 5, 8, 69, 101, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 41, 64, 97, 108, 0, 93, 105, 113, 89, 107, 59, 25, 29, 126, 95, 123, 47, 121, 48, 111, 115, 102, 63, 124, 70, 57, 87, 119, 53, 116, 88, 30, 109, 49, 43, 50, 24, 56, 96, 114, 61, 38, 52, 65, 51, 106, 40, 127, 20, 44, 83, 85, 120, 118, 54, 22, 86, 23, 46, 21, 19, 122, 78, 112, 27, 125, 81, 45, 14, 32, 62, 17, 84, 67, 91, 82, 90, 117, 15, 1, 55, 18, 3, 110, 76, 12, 13, 26, 66, 68, 77, 60, 72, 31, 2, 4, 9, 16, 75, 79, 92, 58, 73, 11, 10, 80, 6, 74, 7, 98, 28, 71, 69, 5, 34, 94, 100, 8, 101, 37, 35, 36, 99], [39, 104, 33, 42, 103, 41, 97, 108, 107, 0, 93, 113, 64, 105, 126, 89, 25, 95, 29, 30, 47, 59, 123, 102, 88, 124, 121, 111, 70, 115, 38, 43, 116, 24, 48, 53, 119, 106, 50, 19, 96, 63, 87, 57, 44, 21, 49, 23, 109, 27, 85, 52, 22, 14, 40, 51, 122, 20, 32, 56, 54, 114, 61, 17, 84, 127, 125, 2, 120, 62, 4, 83, 81, 67, 86, 78, 55, 79, 66, 112, 82, 118, 46, 26, 91, 68, 65, 1, 45, 15, 76, 16, 72, 18, 92, 117, 73, 12, 3, 9, 90, 77, 11, 110, 74, 13, 31, 80, 10, 71, 60, 58, 75, 7, 28, 94, 98, 69, 8, 34, 5, 101, 100, 37, 6, 35, 36, 99], [39, 104, 33, 42, 103, 41, 0, 97, 108, 105, 93, 64, 25, 89, 113, 107, 95, 29, 126, 102, 115, 59, 111, 123, 70, 124, 48, 53, 87, 24, 88, 121, 106, 47, 38, 27, 63, 21, 116, 119, 43, 50, 96, 22, 30, 40, 20, 23, 19, 57, 52, 49, 122, 114, 109, 85, 44, 54, 17, 51, 84, 78, 81, 65, 61, 3, 45, 112, 86, 120, 125, 56, 127, 32, 14, 91, 62, 82, 118, 46, 15, 92, 12, 79, 1, 18, 110, 26, 9, 76, 72, 83, 73, 67, 55, 117, 2, 80, 16, 66, 31, 77, 13, 10, 75, 4, 68, 90, 11, 60, 58, 7, 74, 6, 28, 34, 98, 5, 69, 71, 94, 101, 100, 35, 8, 36, 37, 99], [39, 104, 33, 42, 103, 41, 97, 108, 0, 64, 25, 93, 107, 105, 89, 29, 113, 95, 126, 102, 115, 123, 48, 124, 88, 53, 59, 121, 30, 87, 47, 38, 96, 22, 63, 111, 106, 23, 43, 27, 49, 24, 65, 40, 116, 20, 119, 50, 44, 109, 122, 52, 114, 57, 51, 21, 85, 14, 84, 54, 70, 83, 81, 127, 61, 17, 86, 82, 19, 32, 79, 56, 112, 46, 78, 118, 45, 120, 62, 91, 55, 2, 117, 125, 3, 12, 18, 9, 72, 92, 13, 76, 26, 16, 66, 31, 1, 67, 90, 68, 110, 11, 77, 10, 73, 15, 80, 6, 4, 60, 75, 71, 74, 58, 98, 28, 34, 7, 94, 5, 69, 101, 8, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 97, 41, 93, 64, 0, 108, 25, 107, 113, 89, 105, 102, 126, 95, 29, 47, 30, 123, 59, 111, 88, 115, 24, 48, 121, 38, 124, 53, 119, 63, 49, 43, 57, 96, 50, 27, 52, 23, 116, 21, 87, 19, 106, 51, 20, 61, 32, 84, 44, 22, 109, 40, 78, 85, 114, 127, 14, 118, 83, 125, 17, 86, 112, 66, 122, 45, 120, 2, 54, 81, 62, 56, 1, 3, 68, 65, 4, 70, 46, 26, 79, 55, 12, 82, 6, 91, 15, 92, 72, 18, 80, 76, 117, 13, 9, 77, 90, 16, 67, 110, 31, 11, 10, 73, 60, 58, 75, 74, 71, 7, 28, 5, 98, 69, 34, 94, 8, 101, 100, 35, 37, 99, 36], [39, 104, 33, 42, 103, 41, 97, 108, 64, 93, 105, 95, 29, 107, 0, 89, 25, 126, 88, 113, 102, 123, 124, 47, 121, 48, 53, 30, 115, 111, 59, 38, 43, 87, 106, 116, 40, 96, 22, 50, 109, 6, 24, 63, 23, 49, 119, 27, 21, 54, 44, 32, 52, 61, 83, 114, 122, 57, 20, 86, 56, 85, 81, 112, 45, 127, 19, 84, 51, 14, 125, 120, 1, 78, 118, 46, 62, 82, 17, 91, 15, 65, 18, 3, 117, 90, 67, 79, 26, 31, 92, 12, 66, 2, 77, 68, 55, 76, 16, 110, 13, 73, 58, 4, 80, 10, 9, 75, 74, 11, 72, 98, 60, 70, 7, 71, 34, 28, 94, 69, 8, 5, 101, 100, 36, 35, 37, 99], [39, 104, 33, 42, 103, 41, 97, 0, 64, 108, 93, 25, 113, 29, 89, 105, 59, 126, 107, 95, 115, 111, 102, 48, 123, 57, 47, 87, 124, 53, 121, 6, 63, 116, 38, 52, 88, 24, 96, 109, 30, 43, 119, 61, 106, 21, 49, 127, 44, 50, 54, 19, 23, 40, 51, 84, 20, 125, 27, 114, 56, 112, 14, 86, 85, 22, 83, 45, 122, 17, 65, 32, 120, 46, 81, 66, 78, 55, 1, 91, 3, 18, 79, 82, 62, 118, 26, 67, 12, 110, 90, 2, 76, 13, 77, 92, 31, 117, 10, 4, 15, 16, 9, 80, 75, 60, 73, 58, 68, 11, 74, 8, 71, 7, 72, 28, 70, 69, 98, 5, 101, 34, 94, 100, 35, 37, 36, 99], [39, 33, 104, 42, 103, 41, 97, 108, 0, 93, 25, 89, 107, 64, 113, 105, 126, 29, 95, 102, 59, 111, 123, 48, 124, 47, 30, 38, 53, 24, 43, 121, 6, 57, 88, 27, 115, 63, 96, 23, 44, 106, 116, 87, 21, 83, 49, 119, 109, 85, 52, 20, 32, 22, 50, 40, 14, 122, 19, 51, 17, 86, 78, 54, 114, 127, 81, 61, 45, 46, 84, 65, 62, 91, 56, 125, 112, 66, 120, 18, 2, 12, 15, 68, 67, 26, 82, 90, 3, 55, 118, 4, 79, 92, 13, 31, 1, 117, 76, 9, 8, 73, 80, 11, 77, 10, 16, 110, 75, 60, 74, 7, 71, 28, 58, 72, 34, 94, 98, 69, 5, 70, 101, 100, 37, 35, 36, 99], [104, 39, 33, 42, 103, 64, 0, 41, 108, 97, 93, 113, 107, 126, 25, 89, 59, 29, 105, 121, 111, 63, 115, 47, 102, 123, 95, 57, 88, 48, 124, 6, 53, 119, 21, 24, 61, 87, 116, 43, 96, 30, 106, 49, 51, 20, 83, 50, 114, 52, 67, 38, 109, 27, 65, 54, 125, 56, 122, 1, 112, 14, 4, 85, 44, 22, 120, 127, 23, 78, 19, 32, 81, 45, 40, 55, 86, 46, 118, 84, 17, 62, 2, 3, 12, 15, 82, 66, 79, 68, 26, 76, 18, 91, 9, 117, 110, 73, 8, 77, 92, 13, 16, 90, 7, 75, 10, 11, 80, 74, 71, 60, 31, 58, 70, 69, 28, 5, 98, 101, 94, 72, 34, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 41, 108, 97, 93, 0, 105, 64, 25, 95, 29, 89, 115, 113, 126, 107, 102, 59, 124, 48, 123, 111, 53, 43, 88, 106, 121, 30, 47, 38, 119, 24, 87, 50, 96, 27, 22, 63, 21, 20, 57, 17, 49, 23, 52, 86, 116, 44, 40, 54, 122, 114, 6, 109, 51, 84, 65, 78, 19, 112, 127, 61, 125, 14, 83, 32, 85, 3, 81, 45, 18, 56, 46, 70, 79, 120, 1, 15, 62, 91, 82, 55, 76, 118, 9, 67, 2, 12, 8, 92, 90, 73, 110, 4, 31, 13, 68, 117, 16, 26, 10, 66, 80, 77, 75, 11, 74, 58, 60, 71, 28, 69, 7, 98, 94, 34, 5, 101, 100, 72, 36, 35, 37, 99], [39, 33, 104, 42, 103, 97, 41, 93, 0, 108, 25, 95, 107, 29, 113, 89, 64, 105, 102, 126, 115, 59, 88, 124, 47, 27, 38, 30, 24, 53, 121, 87, 48, 23, 123, 116, 96, 21, 106, 111, 32, 50, 44, 43, 85, 19, 109, 40, 63, 52, 83, 22, 49, 119, 54, 122, 56, 84, 81, 14, 86, 125, 20, 57, 66, 127, 114, 112, 17, 91, 61, 78, 2, 1, 70, 51, 45, 18, 12, 118, 82, 65, 15, 120, 46, 55, 90, 76, 77, 16, 92, 4, 3, 26, 8, 62, 67, 79, 117, 73, 80, 75, 13, 31, 68, 74, 9, 110, 10, 6, 71, 11, 28, 60, 58, 7, 34, 98, 94, 5, 69, 72, 100, 101, 35, 37, 36, 99], [39, 104, 33, 42, 103, 97, 0, 41, 93, 108, 64, 25, 105, 89, 113, 29, 95, 107, 126, 102, 59, 115, 48, 47, 123, 124, 53, 70, 88, 111, 116, 30, 43, 52, 24, 57, 23, 38, 121, 96, 87, 63, 22, 119, 106, 78, 51, 50, 40, 32, 20, 19, 44, 109, 49, 61, 54, 122, 127, 91, 83, 21, 112, 27, 125, 81, 45, 85, 84, 118, 1, 86, 56, 17, 62, 46, 79, 114, 55, 3, 14, 66, 120, 65, 2, 18, 68, 82, 110, 12, 8, 26, 117, 77, 90, 67, 75, 16, 92, 4, 31, 13, 76, 15, 80, 9, 73, 10, 28, 11, 74, 58, 60, 7, 71, 34, 98, 5, 94, 69, 6, 101, 72, 100, 37, 35, 36, 99]], "model.layers.1.self_attn.q_proj": [[103, 12, 107, 104, 46, 33, 42, 105, 49, 123, 50, 117, 62, 48, 57, 122, 19, 6, 59, 31, 29, 52, 54, 56, 5, 44, 115, 23, 92, 63, 8, 76, 70, 80, 66, 79, 72, 20, 116, 85, 120, 111, 84, 65, 78, 83, 25, 3, 22, 18, 127, 55, 90, 9, 27, 7, 126, 11, 87, 74, 37, 58, 82, 16, 77, 124, 4, 67, 2, 35, 13, 109, 15, 125, 88, 102, 71, 75, 17, 34, 94, 21, 100, 86, 14, 61, 24, 81, 114, 69, 45, 1, 98, 112, 73, 0, 51, 10, 39, 113, 101, 38, 32, 30, 36, 26, 121, 89, 41, 43, 97, 108, 106, 47, 40, 28, 93, 96, 64, 60, 91, 99, 118, 53, 119, 68, 110, 95], [103, 107, 104, 42, 105, 117, 46, 123, 50, 62, 24, 48, 33, 3, 67, 57, 122, 111, 59, 70, 56, 7, 127, 25, 65, 29, 49, 63, 115, 31, 79, 58, 13, 124, 16, 74, 44, 6, 66, 0, 4, 11, 116, 22, 9, 92, 82, 54, 5, 77, 101, 23, 125, 45, 84, 52, 1, 90, 78, 12, 98, 20, 94, 27, 126, 71, 2, 83, 120, 17, 73, 109, 38, 18, 87, 37, 19, 86, 32, 36, 15, 75, 64, 80, 55, 35, 81, 10, 14, 102, 100, 8, 113, 99, 76, 61, 121, 69, 26, 85, 53, 72, 97, 21, 30, 112, 89, 68, 118, 39, 43, 88, 41, 106, 114, 34, 110, 91, 95, 93, 60, 47, 108, 40, 51, 28, 96, 119], [100, 44, 102, 113, 37, 38, 59, 115, 35, 57, 122, 116, 34, 104, 114, 103, 109, 123, 46, 107, 50, 48, 62, 120, 117, 54, 42, 105, 52, 49, 125, 63, 112, 111, 98, 61, 56, 60, 53, 126, 121, 101, 33, 25, 99, 110, 127, 32, 84, 86, 58, 47, 82, 124, 88, 108, 21, 94, 40, 119, 41, 45, 36, 43, 89, 29, 118, 7, 51, 78, 77, 96, 17, 13, 30, 72, 106, 81, 95, 70, 92, 97, 93, 27, 11, 4, 79, 14, 9, 16, 55, 71, 74, 26, 83, 20, 67, 65, 75, 5, 0, 87, 66, 19, 12, 76, 85, 80, 91, 39, 3, 90, 24, 31, 28, 18, 6, 2, 22, 1, 64, 8, 15, 23, 68, 69, 73, 10], [103, 107, 104, 42, 33, 105, 46, 123, 117, 50, 62, 57, 122, 48, 49, 9, 29, 125, 111, 63, 31, 92, 90, 115, 10, 56, 4, 18, 54, 83, 69, 35, 7, 84, 44, 20, 23, 71, 22, 59, 100, 79, 75, 14, 5, 58, 127, 25, 113, 77, 8, 74, 86, 87, 27, 82, 70, 66, 65, 99, 120, 24, 13, 2, 73, 6, 12, 52, 78, 85, 55, 124, 114, 38, 1, 47, 108, 116, 72, 121, 11, 45, 88, 89, 81, 126, 112, 32, 3, 80, 93, 21, 15, 37, 16, 30, 97, 26, 94, 19, 53, 28, 101, 60, 118, 102, 17, 109, 91, 36, 39, 119, 98, 61, 110, 0, 34, 76, 95, 41, 51, 43, 40, 106, 96, 67, 68, 64], [104, 38, 97, 103, 42, 43, 48, 56, 90, 45, 87, 110, 47, 125, 50, 120, 53, 121, 49, 124, 61, 126, 58, 123, 30, 6, 31, 122, 57, 62, 68, 12, 119, 84, 52, 67, 81, 29, 16, 4, 105, 70, 112, 75, 117, 77, 78, 28, 63, 127, 7, 51, 82, 59, 13, 10, 41, 76, 20, 66, 85, 1, 18, 9, 8, 83, 15, 55, 64, 100, 60, 89, 113, 108, 23, 115, 116, 69, 72, 88, 98, 24, 73, 3, 34, 2, 91, 118, 54, 17, 79, 65, 5, 14, 109, 35, 11, 21, 80, 86, 107, 101, 37, 32, 19, 71, 74, 106, 99, 44, 25, 0, 92, 93, 36, 22, 26, 96, 94, 27, 111, 114, 33, 46, 40, 102, 95, 39], [104, 38, 103, 97, 42, 43, 87, 47, 45, 110, 49, 53, 50, 123, 126, 61, 124, 6, 29, 57, 90, 120, 121, 48, 12, 66, 31, 13, 117, 64, 28, 59, 81, 56, 10, 68, 58, 122, 62, 78, 7, 44, 75, 105, 52, 67, 20, 125, 51, 16, 1, 72, 119, 54, 15, 2, 9, 55, 18, 69, 115, 113, 82, 127, 100, 80, 11, 109, 91, 99, 4, 35, 96, 8, 85, 71, 63, 60, 5, 116, 70, 17, 118, 22, 101, 3, 36, 84, 37, 89, 107, 14, 23, 46, 73, 79, 106, 112, 74, 27, 77, 30, 25, 76, 65, 21, 41, 98, 108, 24, 19, 88, 0, 114, 26, 32, 34, 111, 86, 83, 94, 33, 92, 93, 40, 95, 39, 102], [104, 103, 97, 38, 42, 43, 13, 49, 110, 47, 53, 50, 121, 56, 126, 59, 45, 87, 123, 124, 61, 6, 122, 48, 31, 117, 120, 29, 37, 78, 81, 90, 127, 28, 118, 119, 67, 51, 12, 68, 16, 55, 105, 70, 113, 58, 52, 82, 10, 35, 19, 72, 125, 8, 69, 24, 71, 66, 79, 75, 1, 23, 115, 112, 57, 54, 11, 62, 91, 73, 116, 18, 7, 85, 60, 25, 80, 44, 101, 20, 98, 30, 114, 41, 84, 17, 15, 76, 108, 88, 32, 63, 89, 34, 77, 46, 96, 27, 4, 64, 74, 14, 36, 100, 22, 21, 5, 107, 94, 9, 86, 92, 99, 2, 3, 65, 106, 83, 111, 26, 93, 109, 33, 0, 95, 102, 40, 39], [120, 49, 97, 124, 57, 103, 61, 104, 59, 87, 38, 18, 105, 121, 42, 122, 43, 54, 53, 50, 23, 31, 47, 35, 45, 48, 110, 101, 92, 126, 82, 84, 123, 117, 113, 56, 118, 115, 127, 51, 62, 41, 55, 109, 90, 12, 27, 89, 28, 114, 94, 91, 108, 81, 19, 26, 102, 44, 58, 16, 77, 78, 60, 88, 37, 15, 116, 95, 80, 119, 100, 107, 75, 6, 10, 29, 52, 20, 36, 111, 72, 46, 13, 73, 79, 74, 25, 22, 85, 33, 63, 106, 9, 86, 68, 30, 32, 11, 17, 98, 4, 93, 67, 71, 7, 125, 66, 1, 24, 76, 21, 96, 112, 3, 99, 69, 83, 34, 14, 8, 64, 70, 5, 39, 65, 2, 40, 0], [107, 42, 41, 40, 103, 33, 46, 118, 95, 24, 126, 109, 116, 125, 18, 14, 82, 115, 114, 63, 53, 56, 93, 113, 119, 111, 112, 58, 84, 54, 108, 117, 85, 59, 55, 49, 83, 60, 121, 20, 79, 80, 61, 87, 13, 27, 75, 76, 25, 50, 21, 99, 110, 48, 51, 124, 90, 92, 26, 7, 38, 89, 86, 77, 71, 94, 123, 8, 81, 74, 29, 23, 72, 120, 10, 15, 127, 22, 37, 32, 91, 96, 98, 73, 30, 52, 16, 34, 12, 122, 28, 11, 106, 47, 19, 31, 17, 35, 45, 102, 78, 97, 43, 9, 101, 88, 70, 36, 69, 67, 44, 66, 57, 3, 4, 65, 100, 62, 68, 104, 5, 105, 39, 0, 6, 1, 2, 64], [101, 98, 120, 108, 112, 127, 114, 47, 113, 34, 61, 49, 115, 62, 59, 122, 32, 111, 30, 118, 58, 36, 121, 119, 46, 48, 60, 123, 42, 50, 55, 53, 63, 94, 124, 41, 40, 125, 116, 37, 35, 126, 107, 56, 109, 44, 117, 54, 100, 88, 103, 38, 89, 22, 91, 45, 21, 68, 84, 102, 33, 57, 39, 85, 51, 25, 1, 93, 92, 6, 11, 64, 4, 52, 77, 69, 67, 8, 0, 71, 86, 99, 12, 24, 83, 95, 96, 97, 10, 81, 78, 80, 28, 9, 18, 110, 16, 7, 26, 2, 66, 104, 13, 79, 72, 31, 15, 82, 20, 3, 5, 19, 14, 65, 23, 73, 27, 70, 75, 76, 90, 43, 87, 29, 74, 17, 106, 105], [40, 42, 107, 41, 33, 112, 95, 46, 103, 116, 93, 20, 115, 114, 18, 121, 109, 27, 122, 15, 118, 78, 60, 16, 19, 72, 58, 54, 92, 25, 126, 125, 53, 17, 81, 26, 23, 119, 108, 12, 111, 13, 62, 83, 22, 56, 10, 49, 117, 48, 77, 55, 124, 11, 68, 120, 79, 14, 34, 87, 71, 113, 50, 29, 86, 59, 21, 74, 3, 57, 75, 127, 67, 5, 76, 80, 8, 104, 70, 89, 66, 84, 6, 61, 36, 82, 91, 9, 51, 98, 73, 38, 37, 90, 106, 85, 97, 69, 99, 28, 24, 52, 105, 1, 39, 31, 110, 63, 32, 2, 45, 4, 123, 94, 101, 88, 96, 43, 7, 47, 44, 30, 102, 65, 100, 35, 0, 64], [107, 41, 42, 103, 40, 109, 33, 85, 112, 108, 46, 95, 126, 54, 24, 66, 49, 68, 111, 117, 113, 38, 4, 114, 62, 122, 119, 125, 56, 22, 27, 1, 93, 61, 7, 118, 32, 36, 53, 18, 0, 17, 121, 10, 71, 6, 23, 8, 94, 78, 67, 2, 120, 98, 115, 116, 127, 59, 25, 81, 92, 60, 11, 21, 110, 37, 9, 15, 55, 80, 30, 57, 69, 77, 48, 64, 75, 12, 100, 3, 20, 19, 88, 63, 82, 123, 13, 16, 73, 34, 52, 5, 96, 124, 87, 70, 51, 101, 26, 35, 45, 99, 14, 79, 74, 43, 65, 58, 102, 29, 50, 83, 76, 86, 72, 84, 47, 28, 90, 91, 106, 44, 89, 97, 31, 105, 39, 104], [52, 55, 102, 57, 117, 111, 56, 95, 109, 54, 51, 101, 114, 63, 97, 53, 50, 127, 59, 104, 48, 122, 24, 86, 61, 98, 103, 90, 100, 112, 88, 26, 93, 33, 115, 41, 80, 110, 29, 31, 42, 116, 96, 125, 108, 16, 113, 85, 120, 27, 124, 126, 121, 60, 46, 49, 44, 22, 123, 74, 99, 37, 119, 30, 39, 17, 62, 81, 118, 107, 47, 58, 38, 43, 32, 40, 94, 10, 25, 20, 28, 21, 36, 45, 23, 91, 35, 34, 105, 84, 19, 92, 12, 76, 87, 106, 14, 89, 11, 18, 82, 15, 69, 78, 7, 13, 3, 75, 77, 72, 9, 8, 83, 5, 67, 79, 73, 6, 71, 4, 66, 2, 70, 68, 65, 0, 64, 1], [103, 104, 42, 41, 0, 33, 90, 44, 114, 109, 111, 63, 56, 59, 122, 102, 29, 61, 86, 127, 53, 101, 125, 121, 54, 51, 3, 95, 66, 16, 52, 10, 4, 112, 107, 48, 46, 83, 65, 50, 120, 84, 77, 49, 7, 76, 15, 69, 55, 71, 19, 115, 74, 12, 57, 25, 28, 78, 17, 72, 6, 85, 14, 91, 5, 9, 100, 123, 118, 80, 1, 126, 64, 124, 47, 21, 70, 13, 18, 2, 22, 75, 117, 23, 81, 68, 82, 11, 106, 87, 73, 113, 88, 67, 94, 96, 79, 105, 116, 108, 35, 110, 8, 20, 34, 39, 40, 98, 26, 24, 38, 30, 60, 36, 62, 99, 58, 89, 27, 37, 43, 45, 119, 32, 93, 92, 31, 97], [46, 44, 33, 52, 24, 111, 104, 103, 109, 124, 41, 102, 123, 57, 125, 107, 120, 108, 51, 114, 122, 61, 127, 59, 54, 90, 56, 85, 48, 29, 42, 53, 63, 95, 86, 55, 116, 76, 101, 50, 27, 80, 12, 22, 58, 60, 37, 110, 113, 26, 88, 119, 47, 21, 97, 81, 74, 93, 126, 96, 112, 34, 19, 117, 83, 7, 23, 3, 20, 15, 25, 10, 49, 77, 18, 43, 31, 89, 4, 84, 100, 0, 115, 17, 69, 98, 99, 78, 121, 87, 16, 9, 38, 28, 66, 14, 75, 92, 30, 11, 32, 91, 6, 62, 94, 105, 65, 118, 45, 106, 13, 8, 36, 35, 70, 79, 72, 67, 82, 71, 2, 68, 64, 73, 5, 1, 40, 39], [103, 57, 44, 101, 104, 41, 42, 33, 52, 109, 90, 55, 114, 46, 12, 111, 29, 54, 63, 56, 59, 127, 122, 61, 53, 51, 125, 95, 85, 72, 112, 81, 76, 48, 17, 3, 50, 11, 4, 100, 7, 121, 73, 115, 15, 120, 47, 69, 82, 19, 66, 80, 65, 23, 86, 78, 28, 67, 123, 102, 107, 0, 20, 22, 13, 16, 84, 88, 70, 10, 91, 30, 9, 126, 94, 5, 18, 14, 71, 83, 110, 25, 34, 58, 21, 32, 49, 6, 89, 2, 26, 8, 124, 77, 93, 64, 68, 75, 62, 45, 79, 117, 113, 87, 96, 43, 92, 24, 60, 98, 38, 106, 99, 27, 105, 1, 74, 36, 119, 108, 118, 31, 116, 35, 37, 97, 40, 39], [104, 101, 105, 45, 110, 97, 44, 106, 43, 64, 114, 95, 89, 113, 56, 116, 57, 93, 48, 119, 47, 58, 50, 68, 1, 15, 6, 70, 9, 84, 2, 76, 81, 16, 13, 74, 3, 14, 69, 11, 80, 20, 55, 0, 65, 60, 28, 72, 78, 7, 63, 17, 115, 71, 36, 46, 66, 8, 19, 121, 5, 109, 52, 73, 4, 118, 25, 40, 75, 34, 108, 41, 126, 82, 112, 59, 107, 111, 122, 61, 51, 42, 124, 123, 35, 125, 99, 62, 98, 67, 79, 88, 54, 85, 117, 49, 103, 27, 102, 38, 53, 32, 96, 91, 100, 90, 10, 21, 24, 12, 30, 22, 26, 127, 29, 86, 87, 83, 120, 77, 94, 23, 39, 18, 37, 92, 33, 31], [114, 104, 101, 97, 105, 45, 69, 44, 89, 110, 106, 43, 63, 95, 72, 75, 3, 93, 113, 56, 4, 57, 14, 55, 76, 119, 116, 58, 50, 48, 84, 47, 74, 20, 70, 67, 1, 9, 81, 80, 13, 60, 28, 118, 123, 16, 115, 18, 73, 64, 17, 11, 15, 22, 126, 78, 12, 7, 121, 46, 68, 77, 112, 62, 109, 65, 85, 61, 2, 83, 24, 23, 52, 127, 125, 108, 122, 6, 40, 51, 41, 79, 107, 10, 54, 26, 96, 42, 66, 21, 49, 19, 86, 25, 103, 38, 120, 37, 36, 124, 111, 90, 8, 102, 35, 87, 92, 27, 88, 91, 5, 82, 34, 100, 39, 117, 98, 94, 0, 71, 53, 32, 59, 29, 30, 99, 31, 33], [104, 64, 45, 110, 101, 105, 44, 106, 97, 43, 89, 113, 56, 57, 1, 119, 116, 0, 48, 58, 50, 47, 95, 93, 114, 68, 3, 84, 70, 123, 13, 2, 5, 63, 76, 28, 55, 60, 127, 59, 118, 14, 74, 122, 9, 81, 80, 15, 20, 46, 66, 8, 51, 16, 126, 36, 109, 112, 26, 40, 52, 4, 67, 41, 75, 108, 12, 69, 7, 11, 65, 107, 42, 77, 17, 53, 18, 78, 117, 61, 88, 6, 39, 35, 115, 72, 98, 111, 124, 32, 82, 71, 121, 91, 99, 86, 22, 85, 38, 62, 34, 10, 125, 73, 90, 49, 120, 19, 87, 30, 25, 21, 102, 83, 96, 27, 23, 100, 94, 24, 79, 92, 54, 103, 29, 31, 37, 33], [55, 63, 104, 97, 101, 72, 105, 45, 44, 110, 106, 43, 114, 52, 89, 93, 95, 77, 113, 58, 57, 56, 119, 116, 78, 15, 48, 60, 47, 83, 13, 62, 70, 50, 84, 115, 126, 80, 81, 118, 75, 8, 90, 17, 79, 38, 28, 91, 73, 20, 18, 12, 121, 112, 22, 125, 6, 69, 21, 16, 25, 86, 85, 46, 19, 37, 124, 29, 23, 9, 109, 76, 74, 102, 14, 61, 3, 10, 4, 59, 127, 111, 123, 108, 68, 71, 53, 92, 94, 36, 120, 24, 103, 35, 82, 54, 99, 27, 107, 51, 100, 11, 122, 117, 5, 88, 26, 41, 34, 96, 42, 87, 40, 32, 33, 67, 31, 30, 1, 49, 39, 98, 2, 7, 64, 65, 66, 0], [104, 0, 45, 106, 43, 103, 112, 1, 97, 102, 127, 2, 64, 116, 60, 93, 117, 67, 113, 4, 88, 115, 65, 50, 70, 71, 66, 57, 77, 78, 49, 63, 59, 69, 16, 76, 52, 62, 20, 48, 15, 73, 122, 74, 109, 22, 51, 58, 114, 123, 11, 81, 82, 8, 3, 124, 125, 107, 53, 87, 120, 17, 56, 83, 42, 19, 61, 84, 118, 18, 7, 24, 55, 40, 72, 39, 121, 86, 119, 6, 14, 54, 13, 12, 75, 10, 5, 38, 85, 9, 26, 21, 79, 92, 80, 89, 126, 111, 110, 44, 23, 68, 47, 32, 27, 25, 105, 98, 95, 108, 90, 28, 41, 91, 30, 100, 29, 37, 36, 96, 31, 34, 46, 94, 101, 33, 99, 35], [104, 103, 106, 45, 43, 97, 112, 102, 5, 0, 116, 3, 93, 68, 69, 127, 88, 20, 66, 4, 67, 60, 114, 117, 11, 71, 57, 70, 113, 49, 73, 115, 83, 2, 76, 122, 16, 82, 77, 78, 63, 51, 59, 48, 124, 8, 6, 22, 1, 15, 74, 81, 50, 65, 19, 58, 62, 52, 118, 123, 84, 85, 56, 120, 87, 75, 125, 61, 121, 109, 53, 55, 7, 64, 24, 17, 86, 54, 79, 107, 110, 80, 18, 72, 13, 42, 119, 10, 12, 14, 23, 47, 9, 111, 92, 108, 126, 29, 40, 27, 89, 44, 38, 21, 90, 46, 39, 28, 95, 25, 33, 26, 91, 31, 36, 41, 100, 30, 105, 94, 99, 37, 32, 96, 34, 35, 98, 101], [103, 114, 104, 116, 122, 59, 51, 97, 106, 48, 49, 43, 20, 124, 58, 18, 62, 88, 45, 57, 118, 63, 19, 55, 22, 102, 86, 108, 92, 123, 61, 79, 52, 112, 47, 125, 14, 121, 80, 126, 110, 120, 29, 33, 113, 17, 56, 75, 127, 84, 53, 82, 117, 24, 13, 115, 12, 50, 90, 46, 23, 93, 81, 26, 60, 119, 83, 91, 44, 10, 87, 27, 72, 54, 9, 32, 89, 15, 41, 111, 95, 94, 25, 28, 30, 36, 39, 38, 98, 77, 31, 85, 7, 37, 100, 6, 16, 99, 96, 105, 101, 5, 109, 35, 76, 78, 74, 21, 34, 73, 40, 11, 68, 8, 3, 71, 70, 66, 69, 4, 0, 1, 65, 67, 107, 2, 42, 64], [114, 118, 51, 103, 104, 49, 55, 53, 110, 125, 20, 123, 106, 122, 120, 121, 102, 56, 61, 43, 63, 124, 108, 126, 111, 88, 45, 19, 62, 119, 47, 54, 17, 58, 97, 18, 92, 22, 112, 79, 14, 80, 57, 41, 36, 99, 101, 100, 86, 116, 35, 90, 30, 98, 37, 34, 75, 59, 31, 44, 95, 32, 105, 96, 12, 13, 117, 127, 89, 91, 10, 50, 81, 33, 60, 24, 93, 72, 83, 94, 38, 9, 115, 46, 27, 52, 26, 7, 82, 113, 23, 84, 6, 87, 48, 29, 15, 5, 85, 109, 21, 28, 68, 25, 16, 11, 107, 3, 78, 39, 8, 77, 69, 66, 76, 42, 70, 65, 71, 74, 73, 4, 67, 40, 2, 0, 1, 64], [104, 102, 97, 44, 4, 103, 24, 107, 106, 29, 50, 126, 120, 55, 21, 62, 116, 59, 125, 95, 48, 3, 71, 121, 78, 67, 119, 45, 76, 0, 112, 82, 15, 65, 91, 123, 8, 92, 9, 49, 13, 1, 66, 10, 11, 68, 52, 122, 53, 80, 117, 51, 18, 110, 83, 6, 7, 58, 73, 77, 127, 75, 57, 113, 47, 101, 85, 108, 54, 81, 5, 20, 70, 16, 12, 111, 19, 43, 115, 74, 37, 32, 2, 69, 26, 42, 30, 56, 124, 64, 114, 46, 61, 72, 88, 96, 17, 118, 99, 22, 87, 89, 14, 98, 34, 63, 40, 41, 84, 27, 25, 94, 109, 79, 100, 60, 86, 23, 105, 36, 90, 35, 28, 93, 39, 33, 31, 38], [97, 102, 104, 44, 103, 105, 107, 106, 29, 24, 53, 50, 48, 126, 120, 116, 55, 119, 95, 57, 21, 121, 117, 125, 45, 82, 16, 123, 61, 78, 63, 15, 51, 62, 111, 11, 127, 92, 52, 13, 56, 9, 115, 5, 91, 72, 68, 12, 7, 59, 3, 2, 83, 6, 76, 118, 23, 108, 0, 49, 18, 54, 65, 30, 60, 75, 17, 80, 79, 89, 90, 77, 43, 98, 10, 32, 112, 74, 42, 19, 86, 81, 70, 14, 34, 84, 26, 25, 94, 35, 100, 27, 22, 85, 96, 36, 66, 124, 99, 37, 40, 28, 87, 20, 4, 8, 122, 113, 110, 109, 101, 47, 58, 71, 39, 31, 69, 67, 33, 73, 46, 64, 114, 41, 88, 38, 1, 93], [104, 102, 97, 44, 103, 29, 107, 106, 24, 50, 21, 126, 55, 120, 68, 82, 53, 52, 95, 11, 117, 78, 5, 13, 0, 125, 9, 59, 12, 62, 2, 6, 48, 92, 65, 15, 7, 45, 58, 51, 72, 3, 70, 116, 105, 18, 14, 123, 91, 46, 8, 121, 75, 57, 119, 109, 113, 122, 66, 76, 54, 26, 49, 110, 85, 88, 73, 17, 67, 71, 112, 30, 108, 114, 47, 60, 74, 80, 63, 23, 101, 1, 64, 79, 61, 43, 10, 22, 77, 41, 4, 42, 19, 111, 56, 37, 35, 124, 16, 90, 118, 84, 87, 34, 20, 69, 127, 100, 96, 115, 89, 36, 40, 32, 33, 94, 98, 83, 99, 25, 86, 81, 27, 28, 93, 38, 31, 39], [103, 102, 104, 105, 48, 97, 59, 44, 106, 107, 24, 116, 29, 117, 78, 21, 11, 120, 50, 126, 52, 16, 62, 55, 82, 53, 57, 74, 13, 51, 9, 18, 123, 95, 110, 119, 121, 75, 2, 15, 54, 85, 68, 125, 19, 122, 7, 88, 6, 70, 14, 112, 72, 92, 8, 80, 3, 45, 113, 30, 109, 66, 0, 17, 26, 76, 71, 65, 49, 58, 5, 1, 115, 114, 91, 47, 60, 22, 4, 77, 108, 67, 64, 10, 111, 32, 63, 124, 12, 87, 25, 43, 93, 56, 84, 86, 79, 83, 96, 42, 23, 41, 118, 94, 20, 61, 27, 69, 100, 127, 73, 46, 101, 98, 37, 40, 34, 81, 89, 90, 38, 36, 35, 99, 39, 28, 31, 33], [114, 103, 98, 44, 43, 49, 57, 48, 97, 105, 122, 95, 40, 14, 127, 63, 121, 107, 124, 83, 19, 24, 21, 120, 42, 78, 119, 94, 54, 32, 90, 58, 117, 110, 111, 74, 51, 92, 53, 84, 25, 26, 62, 29, 36, 18, 86, 109, 38, 115, 50, 46, 56, 112, 116, 15, 59, 123, 113, 85, 81, 33, 87, 99, 125, 88, 10, 17, 22, 31, 60, 100, 104, 118, 126, 82, 70, 93, 91, 52, 61, 28, 37, 55, 101, 108, 30, 35, 79, 47, 45, 6, 8, 102, 23, 16, 39, 13, 76, 11, 89, 77, 96, 12, 20, 34, 80, 27, 7, 9, 72, 4, 106, 75, 3, 68, 66, 73, 2, 41, 71, 65, 5, 67, 1, 69, 64, 0], [44, 63, 98, 62, 60, 111, 51, 49, 103, 124, 43, 59, 48, 127, 42, 97, 86, 17, 105, 22, 24, 120, 95, 55, 114, 92, 29, 50, 40, 99, 113, 110, 45, 90, 102, 25, 18, 57, 37, 36, 54, 77, 61, 47, 85, 28, 88, 94, 33, 74, 21, 87, 84, 101, 93, 100, 31, 26, 27, 108, 123, 125, 14, 91, 118, 35, 116, 115, 38, 117, 10, 52, 112, 121, 20, 56, 15, 109, 30, 81, 89, 13, 82, 78, 122, 104, 70, 83, 39, 126, 119, 96, 58, 32, 8, 19, 53, 71, 66, 7, 16, 4, 79, 34, 23, 107, 2, 46, 72, 67, 11, 68, 6, 41, 106, 80, 12, 0, 75, 1, 69, 64, 3, 73, 65, 9, 5, 76], [103, 105, 97, 42, 44, 111, 40, 121, 89, 95, 9, 119, 22, 58, 49, 57, 4, 48, 120, 124, 94, 62, 122, 65, 50, 60, 67, 11, 109, 51, 13, 59, 56, 92, 43, 99, 18, 79, 53, 127, 55, 16, 0, 70, 84, 63, 54, 5, 1, 73, 36, 81, 24, 26, 114, 46, 20, 71, 6, 98, 2, 82, 90, 112, 23, 66, 80, 86, 3, 77, 74, 85, 113, 100, 126, 12, 72, 76, 10, 7, 32, 38, 21, 115, 91, 78, 17, 118, 83, 125, 123, 27, 19, 29, 69, 68, 25, 47, 45, 102, 88, 64, 75, 8, 61, 14, 93, 116, 28, 52, 110, 34, 31, 37, 117, 87, 30, 15, 35, 101, 33, 96, 41, 107, 39, 108, 104, 106], [40, 109, 43, 44, 49, 97, 42, 103, 48, 53, 124, 105, 37, 118, 98, 120, 127, 62, 57, 56, 36, 95, 29, 51, 60, 59, 74, 92, 114, 22, 90, 84, 121, 55, 78, 81, 63, 10, 89, 54, 122, 16, 77, 58, 108, 24, 15, 115, 70, 38, 126, 119, 85, 86, 111, 125, 13, 47, 61, 82, 21, 25, 123, 8, 101, 93, 94, 83, 87, 45, 11, 102, 76, 19, 79, 18, 99, 20, 28, 88, 46, 116, 68, 80, 117, 32, 52, 100, 106, 9, 14, 17, 23, 67, 1, 4, 75, 96, 110, 26, 41, 2, 112, 33, 91, 113, 27, 30, 35, 50, 71, 107, 34, 72, 6, 12, 0, 3, 7, 64, 73, 69, 66, 39, 5, 31, 104, 65]], "model.layers.1.self_attn.k_proj": [[0, 39, 40, 43, 110, 4, 97, 11, 106, 41, 65, 95, 78, 84, 79, 62, 117, 74, 123, 9, 66, 50, 72, 122, 93, 5, 57, 87, 91, 73, 26, 47, 13, 69, 28, 68, 86, 16, 70, 7, 2, 59, 77, 81, 53, 112, 56, 67, 127, 17, 21, 46, 48, 10, 82, 121, 3, 64, 116, 76, 55, 63, 118, 124, 115, 58, 6, 114, 30, 89, 71, 113, 18, 51, 111, 14, 8, 108, 96, 54, 12, 120, 85, 60, 61, 109, 126, 75, 119, 15, 80, 125, 23, 1, 88, 52, 29, 49, 103, 38, 83, 94, 19, 34, 32, 45, 99, 33, 90, 36, 104, 27, 101, 22, 20, 24, 92, 98, 44, 105, 35, 100, 31, 107, 102, 37, 25, 42], [64, 40, 106, 107, 39, 46, 69, 111, 1, 33, 10, 66, 93, 7, 65, 2, 5, 113, 109, 95, 67, 75, 72, 117, 126, 12, 3, 123, 16, 112, 114, 81, 9, 61, 50, 4, 78, 53, 6, 102, 60, 124, 85, 15, 23, 47, 116, 21, 26, 125, 0, 92, 63, 27, 74, 71, 110, 108, 86, 22, 58, 68, 62, 11, 56, 70, 121, 83, 49, 80, 24, 19, 82, 94, 118, 55, 77, 17, 8, 98, 127, 57, 59, 52, 34, 51, 73, 25, 122, 119, 96, 14, 32, 99, 101, 87, 18, 37, 36, 79, 88, 76, 115, 84, 41, 20, 48, 91, 44, 35, 54, 89, 100, 28, 30, 97, 43, 105, 90, 120, 29, 45, 13, 42, 38, 31, 104, 103], [39, 97, 43, 105, 104, 106, 110, 31, 90, 29, 87, 45, 48, 54, 117, 83, 91, 81, 12, 15, 44, 78, 51, 28, 77, 10, 20, 0, 11, 89, 6, 122, 69, 7, 50, 2, 16, 126, 17, 79, 56, 125, 86, 8, 18, 52, 116, 65, 84, 60, 62, 1, 121, 13, 3, 55, 9, 73, 57, 19, 124, 53, 4, 119, 35, 58, 115, 64, 47, 75, 118, 59, 67, 49, 74, 5, 102, 114, 21, 61, 127, 24, 112, 103, 120, 80, 111, 34, 113, 46, 98, 23, 72, 96, 40, 101, 41, 68, 71, 93, 27, 32, 70, 107, 100, 30, 99, 42, 85, 36, 63, 82, 66, 37, 123, 88, 38, 94, 76, 109, 22, 108, 14, 26, 92, 33, 25, 95], [45, 0, 106, 105, 40, 39, 1, 65, 97, 108, 50, 47, 92, 31, 9, 61, 15, 127, 77, 63, 7, 122, 3, 6, 19, 110, 53, 59, 48, 4, 51, 118, 2, 120, 54, 68, 56, 23, 64, 49, 66, 43, 27, 114, 35, 87, 93, 126, 69, 70, 5, 36, 89, 79, 11, 71, 82, 8, 20, 119, 94, 75, 18, 121, 98, 62, 125, 76, 81, 37, 12, 78, 38, 32, 83, 22, 74, 58, 14, 111, 34, 13, 67, 115, 57, 84, 60, 109, 112, 99, 26, 21, 30, 123, 17, 116, 46, 73, 96, 113, 85, 107, 103, 80, 117, 124, 25, 33, 10, 24, 86, 91, 29, 104, 42, 88, 41, 72, 55, 100, 90, 44, 102, 101, 28, 52, 16, 95], [64, 40, 109, 65, 46, 108, 41, 107, 42, 66, 50, 0, 37, 3, 119, 57, 56, 33, 113, 116, 67, 48, 1, 74, 5, 58, 31, 68, 6, 76, 4, 14, 9, 13, 25, 47, 29, 71, 49, 20, 111, 11, 81, 112, 8, 7, 92, 15, 16, 2, 80, 61, 51, 94, 110, 85, 10, 60, 24, 84, 21, 123, 70, 59, 118, 126, 63, 100, 96, 54, 82, 83, 23, 102, 30, 79, 45, 39, 121, 99, 86, 19, 104, 73, 98, 27, 77, 12, 103, 32, 75, 91, 88, 87, 117, 90, 69, 38, 105, 120, 22, 44, 35, 18, 34, 106, 125, 17, 62, 78, 53, 43, 115, 122, 127, 52, 114, 124, 26, 55, 72, 89, 36, 28, 93, 101, 97, 95], [40, 0, 109, 42, 107, 39, 33, 48, 65, 64, 66, 113, 50, 52, 3, 115, 29, 117, 127, 60, 71, 70, 4, 16, 24, 73, 69, 67, 77, 68, 116, 57, 74, 76, 8, 11, 58, 82, 28, 83, 78, 63, 59, 62, 15, 86, 2, 124, 112, 84, 23, 54, 121, 122, 46, 81, 119, 125, 123, 120, 61, 26, 44, 5, 6, 1, 7, 41, 101, 37, 100, 111, 94, 99, 56, 27, 36, 35, 38, 96, 31, 98, 32, 105, 34, 55, 126, 118, 47, 87, 25, 9, 95, 45, 108, 110, 51, 114, 49, 91, 53, 30, 20, 88, 85, 90, 21, 72, 10, 19, 89, 17, 102, 14, 43, 12, 92, 106, 18, 79, 104, 22, 13, 75, 103, 93, 80, 97], [0, 108, 40, 43, 42, 39, 65, 55, 120, 126, 114, 3, 33, 50, 72, 117, 6, 2, 52, 7, 9, 41, 31, 109, 68, 5, 48, 70, 15, 125, 49, 13, 121, 62, 18, 12, 112, 78, 74, 115, 46, 85, 11, 110, 76, 69, 27, 93, 59, 116, 123, 28, 67, 38, 53, 88, 1, 4, 16, 82, 17, 20, 118, 84, 86, 90, 80, 63, 61, 60, 19, 47, 37, 83, 111, 96, 81, 57, 35, 87, 54, 21, 99, 75, 89, 36, 22, 51, 10, 73, 77, 98, 100, 25, 24, 66, 94, 23, 79, 30, 113, 34, 32, 14, 127, 124, 8, 119, 26, 71, 58, 101, 122, 44, 102, 45, 29, 64, 56, 97, 92, 105, 95, 91, 107, 106, 104, 103], [39, 41, 106, 33, 108, 104, 31, 0, 69, 34, 57, 113, 120, 25, 16, 112, 30, 47, 45, 1, 67, 84, 119, 46, 87, 2, 122, 124, 107, 5, 8, 51, 11, 121, 58, 71, 62, 68, 53, 9, 56, 35, 59, 15, 18, 60, 90, 92, 13, 125, 96, 86, 70, 76, 12, 7, 61, 24, 36, 81, 49, 4, 27, 126, 55, 123, 50, 66, 63, 127, 73, 101, 52, 75, 117, 77, 89, 54, 38, 85, 21, 110, 48, 102, 100, 28, 29, 72, 93, 22, 116, 118, 80, 97, 115, 3, 82, 43, 64, 95, 74, 23, 65, 19, 111, 20, 88, 17, 91, 98, 26, 37, 83, 99, 109, 32, 114, 79, 103, 6, 14, 44, 10, 94, 78, 40, 42, 105]], "model.layers.1.self_attn.qk_proj": [[50, 48, 0, 57, 64, 117, 56, 116, 59, 126, 113, 120, 122, 62, 123, 53, 55, 49, 65, 127, 46, 110, 1, 63, 97, 61, 124, 47, 58, 109, 67, 119, 121, 29, 3, 112, 108, 54, 51, 40, 60, 107, 4, 45, 125, 114, 43, 104, 70, 68, 2, 76, 33, 5, 66, 16, 42, 106, 95, 7, 77, 111, 93, 14, 6, 52, 41, 9, 13, 20, 11, 115, 84, 12, 71, 82, 75, 74, 81, 44, 73, 79, 69, 103, 78, 39, 10, 17, 15, 18, 72, 80, 87, 105, 24, 118, 31, 23, 8, 22, 83, 19, 90, 86, 85, 92, 89, 21, 25, 88, 26, 91, 102, 28, 36, 38, 27, 37, 98, 94, 101, 35, 100, 96, 34, 99, 32, 30], [50, 48, 0, 64, 57, 117, 56, 120, 59, 126, 116, 62, 113, 122, 46, 53, 55, 123, 127, 63, 65, 1, 110, 58, 49, 97, 61, 109, 47, 108, 124, 29, 40, 112, 121, 51, 68, 119, 114, 60, 43, 107, 54, 4, 45, 125, 66, 104, 2, 42, 67, 6, 70, 52, 106, 93, 33, 7, 95, 69, 3, 115, 77, 14, 20, 12, 41, 5, 76, 13, 111, 9, 71, 16, 44, 103, 11, 39, 84, 82, 17, 73, 105, 10, 18, 118, 78, 74, 79, 72, 75, 31, 80, 15, 81, 24, 87, 23, 8, 89, 88, 22, 83, 85, 102, 26, 19, 25, 90, 92, 21, 28, 86, 27, 98, 91, 38, 101, 37, 35, 30, 94, 36, 32, 100, 34, 99, 96], [50, 48, 57, 64, 0, 117, 56, 116, 120, 59, 126, 113, 62, 122, 53, 46, 123, 63, 127, 65, 110, 55, 58, 108, 97, 1, 47, 109, 51, 49, 29, 112, 121, 61, 124, 119, 67, 40, 3, 54, 114, 107, 60, 43, 6, 45, 4, 104, 125, 70, 2, 42, 66, 52, 106, 95, 68, 13, 71, 93, 115, 33, 5, 14, 76, 84, 111, 69, 11, 41, 16, 77, 12, 20, 9, 7, 73, 75, 15, 10, 39, 79, 78, 82, 18, 17, 103, 44, 81, 74, 72, 80, 118, 105, 31, 24, 87, 8, 23, 83, 22, 19, 89, 86, 88, 21, 85, 28, 25, 92, 102, 90, 26, 91, 98, 27, 37, 101, 38, 30, 94, 32, 35, 100, 36, 99, 96, 34], [50, 48, 0, 64, 57, 117, 56, 116, 59, 120, 126, 113, 122, 62, 123, 46, 53, 65, 127, 55, 58, 110, 63, 124, 49, 1, 97, 3, 61, 47, 108, 121, 109, 51, 112, 29, 114, 67, 54, 40, 119, 60, 6, 45, 2, 107, 104, 43, 125, 66, 52, 42, 5, 95, 106, 68, 115, 70, 111, 71, 93, 69, 20, 4, 76, 33, 41, 11, 12, 13, 77, 9, 14, 7, 103, 16, 78, 10, 84, 44, 79, 81, 80, 39, 82, 18, 73, 17, 31, 74, 75, 105, 72, 24, 118, 15, 23, 22, 8, 87, 88, 83, 86, 21, 89, 19, 92, 27, 90, 25, 85, 102, 28, 26, 91, 37, 38, 101, 98, 100, 94, 34, 35, 36, 30, 96, 32, 99], [50, 48, 64, 0, 57, 117, 56, 120, 116, 126, 62, 59, 113, 122, 46, 123, 63, 53, 58, 110, 127, 55, 108, 49, 1, 109, 61, 124, 47, 40, 65, 29, 97, 51, 54, 107, 112, 43, 121, 45, 68, 2, 66, 119, 4, 6, 104, 60, 114, 106, 42, 52, 125, 67, 3, 5, 70, 93, 115, 41, 111, 33, 71, 95, 77, 13, 103, 16, 69, 7, 44, 39, 20, 80, 12, 75, 84, 9, 14, 74, 17, 11, 76, 78, 73, 10, 105, 81, 79, 15, 72, 82, 31, 18, 8, 118, 87, 23, 24, 22, 83, 19, 86, 102, 89, 25, 88, 27, 21, 92, 28, 90, 91, 85, 26, 38, 101, 98, 37, 94, 36, 34, 100, 99, 32, 30, 35, 96], [50, 48, 57, 0, 64, 117, 116, 56, 126, 59, 120, 122, 113, 62, 46, 123, 55, 53, 110, 58, 109, 63, 49, 127, 97, 124, 29, 1, 61, 40, 108, 47, 65, 3, 43, 112, 67, 114, 51, 107, 54, 119, 121, 4, 2, 125, 104, 68, 45, 60, 6, 42, 52, 106, 66, 33, 93, 20, 41, 95, 70, 77, 13, 80, 111, 12, 115, 11, 69, 71, 10, 5, 84, 7, 16, 39, 74, 103, 9, 82, 14, 17, 76, 73, 79, 44, 78, 15, 75, 72, 81, 8, 105, 31, 118, 23, 18, 87, 24, 88, 22, 102, 83, 85, 21, 19, 89, 92, 26, 25, 86, 90, 91, 27, 28, 38, 101, 37, 36, 98, 100, 35, 34, 94, 99, 96, 30, 32], [50, 48, 57, 117, 0, 64, 120, 56, 59, 116, 122, 126, 62, 113, 123, 55, 46, 53, 110, 127, 29, 58, 108, 63, 49, 97, 1, 124, 47, 61, 109, 65, 51, 112, 114, 40, 121, 67, 119, 54, 3, 43, 107, 125, 52, 60, 45, 104, 70, 2, 93, 6, 42, 66, 115, 106, 33, 13, 20, 16, 77, 80, 95, 69, 71, 12, 68, 11, 84, 111, 4, 75, 78, 41, 103, 79, 9, 15, 7, 76, 82, 18, 14, 81, 8, 5, 73, 39, 10, 74, 44, 17, 72, 105, 118, 23, 87, 31, 24, 85, 22, 19, 26, 92, 83, 25, 89, 86, 21, 90, 88, 102, 27, 28, 91, 37, 98, 36, 101, 94, 38, 35, 34, 30, 99, 96, 100, 32], [50, 48, 57, 0, 117, 64, 56, 116, 122, 59, 120, 113, 62, 126, 123, 53, 55, 46, 127, 110, 63, 97, 1, 49, 108, 65, 58, 112, 29, 109, 54, 61, 119, 47, 124, 114, 40, 121, 51, 125, 60, 107, 45, 43, 70, 104, 4, 52, 77, 106, 42, 95, 115, 33, 93, 6, 68, 66, 71, 13, 11, 2, 3, 15, 20, 14, 12, 75, 80, 67, 84, 41, 7, 16, 9, 78, 82, 44, 69, 111, 5, 39, 76, 103, 73, 17, 81, 79, 74, 18, 10, 8, 105, 31, 24, 118, 87, 23, 21, 19, 72, 25, 86, 22, 83, 85, 89, 90, 92, 26, 88, 27, 102, 28, 98, 91, 37, 101, 36, 94, 38, 30, 34, 32, 100, 35, 96, 99], [50, 48, 57, 64, 0, 117, 56, 120, 59, 116, 122, 126, 62, 113, 123, 53, 63, 46, 127, 1, 97, 110, 109, 61, 55, 108, 47, 124, 65, 40, 49, 112, 58, 29, 119, 54, 121, 3, 67, 51, 107, 43, 60, 125, 114, 68, 45, 4, 42, 104, 70, 106, 66, 95, 52, 2, 20, 115, 33, 11, 77, 13, 93, 6, 41, 71, 76, 15, 7, 14, 39, 16, 5, 84, 82, 78, 111, 80, 12, 103, 73, 9, 74, 81, 75, 69, 44, 18, 17, 10, 31, 79, 24, 105, 87, 8, 118, 23, 72, 85, 19, 83, 25, 90, 88, 89, 22, 86, 21, 92, 102, 28, 26, 27, 91, 98, 38, 101, 36, 37, 34, 35, 94, 96, 32, 100, 30, 99], [50, 48, 57, 64, 0, 117, 56, 59, 116, 126, 62, 122, 120, 113, 123, 46, 53, 97, 108, 127, 55, 65, 110, 63, 47, 1, 109, 49, 112, 61, 58, 124, 29, 119, 40, 54, 114, 51, 121, 107, 43, 67, 60, 45, 3, 104, 125, 70, 52, 42, 68, 66, 115, 106, 95, 33, 4, 15, 20, 77, 6, 2, 13, 93, 12, 14, 75, 79, 41, 69, 78, 5, 84, 7, 76, 80, 11, 82, 71, 39, 73, 103, 44, 18, 16, 81, 9, 10, 111, 87, 17, 8, 74, 118, 31, 24, 105, 23, 21, 22, 19, 25, 85, 88, 92, 90, 89, 83, 26, 72, 86, 28, 102, 27, 98, 37, 38, 91, 101, 34, 35, 96, 100, 30, 94, 32, 36, 99], [50, 48, 64, 57, 0, 117, 56, 116, 59, 126, 120, 113, 122, 123, 62, 65, 53, 46, 55, 1, 63, 58, 110, 127, 49, 97, 61, 124, 108, 109, 47, 54, 121, 51, 29, 40, 45, 112, 119, 114, 125, 66, 60, 43, 107, 70, 2, 67, 104, 3, 42, 52, 106, 33, 95, 115, 41, 71, 68, 20, 111, 13, 4, 93, 7, 5, 12, 77, 6, 10, 76, 84, 78, 14, 39, 69, 17, 81, 103, 16, 75, 73, 74, 82, 44, 9, 11, 80, 15, 31, 105, 79, 118, 18, 24, 8, 72, 87, 23, 89, 22, 19, 83, 25, 88, 92, 86, 28, 85, 21, 90, 102, 26, 101, 38, 91, 37, 27, 98, 35, 36, 94, 100, 34, 32, 96, 30, 99], [50, 48, 64, 0, 57, 117, 56, 126, 116, 59, 120, 62, 122, 113, 46, 123, 55, 53, 110, 58, 63, 127, 49, 65, 1, 67, 108, 3, 109, 61, 124, 40, 97, 51, 47, 29, 119, 121, 43, 112, 4, 60, 107, 54, 114, 45, 125, 68, 104, 2, 66, 42, 106, 70, 52, 71, 6, 111, 41, 33, 77, 95, 93, 115, 20, 69, 16, 13, 39, 5, 103, 10, 80, 7, 75, 11, 84, 73, 15, 44, 12, 74, 78, 76, 17, 9, 14, 82, 105, 81, 8, 18, 31, 79, 23, 118, 87, 24, 72, 19, 21, 88, 22, 83, 25, 90, 85, 89, 102, 26, 86, 92, 28, 91, 27, 101, 38, 98, 36, 37, 94, 35, 32, 30, 34, 99, 100, 96], [50, 48, 57, 64, 117, 0, 116, 120, 56, 122, 62, 126, 113, 59, 123, 46, 110, 53, 127, 55, 97, 109, 63, 108, 40, 49, 112, 61, 47, 1, 58, 124, 65, 29, 119, 51, 114, 54, 107, 121, 43, 3, 4, 60, 6, 42, 52, 104, 67, 125, 106, 45, 68, 2, 66, 115, 33, 95, 70, 15, 79, 16, 20, 93, 41, 13, 84, 11, 12, 77, 39, 71, 5, 44, 76, 14, 75, 73, 103, 78, 82, 7, 18, 9, 69, 10, 105, 80, 17, 74, 31, 111, 81, 87, 118, 24, 72, 8, 23, 88, 22, 26, 19, 25, 21, 92, 85, 86, 83, 90, 89, 28, 27, 102, 91, 98, 101, 38, 37, 34, 30, 36, 35, 94, 32, 99, 100, 96], [50, 48, 57, 64, 0, 117, 56, 116, 59, 113, 126, 122, 62, 120, 123, 53, 46, 97, 63, 127, 110, 55, 49, 1, 61, 109, 58, 124, 65, 108, 112, 47, 29, 121, 54, 40, 51, 114, 119, 60, 43, 107, 125, 6, 104, 52, 2, 67, 45, 42, 66, 33, 95, 106, 70, 3, 68, 4, 41, 115, 77, 76, 93, 5, 7, 16, 13, 78, 12, 20, 71, 84, 9, 39, 69, 79, 11, 18, 44, 10, 15, 80, 75, 103, 81, 74, 111, 14, 82, 73, 17, 72, 31, 24, 8, 105, 118, 87, 23, 19, 85, 88, 21, 86, 92, 25, 89, 22, 26, 83, 102, 90, 28, 27, 91, 38, 36, 98, 37, 101, 35, 94, 34, 100, 32, 99, 30, 96], [50, 48, 0, 57, 117, 64, 56, 120, 116, 59, 126, 122, 62, 113, 123, 53, 46, 108, 63, 55, 110, 127, 1, 49, 47, 61, 97, 65, 29, 58, 124, 3, 109, 67, 112, 119, 121, 40, 114, 51, 54, 60, 107, 6, 52, 43, 125, 2, 4, 45, 104, 66, 68, 106, 42, 33, 5, 95, 70, 93, 111, 71, 13, 77, 115, 41, 75, 7, 12, 79, 9, 39, 69, 20, 78, 84, 76, 73, 103, 82, 18, 14, 44, 80, 16, 10, 11, 74, 17, 81, 87, 72, 15, 105, 118, 24, 31, 8, 23, 90, 83, 22, 85, 88, 25, 19, 26, 86, 102, 92, 89, 21, 28, 27, 91, 101, 38, 37, 35, 98, 94, 96, 30, 36, 32, 100, 34, 99], [50, 48, 57, 0, 64, 117, 56, 116, 120, 126, 59, 122, 113, 62, 53, 63, 55, 46, 123, 58, 127, 110, 108, 65, 47, 61, 109, 49, 1, 40, 124, 97, 29, 121, 51, 119, 112, 114, 43, 54, 4, 60, 67, 107, 3, 68, 6, 45, 104, 125, 42, 2, 106, 66, 33, 52, 93, 115, 7, 41, 95, 20, 13, 111, 77, 70, 39, 9, 78, 71, 103, 75, 44, 84, 74, 12, 5, 73, 69, 80, 72, 76, 11, 79, 18, 81, 82, 10, 17, 16, 31, 14, 15, 105, 118, 8, 23, 87, 24, 83, 90, 102, 85, 88, 89, 25, 26, 22, 91, 19, 28, 98, 86, 21, 27, 92, 37, 38, 101, 94, 36, 35, 96, 30, 99, 100, 32, 34], [50, 48, 57, 117, 64, 0, 56, 116, 59, 62, 120, 126, 122, 113, 53, 46, 55, 123, 110, 127, 49, 58, 63, 109, 47, 97, 29, 108, 124, 54, 61, 65, 112, 114, 121, 40, 1, 51, 119, 60, 43, 107, 104, 66, 125, 52, 68, 2, 42, 45, 4, 67, 6, 33, 3, 70, 106, 115, 93, 95, 79, 13, 7, 5, 20, 16, 12, 11, 75, 78, 41, 77, 80, 9, 84, 82, 15, 14, 10, 103, 72, 44, 76, 69, 81, 73, 39, 71, 18, 111, 17, 118, 87, 105, 74, 31, 24, 23, 83, 8, 85, 21, 88, 26, 89, 86, 22, 92, 90, 25, 19, 102, 28, 27, 98, 91, 38, 101, 37, 94, 35, 30, 36, 34, 32, 100, 99, 96], [50, 48, 57, 117, 0, 64, 56, 59, 116, 126, 120, 122, 62, 113, 123, 53, 46, 127, 63, 110, 97, 55, 49, 58, 1, 61, 29, 124, 109, 47, 112, 108, 121, 67, 65, 51, 54, 119, 3, 114, 40, 107, 60, 66, 45, 125, 43, 2, 104, 70, 52, 6, 95, 13, 93, 106, 42, 4, 33, 5, 12, 115, 16, 68, 9, 77, 20, 14, 76, 84, 79, 7, 111, 75, 78, 71, 69, 82, 18, 15, 103, 41, 17, 39, 44, 81, 73, 11, 10, 80, 72, 31, 118, 74, 105, 24, 22, 87, 23, 85, 83, 25, 88, 21, 89, 86, 8, 19, 90, 92, 26, 28, 27, 102, 98, 91, 94, 38, 37, 101, 36, 30, 35, 96, 34, 100, 32, 99], [50, 48, 64, 57, 0, 117, 56, 59, 116, 120, 126, 113, 62, 122, 123, 55, 53, 46, 63, 127, 1, 58, 110, 97, 65, 61, 49, 121, 108, 109, 47, 119, 124, 29, 112, 40, 51, 114, 54, 60, 2, 107, 3, 125, 4, 68, 43, 70, 104, 45, 66, 67, 42, 52, 6, 106, 115, 95, 41, 33, 93, 5, 77, 13, 16, 20, 7, 79, 82, 12, 76, 71, 69, 9, 84, 111, 81, 103, 75, 73, 78, 11, 14, 10, 39, 44, 80, 17, 72, 118, 31, 18, 74, 15, 87, 23, 105, 8, 24, 22, 88, 19, 21, 83, 89, 85, 25, 102, 90, 86, 92, 28, 26, 27, 91, 98, 38, 94, 101, 37, 100, 36, 32, 35, 30, 96, 34, 99], [50, 48, 57, 0, 64, 117, 120, 56, 126, 116, 59, 62, 122, 113, 53, 123, 46, 63, 55, 127, 110, 58, 108, 1, 109, 61, 97, 65, 40, 49, 47, 112, 124, 29, 121, 51, 107, 68, 119, 114, 43, 60, 70, 4, 54, 2, 104, 42, 125, 66, 106, 52, 45, 3, 67, 33, 93, 95, 6, 115, 5, 69, 41, 71, 77, 79, 75, 84, 39, 13, 78, 12, 20, 44, 9, 103, 111, 7, 73, 82, 17, 76, 16, 10, 14, 18, 80, 105, 74, 72, 11, 31, 8, 15, 81, 118, 24, 23, 87, 83, 88, 22, 85, 89, 102, 19, 27, 92, 25, 21, 86, 91, 90, 26, 28, 38, 101, 98, 37, 30, 96, 94, 35, 34, 36, 100, 32, 99], [50, 48, 57, 64, 0, 117, 116, 120, 56, 126, 122, 59, 113, 62, 53, 123, 46, 63, 127, 55, 49, 108, 58, 110, 97, 124, 29, 67, 47, 109, 61, 112, 3, 51, 1, 121, 114, 40, 119, 54, 65, 107, 43, 60, 70, 33, 125, 104, 45, 2, 42, 106, 52, 95, 93, 66, 68, 71, 4, 13, 84, 77, 69, 6, 20, 111, 115, 17, 75, 76, 82, 41, 9, 80, 39, 7, 16, 103, 79, 5, 78, 74, 14, 44, 11, 12, 73, 18, 10, 15, 31, 81, 105, 87, 24, 118, 23, 72, 83, 8, 22, 88, 86, 19, 89, 90, 28, 25, 85, 21, 102, 92, 26, 91, 98, 27, 37, 36, 38, 101, 94, 32, 30, 99, 35, 100, 96, 34], [50, 48, 57, 64, 0, 117, 56, 120, 116, 59, 126, 62, 113, 122, 123, 55, 53, 1, 46, 63, 110, 58, 127, 65, 49, 61, 121, 97, 108, 112, 47, 29, 119, 124, 109, 51, 54, 114, 40, 60, 2, 66, 3, 125, 45, 107, 70, 104, 52, 43, 4, 6, 5, 115, 67, 95, 42, 93, 106, 68, 71, 69, 33, 77, 13, 76, 75, 9, 12, 78, 20, 80, 15, 111, 41, 18, 82, 7, 84, 103, 11, 73, 10, 39, 14, 17, 79, 8, 44, 118, 16, 81, 74, 31, 87, 105, 23, 72, 24, 25, 22, 21, 102, 19, 88, 28, 85, 89, 86, 83, 90, 92, 26, 27, 91, 98, 37, 38, 101, 30, 94, 96, 35, 34, 32, 100, 36, 99], [50, 48, 57, 0, 117, 64, 56, 116, 120, 126, 122, 59, 113, 62, 123, 46, 63, 53, 110, 127, 55, 58, 124, 97, 108, 49, 109, 47, 61, 40, 65, 112, 114, 29, 51, 119, 54, 107, 121, 1, 60, 4, 43, 125, 104, 68, 52, 42, 6, 3, 106, 33, 45, 70, 2, 95, 66, 93, 115, 67, 20, 77, 9, 5, 7, 71, 41, 13, 11, 16, 111, 103, 80, 14, 84, 15, 82, 12, 39, 69, 75, 10, 79, 17, 78, 44, 76, 73, 8, 74, 18, 105, 81, 118, 31, 24, 87, 72, 19, 23, 25, 22, 21, 27, 88, 26, 28, 83, 91, 86, 85, 92, 89, 90, 102, 38, 37, 98, 101, 30, 94, 34, 35, 96, 32, 36, 100, 99], [50, 48, 57, 0, 64, 117, 56, 120, 126, 116, 59, 122, 113, 62, 123, 46, 53, 110, 63, 55, 58, 49, 109, 127, 124, 97, 65, 112, 3, 108, 61, 47, 40, 1, 29, 67, 114, 43, 119, 121, 54, 51, 107, 60, 125, 104, 66, 42, 6, 68, 2, 33, 106, 4, 45, 52, 93, 95, 70, 13, 69, 41, 115, 12, 71, 17, 39, 44, 20, 79, 84, 9, 77, 16, 103, 76, 14, 8, 82, 74, 7, 5, 111, 80, 75, 78, 73, 11, 18, 15, 105, 10, 118, 31, 81, 87, 24, 23, 72, 88, 22, 19, 92, 26, 102, 89, 25, 28, 83, 21, 85, 86, 90, 27, 91, 38, 98, 101, 37, 36, 35, 94, 30, 100, 96, 99, 34, 32], [50, 48, 57, 64, 117, 0, 116, 56, 120, 126, 59, 122, 113, 62, 123, 55, 53, 110, 46, 63, 108, 58, 127, 1, 29, 109, 97, 49, 124, 114, 61, 40, 119, 51, 121, 47, 112, 54, 65, 125, 43, 107, 52, 60, 6, 45, 104, 2, 66, 67, 42, 106, 93, 70, 33, 3, 115, 95, 4, 69, 68, 80, 13, 11, 76, 41, 84, 71, 7, 77, 5, 74, 75, 103, 111, 14, 8, 79, 9, 20, 16, 17, 39, 73, 15, 82, 12, 78, 18, 44, 10, 31, 105, 118, 23, 87, 81, 72, 85, 24, 83, 25, 92, 22, 88, 86, 19, 89, 26, 102, 21, 28, 90, 91, 101, 38, 37, 27, 98, 94, 35, 30, 36, 34, 99, 96, 100, 32], [50, 48, 64, 57, 0, 117, 116, 120, 56, 126, 59, 113, 62, 122, 46, 123, 53, 63, 55, 110, 127, 58, 97, 49, 108, 109, 47, 1, 65, 112, 124, 61, 121, 29, 40, 114, 119, 51, 54, 60, 43, 107, 67, 125, 66, 3, 45, 6, 104, 42, 33, 2, 106, 52, 95, 68, 93, 115, 20, 4, 70, 77, 13, 41, 16, 71, 5, 7, 79, 111, 84, 69, 44, 11, 12, 39, 17, 76, 73, 75, 80, 82, 18, 14, 9, 103, 15, 10, 78, 31, 105, 74, 81, 24, 118, 8, 23, 72, 87, 19, 86, 92, 85, 88, 89, 25, 83, 28, 26, 22, 102, 21, 90, 37, 27, 98, 91, 101, 38, 30, 94, 35, 99, 34, 36, 32, 100, 96], [50, 48, 57, 0, 117, 64, 116, 56, 126, 113, 120, 59, 62, 122, 123, 53, 46, 63, 65, 127, 110, 49, 47, 97, 108, 61, 109, 124, 55, 112, 58, 29, 67, 1, 3, 51, 40, 54, 114, 119, 121, 43, 107, 68, 4, 45, 104, 60, 125, 6, 66, 2, 42, 33, 106, 95, 52, 93, 115, 20, 111, 77, 44, 11, 12, 13, 76, 84, 39, 7, 70, 71, 69, 41, 5, 73, 17, 78, 82, 103, 79, 9, 18, 10, 80, 16, 75, 15, 14, 74, 105, 81, 31, 24, 118, 8, 87, 23, 72, 83, 22, 86, 25, 88, 21, 92, 28, 85, 19, 26, 90, 102, 89, 27, 98, 37, 91, 38, 101, 100, 35, 94, 36, 32, 30, 99, 34, 96], [50, 48, 0, 57, 64, 117, 56, 120, 116, 59, 62, 113, 126, 122, 123, 46, 53, 63, 55, 110, 1, 58, 65, 108, 127, 109, 61, 97, 47, 49, 40, 121, 29, 124, 112, 51, 107, 119, 66, 114, 43, 54, 125, 45, 104, 60, 52, 42, 70, 106, 4, 2, 6, 68, 95, 3, 33, 67, 69, 93, 115, 77, 5, 41, 20, 13, 76, 12, 71, 7, 111, 11, 73, 103, 78, 80, 39, 9, 10, 79, 16, 14, 75, 84, 74, 18, 8, 44, 15, 82, 81, 31, 17, 105, 118, 87, 23, 72, 24, 19, 22, 86, 21, 88, 92, 102, 83, 85, 89, 25, 26, 90, 27, 28, 91, 38, 98, 37, 101, 94, 35, 34, 32, 36, 99, 30, 100, 96], [50, 48, 57, 0, 64, 117, 116, 120, 56, 59, 126, 113, 122, 62, 123, 55, 46, 58, 53, 63, 127, 49, 1, 108, 110, 109, 61, 97, 65, 47, 124, 112, 29, 119, 121, 40, 3, 51, 67, 54, 114, 107, 43, 104, 60, 2, 45, 70, 125, 42, 66, 106, 33, 13, 6, 95, 52, 4, 41, 93, 7, 68, 69, 111, 115, 71, 20, 5, 76, 78, 77, 74, 73, 11, 103, 14, 9, 12, 79, 84, 16, 39, 80, 75, 18, 81, 82, 44, 105, 17, 72, 8, 10, 15, 31, 118, 87, 24, 23, 25, 88, 22, 19, 83, 102, 86, 21, 89, 26, 85, 90, 28, 92, 91, 101, 27, 37, 38, 98, 94, 36, 35, 100, 30, 34, 96, 99, 32], [50, 48, 0, 57, 117, 64, 120, 116, 56, 59, 126, 113, 122, 62, 46, 53, 123, 127, 58, 110, 1, 55, 63, 109, 29, 97, 124, 49, 108, 65, 40, 3, 61, 47, 67, 51, 114, 112, 54, 121, 43, 60, 107, 119, 68, 70, 104, 2, 125, 42, 4, 45, 66, 33, 106, 52, 115, 95, 93, 6, 71, 77, 20, 41, 73, 5, 69, 13, 44, 111, 80, 16, 78, 7, 76, 12, 103, 74, 11, 9, 79, 39, 84, 81, 75, 10, 18, 14, 105, 72, 15, 31, 17, 8, 82, 118, 24, 87, 23, 22, 88, 86, 83, 19, 25, 26, 21, 92, 89, 85, 102, 28, 90, 91, 27, 38, 101, 98, 34, 37, 36, 94, 32, 100, 30, 99, 96, 35], [50, 48, 57, 117, 64, 0, 116, 59, 126, 56, 122, 120, 62, 113, 123, 46, 53, 49, 55, 127, 58, 110, 97, 108, 63, 109, 47, 124, 29, 61, 54, 112, 119, 114, 121, 40, 65, 1, 51, 107, 104, 70, 43, 125, 4, 60, 52, 42, 45, 33, 68, 93, 95, 66, 11, 13, 106, 20, 2, 6, 15, 79, 76, 77, 115, 9, 5, 18, 84, 74, 80, 44, 14, 78, 16, 103, 81, 67, 41, 75, 69, 72, 82, 12, 3, 73, 71, 7, 39, 10, 105, 111, 17, 118, 87, 31, 23, 24, 8, 85, 22, 88, 25, 19, 83, 26, 92, 21, 89, 86, 102, 90, 27, 28, 91, 38, 98, 37, 101, 94, 36, 34, 99, 30, 100, 35, 96, 32], [50, 48, 57, 64, 117, 0, 116, 120, 56, 126, 113, 62, 122, 59, 123, 53, 46, 55, 63, 127, 110, 97, 109, 108, 58, 124, 61, 49, 112, 47, 65, 29, 40, 114, 125, 119, 1, 121, 43, 54, 51, 60, 3, 107, 67, 45, 66, 70, 104, 33, 52, 42, 95, 106, 115, 2, 6, 93, 13, 5, 16, 20, 77, 14, 7, 76, 41, 84, 9, 11, 12, 44, 4, 71, 111, 81, 79, 103, 68, 18, 80, 39, 17, 75, 15, 82, 74, 10, 105, 69, 78, 73, 72, 23, 87, 31, 118, 24, 8, 19, 85, 22, 86, 88, 83, 92, 89, 25, 28, 26, 102, 90, 21, 27, 91, 38, 98, 37, 101, 94, 36, 35, 30, 34, 100, 99, 96, 32]], "model.layers.2.self_attn.q_proj": [[104, 127, 46, 34, 49, 88, 113, 82, 21, 79, 75, 76, 24, 14, 71, 7, 5, 9, 92, 40, 107, 3, 15, 19, 52, 125, 62, 11, 119, 27, 121, 12, 78, 120, 20, 116, 66, 8, 93, 53, 50, 77, 109, 57, 47, 10, 56, 69, 2, 96, 18, 65, 84, 86, 85, 73, 60, 30, 91, 115, 72, 63, 41, 83, 54, 1, 45, 95, 17, 51, 74, 99, 29, 81, 25, 59, 28, 118, 16, 108, 55, 103, 31, 101, 64, 106, 48, 32, 97, 102, 13, 90, 87, 61, 80, 35, 112, 124, 94, 39, 43, 122, 42, 23, 67, 26, 38, 117, 4, 58, 100, 33, 111, 126, 6, 105, 22, 44, 37, 70, 36, 110, 123, 114, 89, 98, 68, 0], [104, 49, 46, 34, 127, 113, 21, 108, 24, 82, 79, 105, 40, 57, 9, 88, 75, 107, 76, 121, 56, 80, 5, 27, 120, 51, 90, 14, 30, 71, 92, 62, 26, 87, 48, 16, 22, 83, 35, 28, 52, 43, 17, 50, 125, 63, 84, 106, 94, 4, 117, 23, 111, 109, 61, 89, 59, 29, 3, 100, 45, 114, 119, 126, 42, 118, 19, 36, 47, 54, 67, 38, 110, 70, 116, 115, 10, 123, 96, 72, 98, 68, 33, 122, 11, 41, 44, 124, 93, 99, 20, 112, 102, 39, 103, 55, 95, 101, 77, 53, 37, 60, 7, 31, 18, 66, 32, 81, 58, 85, 86, 91, 97, 69, 78, 73, 65, 15, 25, 1, 13, 6, 8, 64, 12, 74, 0, 2], [104, 34, 49, 127, 46, 21, 82, 71, 88, 79, 113, 76, 9, 75, 3, 5, 40, 14, 26, 121, 7, 53, 64, 1, 125, 66, 23, 119, 8, 52, 67, 10, 107, 24, 11, 81, 73, 4, 15, 17, 57, 120, 69, 77, 50, 19, 2, 20, 85, 22, 18, 32, 94, 70, 84, 6, 74, 72, 62, 92, 33, 12, 65, 16, 93, 89, 90, 63, 87, 56, 96, 45, 30, 126, 38, 31, 27, 51, 35, 13, 78, 42, 80, 108, 44, 43, 118, 103, 47, 54, 124, 48, 100, 86, 58, 95, 115, 91, 60, 28, 36, 83, 25, 37, 29, 117, 123, 102, 116, 39, 114, 122, 101, 111, 97, 112, 99, 109, 105, 41, 55, 68, 0, 59, 61, 106, 110, 98], [104, 46, 49, 34, 127, 113, 82, 21, 107, 88, 75, 125, 14, 52, 90, 53, 9, 22, 28, 79, 26, 24, 92, 5, 38, 121, 40, 19, 105, 94, 77, 76, 120, 119, 84, 27, 48, 51, 103, 50, 111, 39, 93, 29, 62, 16, 41, 60, 30, 98, 17, 124, 71, 91, 36, 56, 57, 126, 99, 118, 3, 81, 47, 106, 33, 123, 110, 109, 117, 115, 54, 44, 112, 63, 86, 37, 80, 59, 116, 100, 42, 35, 108, 20, 87, 55, 31, 122, 102, 83, 73, 58, 101, 70, 114, 96, 32, 97, 11, 15, 78, 23, 95, 43, 69, 72, 45, 61, 10, 89, 66, 6, 1, 7, 13, 18, 85, 65, 64, 67, 8, 25, 4, 68, 0, 74, 12, 2], [104, 107, 91, 35, 23, 84, 17, 27, 15, 13, 81, 47, 20, 94, 112, 113, 82, 114, 61, 43, 55, 99, 75, 77, 40, 56, 89, 53, 86, 111, 10, 96, 79, 92, 25, 117, 119, 123, 11, 28, 125, 51, 21, 115, 97, 118, 30, 124, 9, 73, 80, 93, 12, 16, 57, 120, 29, 48, 87, 122, 46, 76, 42, 70, 6, 102, 24, 26, 72, 18, 88, 116, 7, 110, 49, 108, 85, 38, 83, 14, 98, 105, 74, 22, 50, 68, 67, 60, 101, 31, 90, 41, 59, 71, 106, 8, 37, 32, 33, 78, 39, 44, 19, 63, 58, 127, 126, 103, 34, 45, 54, 69, 121, 36, 100, 95, 4, 62, 52, 109, 2, 3, 0, 65, 66, 5, 1, 64], [104, 107, 35, 91, 27, 23, 99, 94, 84, 114, 47, 43, 113, 89, 112, 61, 9, 55, 119, 123, 53, 15, 40, 73, 111, 92, 17, 12, 82, 81, 120, 28, 98, 77, 56, 46, 105, 117, 125, 97, 25, 67, 86, 116, 70, 48, 115, 122, 7, 124, 42, 13, 36, 102, 118, 51, 32, 14, 90, 29, 57, 22, 79, 11, 49, 37, 10, 85, 63, 16, 41, 109, 68, 126, 6, 103, 44, 110, 54, 121, 72, 76, 106, 58, 75, 108, 34, 93, 45, 96, 101, 52, 59, 69, 39, 38, 24, 87, 30, 95, 100, 80, 50, 18, 60, 31, 127, 74, 26, 33, 71, 62, 88, 21, 19, 83, 65, 20, 2, 78, 4, 0, 3, 8, 66, 64, 5, 1], [107, 104, 47, 35, 99, 124, 94, 89, 96, 27, 55, 91, 43, 61, 119, 112, 113, 46, 25, 123, 85, 63, 105, 48, 114, 120, 53, 116, 125, 111, 23, 54, 40, 84, 102, 52, 28, 56, 117, 97, 37, 51, 115, 82, 121, 126, 92, 42, 109, 62, 22, 41, 32, 106, 60, 29, 118, 44, 110, 95, 49, 57, 86, 122, 98, 38, 108, 100, 58, 127, 36, 34, 50, 12, 101, 33, 103, 17, 59, 67, 83, 26, 45, 31, 93, 39, 14, 78, 90, 30, 21, 15, 73, 81, 16, 24, 77, 6, 88, 18, 7, 19, 80, 79, 9, 10, 20, 68, 66, 11, 71, 70, 3, 76, 75, 13, 87, 2, 74, 72, 65, 4, 0, 8, 64, 5, 1, 69], [104, 107, 35, 15, 75, 8, 13, 17, 91, 5, 84, 4, 23, 99, 73, 40, 43, 3, 1, 71, 72, 87, 112, 114, 69, 61, 68, 6, 113, 55, 67, 20, 2, 47, 53, 123, 7, 66, 0, 56, 65, 11, 120, 70, 115, 111, 10, 119, 89, 9, 25, 110, 77, 30, 64, 125, 57, 21, 16, 46, 124, 12, 94, 18, 51, 117, 86, 79, 74, 122, 108, 81, 76, 58, 14, 92, 49, 78, 85, 83, 82, 96, 127, 116, 121, 19, 90, 28, 88, 26, 102, 118, 31, 29, 80, 97, 105, 59, 60, 48, 32, 95, 93, 24, 22, 39, 37, 62, 63, 34, 50, 106, 33, 44, 42, 38, 101, 36, 27, 98, 100, 103, 54, 41, 109, 52, 45, 126], [110, 38, 97, 112, 46, 16, 89, 78, 10, 19, 8, 4, 12, 70, 68, 48, 25, 2, 69, 7, 6, 86, 82, 56, 1, 31, 20, 64, 50, 5, 102, 71, 22, 75, 125, 119, 11, 113, 3, 111, 13, 83, 67, 0, 127, 77, 61, 66, 47, 109, 51, 80, 14, 85, 73, 52, 118, 74, 72, 84, 94, 9, 81, 116, 117, 41, 93, 40, 120, 123, 18, 76, 87, 124, 107, 63, 15, 23, 21, 115, 65, 30, 104, 79, 36, 37, 43, 28, 34, 55, 88, 53, 91, 62, 24, 59, 17, 98, 108, 27, 60, 126, 99, 96, 45, 103, 32, 44, 57, 100, 101, 121, 58, 42, 122, 106, 29, 39, 92, 54, 35, 49, 95, 114, 26, 90, 105, 33], [110, 38, 112, 97, 46, 19, 78, 16, 12, 10, 8, 89, 4, 68, 69, 48, 71, 25, 5, 9, 70, 6, 102, 13, 24, 50, 2, 1, 88, 51, 62, 111, 66, 77, 65, 56, 109, 31, 118, 64, 113, 21, 15, 75, 61, 22, 127, 85, 73, 72, 3, 7, 67, 55, 14, 30, 74, 17, 27, 0, 34, 76, 83, 79, 124, 86, 115, 125, 20, 11, 100, 43, 94, 52, 122, 104, 107, 116, 87, 93, 99, 23, 105, 80, 84, 117, 95, 119, 47, 126, 121, 35, 82, 18, 91, 98, 44, 58, 120, 81, 123, 26, 40, 54, 42, 49, 45, 37, 101, 32, 92, 53, 39, 59, 36, 41, 106, 96, 108, 60, 29, 114, 103, 90, 28, 63, 57, 33], [38, 97, 110, 94, 112, 46, 25, 19, 83, 89, 78, 48, 93, 118, 85, 16, 50, 11, 96, 62, 39, 56, 21, 111, 127, 36, 77, 61, 27, 109, 90, 106, 80, 101, 17, 10, 88, 30, 12, 125, 60, 87, 95, 44, 86, 40, 105, 35, 121, 34, 31, 24, 104, 59, 116, 108, 115, 120, 57, 43, 54, 26, 123, 58, 81, 103, 126, 117, 20, 99, 23, 119, 13, 45, 14, 52, 124, 51, 37, 122, 29, 73, 100, 55, 42, 91, 92, 47, 107, 63, 22, 75, 32, 41, 114, 98, 53, 113, 9, 28, 33, 49, 15, 84, 6, 8, 82, 7, 18, 79, 69, 71, 76, 74, 70, 72, 5, 68, 4, 102, 2, 66, 3, 67, 65, 1, 0, 64], [110, 38, 97, 112, 46, 89, 78, 16, 10, 77, 25, 17, 19, 12, 8, 48, 6, 69, 11, 71, 102, 94, 51, 15, 50, 4, 75, 68, 2, 86, 21, 83, 62, 76, 31, 70, 52, 5, 72, 111, 81, 9, 127, 1, 22, 108, 99, 29, 121, 118, 109, 84, 124, 14, 7, 116, 82, 92, 36, 45, 37, 117, 74, 123, 90, 23, 79, 88, 80, 104, 35, 64, 105, 95, 61, 56, 66, 34, 58, 65, 103, 73, 93, 42, 44, 33, 107, 122, 28, 126, 20, 18, 55, 106, 87, 27, 113, 57, 47, 125, 53, 98, 85, 115, 41, 24, 54, 114, 26, 120, 30, 39, 101, 59, 40, 13, 96, 119, 67, 60, 100, 91, 43, 32, 49, 63, 3, 0], [40, 98, 121, 125, 53, 51, 17, 20, 82, 80, 18, 115, 27, 19, 21, 88, 104, 77, 22, 42, 119, 12, 89, 26, 95, 86, 30, 75, 92, 28, 84, 78, 25, 117, 96, 94, 29, 32, 120, 23, 93, 90, 57, 102, 116, 52, 79, 38, 127, 41, 24, 97, 50, 87, 123, 105, 56, 122, 10, 109, 113, 16, 31, 63, 101, 110, 43, 44, 48, 107, 33, 61, 49, 111, 36, 58, 62, 76, 124, 100, 91, 39, 114, 59, 45, 81, 108, 83, 46, 37, 35, 9, 103, 99, 60, 112, 47, 55, 85, 8, 126, 118, 54, 74, 14, 34, 71, 73, 15, 106, 7, 13, 11, 6, 72, 5, 70, 2, 69, 65, 68, 3, 64, 4, 67, 66, 1, 0], [40, 125, 121, 53, 51, 98, 27, 20, 82, 16, 84, 81, 12, 21, 42, 19, 115, 10, 119, 88, 8, 78, 74, 29, 50, 85, 86, 35, 72, 91, 36, 95, 5, 6, 117, 14, 94, 69, 104, 17, 18, 34, 93, 24, 76, 22, 65, 67, 97, 26, 79, 68, 75, 120, 38, 116, 15, 23, 30, 122, 44, 57, 63, 66, 107, 89, 58, 80, 71, 83, 109, 87, 7, 106, 9, 99, 108, 96, 111, 41, 33, 118, 13, 105, 103, 101, 48, 127, 114, 56, 100, 37, 124, 52, 28, 45, 90, 3, 113, 49, 92, 43, 77, 39, 32, 123, 59, 102, 110, 46, 31, 60, 126, 54, 70, 25, 61, 112, 62, 64, 47, 0, 55, 11, 4, 73, 1, 2], [40, 121, 42, 53, 125, 51, 98, 91, 123, 50, 110, 52, 111, 113, 58, 63, 105, 126, 47, 33, 115, 27, 54, 60, 118, 93, 117, 107, 97, 59, 122, 127, 48, 108, 61, 56, 15, 44, 124, 109, 57, 112, 116, 43, 45, 114, 62, 46, 120, 106, 38, 119, 49, 55, 41, 95, 23, 103, 39, 36, 101, 31, 100, 24, 86, 37, 99, 32, 35, 102, 30, 21, 12, 29, 96, 18, 85, 34, 89, 104, 94, 77, 92, 17, 81, 84, 83, 78, 79, 20, 74, 87, 9, 28, 69, 72, 13, 26, 88, 19, 75, 90, 25, 82, 22, 7, 10, 80, 2, 65, 70, 4, 76, 8, 11, 64, 3, 16, 6, 68, 1, 73, 67, 14, 5, 0, 71, 66], [53, 125, 51, 121, 42, 40, 115, 98, 119, 52, 108, 57, 123, 116, 91, 107, 50, 114, 117, 122, 127, 124, 120, 58, 111, 59, 118, 113, 63, 15, 55, 41, 60, 110, 48, 49, 9, 109, 56, 61, 100, 62, 44, 43, 46, 7, 105, 99, 112, 45, 54, 77, 126, 47, 102, 72, 39, 38, 33, 74, 12, 103, 69, 106, 93, 18, 101, 35, 30, 17, 95, 75, 78, 2, 70, 31, 11, 37, 27, 32, 80, 36, 84, 3, 24, 104, 34, 23, 96, 97, 4, 86, 68, 94, 83, 79, 64, 8, 29, 92, 10, 89, 85, 6, 65, 13, 0, 73, 76, 71, 26, 20, 87, 28, 21, 19, 88, 25, 90, 5, 1, 66, 16, 81, 67, 14, 22, 82], [102, 49, 98, 56, 116, 121, 113, 89, 20, 11, 81, 9, 77, 22, 15, 124, 52, 25, 2, 6, 73, 71, 18, 96, 75, 86, 110, 72, 88, 79, 70, 101, 19, 51, 10, 74, 100, 4, 3, 29, 67, 91, 1, 31, 83, 87, 92, 78, 17, 64, 13, 60, 76, 16, 120, 24, 84, 59, 61, 8, 30, 41, 80, 14, 21, 7, 58, 93, 47, 37, 12, 26, 28, 117, 62, 115, 90, 44, 46, 82, 32, 94, 27, 112, 63, 68, 103, 40, 105, 109, 33, 39, 42, 85, 95, 114, 97, 111, 35, 122, 48, 107, 23, 118, 53, 55, 54, 45, 127, 43, 123, 50, 104, 106, 119, 34, 126, 57, 99, 36, 108, 0, 125, 5, 69, 66, 65, 38], [102, 49, 116, 56, 98, 121, 113, 89, 81, 124, 20, 11, 15, 77, 68, 9, 71, 52, 70, 25, 29, 2, 64, 91, 19, 85, 4, 0, 105, 88, 1, 65, 66, 8, 3, 73, 101, 41, 46, 75, 12, 127, 110, 62, 60, 51, 72, 80, 93, 21, 67, 7, 27, 74, 14, 86, 96, 76, 47, 117, 13, 126, 18, 84, 44, 87, 6, 111, 115, 120, 79, 17, 22, 82, 90, 112, 10, 95, 107, 5, 61, 39, 24, 23, 16, 103, 30, 122, 92, 57, 83, 94, 32, 97, 118, 45, 78, 33, 26, 59, 36, 69, 43, 123, 109, 53, 37, 40, 100, 54, 114, 28, 50, 125, 99, 42, 108, 106, 48, 55, 35, 104, 58, 31, 63, 119, 34, 38], [102, 49, 116, 56, 98, 113, 121, 89, 77, 11, 81, 20, 9, 71, 15, 52, 68, 124, 110, 29, 19, 91, 86, 10, 101, 25, 6, 70, 2, 41, 66, 74, 47, 7, 4, 14, 46, 59, 67, 117, 82, 75, 72, 80, 64, 18, 96, 92, 73, 0, 31, 3, 95, 79, 115, 126, 12, 8, 21, 16, 93, 1, 22, 13, 87, 17, 60, 39, 88, 36, 107, 127, 50, 97, 32, 35, 120, 33, 122, 69, 53, 105, 85, 30, 61, 65, 84, 111, 62, 104, 5, 26, 42, 58, 118, 76, 94, 114, 23, 83, 112, 51, 24, 109, 54, 27, 44, 28, 40, 90, 125, 119, 100, 99, 78, 43, 57, 55, 48, 106, 123, 108, 103, 63, 45, 37, 34, 38], [102, 121, 110, 49, 98, 56, 116, 113, 101, 89, 25, 62, 93, 29, 31, 52, 91, 41, 51, 46, 107, 27, 45, 126, 34, 40, 100, 38, 115, 58, 106, 60, 112, 20, 105, 63, 96, 85, 39, 48, 103, 22, 57, 19, 61, 119, 86, 50, 47, 108, 53, 18, 123, 109, 114, 55, 127, 54, 120, 84, 104, 28, 111, 78, 94, 44, 118, 43, 83, 122, 81, 21, 42, 125, 124, 59, 92, 80, 30, 24, 16, 117, 35, 95, 37, 33, 26, 32, 88, 99, 36, 15, 97, 23, 77, 76, 14, 87, 90, 82, 10, 12, 5, 17, 71, 79, 2, 11, 69, 66, 8, 13, 9, 74, 7, 72, 70, 68, 75, 67, 4, 6, 64, 73, 65, 0, 3, 1], [126, 61, 56, 60, 54, 51, 123, 63, 124, 119, 100, 58, 117, 32, 127, 108, 112, 49, 39, 97, 114, 47, 104, 42, 125, 44, 106, 46, 40, 41, 48, 118, 99, 62, 110, 53, 120, 34, 52, 105, 121, 57, 107, 115, 116, 102, 55, 103, 43, 122, 50, 45, 113, 59, 111, 109, 38, 36, 33, 101, 96, 37, 94, 98, 30, 85, 31, 35, 8, 28, 25, 67, 70, 7, 5, 91, 29, 73, 2, 74, 27, 68, 92, 75, 77, 95, 93, 83, 10, 22, 4, 21, 65, 11, 76, 26, 71, 13, 89, 64, 12, 0, 9, 90, 19, 1, 78, 86, 23, 72, 66, 80, 88, 87, 15, 24, 6, 79, 14, 20, 69, 82, 81, 84, 3, 16, 17, 18], [55, 119, 123, 127, 115, 124, 60, 58, 56, 54, 63, 51, 61, 117, 62, 113, 126, 59, 96, 57, 52, 109, 48, 53, 121, 116, 120, 107, 114, 45, 108, 43, 50, 122, 110, 112, 42, 47, 125, 49, 118, 111, 44, 94, 46, 106, 105, 41, 32, 93, 104, 40, 103, 95, 39, 30, 23, 102, 98, 91, 34, 38, 90, 24, 92, 37, 17, 16, 101, 88, 22, 18, 100, 79, 7, 31, 35, 10, 78, 99, 75, 89, 8, 29, 84, 13, 26, 70, 36, 73, 5, 76, 19, 81, 15, 21, 20, 97, 80, 33, 9, 4, 27, 65, 0, 2, 87, 67, 85, 68, 66, 77, 25, 83, 28, 72, 14, 3, 69, 82, 12, 86, 1, 71, 74, 6, 11, 64], [119, 123, 55, 100, 124, 127, 56, 60, 58, 54, 115, 51, 63, 61, 36, 117, 126, 62, 32, 98, 59, 113, 52, 57, 121, 48, 45, 109, 120, 116, 53, 43, 107, 50, 30, 122, 114, 47, 106, 108, 110, 111, 112, 125, 118, 42, 49, 44, 46, 34, 41, 105, 39, 38, 103, 101, 104, 102, 40, 35, 37, 31, 96, 33, 99, 93, 97, 95, 28, 22, 88, 91, 94, 29, 19, 24, 68, 77, 64, 85, 90, 92, 27, 12, 26, 86, 2, 5, 83, 14, 74, 1, 13, 6, 25, 11, 72, 71, 67, 7, 8, 21, 69, 3, 73, 79, 78, 10, 9, 16, 81, 76, 20, 70, 89, 66, 80, 75, 87, 15, 65, 82, 18, 0, 4, 23, 17, 84], [126, 61, 100, 26, 54, 60, 28, 20, 63, 119, 51, 58, 123, 56, 124, 87, 55, 96, 18, 93, 24, 127, 25, 115, 86, 117, 85, 92, 30, 27, 94, 17, 97, 90, 33, 91, 62, 31, 32, 49, 112, 108, 106, 125, 114, 34, 104, 42, 39, 47, 120, 46, 44, 48, 53, 110, 121, 57, 41, 118, 40, 52, 105, 59, 43, 116, 107, 103, 50, 113, 122, 45, 109, 36, 101, 80, 102, 111, 98, 23, 81, 38, 15, 16, 89, 21, 99, 37, 83, 88, 95, 29, 35, 82, 19, 22, 14, 79, 78, 84, 9, 13, 75, 10, 71, 74, 12, 76, 8, 7, 67, 5, 70, 66, 72, 4, 77, 69, 65, 73, 11, 2, 0, 64, 1, 3, 6, 68], [33, 40, 108, 117, 57, 54, 22, 19, 118, 122, 82, 24, 16, 121, 77, 89, 127, 25, 76, 14, 52, 7, 125, 49, 44, 12, 123, 92, 109, 93, 106, 107, 3, 13, 69, 73, 11, 115, 46, 29, 43, 84, 62, 21, 53, 104, 120, 32, 119, 39, 28, 20, 41, 94, 124, 98, 61, 67, 87, 86, 88, 30, 111, 83, 34, 112, 59, 2, 15, 74, 64, 70, 65, 95, 48, 85, 71, 47, 60, 91, 105, 63, 96, 6, 17, 38, 68, 27, 81, 58, 42, 113, 116, 126, 110, 8, 23, 45, 72, 4, 114, 97, 79, 50, 90, 78, 5, 18, 10, 37, 26, 31, 1, 51, 101, 35, 103, 99, 102, 100, 55, 56, 80, 36, 66, 0, 9, 75], [108, 40, 33, 57, 54, 86, 25, 44, 118, 18, 117, 82, 80, 52, 91, 29, 104, 62, 121, 94, 115, 90, 92, 122, 89, 31, 56, 30, 126, 27, 95, 93, 48, 28, 88, 63, 34, 51, 110, 99, 124, 35, 59, 101, 112, 123, 26, 98, 113, 36, 60, 37, 43, 116, 103, 109, 41, 32, 39, 47, 58, 46, 42, 105, 119, 102, 100, 96, 45, 111, 106, 120, 49, 125, 38, 50, 107, 14, 53, 127, 55, 61, 23, 84, 87, 16, 114, 24, 85, 20, 83, 77, 21, 22, 76, 19, 78, 97, 81, 79, 17, 75, 11, 13, 66, 70, 12, 15, 74, 5, 4, 6, 7, 3, 67, 0, 9, 68, 8, 72, 65, 71, 2, 1, 10, 64, 69, 73], [33, 40, 108, 57, 54, 16, 14, 89, 118, 44, 117, 78, 82, 22, 13, 104, 52, 24, 121, 27, 71, 19, 122, 12, 75, 25, 90, 87, 28, 62, 109, 124, 93, 99, 94, 48, 115, 30, 31, 29, 123, 110, 101, 26, 126, 56, 91, 127, 112, 95, 53, 63, 119, 96, 46, 59, 37, 73, 21, 120, 51, 86, 36, 98, 92, 77, 43, 116, 102, 105, 85, 41, 103, 42, 38, 34, 106, 35, 60, 32, 125, 88, 111, 100, 47, 113, 58, 23, 39, 45, 76, 49, 9, 50, 61, 107, 114, 55, 5, 17, 20, 83, 6, 18, 81, 84, 79, 68, 15, 80, 10, 74, 7, 3, 11, 8, 67, 72, 66, 70, 2, 97, 4, 1, 69, 65, 0, 64], [40, 108, 33, 57, 54, 11, 12, 89, 82, 117, 118, 76, 44, 16, 14, 22, 19, 73, 7, 9, 24, 104, 86, 77, 10, 71, 78, 75, 121, 27, 13, 127, 52, 122, 93, 70, 85, 74, 123, 48, 109, 23, 87, 66, 124, 43, 126, 106, 90, 21, 68, 101, 115, 53, 110, 28, 112, 30, 81, 26, 80, 31, 15, 119, 79, 99, 4, 63, 2, 64, 84, 25, 46, 94, 83, 62, 51, 3, 56, 96, 114, 95, 29, 98, 111, 120, 116, 36, 37, 59, 35, 32, 91, 92, 42, 102, 1, 41, 100, 47, 113, 60, 38, 125, 65, 88, 49, 20, 34, 103, 39, 17, 105, 6, 45, 50, 58, 107, 55, 69, 61, 72, 18, 67, 0, 5, 8, 97], [110, 103, 33, 31, 46, 21, 78, 76, 80, 10, 8, 19, 114, 120, 79, 13, 112, 70, 4, 82, 17, 116, 49, 12, 6, 123, 16, 61, 54, 57, 89, 124, 58, 121, 25, 73, 125, 75, 47, 39, 22, 107, 18, 67, 117, 1, 118, 94, 27, 83, 56, 43, 85, 108, 111, 115, 69, 77, 48, 2, 14, 101, 63, 24, 32, 50, 51, 29, 106, 86, 93, 44, 74, 113, 23, 62, 41, 15, 28, 119, 71, 40, 91, 68, 36, 104, 127, 55, 30, 84, 11, 20, 59, 9, 92, 38, 42, 102, 72, 88, 35, 122, 37, 34, 99, 60, 52, 81, 26, 90, 45, 87, 7, 96, 98, 105, 100, 53, 126, 109, 97, 5, 66, 0, 95, 3, 65, 64], [103, 110, 33, 31, 46, 21, 19, 114, 49, 78, 120, 116, 23, 76, 12, 24, 57, 112, 79, 80, 72, 58, 16, 61, 118, 14, 4, 8, 6, 123, 48, 54, 50, 18, 67, 111, 47, 83, 98, 10, 85, 95, 45, 89, 17, 15, 108, 117, 121, 127, 73, 91, 109, 41, 63, 20, 1, 43, 90, 104, 37, 88, 81, 55, 102, 13, 39, 32, 119, 28, 113, 94, 2, 115, 70, 84, 44, 60, 51, 126, 42, 38, 96, 53, 105, 125, 122, 35, 34, 124, 106, 59, 97, 27, 29, 100, 74, 82, 9, 36, 107, 99, 93, 75, 56, 52, 40, 101, 71, 69, 86, 92, 30, 65, 3, 87, 22, 26, 62, 5, 25, 68, 11, 77, 0, 66, 7, 64], [110, 103, 33, 31, 46, 80, 21, 10, 8, 76, 78, 4, 114, 70, 68, 57, 120, 5, 49, 123, 58, 116, 112, 16, 17, 69, 3, 61, 1, 39, 65, 66, 12, 117, 121, 67, 74, 118, 54, 124, 9, 73, 7, 71, 127, 72, 14, 50, 32, 15, 11, 75, 107, 47, 125, 0, 51, 84, 111, 64, 45, 48, 79, 25, 77, 2, 85, 6, 91, 89, 87, 13, 93, 82, 60, 24, 63, 19, 109, 23, 81, 104, 18, 88, 122, 20, 83, 56, 92, 29, 38, 37, 115, 105, 86, 55, 119, 113, 35, 102, 36, 106, 53, 28, 44, 41, 62, 94, 42, 22, 30, 108, 95, 52, 34, 26, 96, 99, 43, 40, 101, 59, 100, 98, 90, 97, 27, 126], [110, 103, 33, 31, 46, 21, 10, 80, 78, 79, 69, 8, 76, 120, 114, 70, 82, 2, 6, 67, 49, 57, 112, 73, 4, 123, 58, 12, 61, 39, 5, 66, 1, 116, 68, 23, 71, 3, 16, 13, 0, 19, 17, 127, 56, 117, 111, 72, 74, 118, 121, 7, 107, 125, 14, 51, 50, 64, 54, 87, 11, 9, 45, 75, 124, 65, 85, 24, 48, 88, 81, 43, 20, 77, 15, 89, 84, 126, 119, 94, 59, 42, 18, 32, 47, 92, 26, 37, 115, 63, 86, 83, 25, 109, 36, 62, 28, 104, 53, 44, 106, 91, 122, 60, 55, 40, 34, 27, 41, 108, 99, 30, 113, 105, 102, 22, 52, 97, 90, 35, 98, 96, 29, 100, 95, 38, 93, 101]], "model.layers.2.self_attn.k_proj": [[40, 49, 110, 127, 46, 98, 76, 9, 21, 82, 24, 79, 71, 14, 5, 75, 67, 66, 0, 64, 65, 70, 125, 88, 53, 2, 92, 81, 50, 69, 26, 93, 121, 4, 27, 116, 77, 30, 112, 52, 105, 74, 80, 119, 118, 97, 62, 13, 51, 56, 72, 57, 25, 68, 31, 109, 23, 106, 1, 126, 54, 120, 113, 83, 28, 84, 63, 48, 107, 29, 124, 44, 43, 59, 89, 20, 99, 45, 73, 47, 86, 60, 42, 108, 87, 61, 16, 95, 8, 115, 90, 122, 117, 32, 39, 17, 19, 36, 96, 22, 114, 100, 41, 33, 103, 91, 123, 35, 58, 55, 37, 111, 10, 94, 102, 38, 101, 6, 104, 15, 3, 78, 7, 12, 85, 11, 34, 18], [43, 40, 99, 56, 48, 50, 119, 23, 117, 61, 84, 13, 113, 15, 17, 75, 123, 5, 51, 47, 0, 8, 111, 4, 57, 91, 55, 53, 46, 3, 104, 94, 65, 49, 6, 124, 73, 114, 44, 110, 125, 2, 89, 80, 10, 71, 60, 120, 64, 107, 122, 108, 121, 1, 12, 58, 118, 14, 86, 42, 85, 66, 25, 82, 70, 59, 52, 62, 28, 112, 76, 109, 127, 106, 27, 92, 126, 63, 83, 88, 97, 54, 22, 41, 19, 69, 33, 18, 39, 29, 115, 101, 38, 116, 21, 74, 78, 26, 87, 45, 105, 34, 37, 103, 7, 32, 98, 36, 90, 96, 31, 93, 102, 67, 16, 30, 24, 68, 95, 100, 72, 79, 35, 20, 81, 9, 11, 77], [46, 112, 102, 33, 110, 0, 12, 48, 16, 8, 10, 78, 25, 70, 69, 65, 66, 19, 2, 4, 64, 5, 77, 89, 50, 85, 17, 67, 47, 1, 7, 68, 22, 83, 30, 20, 95, 93, 88, 82, 125, 62, 56, 124, 116, 54, 87, 51, 9, 43, 27, 127, 55, 49, 118, 23, 24, 73, 3, 11, 114, 15, 13, 84, 106, 40, 92, 79, 32, 121, 91, 103, 41, 36, 61, 99, 126, 98, 101, 113, 109, 81, 29, 31, 107, 123, 105, 45, 57, 44, 108, 94, 104, 39, 59, 52, 96, 28, 120, 100, 119, 26, 63, 60, 35, 21, 18, 71, 37, 115, 58, 6, 90, 111, 53, 42, 86, 122, 117, 34, 74, 76, 75, 14, 80, 72, 97, 38], [104, 121, 125, 53, 51, 20, 16, 34, 78, 81, 82, 119, 19, 75, 42, 43, 10, 122, 88, 50, 61, 63, 127, 93, 70, 21, 13, 109, 118, 106, 116, 44, 113, 30, 7, 120, 8, 48, 57, 124, 114, 111, 62, 47, 60, 56, 58, 107, 54, 123, 115, 31, 76, 126, 9, 49, 23, 55, 38, 105, 110, 97, 22, 112, 39, 46, 41, 100, 45, 59, 91, 52, 102, 108, 37, 117, 36, 4, 3, 99, 86, 26, 89, 35, 95, 101, 32, 92, 28, 27, 33, 103, 94, 96, 24, 25, 2, 69, 90, 29, 14, 18, 87, 73, 17, 85, 15, 65, 5, 68, 83, 64, 80, 1, 84, 98, 0, 12, 72, 79, 74, 71, 11, 77, 40, 66, 6, 67], [113, 38, 116, 56, 34, 121, 49, 77, 11, 9, 81, 15, 71, 25, 0, 93, 68, 70, 20, 46, 65, 86, 67, 18, 66, 91, 16, 30, 124, 112, 102, 115, 31, 83, 61, 22, 19, 87, 72, 5, 118, 88, 76, 27, 85, 29, 1, 45, 78, 8, 82, 89, 23, 7, 6, 111, 48, 117, 36, 44, 74, 41, 10, 58, 105, 59, 99, 47, 40, 108, 51, 106, 119, 42, 127, 101, 62, 43, 92, 50, 123, 104, 114, 103, 120, 57, 84, 37, 90, 95, 97, 125, 122, 24, 32, 55, 63, 69, 35, 109, 96, 53, 126, 60, 54, 3, 28, 33, 94, 39, 12, 107, 26, 100, 79, 13, 80, 52, 4, 73, 14, 110, 21, 75, 17, 2, 64, 98], [126, 36, 124, 60, 61, 54, 63, 58, 119, 56, 115, 51, 123, 55, 127, 32, 117, 125, 118, 49, 122, 110, 112, 114, 46, 30, 50, 53, 52, 120, 121, 116, 48, 59, 111, 62, 113, 47, 57, 34, 108, 44, 45, 109, 106, 95, 42, 93, 40, 41, 107, 104, 105, 43, 98, 20, 24, 23, 39, 103, 96, 33, 102, 97, 90, 94, 38, 82, 81, 92, 101, 26, 99, 27, 91, 89, 37, 35, 100, 22, 29, 14, 80, 87, 15, 31, 85, 16, 25, 86, 28, 79, 12, 9, 19, 13, 11, 83, 88, 7, 4, 74, 66, 72, 5, 70, 75, 10, 69, 0, 3, 21, 18, 1, 65, 6, 71, 76, 8, 77, 17, 67, 78, 64, 84, 73, 68, 2], [44, 104, 54, 57, 97, 117, 82, 16, 22, 14, 77, 73, 25, 11, 76, 9, 127, 53, 108, 89, 7, 29, 122, 40, 92, 123, 12, 27, 19, 4, 24, 52, 91, 112, 93, 95, 90, 26, 31, 30, 94, 32, 88, 115, 110, 120, 119, 124, 21, 87, 113, 96, 28, 118, 45, 62, 46, 69, 121, 126, 47, 10, 125, 116, 98, 39, 34, 58, 63, 67, 37, 111, 107, 49, 41, 43, 99, 35, 51, 36, 60, 42, 103, 105, 102, 106, 59, 70, 101, 5, 100, 23, 50, 1, 38, 109, 55, 61, 2, 20, 56, 85, 48, 17, 8, 15, 114, 75, 78, 0, 84, 72, 81, 80, 74, 71, 66, 79, 83, 18, 64, 6, 86, 13, 3, 33, 68, 65], [46, 39, 110, 97, 95, 64, 78, 76, 80, 10, 8, 21, 70, 68, 79, 66, 73, 3, 65, 19, 120, 58, 50, 57, 123, 9, 5, 48, 1, 2, 82, 7, 61, 23, 69, 113, 116, 118, 114, 67, 125, 51, 0, 124, 24, 121, 127, 77, 117, 112, 53, 49, 81, 47, 84, 52, 59, 56, 109, 72, 6, 17, 60, 87, 119, 4, 91, 75, 122, 107, 20, 22, 106, 45, 89, 96, 54, 40, 90, 29, 11, 126, 25, 63, 43, 42, 62, 92, 86, 30, 101, 98, 105, 44, 14, 115, 37, 104, 26, 27, 102, 108, 93, 36, 38, 55, 111, 12, 32, 41, 99, 100, 15, 18, 34, 83, 35, 85, 88, 28, 94, 74, 13, 71, 16, 33, 31, 103]], "model.layers.2.self_attn.qk_proj": [[46, 110, 49, 121, 112, 54, 40, 51, 127, 104, 57, 56, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 60, 119, 123, 117, 108, 48, 89, 39, 33, 25, 16, 14, 80, 115, 78, 63, 99, 107, 12, 76, 55, 18, 50, 85, 47, 98, 120, 21, 114, 58, 75, 97, 88, 86, 74, 8, 118, 82, 13, 11, 81, 95, 83, 19, 34, 20, 27, 84, 79, 15, 91, 77, 122, 17, 9, 87, 42, 62, 10, 94, 73, 29, 24, 31, 22, 38, 7, 52, 23, 36, 45, 41, 59, 92, 71, 6, 72, 111, 4, 69, 5, 30, 68, 105, 101, 35, 93, 32, 109, 70, 96, 28, 66, 2, 0, 64, 103, 90, 106, 1, 100, 67, 3, 65, 37, 26], [46, 110, 49, 121, 112, 54, 40, 56, 104, 51, 57, 53, 125, 127, 113, 116, 43, 44, 126, 123, 61, 119, 124, 102, 60, 117, 48, 39, 108, 89, 25, 16, 107, 58, 120, 80, 33, 14, 115, 63, 12, 99, 47, 78, 76, 8, 118, 55, 98, 50, 85, 82, 27, 21, 114, 18, 20, 95, 75, 62, 83, 91, 15, 97, 19, 122, 77, 81, 74, 34, 13, 79, 17, 11, 94, 86, 84, 10, 9, 87, 42, 24, 31, 73, 70, 88, 23, 36, 111, 22, 4, 29, 38, 5, 7, 71, 68, 109, 52, 69, 0, 96, 28, 106, 64, 45, 103, 32, 1, 92, 105, 93, 66, 59, 30, 35, 67, 2, 72, 6, 101, 3, 65, 90, 26, 100, 41, 37], [46, 110, 49, 121, 112, 54, 56, 40, 104, 53, 51, 57, 127, 113, 125, 116, 43, 44, 61, 126, 124, 102, 119, 123, 117, 60, 108, 63, 33, 107, 55, 48, 89, 16, 25, 39, 120, 47, 58, 80, 76, 14, 8, 115, 12, 99, 118, 78, 97, 85, 50, 21, 70, 18, 98, 19, 82, 27, 114, 74, 11, 20, 77, 84, 15, 79, 94, 9, 17, 75, 42, 95, 91, 34, 13, 86, 52, 87, 10, 73, 81, 5, 38, 23, 24, 36, 7, 71, 68, 69, 122, 83, 4, 29, 62, 59, 88, 31, 111, 32, 109, 22, 66, 30, 106, 45, 64, 96, 105, 2, 1, 28, 92, 93, 100, 0, 103, 67, 41, 72, 3, 26, 65, 35, 101, 90, 6, 37], [46, 110, 49, 121, 112, 54, 127, 51, 57, 40, 56, 104, 113, 125, 53, 116, 43, 126, 44, 61, 124, 123, 102, 117, 60, 58, 119, 48, 33, 25, 108, 63, 16, 39, 89, 80, 55, 14, 8, 107, 99, 76, 118, 12, 78, 98, 21, 19, 120, 47, 18, 50, 91, 114, 115, 70, 97, 84, 11, 82, 27, 95, 52, 85, 77, 10, 74, 34, 23, 79, 20, 75, 42, 15, 86, 94, 83, 87, 62, 81, 31, 13, 17, 5, 9, 71, 73, 88, 36, 4, 69, 38, 24, 122, 68, 109, 106, 29, 30, 22, 7, 92, 41, 66, 111, 67, 59, 28, 93, 32, 2, 103, 65, 101, 0, 45, 1, 96, 105, 3, 64, 100, 26, 90, 35, 72, 6, 37], [46, 110, 49, 121, 112, 54, 40, 127, 51, 57, 56, 125, 104, 113, 53, 116, 43, 61, 44, 126, 119, 102, 124, 123, 117, 58, 108, 25, 63, 33, 55, 16, 76, 48, 120, 8, 107, 12, 99, 118, 14, 60, 89, 80, 78, 39, 47, 115, 21, 82, 114, 98, 50, 18, 91, 95, 84, 97, 11, 20, 70, 9, 79, 94, 27, 74, 85, 17, 52, 75, 73, 15, 19, 69, 13, 77, 31, 24, 10, 88, 7, 87, 81, 68, 86, 42, 23, 62, 36, 5, 34, 38, 83, 71, 59, 4, 111, 22, 2, 67, 3, 122, 66, 28, 64, 65, 41, 0, 92, 29, 109, 30, 32, 35, 96, 103, 100, 101, 106, 93, 1, 72, 6, 105, 45, 90, 26, 37], [46, 110, 49, 121, 112, 51, 54, 56, 40, 104, 57, 127, 113, 125, 53, 116, 43, 44, 61, 126, 102, 124, 123, 119, 48, 60, 58, 25, 115, 108, 117, 33, 89, 39, 16, 107, 55, 80, 14, 12, 120, 8, 78, 76, 47, 99, 98, 18, 21, 50, 114, 82, 63, 85, 75, 52, 20, 19, 95, 27, 9, 31, 15, 11, 118, 111, 74, 84, 79, 73, 97, 34, 91, 122, 70, 88, 17, 42, 86, 13, 62, 94, 87, 24, 77, 23, 10, 83, 7, 4, 22, 36, 81, 68, 69, 38, 5, 71, 28, 32, 106, 6, 29, 93, 103, 72, 67, 96, 105, 30, 66, 41, 92, 35, 1, 59, 0, 65, 45, 64, 109, 101, 2, 3, 100, 90, 26, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 127, 57, 51, 113, 125, 116, 53, 43, 44, 61, 126, 119, 124, 123, 102, 89, 60, 48, 117, 108, 115, 16, 107, 39, 14, 58, 33, 25, 12, 47, 55, 76, 80, 78, 99, 8, 120, 63, 85, 11, 18, 50, 9, 21, 83, 82, 118, 79, 52, 84, 75, 98, 15, 42, 17, 20, 95, 77, 91, 19, 111, 27, 13, 114, 97, 73, 74, 81, 24, 94, 88, 31, 86, 36, 122, 10, 34, 38, 87, 71, 23, 109, 22, 4, 6, 7, 29, 68, 62, 69, 5, 32, 106, 45, 72, 105, 70, 96, 30, 35, 93, 59, 103, 92, 101, 41, 28, 66, 3, 0, 1, 64, 2, 26, 67, 65, 100, 37, 90], [46, 110, 49, 121, 112, 54, 40, 104, 56, 57, 51, 127, 113, 125, 53, 116, 43, 44, 61, 119, 126, 102, 124, 123, 60, 89, 108, 117, 25, 33, 63, 55, 48, 39, 78, 107, 80, 14, 16, 12, 85, 99, 76, 115, 47, 118, 58, 18, 21, 82, 120, 50, 11, 15, 97, 74, 9, 83, 91, 31, 84, 98, 52, 20, 75, 79, 27, 19, 24, 94, 114, 17, 77, 81, 6, 10, 86, 29, 13, 73, 87, 8, 38, 111, 95, 34, 122, 42, 62, 23, 88, 72, 36, 7, 68, 69, 22, 59, 71, 30, 4, 5, 32, 103, 105, 106, 45, 28, 96, 41, 0, 3, 2, 35, 93, 64, 109, 67, 26, 66, 92, 65, 70, 101, 100, 1, 90, 37], [46, 110, 49, 121, 112, 40, 51, 56, 54, 104, 57, 127, 53, 125, 113, 116, 43, 44, 126, 61, 102, 119, 117, 124, 60, 123, 25, 39, 108, 89, 48, 16, 115, 58, 63, 55, 21, 14, 80, 78, 33, 107, 99, 12, 76, 120, 85, 62, 98, 82, 47, 18, 15, 19, 31, 97, 27, 50, 42, 72, 20, 84, 95, 6, 11, 91, 75, 114, 94, 81, 23, 9, 79, 83, 77, 74, 86, 17, 29, 10, 52, 34, 13, 118, 111, 24, 88, 122, 73, 87, 36, 38, 45, 22, 68, 41, 4, 7, 93, 105, 71, 30, 96, 106, 69, 8, 5, 28, 66, 109, 32, 59, 100, 64, 90, 0, 103, 3, 92, 35, 67, 65, 26, 2, 1, 101, 70, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 57, 127, 125, 53, 51, 113, 116, 43, 44, 126, 61, 119, 102, 124, 60, 117, 123, 25, 108, 16, 89, 107, 39, 115, 48, 58, 80, 78, 14, 120, 33, 63, 12, 99, 76, 21, 98, 82, 27, 55, 97, 19, 47, 42, 84, 18, 72, 91, 11, 94, 6, 85, 15, 20, 83, 9, 75, 13, 17, 77, 79, 52, 24, 114, 95, 62, 10, 81, 50, 34, 118, 74, 87, 31, 88, 86, 23, 73, 111, 22, 36, 7, 71, 109, 69, 4, 68, 29, 105, 38, 5, 122, 106, 59, 96, 30, 41, 92, 8, 103, 32, 66, 28, 35, 45, 93, 2, 90, 26, 3, 70, 100, 101, 0, 67, 64, 1, 65, 37], [46, 110, 49, 121, 112, 40, 51, 54, 57, 56, 127, 104, 53, 113, 125, 116, 43, 126, 44, 61, 124, 102, 117, 63, 60, 119, 123, 108, 16, 89, 25, 33, 48, 39, 58, 80, 72, 78, 55, 12, 14, 98, 99, 107, 120, 115, 76, 50, 114, 95, 85, 82, 118, 42, 91, 21, 97, 18, 27, 47, 20, 15, 11, 19, 84, 79, 87, 94, 77, 74, 75, 10, 24, 17, 6, 73, 66, 86, 13, 83, 52, 62, 31, 69, 5, 9, 23, 111, 38, 88, 81, 4, 36, 68, 34, 29, 22, 106, 7, 71, 122, 2, 30, 64, 32, 0, 92, 70, 1, 59, 67, 3, 105, 65, 93, 96, 103, 109, 41, 28, 35, 45, 100, 90, 101, 8, 26, 37], [46, 110, 49, 121, 112, 54, 40, 127, 51, 57, 56, 104, 125, 53, 113, 116, 43, 44, 61, 126, 123, 119, 124, 102, 117, 63, 108, 33, 48, 89, 16, 72, 107, 115, 99, 58, 25, 60, 98, 39, 14, 55, 47, 114, 12, 78, 80, 76, 50, 21, 18, 82, 120, 27, 85, 118, 97, 91, 15, 84, 20, 95, 10, 111, 19, 11, 52, 87, 73, 75, 24, 42, 86, 36, 94, 31, 17, 34, 79, 83, 23, 74, 88, 81, 62, 77, 69, 9, 4, 70, 59, 5, 7, 13, 68, 22, 38, 71, 32, 30, 122, 29, 1, 3, 0, 6, 66, 105, 96, 92, 93, 106, 28, 65, 2, 109, 41, 45, 67, 35, 101, 64, 100, 103, 26, 90, 8, 37], [46, 110, 49, 121, 112, 54, 40, 104, 56, 57, 51, 125, 127, 53, 113, 116, 43, 44, 119, 61, 126, 123, 102, 124, 117, 89, 25, 108, 107, 48, 58, 63, 60, 39, 33, 16, 115, 99, 78, 80, 14, 12, 47, 21, 72, 76, 120, 18, 82, 77, 20, 85, 114, 79, 97, 55, 94, 27, 15, 111, 75, 11, 52, 118, 91, 19, 42, 84, 81, 17, 31, 73, 34, 10, 50, 83, 86, 87, 98, 24, 88, 95, 9, 70, 74, 13, 23, 38, 36, 29, 62, 7, 68, 22, 30, 122, 96, 5, 59, 41, 93, 105, 109, 69, 4, 71, 92, 28, 32, 106, 35, 45, 101, 100, 67, 66, 6, 26, 3, 8, 103, 37, 64, 1, 2, 90, 0, 65], [46, 110, 49, 121, 112, 56, 54, 51, 127, 40, 57, 104, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 119, 60, 48, 123, 25, 108, 89, 117, 16, 39, 47, 33, 58, 21, 120, 99, 80, 107, 55, 12, 72, 115, 78, 14, 63, 76, 18, 97, 19, 82, 27, 85, 50, 70, 15, 42, 114, 11, 20, 84, 98, 75, 95, 118, 79, 77, 34, 13, 91, 86, 122, 17, 94, 10, 74, 73, 81, 83, 31, 59, 9, 62, 24, 88, 87, 52, 22, 4, 38, 5, 41, 23, 111, 29, 69, 71, 36, 68, 7, 105, 45, 92, 93, 64, 30, 35, 8, 28, 96, 101, 1, 100, 103, 32, 66, 2, 109, 3, 106, 67, 65, 26, 0, 6, 37, 90], [46, 110, 49, 112, 121, 54, 40, 104, 56, 57, 51, 127, 113, 125, 53, 116, 43, 44, 61, 126, 124, 123, 102, 119, 117, 89, 33, 60, 48, 25, 39, 63, 16, 108, 120, 78, 80, 47, 118, 99, 55, 107, 12, 14, 21, 76, 70, 58, 72, 115, 82, 18, 114, 98, 95, 85, 50, 42, 97, 91, 11, 27, 94, 20, 73, 79, 52, 74, 75, 31, 17, 15, 84, 88, 19, 86, 13, 83, 34, 81, 24, 62, 77, 23, 10, 22, 9, 71, 5, 68, 38, 4, 69, 122, 36, 8, 87, 29, 7, 32, 109, 105, 111, 93, 30, 59, 41, 67, 2, 92, 96, 66, 65, 45, 106, 1, 103, 0, 28, 101, 3, 100, 90, 64, 35, 26, 37, 6], [46, 110, 49, 112, 121, 54, 51, 40, 104, 127, 125, 56, 57, 113, 53, 116, 43, 44, 126, 61, 124, 117, 102, 123, 119, 63, 108, 107, 33, 60, 25, 48, 55, 115, 16, 39, 89, 120, 80, 58, 12, 78, 114, 47, 99, 118, 21, 52, 14, 50, 76, 18, 98, 97, 27, 42, 82, 95, 84, 85, 24, 72, 74, 11, 91, 20, 19, 88, 94, 70, 75, 36, 87, 73, 15, 8, 79, 23, 17, 86, 38, 31, 10, 111, 34, 13, 22, 4, 9, 81, 83, 77, 5, 29, 68, 122, 71, 69, 62, 41, 105, 7, 109, 28, 92, 65, 30, 59, 2, 32, 106, 96, 0, 3, 103, 67, 45, 93, 66, 64, 6, 35, 1, 26, 101, 90, 100, 37], [46, 110, 49, 121, 112, 54, 40, 104, 56, 51, 57, 125, 127, 113, 53, 116, 43, 44, 61, 126, 119, 102, 124, 107, 123, 117, 60, 25, 108, 89, 48, 58, 33, 63, 16, 115, 80, 76, 39, 78, 12, 14, 118, 99, 47, 55, 21, 120, 18, 98, 82, 15, 75, 84, 11, 9, 85, 19, 79, 91, 42, 52, 114, 20, 27, 74, 94, 77, 8, 31, 97, 73, 81, 86, 13, 17, 24, 50, 88, 10, 87, 83, 111, 70, 95, 36, 38, 72, 62, 23, 4, 34, 29, 71, 22, 68, 7, 122, 5, 69, 41, 109, 45, 28, 106, 6, 35, 59, 32, 92, 105, 93, 30, 96, 101, 103, 67, 100, 64, 2, 66, 3, 90, 26, 1, 0, 65, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 57, 51, 53, 127, 113, 125, 116, 43, 44, 126, 61, 124, 102, 60, 119, 123, 117, 25, 108, 39, 16, 33, 89, 107, 48, 58, 118, 55, 21, 80, 78, 76, 14, 12, 115, 8, 99, 63, 18, 97, 82, 120, 91, 85, 98, 79, 11, 47, 19, 81, 20, 84, 50, 86, 77, 114, 15, 42, 95, 10, 122, 52, 75, 27, 88, 17, 73, 13, 34, 94, 62, 6, 74, 9, 83, 22, 31, 87, 24, 111, 5, 23, 68, 38, 45, 36, 71, 29, 41, 93, 7, 59, 4, 105, 69, 32, 30, 28, 106, 103, 72, 70, 66, 92, 96, 35, 0, 2, 64, 100, 109, 65, 101, 1, 67, 90, 26, 3, 37], [46, 110, 49, 121, 112, 127, 54, 40, 56, 104, 51, 57, 125, 113, 116, 53, 43, 126, 44, 61, 102, 124, 119, 117, 123, 60, 48, 108, 33, 25, 80, 39, 63, 58, 89, 16, 14, 8, 21, 12, 107, 78, 76, 120, 55, 115, 118, 99, 18, 19, 91, 97, 82, 47, 114, 50, 85, 20, 6, 62, 98, 42, 27, 11, 84, 75, 86, 79, 10, 81, 95, 77, 15, 13, 88, 9, 34, 23, 52, 24, 74, 73, 94, 17, 87, 122, 83, 5, 31, 22, 36, 4, 109, 29, 105, 71, 68, 111, 38, 45, 69, 7, 32, 0, 59, 92, 30, 106, 28, 2, 41, 67, 93, 66, 65, 64, 96, 3, 103, 101, 35, 1, 90, 100, 72, 70, 26, 37], [46, 110, 49, 121, 112, 54, 40, 104, 51, 56, 127, 57, 113, 53, 116, 125, 43, 44, 126, 61, 124, 123, 119, 102, 25, 108, 63, 117, 60, 89, 48, 8, 80, 16, 39, 14, 115, 107, 78, 33, 55, 120, 12, 76, 58, 47, 6, 99, 98, 82, 18, 50, 21, 95, 20, 75, 97, 114, 85, 11, 9, 91, 13, 27, 84, 10, 15, 73, 19, 79, 118, 17, 74, 42, 86, 24, 62, 31, 94, 52, 5, 111, 81, 77, 68, 59, 87, 23, 36, 7, 83, 4, 88, 34, 69, 45, 71, 22, 106, 122, 38, 29, 96, 105, 92, 35, 30, 66, 3, 32, 100, 67, 109, 93, 28, 103, 41, 64, 26, 1, 101, 65, 2, 0, 90, 72, 70, 37], [46, 110, 49, 121, 112, 54, 40, 51, 56, 104, 57, 127, 113, 53, 116, 125, 43, 44, 124, 61, 126, 117, 102, 119, 48, 123, 33, 25, 58, 107, 108, 47, 16, 39, 60, 63, 80, 89, 55, 8, 12, 99, 14, 98, 78, 76, 115, 114, 97, 82, 120, 18, 21, 6, 50, 91, 95, 75, 79, 27, 85, 10, 20, 42, 94, 84, 74, 81, 11, 17, 83, 15, 19, 9, 31, 87, 52, 36, 73, 38, 34, 86, 88, 77, 4, 13, 71, 24, 22, 118, 111, 62, 23, 29, 5, 122, 69, 7, 105, 68, 106, 59, 30, 32, 28, 93, 66, 45, 96, 103, 92, 109, 64, 0, 100, 41, 2, 101, 35, 72, 3, 67, 90, 70, 1, 26, 65, 37], [46, 110, 49, 121, 112, 54, 56, 40, 57, 104, 127, 51, 113, 53, 116, 125, 43, 61, 44, 126, 123, 102, 119, 117, 124, 60, 39, 48, 25, 107, 108, 63, 33, 89, 80, 115, 114, 78, 16, 58, 55, 14, 99, 47, 12, 98, 8, 76, 118, 21, 50, 120, 18, 82, 97, 84, 27, 95, 85, 83, 75, 11, 79, 52, 9, 20, 34, 19, 74, 15, 81, 91, 13, 87, 6, 42, 73, 10, 36, 77, 94, 17, 69, 86, 31, 38, 29, 71, 4, 88, 111, 5, 24, 23, 59, 22, 68, 7, 122, 28, 64, 109, 30, 45, 41, 62, 66, 96, 106, 70, 35, 93, 100, 103, 72, 92, 67, 1, 105, 32, 2, 3, 0, 101, 65, 26, 90, 37], [46, 110, 49, 121, 112, 51, 54, 40, 104, 57, 127, 113, 56, 125, 53, 116, 43, 44, 126, 61, 119, 117, 124, 102, 123, 25, 33, 58, 55, 89, 63, 107, 16, 108, 47, 48, 39, 60, 115, 14, 80, 78, 12, 76, 99, 21, 85, 82, 8, 18, 97, 114, 98, 120, 75, 27, 20, 84, 17, 91, 50, 118, 13, 19, 79, 10, 94, 95, 15, 74, 31, 87, 11, 77, 81, 9, 24, 34, 88, 23, 83, 73, 86, 68, 22, 29, 62, 52, 72, 4, 38, 36, 70, 42, 71, 111, 5, 109, 69, 45, 59, 30, 7, 28, 122, 41, 106, 32, 96, 6, 93, 92, 105, 35, 67, 100, 2, 1, 103, 3, 64, 0, 101, 90, 66, 26, 65, 37], [46, 110, 49, 121, 112, 51, 40, 57, 56, 54, 104, 127, 113, 125, 53, 116, 43, 44, 126, 61, 124, 102, 119, 123, 60, 117, 25, 108, 58, 107, 89, 48, 33, 80, 39, 16, 63, 78, 115, 47, 12, 76, 99, 55, 14, 98, 18, 120, 27, 19, 85, 97, 94, 114, 79, 21, 20, 83, 95, 82, 42, 91, 15, 75, 8, 52, 11, 84, 17, 74, 70, 50, 118, 24, 77, 73, 13, 86, 10, 72, 87, 22, 9, 36, 81, 34, 88, 31, 23, 38, 111, 4, 62, 122, 69, 68, 29, 7, 30, 71, 5, 103, 96, 45, 28, 41, 92, 106, 93, 67, 2, 109, 32, 105, 66, 101, 35, 59, 100, 0, 64, 1, 6, 26, 65, 90, 3, 37], [46, 110, 49, 121, 112, 40, 54, 104, 56, 57, 127, 51, 125, 113, 53, 116, 43, 44, 61, 126, 119, 123, 124, 117, 102, 33, 108, 39, 48, 63, 60, 89, 107, 25, 58, 16, 78, 115, 55, 99, 80, 14, 47, 76, 12, 18, 70, 120, 97, 118, 114, 85, 98, 21, 27, 20, 75, 84, 72, 11, 52, 73, 9, 82, 17, 15, 91, 42, 79, 50, 94, 87, 13, 62, 81, 74, 10, 19, 95, 24, 34, 83, 88, 77, 86, 5, 111, 31, 36, 7, 8, 122, 71, 4, 68, 22, 69, 23, 38, 109, 30, 29, 106, 32, 105, 45, 35, 2, 103, 92, 67, 96, 28, 101, 3, 93, 64, 66, 59, 41, 100, 65, 26, 1, 0, 90, 6, 37], [46, 110, 49, 121, 112, 54, 40, 51, 127, 56, 104, 57, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 123, 119, 117, 89, 39, 107, 60, 48, 25, 55, 33, 108, 58, 47, 16, 78, 99, 80, 115, 114, 76, 63, 12, 14, 118, 85, 120, 18, 72, 27, 50, 70, 97, 98, 79, 91, 21, 19, 20, 82, 95, 11, 84, 83, 75, 15, 87, 74, 13, 52, 34, 9, 17, 122, 77, 81, 94, 10, 31, 42, 73, 22, 38, 24, 62, 86, 111, 36, 7, 5, 69, 88, 29, 23, 68, 30, 64, 4, 71, 32, 103, 106, 35, 41, 45, 28, 105, 109, 59, 92, 8, 67, 93, 1, 96, 66, 2, 26, 100, 0, 101, 3, 65, 90, 37, 6], [46, 110, 49, 121, 112, 40, 57, 104, 54, 56, 51, 127, 125, 53, 113, 116, 43, 44, 126, 61, 124, 117, 102, 119, 123, 60, 39, 25, 89, 108, 48, 63, 107, 33, 80, 16, 99, 115, 58, 78, 12, 55, 14, 76, 21, 47, 72, 50, 97, 82, 91, 20, 85, 98, 27, 18, 94, 83, 114, 79, 120, 118, 38, 24, 73, 19, 11, 95, 75, 77, 34, 86, 13, 74, 52, 15, 42, 88, 17, 81, 84, 62, 70, 87, 31, 10, 36, 45, 9, 111, 23, 5, 122, 4, 22, 71, 29, 68, 7, 69, 92, 30, 106, 93, 41, 32, 96, 103, 28, 59, 2, 109, 6, 101, 35, 105, 8, 90, 3, 1, 26, 67, 66, 0, 100, 65, 64, 37], [46, 110, 49, 121, 112, 40, 54, 56, 104, 51, 57, 113, 53, 125, 127, 116, 43, 44, 61, 126, 119, 102, 60, 124, 39, 123, 117, 108, 89, 72, 58, 25, 16, 33, 48, 80, 107, 78, 63, 76, 120, 12, 99, 14, 98, 55, 115, 47, 50, 118, 11, 18, 79, 85, 82, 73, 42, 114, 95, 21, 97, 91, 9, 75, 84, 10, 27, 20, 77, 17, 19, 13, 15, 83, 34, 38, 74, 52, 122, 94, 69, 24, 4, 31, 81, 23, 68, 22, 70, 88, 7, 71, 62, 36, 29, 86, 87, 6, 5, 111, 45, 109, 32, 3, 28, 59, 2, 30, 0, 1, 90, 103, 105, 67, 65, 96, 66, 35, 93, 64, 92, 101, 106, 100, 41, 26, 8, 37], [46, 110, 49, 121, 112, 54, 40, 57, 56, 104, 127, 51, 113, 125, 53, 116, 43, 126, 44, 61, 123, 124, 102, 119, 60, 117, 108, 58, 115, 33, 89, 48, 25, 72, 16, 107, 39, 63, 80, 76, 78, 99, 14, 120, 47, 55, 118, 12, 98, 50, 114, 82, 27, 21, 18, 52, 85, 97, 20, 9, 83, 11, 91, 84, 74, 75, 79, 15, 10, 95, 86, 87, 19, 122, 42, 13, 34, 81, 94, 24, 17, 73, 6, 77, 36, 5, 23, 88, 38, 69, 7, 111, 22, 45, 31, 68, 4, 71, 109, 30, 62, 29, 59, 28, 106, 92, 105, 3, 103, 32, 2, 70, 96, 65, 1, 64, 100, 0, 66, 90, 41, 93, 67, 8, 35, 101, 37, 26], [46, 110, 49, 121, 112, 54, 40, 51, 56, 57, 104, 127, 113, 125, 116, 53, 43, 44, 61, 126, 124, 102, 123, 119, 117, 108, 39, 107, 60, 25, 58, 48, 16, 89, 72, 33, 63, 80, 12, 76, 55, 99, 78, 14, 115, 98, 47, 50, 18, 82, 21, 6, 85, 95, 9, 114, 120, 11, 27, 97, 20, 79, 91, 118, 75, 84, 83, 73, 10, 42, 15, 34, 36, 86, 13, 77, 19, 24, 74, 88, 31, 87, 62, 94, 52, 17, 7, 4, 81, 68, 5, 111, 38, 71, 69, 103, 45, 23, 29, 122, 22, 93, 106, 32, 105, 109, 59, 30, 41, 35, 8, 28, 3, 92, 1, 64, 2, 67, 66, 70, 0, 65, 96, 100, 90, 101, 26, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 51, 127, 57, 125, 113, 116, 53, 43, 44, 61, 126, 119, 102, 124, 123, 117, 60, 108, 25, 89, 39, 107, 48, 80, 115, 16, 12, 58, 99, 76, 78, 14, 33, 47, 18, 55, 72, 19, 21, 27, 6, 63, 75, 120, 94, 20, 82, 9, 50, 98, 83, 91, 73, 11, 15, 34, 42, 84, 13, 88, 85, 79, 81, 52, 97, 77, 74, 10, 95, 86, 114, 118, 17, 68, 87, 24, 31, 111, 36, 71, 106, 22, 62, 23, 5, 7, 38, 29, 109, 93, 4, 30, 8, 103, 122, 35, 69, 45, 92, 41, 59, 105, 32, 66, 67, 100, 28, 64, 0, 65, 3, 96, 2, 101, 90, 26, 70, 1, 37], [46, 110, 49, 112, 121, 54, 51, 40, 57, 56, 104, 127, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 60, 123, 119, 117, 89, 58, 108, 48, 55, 39, 63, 33, 25, 80, 107, 99, 50, 16, 14, 47, 78, 76, 12, 18, 114, 115, 120, 42, 21, 85, 118, 83, 19, 98, 95, 27, 34, 97, 52, 82, 79, 91, 20, 84, 75, 6, 77, 94, 15, 11, 73, 17, 10, 13, 31, 81, 38, 72, 9, 88, 86, 62, 87, 74, 24, 8, 22, 59, 106, 36, 23, 122, 29, 5, 68, 7, 71, 4, 111, 93, 41, 30, 92, 103, 69, 109, 45, 32, 96, 100, 66, 35, 65, 90, 1, 64, 0, 28, 105, 101, 3, 67, 26, 37, 70, 2]], "model.layers.3.self_attn.q_proj": [[40, 60, 98, 56, 52, 57, 119, 62, 32, 58, 49, 37, 113, 118, 53, 59, 95, 123, 48, 105, 89, 50, 24, 45, 122, 63, 120, 25, 101, 26, 30, 93, 39, 86, 51, 61, 85, 42, 34, 17, 38, 109, 111, 102, 35, 107, 36, 121, 114, 43, 116, 21, 126, 103, 54, 83, 84, 125, 29, 112, 108, 55, 46, 97, 96, 115, 92, 127, 110, 124, 31, 99, 47, 44, 33, 100, 41, 91, 88, 94, 15, 106, 20, 18, 117, 90, 27, 28, 82, 87, 23, 22, 81, 104, 80, 11, 78, 75, 76, 79, 74, 77, 10, 19, 8, 72, 14, 12, 68, 13, 65, 71, 5, 73, 16, 4, 2, 67, 69, 6, 9, 7, 70, 3, 1, 66, 0, 64], [119, 62, 40, 98, 121, 52, 123, 58, 53, 48, 50, 118, 38, 25, 49, 30, 29, 54, 39, 63, 60, 57, 84, 56, 122, 59, 42, 115, 86, 102, 112, 33, 27, 37, 93, 113, 116, 28, 32, 111, 107, 89, 83, 15, 46, 108, 114, 120, 103, 110, 125, 101, 80, 18, 88, 36, 45, 41, 34, 47, 21, 43, 127, 24, 55, 51, 117, 44, 23, 109, 14, 124, 61, 100, 105, 97, 92, 106, 126, 73, 35, 26, 104, 12, 72, 19, 99, 31, 94, 17, 22, 95, 85, 90, 91, 96, 74, 20, 67, 70, 76, 79, 87, 81, 11, 10, 75, 71, 2, 13, 9, 5, 4, 65, 82, 68, 78, 7, 8, 0, 77, 3, 1, 66, 6, 64, 16, 69], [40, 98, 56, 60, 62, 119, 86, 29, 52, 58, 83, 14, 80, 57, 12, 54, 59, 10, 48, 93, 17, 15, 50, 63, 24, 25, 125, 4, 123, 118, 121, 101, 9, 120, 74, 113, 72, 26, 5, 85, 114, 51, 106, 33, 99, 53, 70, 45, 122, 95, 73, 71, 78, 76, 11, 16, 30, 81, 19, 22, 89, 104, 32, 103, 108, 21, 20, 88, 66, 67, 90, 84, 7, 2, 41, 49, 18, 111, 13, 1, 39, 23, 38, 87, 110, 75, 43, 36, 27, 6, 0, 82, 77, 79, 69, 37, 47, 8, 3, 97, 100, 102, 94, 92, 126, 96, 127, 34, 91, 42, 61, 28, 109, 115, 117, 35, 31, 116, 64, 105, 112, 68, 46, 55, 107, 65, 44, 124], [58, 40, 98, 119, 62, 60, 52, 29, 57, 86, 56, 48, 12, 80, 83, 59, 14, 24, 50, 51, 10, 9, 54, 72, 95, 73, 4, 19, 11, 71, 114, 116, 93, 66, 100, 76, 69, 63, 123, 32, 2, 49, 70, 106, 0, 125, 113, 16, 118, 68, 91, 5, 26, 7, 67, 75, 82, 53, 39, 90, 127, 79, 28, 120, 99, 25, 23, 103, 109, 27, 94, 22, 55, 111, 81, 13, 77, 108, 89, 18, 61, 117, 47, 46, 85, 87, 8, 121, 78, 88, 38, 36, 21, 31, 107, 3, 17, 37, 20, 30, 42, 15, 35, 122, 124, 96, 112, 84, 92, 45, 102, 43, 105, 101, 97, 33, 44, 64, 104, 126, 74, 6, 41, 115, 1, 110, 34, 65], [38, 97, 111, 120, 123, 122, 47, 84, 81, 10, 14, 71, 12, 8, 2, 67, 69, 58, 1, 0, 87, 88, 76, 19, 65, 83, 93, 106, 17, 86, 57, 34, 53, 11, 42, 32, 25, 118, 26, 20, 85, 99, 96, 56, 80, 49, 66, 31, 15, 78, 3, 105, 95, 89, 112, 109, 94, 52, 124, 101, 98, 110, 30, 126, 90, 92, 37, 22, 64, 55, 115, 82, 91, 103, 79, 44, 48, 74, 73, 119, 113, 60, 27, 45, 36, 4, 39, 29, 54, 62, 41, 61, 100, 21, 121, 108, 35, 63, 28, 51, 72, 46, 7, 18, 114, 127, 107, 125, 75, 77, 117, 16, 59, 23, 50, 43, 116, 104, 40, 13, 70, 5, 9, 24, 6, 68, 33, 102], [38, 97, 111, 122, 123, 120, 47, 81, 23, 84, 14, 71, 12, 10, 8, 69, 67, 75, 19, 73, 25, 7, 3, 1, 56, 76, 21, 105, 13, 78, 58, 5, 65, 2, 49, 79, 64, 4, 94, 70, 66, 30, 113, 0, 31, 98, 6, 112, 72, 16, 126, 86, 61, 114, 87, 52, 15, 88, 43, 35, 74, 42, 124, 62, 119, 63, 68, 37, 11, 104, 121, 45, 118, 17, 99, 20, 110, 57, 93, 82, 24, 26, 90, 77, 80, 32, 28, 22, 103, 85, 117, 46, 36, 9, 51, 106, 125, 18, 108, 92, 59, 83, 55, 115, 89, 60, 53, 41, 29, 44, 107, 33, 100, 95, 39, 127, 48, 54, 27, 101, 116, 50, 40, 91, 109, 96, 102, 34], [38, 111, 97, 123, 120, 122, 47, 81, 14, 84, 23, 12, 10, 8, 69, 71, 59, 75, 58, 3, 1, 86, 67, 16, 11, 40, 56, 127, 88, 25, 24, 105, 50, 31, 32, 87, 51, 9, 90, 45, 116, 76, 43, 73, 110, 74, 5, 125, 22, 21, 106, 4, 2, 62, 82, 19, 91, 78, 13, 98, 72, 113, 26, 93, 27, 28, 48, 100, 20, 83, 99, 66, 94, 15, 53, 117, 39, 17, 95, 42, 80, 49, 104, 108, 63, 65, 0, 118, 96, 126, 55, 60, 33, 92, 61, 77, 37, 85, 114, 30, 107, 35, 52, 119, 46, 44, 79, 103, 6, 41, 34, 109, 7, 29, 57, 18, 36, 115, 112, 64, 54, 101, 121, 89, 70, 102, 124, 68], [38, 97, 111, 123, 122, 120, 47, 84, 81, 14, 23, 12, 10, 71, 24, 8, 69, 67, 3, 1, 55, 105, 93, 76, 44, 59, 78, 86, 7, 56, 58, 70, 73, 16, 61, 107, 25, 30, 2, 21, 11, 57, 60, 126, 6, 51, 68, 4, 89, 80, 88, 79, 17, 104, 72, 32, 35, 20, 19, 74, 66, 95, 46, 41, 42, 26, 96, 43, 94, 75, 0, 98, 116, 106, 40, 15, 22, 48, 5, 50, 54, 9, 29, 63, 87, 124, 109, 114, 118, 110, 82, 37, 53, 112, 113, 27, 125, 45, 92, 121, 91, 85, 77, 34, 117, 62, 39, 49, 101, 65, 103, 119, 31, 90, 83, 28, 100, 115, 99, 36, 64, 52, 127, 18, 13, 108, 33, 102], [41, 34, 18, 20, 52, 76, 89, 79, 22, 16, 14, 120, 114, 62, 8, 10, 105, 46, 47, 63, 81, 126, 116, 69, 13, 125, 117, 7, 9, 127, 3, 59, 123, 72, 71, 5, 80, 23, 28, 73, 53, 70, 112, 12, 74, 50, 119, 24, 78, 19, 113, 6, 49, 66, 58, 15, 2, 110, 25, 75, 82, 54, 17, 109, 107, 95, 85, 86, 67, 11, 83, 94, 111, 118, 104, 60, 29, 37, 55, 61, 32, 93, 122, 27, 88, 68, 1, 39, 56, 115, 84, 64, 38, 42, 31, 43, 33, 101, 102, 87, 21, 26, 44, 103, 100, 0, 97, 35, 96, 121, 57, 48, 91, 92, 77, 45, 108, 106, 124, 4, 99, 51, 30, 90, 40, 65, 36, 98], [41, 34, 89, 52, 20, 22, 120, 18, 47, 46, 63, 114, 126, 80, 116, 76, 62, 105, 72, 14, 28, 78, 125, 117, 11, 79, 123, 82, 127, 13, 85, 7, 59, 74, 77, 10, 12, 19, 84, 58, 21, 15, 49, 9, 107, 119, 53, 54, 90, 24, 86, 81, 113, 5, 109, 73, 25, 95, 112, 29, 8, 23, 92, 103, 27, 104, 30, 37, 50, 61, 102, 39, 16, 88, 69, 96, 35, 32, 94, 3, 31, 71, 26, 110, 111, 101, 55, 33, 60, 124, 51, 93, 36, 83, 17, 38, 66, 57, 118, 43, 87, 42, 115, 98, 91, 75, 40, 4, 56, 99, 1, 108, 44, 122, 97, 100, 121, 6, 67, 68, 65, 45, 48, 106, 70, 2, 64, 0], [41, 34, 52, 81, 22, 18, 114, 20, 126, 76, 79, 120, 14, 10, 62, 69, 47, 116, 8, 3, 105, 63, 5, 46, 117, 125, 127, 72, 7, 9, 89, 54, 66, 123, 24, 1, 70, 6, 59, 74, 53, 71, 16, 113, 49, 28, 2, 64, 58, 77, 86, 110, 32, 73, 0, 119, 27, 68, 85, 50, 107, 112, 78, 12, 67, 17, 87, 83, 15, 13, 80, 84, 26, 104, 109, 61, 25, 92, 90, 60, 96, 118, 56, 38, 65, 57, 75, 95, 19, 45, 43, 82, 11, 111, 91, 99, 88, 35, 93, 33, 101, 31, 97, 122, 21, 29, 30, 37, 102, 39, 40, 100, 44, 42, 23, 106, 94, 55, 36, 124, 121, 115, 48, 103, 51, 4, 108, 98], [41, 34, 20, 18, 22, 52, 10, 76, 14, 6, 67, 62, 46, 120, 79, 126, 125, 114, 3, 116, 47, 105, 7, 8, 63, 9, 69, 127, 54, 2, 28, 117, 59, 58, 25, 0, 13, 123, 70, 110, 72, 80, 77, 112, 12, 49, 24, 90, 65, 75, 44, 113, 11, 119, 53, 71, 74, 23, 73, 91, 104, 35, 32, 78, 85, 15, 64, 87, 86, 89, 84, 88, 45, 66, 61, 21, 26, 5, 17, 68, 95, 60, 1, 42, 121, 83, 51, 122, 92, 100, 107, 56, 103, 109, 36, 106, 111, 94, 81, 48, 82, 38, 50, 55, 43, 16, 37, 19, 31, 102, 27, 29, 108, 124, 57, 97, 39, 93, 101, 99, 40, 115, 33, 96, 118, 30, 4, 98], [103, 117, 125, 109, 46, 56, 122, 97, 58, 52, 63, 57, 47, 28, 126, 59, 115, 89, 39, 21, 124, 119, 51, 49, 116, 62, 110, 60, 113, 55, 114, 111, 54, 86, 50, 87, 112, 123, 127, 61, 121, 53, 81, 33, 82, 48, 118, 108, 45, 120, 24, 107, 20, 44, 41, 31, 42, 19, 105, 106, 79, 43, 13, 96, 37, 30, 40, 100, 102, 104, 101, 99, 38, 36, 94, 34, 35, 95, 98, 32, 73, 80, 92, 93, 90, 17, 22, 91, 29, 76, 12, 26, 78, 88, 25, 71, 70, 27, 16, 83, 7, 9, 10, 85, 77, 4, 23, 68, 18, 84, 74, 5, 6, 8, 15, 3, 14, 67, 66, 0, 11, 65, 75, 69, 1, 64, 72, 2], [103, 117, 52, 109, 125, 56, 97, 122, 46, 119, 57, 28, 63, 58, 89, 21, 82, 115, 9, 47, 12, 78, 87, 126, 80, 20, 124, 33, 49, 116, 13, 62, 19, 110, 31, 83, 114, 59, 86, 54, 17, 111, 92, 61, 123, 127, 50, 60, 51, 94, 30, 55, 25, 6, 112, 74, 107, 81, 79, 71, 118, 77, 90, 11, 108, 121, 7, 24, 113, 53, 39, 48, 16, 101, 15, 18, 44, 32, 88, 95, 73, 4, 67, 22, 36, 45, 29, 84, 76, 69, 43, 41, 8, 68, 105, 37, 23, 26, 91, 85, 70, 5, 104, 93, 99, 14, 34, 75, 10, 100, 96, 120, 42, 106, 35, 3, 65, 38, 102, 66, 0, 27, 40, 98, 1, 2, 72, 64], [103, 117, 52, 125, 109, 97, 119, 122, 21, 89, 56, 28, 46, 59, 60, 20, 78, 82, 79, 87, 58, 47, 80, 55, 9, 124, 113, 50, 126, 121, 57, 12, 114, 123, 111, 62, 7, 74, 11, 63, 25, 118, 51, 53, 61, 92, 15, 54, 4, 22, 90, 116, 31, 112, 19, 110, 30, 115, 127, 17, 107, 33, 83, 70, 49, 95, 8, 13, 91, 98, 93, 6, 69, 24, 81, 44, 94, 77, 32, 71, 48, 67, 88, 73, 45, 38, 23, 99, 43, 42, 86, 29, 26, 66, 104, 84, 37, 5, 76, 101, 96, 102, 120, 39, 105, 41, 106, 34, 108, 100, 40, 1, 27, 16, 36, 10, 85, 68, 35, 18, 14, 0, 75, 2, 65, 64, 72, 3], [103, 117, 52, 46, 97, 125, 109, 119, 56, 122, 21, 63, 82, 78, 80, 87, 58, 9, 83, 89, 20, 12, 28, 92, 57, 8, 11, 6, 126, 15, 59, 75, 25, 79, 67, 124, 54, 74, 16, 13, 81, 114, 49, 62, 76, 115, 71, 123, 31, 53, 110, 68, 69, 60, 85, 90, 18, 19, 77, 95, 30, 14, 23, 127, 66, 111, 50, 72, 48, 33, 24, 84, 113, 10, 64, 70, 73, 51, 5, 3, 47, 112, 61, 4, 26, 22, 86, 55, 17, 93, 1, 2, 88, 27, 29, 0, 94, 118, 107, 91, 32, 96, 44, 99, 34, 65, 121, 108, 116, 101, 7, 41, 98, 100, 36, 35, 102, 106, 104, 120, 37, 43, 42, 38, 40, 105, 45, 39], [104, 33, 108, 20, 18, 23, 16, 13, 10, 44, 72, 68, 70, 48, 56, 107, 63, 55, 40, 117, 54, 84, 116, 57, 90, 21, 50, 12, 66, 14, 110, 31, 115, 51, 11, 9, 58, 73, 78, 61, 109, 114, 87, 111, 96, 121, 123, 25, 52, 120, 79, 1, 5, 82, 26, 100, 86, 43, 74, 125, 127, 59, 94, 119, 83, 118, 3, 76, 15, 126, 2, 38, 92, 45, 77, 34, 6, 8, 22, 102, 41, 62, 24, 88, 95, 71, 112, 36, 7, 60, 105, 106, 93, 80, 89, 42, 99, 85, 113, 47, 67, 101, 27, 39, 17, 98, 81, 46, 29, 103, 69, 53, 28, 30, 19, 35, 4, 91, 49, 124, 75, 122, 37, 32, 64, 65, 97, 0], [104, 108, 33, 65, 10, 13, 68, 18, 70, 1, 16, 64, 7, 44, 23, 67, 66, 87, 40, 0, 72, 20, 4, 56, 110, 69, 111, 116, 120, 3, 55, 5, 50, 12, 115, 6, 57, 61, 11, 123, 71, 54, 58, 84, 113, 77, 112, 48, 8, 2, 19, 62, 125, 114, 117, 51, 100, 43, 124, 52, 118, 78, 38, 14, 26, 9, 99, 80, 97, 63, 31, 96, 45, 24, 83, 49, 79, 21, 59, 107, 127, 29, 119, 74, 30, 60, 88, 109, 39, 122, 121, 89, 15, 85, 92, 36, 91, 41, 86, 32, 101, 25, 106, 94, 37, 93, 126, 34, 42, 90, 75, 82, 105, 53, 17, 73, 98, 103, 76, 47, 22, 27, 46, 28, 35, 81, 102, 95], [104, 108, 33, 64, 0, 44, 70, 13, 68, 10, 7, 66, 67, 16, 20, 18, 87, 1, 40, 23, 65, 72, 111, 56, 50, 110, 57, 55, 5, 120, 48, 116, 4, 84, 113, 115, 11, 123, 12, 61, 71, 51, 114, 100, 3, 117, 58, 21, 6, 118, 43, 125, 2, 80, 62, 45, 77, 52, 96, 75, 54, 69, 109, 86, 8, 126, 41, 38, 124, 121, 39, 47, 76, 101, 74, 27, 59, 89, 25, 99, 31, 35, 28, 79, 24, 81, 107, 83, 82, 34, 78, 92, 90, 60, 95, 37, 19, 94, 14, 9, 98, 127, 49, 17, 91, 29, 53, 30, 88, 106, 63, 85, 26, 112, 42, 36, 102, 32, 105, 46, 122, 103, 22, 93, 15, 119, 73, 97], [104, 108, 33, 23, 16, 18, 10, 44, 20, 13, 7, 72, 12, 116, 55, 87, 40, 110, 61, 67, 68, 107, 115, 117, 56, 70, 48, 114, 69, 50, 57, 125, 109, 1, 66, 118, 123, 120, 60, 54, 124, 111, 21, 81, 83, 25, 58, 126, 78, 64, 11, 59, 29, 63, 9, 88, 28, 90, 43, 32, 52, 46, 42, 62, 2, 8, 85, 47, 101, 73, 27, 89, 15, 113, 84, 76, 49, 103, 91, 99, 31, 24, 14, 3, 119, 86, 22, 98, 105, 122, 121, 36, 75, 79, 41, 106, 45, 30, 94, 127, 92, 80, 82, 93, 19, 96, 112, 77, 53, 17, 38, 6, 26, 95, 102, 39, 34, 100, 37, 4, 51, 71, 35, 5, 74, 97, 65, 0], [41, 46, 53, 127, 40, 97, 60, 105, 91, 31, 119, 39, 122, 47, 27, 112, 38, 24, 116, 99, 36, 115, 118, 42, 93, 57, 117, 33, 43, 125, 51, 126, 35, 110, 85, 55, 18, 114, 90, 87, 102, 88, 92, 96, 111, 100, 29, 124, 20, 109, 59, 49, 52, 95, 123, 63, 50, 56, 81, 44, 23, 30, 106, 48, 104, 79, 120, 108, 98, 25, 80, 107, 62, 121, 34, 37, 22, 94, 77, 82, 19, 113, 14, 26, 9, 58, 61, 28, 45, 54, 101, 75, 21, 83, 103, 13, 16, 86, 32, 84, 89, 74, 11, 76, 17, 5, 72, 15, 6, 66, 4, 73, 71, 2, 69, 10, 67, 70, 1, 68, 78, 64, 12, 0, 7, 8, 65, 3], [40, 97, 53, 127, 46, 24, 85, 31, 110, 14, 81, 75, 88, 60, 19, 90, 93, 41, 76, 47, 115, 71, 80, 74, 9, 4, 122, 82, 83, 119, 21, 91, 43, 38, 6, 104, 96, 55, 105, 42, 112, 27, 3, 72, 73, 33, 68, 101, 100, 37, 22, 26, 63, 124, 108, 13, 29, 54, 62, 126, 69, 30, 25, 125, 98, 114, 84, 15, 66, 117, 23, 45, 10, 77, 70, 118, 51, 16, 35, 106, 79, 87, 113, 17, 92, 34, 116, 1, 102, 89, 52, 44, 103, 12, 28, 48, 99, 56, 2, 20, 64, 111, 78, 36, 18, 123, 7, 61, 11, 94, 39, 65, 32, 50, 95, 86, 49, 8, 5, 59, 109, 57, 67, 121, 0, 107, 58, 120], [41, 110, 40, 53, 97, 127, 60, 43, 46, 38, 31, 47, 112, 91, 92, 35, 116, 52, 19, 122, 113, 33, 124, 119, 117, 26, 93, 59, 39, 27, 24, 125, 108, 63, 86, 56, 123, 95, 118, 57, 115, 49, 29, 126, 48, 45, 58, 88, 28, 76, 100, 96, 55, 42, 114, 18, 80, 85, 121, 99, 13, 87, 103, 37, 36, 22, 106, 51, 50, 111, 61, 90, 98, 109, 94, 12, 62, 105, 54, 14, 25, 107, 21, 20, 30, 44, 83, 82, 77, 101, 104, 70, 89, 6, 34, 102, 120, 84, 32, 81, 79, 23, 74, 16, 75, 15, 72, 9, 67, 3, 10, 78, 5, 71, 2, 0, 66, 73, 4, 65, 11, 8, 1, 69, 7, 68, 17, 64], [40, 46, 127, 97, 53, 43, 31, 38, 67, 41, 60, 75, 5, 14, 85, 71, 81, 65, 110, 24, 0, 9, 39, 90, 1, 19, 104, 47, 115, 112, 35, 91, 77, 21, 27, 88, 26, 17, 72, 7, 59, 102, 121, 124, 80, 93, 23, 79, 10, 76, 125, 48, 44, 2, 70, 8, 105, 100, 118, 54, 3, 56, 52, 103, 92, 64, 69, 116, 4, 12, 82, 57, 96, 87, 78, 49, 99, 36, 119, 42, 68, 74, 98, 11, 15, 117, 114, 18, 25, 86, 126, 37, 63, 28, 106, 122, 66, 34, 108, 120, 83, 84, 32, 62, 50, 94, 73, 107, 95, 22, 111, 30, 51, 113, 55, 16, 58, 61, 123, 109, 101, 20, 13, 45, 33, 89, 6, 29], [39, 124, 34, 117, 119, 47, 24, 62, 109, 20, 15, 13, 48, 26, 17, 11, 53, 70, 9, 72, 55, 19, 1, 65, 73, 28, 10, 78, 69, 123, 5, 21, 118, 68, 86, 83, 106, 81, 71, 49, 95, 112, 122, 25, 87, 31, 79, 116, 12, 92, 85, 18, 88, 27, 35, 115, 30, 90, 107, 14, 75, 8, 89, 7, 77, 66, 74, 22, 23, 94, 84, 45, 80, 91, 33, 3, 61, 6, 67, 76, 110, 52, 16, 82, 121, 2, 64, 4, 50, 93, 54, 29, 32, 43, 0, 97, 99, 41, 51, 111, 100, 63, 46, 37, 101, 36, 38, 56, 96, 59, 44, 127, 60, 40, 120, 114, 102, 42, 98, 113, 57, 105, 58, 108, 126, 104, 103, 125], [109, 119, 39, 62, 117, 34, 48, 124, 45, 61, 20, 30, 26, 94, 86, 110, 92, 24, 115, 17, 112, 116, 28, 122, 103, 19, 118, 114, 98, 15, 57, 123, 78, 51, 58, 125, 42, 63, 27, 53, 91, 55, 22, 47, 49, 126, 87, 95, 83, 54, 43, 127, 11, 59, 9, 44, 113, 111, 33, 121, 14, 38, 60, 50, 56, 52, 46, 108, 16, 68, 76, 41, 29, 32, 99, 89, 40, 13, 107, 105, 71, 106, 104, 102, 73, 90, 25, 21, 96, 37, 23, 100, 120, 72, 93, 35, 36, 101, 97, 31, 82, 80, 10, 85, 18, 84, 4, 70, 88, 69, 81, 7, 67, 75, 64, 0, 77, 74, 79, 2, 12, 8, 1, 3, 66, 5, 65, 6], [48, 39, 62, 34, 119, 109, 124, 117, 24, 47, 20, 11, 15, 13, 17, 94, 70, 73, 85, 10, 9, 68, 7, 86, 53, 80, 115, 19, 61, 118, 123, 45, 28, 22, 72, 121, 25, 55, 1, 3, 110, 93, 54, 92, 29, 89, 106, 30, 31, 69, 75, 112, 87, 95, 88, 66, 50, 12, 79, 26, 27, 81, 77, 16, 71, 82, 14, 60, 91, 64, 4, 43, 122, 84, 21, 40, 114, 90, 8, 23, 96, 0, 67, 18, 116, 76, 107, 57, 46, 83, 74, 97, 2, 32, 33, 102, 6, 100, 52, 111, 108, 37, 99, 105, 63, 78, 35, 44, 56, 49, 41, 36, 38, 65, 126, 127, 101, 42, 120, 51, 104, 125, 113, 58, 59, 5, 98, 103], [48, 39, 62, 124, 119, 109, 117, 34, 47, 53, 26, 30, 49, 86, 116, 45, 42, 112, 123, 20, 115, 63, 61, 55, 50, 125, 24, 111, 92, 110, 58, 33, 43, 122, 107, 118, 80, 54, 29, 114, 44, 41, 94, 46, 93, 96, 57, 40, 106, 52, 25, 120, 56, 60, 37, 113, 127, 73, 121, 51, 108, 36, 126, 104, 35, 22, 28, 98, 100, 105, 87, 82, 17, 83, 32, 102, 38, 99, 31, 59, 101, 91, 97, 95, 19, 23, 15, 27, 103, 89, 90, 85, 16, 76, 13, 21, 71, 78, 84, 74, 81, 9, 6, 14, 79, 18, 75, 72, 88, 11, 12, 10, 5, 77, 67, 68, 4, 8, 7, 70, 3, 2, 1, 69, 0, 65, 66, 64], [40, 97, 119, 31, 81, 23, 19, 52, 76, 85, 79, 111, 112, 93, 53, 110, 7, 73, 59, 61, 57, 104, 113, 117, 5, 126, 60, 58, 27, 3, 44, 54, 107, 48, 86, 13, 43, 24, 91, 114, 1, 10, 55, 67, 28, 115, 127, 78, 123, 50, 30, 90, 71, 15, 116, 56, 109, 82, 121, 21, 83, 94, 88, 108, 101, 12, 80, 87, 74, 103, 75, 29, 72, 77, 96, 47, 42, 17, 35, 16, 39, 84, 25, 11, 37, 100, 45, 26, 33, 122, 6, 22, 118, 124, 0, 34, 92, 63, 46, 8, 70, 65, 125, 36, 68, 106, 120, 18, 98, 69, 9, 99, 4, 89, 62, 32, 102, 105, 51, 38, 64, 20, 49, 14, 41, 95, 66, 2], [40, 97, 119, 31, 23, 81, 52, 13, 111, 53, 19, 79, 58, 112, 85, 25, 110, 60, 61, 117, 55, 123, 54, 24, 76, 57, 51, 126, 84, 20, 93, 113, 59, 73, 114, 104, 121, 88, 27, 91, 33, 108, 29, 47, 82, 18, 43, 26, 7, 28, 41, 86, 90, 72, 99, 103, 44, 109, 63, 107, 45, 77, 94, 122, 75, 30, 87, 70, 34, 105, 5, 21, 100, 42, 78, 74, 22, 127, 120, 50, 35, 101, 116, 32, 69, 96, 83, 56, 48, 14, 1, 115, 8, 15, 89, 38, 46, 124, 125, 17, 80, 37, 68, 49, 98, 12, 92, 16, 11, 36, 6, 118, 39, 9, 62, 102, 106, 2, 67, 95, 71, 0, 3, 66, 10, 4, 64, 65], [40, 97, 119, 31, 81, 85, 52, 23, 19, 111, 24, 79, 76, 73, 126, 57, 112, 54, 5, 93, 58, 53, 44, 104, 110, 117, 13, 48, 7, 59, 108, 107, 60, 15, 70, 123, 61, 113, 120, 34, 55, 114, 43, 25, 29, 3, 89, 1, 50, 109, 12, 30, 28, 6, 88, 82, 83, 116, 22, 39, 78, 91, 42, 56, 87, 2, 80, 121, 74, 69, 75, 127, 26, 51, 27, 115, 21, 68, 37, 103, 32, 46, 101, 94, 45, 47, 118, 99, 66, 62, 38, 122, 100, 124, 35, 20, 102, 77, 86, 64, 14, 41, 11, 96, 90, 92, 17, 33, 106, 105, 98, 16, 9, 63, 36, 10, 65, 8, 84, 125, 0, 49, 4, 71, 72, 18, 95, 67], [40, 97, 119, 31, 26, 81, 52, 19, 85, 79, 23, 112, 126, 111, 73, 110, 13, 76, 53, 57, 25, 58, 93, 55, 104, 114, 54, 78, 5, 117, 96, 51, 60, 123, 59, 7, 113, 99, 61, 42, 20, 27, 88, 84, 82, 45, 77, 43, 1, 125, 91, 87, 48, 44, 116, 24, 107, 109, 3, 70, 80, 102, 16, 94, 15, 69, 106, 21, 2, 9, 90, 56, 127, 121, 115, 83, 75, 12, 89, 33, 47, 122, 28, 101, 120, 92, 36, 34, 22, 17, 29, 6, 74, 41, 103, 108, 49, 32, 30, 63, 62, 14, 11, 86, 100, 105, 64, 0, 10, 67, 46, 35, 68, 118, 50, 39, 98, 37, 124, 66, 72, 38, 71, 4, 65, 18, 8, 95]], "model.layers.3.self_attn.k_proj": [[104, 34, 93, 56, 62, 119, 60, 58, 80, 83, 24, 14, 86, 12, 10, 0, 26, 70, 5, 2, 67, 125, 72, 54, 116, 114, 77, 25, 85, 73, 22, 9, 6, 71, 118, 115, 18, 13, 42, 112, 96, 32, 37, 68, 95, 117, 8, 92, 29, 49, 41, 51, 65, 84, 59, 7, 113, 61, 111, 100, 43, 105, 17, 103, 45, 127, 35, 121, 108, 55, 99, 110, 57, 109, 124, 30, 89, 75, 106, 122, 48, 107, 33, 63, 88, 50, 91, 28, 21, 120, 31, 76, 87, 101, 46, 102, 44, 47, 38, 82, 1, 15, 20, 78, 53, 27, 126, 97, 79, 4, 64, 123, 39, 36, 23, 52, 69, 94, 11, 90, 74, 81, 66, 3, 16, 19, 98, 40], [47, 122, 102, 123, 120, 64, 65, 69, 33, 8, 10, 12, 71, 81, 84, 67, 14, 23, 68, 66, 0, 2, 1, 75, 5, 3, 70, 24, 86, 113, 25, 13, 6, 19, 73, 43, 72, 88, 124, 87, 9, 42, 21, 40, 118, 59, 119, 111, 41, 38, 61, 103, 105, 39, 83, 79, 117, 49, 100, 114, 27, 15, 18, 16, 116, 60, 104, 82, 125, 121, 45, 50, 110, 53, 11, 108, 52, 37, 85, 55, 4, 109, 106, 127, 48, 115, 92, 96, 98, 58, 63, 28, 30, 89, 26, 107, 90, 31, 54, 80, 36, 77, 46, 95, 99, 56, 62, 51, 57, 112, 91, 29, 34, 35, 126, 94, 44, 22, 101, 20, 93, 74, 7, 17, 78, 32, 76, 97], [105, 98, 52, 79, 18, 9, 0, 20, 22, 76, 7, 111, 69, 110, 14, 120, 10, 2, 62, 50, 63, 67, 1, 4, 126, 53, 6, 127, 41, 59, 114, 125, 24, 113, 65, 43, 11, 8, 46, 48, 89, 123, 19, 77, 117, 119, 54, 58, 81, 90, 70, 17, 47, 60, 68, 95, 28, 64, 3, 40, 122, 112, 72, 91, 71, 108, 75, 85, 61, 44, 83, 51, 121, 124, 73, 115, 32, 93, 116, 66, 30, 74, 97, 29, 45, 87, 57, 33, 94, 38, 16, 55, 99, 39, 23, 21, 100, 31, 26, 88, 109, 118, 96, 56, 27, 101, 42, 103, 49, 104, 102, 78, 106, 35, 92, 37, 107, 36, 80, 25, 86, 82, 12, 15, 5, 13, 84, 34], [39, 52, 33, 45, 125, 110, 28, 74, 119, 87, 117, 46, 78, 89, 4, 21, 12, 80, 0, 66, 11, 86, 8, 17, 71, 82, 20, 1, 7, 58, 62, 65, 79, 24, 6, 53, 67, 57, 95, 3, 90, 94, 64, 72, 127, 5, 56, 50, 54, 9, 116, 126, 63, 48, 108, 55, 123, 19, 75, 43, 109, 61, 92, 69, 68, 15, 118, 49, 73, 59, 13, 60, 106, 122, 51, 77, 88, 41, 120, 113, 124, 115, 101, 112, 107, 44, 84, 32, 42, 105, 37, 121, 96, 114, 83, 102, 104, 30, 14, 111, 10, 16, 100, 40, 38, 29, 99, 35, 34, 47, 36, 2, 98, 31, 91, 27, 81, 76, 93, 26, 18, 103, 22, 70, 97, 23, 85, 25], [44, 40, 64, 97, 108, 13, 1, 112, 16, 10, 56, 23, 46, 50, 67, 18, 70, 66, 47, 7, 68, 69, 123, 2, 104, 61, 72, 20, 52, 55, 57, 120, 51, 12, 3, 125, 116, 118, 58, 119, 117, 54, 49, 109, 4, 124, 14, 73, 11, 39, 53, 6, 43, 115, 114, 86, 75, 21, 5, 85, 45, 60, 126, 62, 78, 22, 65, 107, 41, 90, 83, 42, 81, 36, 15, 76, 27, 122, 38, 19, 127, 99, 71, 95, 25, 106, 91, 89, 111, 110, 88, 79, 121, 103, 92, 17, 63, 29, 98, 31, 24, 96, 105, 26, 93, 30, 28, 59, 113, 94, 32, 84, 102, 101, 37, 77, 35, 87, 48, 9, 34, 100, 74, 82, 0, 80, 8, 33], [104, 127, 53, 110, 95, 33, 90, 85, 122, 35, 83, 64, 88, 111, 81, 19, 115, 91, 14, 87, 102, 106, 75, 70, 76, 9, 29, 71, 66, 80, 93, 24, 18, 61, 99, 48, 45, 63, 20, 92, 126, 54, 107, 103, 125, 13, 4, 97, 105, 1, 60, 108, 72, 121, 77, 2, 59, 57, 123, 6, 3, 62, 117, 49, 114, 56, 67, 5, 44, 42, 36, 50, 43, 51, 109, 28, 58, 47, 113, 116, 37, 74, 98, 55, 112, 101, 27, 124, 89, 118, 34, 46, 120, 119, 79, 32, 52, 25, 94, 69, 78, 86, 96, 30, 10, 15, 39, 82, 100, 16, 23, 84, 0, 8, 22, 41, 26, 68, 65, 21, 73, 31, 38, 11, 17, 12, 40, 7], [103, 98, 124, 119, 62, 48, 11, 24, 17, 117, 20, 15, 13, 72, 112, 26, 0, 70, 68, 1, 30, 69, 53, 2, 3, 45, 9, 47, 111, 109, 78, 7, 54, 43, 10, 67, 19, 22, 76, 52, 86, 115, 64, 87, 49, 125, 57, 46, 126, 51, 56, 122, 66, 61, 118, 63, 127, 60, 71, 44, 120, 113, 80, 107, 55, 92, 121, 101, 106, 40, 105, 110, 38, 74, 108, 95, 73, 100, 42, 35, 58, 114, 32, 29, 104, 41, 50, 28, 116, 21, 37, 33, 123, 102, 4, 31, 93, 94, 14, 25, 82, 36, 97, 59, 96, 27, 23, 83, 85, 99, 39, 5, 8, 84, 89, 91, 34, 90, 18, 6, 16, 75, 88, 12, 77, 65, 81, 79], [104, 33, 119, 23, 19, 79, 95, 47, 85, 76, 112, 81, 55, 52, 53, 73, 113, 7, 65, 75, 59, 58, 126, 5, 46, 60, 57, 29, 61, 108, 13, 64, 49, 3, 123, 107, 54, 71, 14, 44, 117, 127, 10, 69, 80, 109, 43, 68, 114, 67, 37, 86, 25, 48, 77, 24, 28, 91, 2, 56, 11, 27, 18, 111, 4, 118, 124, 72, 30, 9, 115, 66, 40, 121, 62, 42, 16, 106, 103, 22, 38, 125, 70, 63, 74, 120, 93, 50, 39, 20, 6, 105, 35, 98, 90, 17, 101, 34, 32, 94, 96, 88, 92, 26, 110, 100, 99, 102, 89, 116, 87, 45, 51, 82, 41, 122, 36, 78, 21, 84, 0, 8, 1, 12, 15, 97, 83, 31]], "model.layers.3.self_attn.qk_proj": [[119, 104, 47, 62, 120, 52, 44, 123, 122, 53, 117, 127, 105, 40, 108, 125, 97, 110, 124, 48, 46, 56, 84, 20, 58, 60, 12, 87, 41, 109, 23, 74, 17, 76, 81, 78, 111, 112, 82, 10, 7, 14, 126, 64, 39, 15, 0, 79, 71, 103, 77, 57, 18, 13, 16, 33, 54, 114, 63, 24, 72, 88, 38, 83, 22, 86, 70, 55, 3, 19, 80, 21, 67, 9, 85, 29, 8, 45, 50, 59, 65, 5, 73, 69, 34, 89, 11, 92, 66, 90, 4, 116, 1, 25, 61, 2, 95, 113, 93, 68, 98, 75, 31, 43, 118, 26, 49, 6, 27, 115, 51, 121, 94, 28, 102, 35, 42, 30, 107, 32, 37, 106, 99, 91, 36, 96, 101, 100], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 48, 46, 60, 58, 20, 84, 41, 76, 87, 109, 23, 17, 12, 74, 81, 10, 0, 14, 111, 112, 78, 64, 82, 15, 7, 103, 71, 70, 18, 86, 33, 39, 114, 63, 22, 19, 24, 80, 72, 77, 79, 57, 38, 83, 88, 13, 54, 3, 1, 65, 59, 16, 50, 8, 5, 21, 126, 69, 85, 29, 55, 92, 73, 9, 67, 2, 90, 66, 4, 68, 31, 45, 34, 11, 113, 98, 25, 116, 61, 95, 43, 89, 93, 75, 26, 51, 27, 118, 49, 6, 115, 102, 28, 94, 107, 121, 35, 30, 37, 42, 32, 96, 91, 99, 106, 101, 36, 100], [119, 104, 47, 62, 52, 120, 44, 123, 122, 53, 117, 127, 40, 105, 125, 108, 97, 56, 110, 124, 48, 58, 46, 20, 60, 84, 87, 23, 41, 12, 76, 109, 17, 74, 81, 64, 111, 82, 78, 10, 0, 14, 7, 112, 39, 18, 33, 71, 15, 38, 79, 86, 70, 67, 13, 3, 22, 24, 103, 57, 77, 88, 83, 5, 80, 16, 21, 19, 54, 126, 72, 69, 8, 1, 59, 65, 9, 55, 73, 114, 66, 4, 63, 50, 34, 29, 85, 2, 68, 92, 11, 45, 113, 93, 25, 89, 98, 90, 31, 116, 95, 61, 43, 75, 6, 26, 28, 49, 27, 51, 94, 115, 102, 118, 107, 121, 35, 42, 32, 30, 37, 91, 99, 106, 36, 96, 100, 101], [119, 104, 47, 62, 52, 120, 123, 122, 53, 44, 127, 117, 105, 40, 125, 108, 97, 48, 110, 124, 56, 46, 58, 60, 84, 20, 87, 41, 12, 23, 74, 112, 17, 109, 76, 81, 111, 82, 0, 78, 10, 14, 18, 64, 79, 86, 7, 38, 71, 57, 39, 67, 24, 15, 103, 33, 19, 54, 13, 5, 8, 22, 65, 80, 114, 70, 55, 63, 88, 3, 83, 69, 126, 72, 1, 16, 9, 29, 50, 77, 85, 59, 45, 4, 92, 66, 31, 73, 2, 93, 68, 116, 90, 21, 113, 98, 89, 25, 6, 95, 34, 61, 75, 11, 43, 26, 27, 115, 118, 51, 102, 49, 28, 94, 107, 30, 121, 42, 32, 106, 91, 37, 35, 99, 96, 101, 100, 36], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 117, 40, 105, 125, 108, 97, 48, 124, 110, 56, 60, 46, 58, 84, 12, 20, 87, 17, 76, 41, 112, 10, 23, 81, 78, 74, 109, 111, 0, 7, 18, 64, 71, 79, 14, 82, 24, 86, 33, 8, 77, 5, 19, 63, 22, 114, 39, 103, 57, 126, 67, 69, 72, 15, 38, 13, 9, 54, 59, 16, 88, 55, 80, 68, 3, 1, 65, 83, 6, 50, 29, 113, 21, 85, 73, 4, 45, 66, 92, 70, 90, 2, 93, 98, 89, 11, 95, 116, 75, 31, 61, 43, 34, 25, 26, 94, 115, 27, 51, 118, 49, 121, 28, 102, 107, 42, 32, 106, 30, 35, 91, 37, 96, 36, 100, 99, 101], [119, 104, 47, 52, 62, 120, 123, 44, 53, 122, 127, 117, 105, 40, 125, 108, 97, 124, 110, 56, 46, 48, 84, 87, 58, 20, 60, 12, 76, 41, 17, 74, 23, 78, 109, 81, 112, 10, 111, 7, 0, 64, 14, 71, 18, 79, 82, 24, 86, 19, 39, 69, 15, 13, 54, 8, 114, 77, 67, 3, 9, 57, 63, 83, 103, 88, 5, 6, 33, 126, 55, 16, 38, 72, 22, 80, 85, 59, 65, 29, 21, 50, 73, 2, 66, 113, 68, 1, 116, 45, 11, 4, 92, 95, 90, 61, 31, 75, 89, 98, 34, 93, 25, 70, 43, 115, 26, 118, 27, 51, 49, 42, 94, 102, 121, 28, 107, 30, 37, 91, 106, 32, 99, 35, 101, 96, 36, 100], [119, 104, 47, 52, 120, 62, 123, 122, 44, 53, 105, 117, 40, 127, 108, 125, 97, 110, 56, 124, 58, 87, 84, 46, 41, 20, 60, 48, 76, 12, 74, 23, 17, 81, 112, 109, 82, 71, 14, 10, 78, 103, 18, 79, 39, 7, 64, 0, 111, 15, 6, 22, 13, 16, 24, 54, 19, 77, 88, 86, 126, 8, 69, 57, 85, 59, 83, 33, 5, 63, 9, 3, 72, 38, 21, 80, 67, 29, 61, 113, 114, 1, 50, 73, 55, 66, 34, 11, 68, 65, 116, 75, 2, 25, 93, 95, 90, 4, 92, 89, 45, 98, 31, 43, 115, 27, 118, 26, 94, 51, 49, 70, 121, 42, 102, 30, 107, 35, 28, 99, 32, 37, 91, 106, 101, 36, 96, 100], [119, 104, 52, 47, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 125, 108, 97, 110, 56, 124, 58, 48, 87, 84, 20, 46, 41, 76, 12, 60, 17, 23, 81, 78, 74, 10, 109, 14, 79, 18, 0, 6, 111, 112, 7, 82, 64, 13, 15, 19, 86, 71, 39, 16, 103, 22, 77, 80, 88, 33, 9, 24, 8, 38, 54, 83, 63, 5, 57, 85, 21, 126, 73, 72, 59, 65, 3, 69, 55, 114, 1, 2, 29, 66, 50, 61, 34, 45, 93, 75, 68, 67, 90, 89, 95, 4, 11, 92, 98, 113, 25, 116, 31, 26, 118, 49, 70, 51, 115, 28, 27, 43, 42, 121, 94, 35, 102, 107, 32, 37, 30, 101, 91, 106, 36, 99, 96, 100], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 127, 117, 105, 40, 125, 108, 97, 124, 110, 56, 58, 48, 46, 20, 60, 84, 87, 41, 76, 12, 81, 23, 17, 109, 78, 111, 10, 74, 14, 112, 79, 64, 7, 39, 82, 18, 0, 71, 63, 86, 13, 16, 15, 103, 19, 33, 6, 8, 24, 80, 67, 114, 22, 83, 38, 77, 88, 21, 3, 5, 50, 126, 1, 57, 59, 54, 72, 55, 29, 9, 85, 69, 65, 45, 2, 4, 89, 66, 116, 31, 73, 98, 34, 90, 92, 68, 25, 11, 75, 113, 93, 95, 61, 26, 70, 43, 49, 28, 27, 115, 121, 51, 118, 94, 102, 107, 35, 42, 30, 37, 101, 99, 32, 91, 106, 96, 100, 36], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 105, 117, 40, 127, 108, 125, 97, 124, 56, 110, 58, 46, 48, 84, 87, 20, 12, 41, 23, 60, 81, 17, 109, 74, 76, 18, 10, 14, 78, 82, 15, 33, 111, 112, 7, 79, 86, 64, 22, 0, 103, 39, 80, 19, 83, 24, 71, 38, 13, 16, 77, 88, 57, 21, 63, 59, 55, 85, 8, 54, 29, 5, 114, 45, 69, 6, 9, 72, 3, 73, 92, 126, 67, 50, 2, 113, 66, 98, 34, 93, 89, 116, 1, 65, 61, 90, 68, 43, 95, 4, 75, 25, 26, 31, 70, 11, 49, 27, 115, 118, 28, 30, 102, 51, 94, 35, 42, 121, 107, 32, 37, 99, 106, 96, 91, 101, 36, 100], [119, 104, 47, 62, 120, 52, 123, 122, 44, 53, 127, 40, 117, 105, 125, 108, 97, 48, 124, 56, 110, 46, 60, 20, 84, 41, 58, 12, 87, 76, 17, 111, 109, 10, 23, 112, 81, 0, 14, 18, 74, 78, 64, 7, 71, 39, 79, 15, 103, 82, 63, 33, 1, 8, 24, 19, 38, 86, 80, 69, 77, 13, 57, 55, 114, 50, 85, 22, 83, 3, 59, 88, 72, 16, 54, 5, 73, 29, 65, 126, 4, 21, 45, 67, 70, 116, 66, 31, 2, 9, 95, 90, 93, 68, 75, 61, 26, 92, 98, 113, 6, 34, 25, 89, 11, 118, 49, 94, 43, 115, 51, 27, 102, 28, 121, 107, 42, 91, 106, 30, 35, 37, 32, 99, 100, 101, 96, 36], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 40, 117, 125, 105, 108, 97, 124, 56, 110, 48, 46, 20, 60, 58, 84, 87, 41, 76, 112, 12, 81, 17, 14, 23, 10, 109, 111, 74, 7, 0, 3, 39, 78, 82, 64, 71, 24, 67, 103, 18, 79, 86, 114, 15, 88, 70, 33, 5, 57, 69, 8, 38, 72, 54, 22, 19, 63, 77, 80, 1, 13, 126, 55, 83, 16, 65, 85, 59, 21, 50, 29, 73, 9, 61, 68, 66, 2, 90, 92, 4, 116, 75, 95, 34, 45, 113, 11, 25, 98, 89, 93, 31, 26, 6, 27, 118, 51, 94, 115, 43, 49, 102, 28, 121, 42, 30, 107, 32, 35, 37, 106, 99, 91, 100, 36, 101, 96], [119, 104, 52, 47, 62, 120, 123, 44, 122, 117, 53, 105, 40, 127, 108, 125, 97, 56, 110, 124, 58, 20, 84, 87, 48, 41, 46, 76, 23, 60, 12, 17, 81, 10, 14, 74, 109, 78, 71, 82, 18, 15, 79, 112, 7, 80, 86, 33, 83, 70, 39, 103, 19, 24, 22, 64, 13, 77, 111, 38, 0, 88, 63, 5, 21, 16, 57, 69, 72, 85, 8, 9, 54, 55, 29, 67, 126, 92, 116, 93, 50, 59, 113, 3, 1, 2, 114, 25, 61, 73, 66, 75, 34, 89, 4, 65, 11, 45, 90, 95, 68, 98, 31, 27, 118, 26, 43, 49, 94, 28, 115, 51, 30, 107, 102, 6, 35, 42, 32, 121, 96, 37, 91, 99, 106, 101, 100, 36], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 127, 117, 105, 40, 125, 108, 97, 110, 124, 56, 20, 48, 58, 46, 84, 60, 23, 41, 76, 17, 12, 109, 87, 74, 111, 81, 10, 14, 71, 112, 64, 82, 15, 18, 78, 103, 7, 70, 0, 39, 79, 86, 22, 13, 33, 19, 69, 24, 57, 83, 16, 38, 80, 114, 72, 54, 59, 8, 63, 77, 126, 88, 1, 73, 5, 85, 50, 55, 66, 29, 9, 21, 67, 2, 3, 45, 25, 65, 68, 90, 92, 113, 4, 34, 116, 11, 61, 93, 95, 98, 75, 31, 89, 43, 49, 26, 115, 51, 118, 6, 27, 28, 94, 121, 102, 30, 107, 42, 32, 99, 91, 37, 35, 106, 36, 101, 100, 96], [119, 104, 47, 62, 120, 52, 123, 44, 122, 53, 117, 127, 105, 40, 125, 108, 97, 110, 124, 58, 56, 48, 20, 84, 60, 87, 41, 46, 23, 12, 76, 81, 111, 14, 74, 17, 71, 10, 112, 0, 109, 64, 7, 79, 39, 18, 67, 72, 103, 78, 15, 69, 3, 86, 82, 126, 1, 33, 24, 13, 55, 5, 38, 70, 22, 80, 54, 19, 16, 57, 83, 114, 88, 9, 73, 85, 77, 68, 113, 8, 63, 65, 59, 2, 21, 29, 50, 66, 90, 31, 98, 116, 6, 4, 75, 92, 61, 93, 25, 45, 89, 34, 11, 95, 26, 49, 118, 51, 27, 115, 28, 102, 43, 42, 94, 121, 106, 35, 30, 99, 107, 32, 91, 101, 37, 96, 36, 100], [119, 104, 47, 62, 52, 120, 123, 122, 44, 53, 127, 117, 105, 40, 125, 108, 97, 48, 110, 56, 124, 46, 20, 58, 84, 41, 60, 87, 76, 12, 17, 81, 23, 111, 112, 109, 71, 10, 74, 14, 78, 39, 7, 15, 0, 82, 72, 18, 103, 86, 64, 63, 24, 33, 57, 13, 55, 38, 79, 69, 114, 67, 83, 126, 22, 88, 3, 21, 80, 16, 59, 50, 29, 54, 19, 77, 9, 5, 85, 65, 73, 8, 1, 116, 92, 90, 113, 70, 68, 4, 45, 6, 98, 95, 93, 2, 31, 61, 66, 26, 11, 34, 43, 75, 25, 89, 118, 51, 49, 115, 94, 102, 27, 121, 28, 42, 30, 106, 107, 35, 91, 32, 99, 37, 100, 101, 96, 36], [119, 104, 52, 47, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 108, 125, 97, 110, 124, 56, 20, 48, 46, 58, 84, 87, 23, 41, 76, 12, 60, 10, 17, 81, 74, 78, 18, 109, 14, 7, 111, 71, 15, 112, 82, 79, 22, 80, 13, 86, 64, 69, 19, 83, 0, 57, 33, 39, 24, 77, 9, 38, 103, 88, 6, 5, 21, 72, 29, 63, 85, 54, 55, 16, 73, 8, 126, 50, 3, 114, 75, 92, 67, 59, 68, 65, 66, 2, 89, 1, 113, 116, 95, 4, 34, 93, 45, 11, 25, 90, 98, 61, 115, 31, 70, 26, 49, 27, 43, 51, 28, 118, 102, 94, 30, 42, 121, 107, 37, 91, 35, 32, 99, 106, 100, 96, 101, 36], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 117, 105, 127, 40, 108, 125, 97, 56, 124, 110, 20, 58, 84, 48, 41, 23, 76, 60, 87, 46, 17, 12, 10, 109, 74, 81, 82, 14, 78, 112, 6, 64, 15, 0, 71, 7, 111, 18, 22, 79, 69, 39, 16, 57, 86, 33, 19, 24, 103, 3, 13, 83, 77, 80, 38, 72, 67, 1, 114, 59, 63, 54, 21, 55, 88, 8, 73, 85, 29, 9, 2, 126, 93, 66, 113, 45, 4, 116, 75, 65, 50, 5, 92, 95, 98, 90, 61, 25, 31, 68, 34, 89, 11, 26, 70, 28, 43, 51, 49, 27, 118, 115, 102, 30, 42, 94, 37, 121, 107, 32, 91, 35, 106, 99, 101, 96, 36, 100], [119, 104, 47, 52, 62, 120, 123, 122, 53, 44, 127, 117, 105, 40, 125, 97, 108, 110, 124, 56, 48, 58, 60, 46, 20, 84, 76, 87, 23, 41, 12, 81, 17, 10, 74, 14, 112, 78, 111, 109, 64, 82, 71, 7, 15, 79, 39, 18, 0, 6, 86, 33, 103, 57, 126, 38, 16, 114, 69, 24, 1, 77, 19, 8, 67, 22, 13, 83, 80, 88, 54, 85, 63, 72, 55, 73, 59, 3, 29, 5, 9, 116, 50, 2, 75, 93, 4, 21, 66, 61, 92, 65, 98, 34, 90, 45, 68, 31, 95, 113, 89, 25, 11, 26, 70, 49, 43, 51, 118, 115, 27, 94, 28, 102, 42, 107, 121, 32, 30, 35, 37, 91, 99, 106, 101, 36, 96, 100], [119, 104, 47, 62, 52, 120, 122, 123, 44, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 58, 60, 46, 48, 84, 87, 20, 41, 12, 76, 17, 74, 81, 10, 109, 111, 23, 14, 112, 71, 7, 78, 79, 18, 82, 86, 69, 64, 15, 19, 33, 0, 24, 57, 39, 6, 13, 83, 22, 103, 63, 77, 38, 73, 80, 114, 72, 1, 88, 8, 55, 16, 67, 9, 5, 126, 54, 29, 21, 2, 3, 50, 85, 116, 59, 113, 45, 92, 68, 75, 4, 90, 93, 98, 31, 34, 65, 95, 66, 61, 89, 43, 26, 70, 11, 25, 49, 51, 102, 28, 115, 118, 27, 94, 42, 30, 107, 121, 35, 37, 32, 91, 101, 99, 96, 106, 100, 36], [119, 104, 62, 52, 47, 120, 123, 53, 122, 44, 127, 117, 40, 105, 125, 108, 97, 110, 48, 124, 56, 46, 20, 84, 60, 58, 87, 41, 109, 17, 12, 23, 112, 76, 111, 81, 74, 10, 82, 78, 14, 71, 39, 7, 0, 79, 57, 18, 38, 64, 24, 3, 22, 33, 15, 16, 126, 67, 54, 19, 55, 80, 114, 77, 86, 63, 59, 72, 9, 88, 69, 103, 83, 13, 8, 29, 5, 50, 1, 85, 21, 116, 4, 95, 68, 89, 92, 113, 90, 6, 45, 73, 70, 65, 98, 31, 66, 93, 75, 2, 11, 61, 34, 25, 26, 51, 115, 49, 94, 118, 42, 43, 121, 28, 27, 107, 30, 102, 37, 35, 91, 99, 106, 100, 32, 101, 36, 96], [119, 104, 47, 52, 120, 62, 123, 122, 53, 44, 117, 127, 105, 40, 125, 108, 97, 110, 58, 124, 56, 84, 46, 87, 48, 20, 60, 41, 12, 17, 23, 76, 112, 111, 81, 109, 10, 0, 64, 74, 7, 14, 18, 39, 71, 57, 79, 33, 82, 15, 78, 86, 24, 54, 70, 22, 126, 38, 77, 103, 69, 63, 1, 80, 8, 16, 114, 5, 55, 67, 2, 19, 9, 72, 83, 29, 50, 68, 3, 88, 13, 66, 73, 85, 21, 116, 4, 34, 65, 59, 31, 113, 93, 45, 61, 95, 89, 90, 92, 98, 11, 26, 75, 6, 43, 25, 118, 115, 49, 27, 51, 102, 28, 42, 94, 107, 121, 30, 35, 32, 91, 99, 106, 101, 37, 96, 36, 100], [119, 104, 52, 47, 62, 120, 123, 122, 44, 53, 117, 127, 40, 105, 125, 97, 108, 110, 48, 124, 56, 46, 84, 58, 60, 20, 87, 23, 41, 12, 76, 17, 74, 81, 109, 14, 10, 111, 112, 18, 7, 78, 71, 82, 79, 22, 15, 33, 86, 70, 0, 64, 24, 39, 19, 8, 38, 83, 57, 16, 103, 29, 88, 13, 5, 63, 80, 85, 77, 9, 59, 67, 54, 69, 114, 72, 55, 21, 65, 126, 50, 68, 116, 73, 4, 1, 113, 61, 3, 45, 92, 89, 98, 2, 93, 34, 31, 90, 75, 11, 66, 95, 25, 26, 118, 6, 43, 27, 94, 28, 49, 51, 102, 115, 42, 107, 30, 121, 32, 91, 35, 106, 96, 37, 99, 101, 36, 100], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 117, 105, 40, 125, 97, 108, 110, 48, 124, 56, 46, 20, 84, 58, 60, 87, 41, 12, 76, 74, 23, 17, 112, 81, 111, 78, 109, 10, 82, 18, 14, 15, 7, 86, 39, 71, 103, 57, 0, 24, 8, 70, 16, 19, 126, 64, 38, 67, 33, 13, 22, 83, 79, 63, 77, 114, 88, 55, 80, 50, 3, 59, 9, 5, 92, 54, 116, 29, 85, 1, 73, 113, 69, 21, 34, 68, 72, 89, 45, 98, 61, 95, 4, 65, 2, 11, 90, 31, 75, 66, 93, 43, 25, 26, 51, 115, 118, 49, 27, 94, 6, 28, 102, 42, 107, 30, 121, 37, 106, 91, 32, 35, 99, 101, 36, 100, 96], [119, 104, 47, 52, 62, 120, 123, 122, 44, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 58, 48, 60, 20, 84, 46, 87, 41, 12, 76, 23, 17, 109, 81, 10, 78, 74, 112, 7, 111, 82, 39, 15, 14, 57, 18, 79, 126, 71, 64, 0, 103, 33, 86, 24, 88, 16, 54, 83, 22, 5, 8, 13, 77, 85, 80, 38, 69, 19, 70, 114, 59, 21, 72, 29, 63, 3, 1, 67, 61, 9, 55, 92, 50, 73, 95, 31, 93, 90, 4, 2, 66, 65, 25, 113, 34, 45, 75, 11, 89, 116, 98, 68, 118, 26, 6, 51, 49, 43, 27, 94, 115, 42, 102, 28, 30, 35, 121, 32, 107, 37, 106, 99, 91, 96, 36, 101, 100], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 127, 117, 40, 105, 125, 108, 97, 56, 110, 48, 124, 20, 46, 58, 87, 60, 84, 41, 12, 76, 23, 81, 10, 112, 17, 74, 109, 111, 7, 39, 14, 18, 0, 78, 82, 64, 15, 71, 103, 79, 57, 33, 19, 22, 83, 16, 13, 86, 126, 77, 38, 69, 80, 8, 88, 3, 24, 5, 50, 67, 54, 116, 85, 70, 72, 9, 63, 1, 114, 21, 55, 59, 92, 73, 2, 29, 34, 65, 25, 68, 61, 113, 89, 45, 4, 66, 90, 95, 31, 6, 11, 98, 118, 93, 51, 75, 28, 26, 115, 27, 49, 43, 30, 107, 94, 42, 102, 121, 106, 37, 35, 91, 32, 36, 101, 100, 99, 96], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 117, 127, 105, 40, 108, 125, 97, 56, 110, 124, 20, 58, 48, 84, 41, 46, 60, 23, 12, 87, 109, 76, 74, 17, 81, 111, 10, 112, 18, 78, 14, 7, 33, 71, 0, 15, 39, 64, 83, 82, 57, 79, 77, 86, 38, 22, 103, 88, 16, 19, 24, 3, 80, 126, 67, 59, 8, 72, 54, 13, 55, 21, 63, 65, 85, 50, 69, 45, 5, 29, 92, 68, 6, 73, 1, 25, 89, 9, 114, 34, 2, 95, 113, 4, 93, 98, 11, 90, 66, 116, 75, 49, 70, 31, 61, 26, 27, 121, 43, 115, 118, 102, 51, 28, 94, 35, 42, 107, 30, 32, 91, 37, 106, 100, 99, 96, 36, 101], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 125, 108, 97, 110, 124, 56, 48, 58, 84, 46, 76, 20, 87, 60, 41, 12, 74, 81, 23, 10, 17, 112, 0, 7, 71, 109, 14, 64, 78, 111, 18, 15, 57, 82, 79, 77, 126, 6, 39, 72, 69, 103, 86, 33, 8, 114, 59, 88, 1, 38, 80, 5, 16, 83, 13, 73, 22, 24, 63, 54, 67, 19, 3, 55, 9, 2, 85, 50, 21, 29, 65, 68, 66, 113, 93, 92, 34, 45, 75, 4, 98, 116, 31, 90, 61, 11, 95, 49, 25, 26, 89, 115, 27, 43, 118, 51, 70, 102, 94, 42, 28, 37, 121, 30, 107, 35, 99, 91, 32, 100, 106, 96, 36, 101], [119, 104, 47, 120, 62, 52, 123, 53, 122, 44, 127, 117, 40, 105, 125, 108, 97, 110, 124, 56, 20, 48, 60, 58, 46, 87, 84, 41, 76, 12, 74, 17, 23, 112, 111, 109, 10, 81, 78, 14, 7, 39, 71, 18, 82, 0, 79, 33, 86, 64, 114, 15, 103, 24, 77, 6, 83, 38, 8, 1, 72, 19, 22, 13, 57, 3, 126, 16, 54, 88, 55, 59, 5, 9, 63, 80, 50, 73, 67, 21, 69, 29, 65, 92, 116, 85, 113, 31, 95, 93, 68, 98, 4, 90, 66, 2, 45, 11, 34, 61, 75, 43, 25, 89, 118, 26, 115, 49, 70, 102, 27, 51, 121, 94, 28, 107, 30, 42, 37, 32, 35, 91, 106, 99, 100, 36, 101, 96], [119, 104, 47, 62, 52, 120, 123, 44, 122, 53, 127, 40, 105, 117, 125, 97, 108, 124, 48, 56, 46, 110, 20, 84, 60, 12, 58, 41, 87, 76, 23, 17, 74, 111, 112, 81, 109, 7, 10, 14, 78, 0, 39, 82, 71, 18, 15, 64, 79, 67, 6, 77, 103, 83, 72, 5, 22, 57, 33, 54, 59, 38, 126, 24, 86, 88, 13, 69, 16, 3, 63, 19, 8, 114, 29, 80, 9, 73, 55, 85, 65, 50, 68, 21, 1, 116, 2, 92, 11, 113, 4, 45, 89, 95, 31, 66, 61, 90, 34, 98, 43, 93, 25, 70, 118, 27, 75, 26, 115, 121, 51, 49, 102, 30, 94, 42, 91, 107, 28, 106, 37, 32, 35, 99, 101, 96, 100, 36], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 117, 105, 40, 127, 108, 125, 97, 124, 56, 110, 58, 20, 87, 84, 46, 41, 76, 60, 23, 48, 12, 74, 17, 10, 81, 111, 0, 78, 14, 109, 18, 64, 82, 79, 71, 86, 7, 112, 103, 15, 33, 39, 83, 5, 22, 57, 72, 77, 38, 19, 13, 59, 24, 54, 16, 80, 6, 88, 9, 55, 126, 21, 69, 8, 3, 63, 73, 65, 29, 2, 85, 34, 4, 50, 113, 67, 114, 92, 66, 1, 89, 45, 75, 70, 95, 25, 68, 11, 116, 61, 98, 93, 31, 90, 43, 26, 115, 118, 27, 49, 102, 28, 51, 30, 94, 121, 35, 32, 42, 99, 37, 107, 91, 106, 96, 101, 36, 100], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 127, 40, 105, 117, 125, 108, 97, 56, 46, 110, 124, 48, 60, 20, 87, 41, 84, 58, 76, 12, 17, 109, 23, 81, 10, 74, 111, 112, 14, 78, 82, 103, 7, 71, 64, 15, 18, 39, 79, 72, 0, 13, 22, 80, 33, 63, 86, 83, 5, 38, 24, 126, 3, 57, 88, 114, 16, 55, 77, 67, 19, 59, 85, 116, 69, 9, 8, 54, 29, 65, 21, 2, 50, 73, 1, 34, 70, 113, 66, 4, 89, 92, 98, 6, 61, 25, 45, 95, 11, 31, 90, 75, 68, 93, 118, 115, 43, 26, 51, 27, 49, 28, 102, 121, 42, 30, 94, 107, 32, 37, 35, 106, 101, 36, 99, 91, 100, 96]], "model.layers.4.self_attn.q_proj": [[42, 122, 117, 56, 53, 123, 101, 124, 102, 121, 33, 61, 27, 107, 104, 119, 59, 48, 110, 39, 116, 83, 55, 109, 127, 115, 37, 114, 113, 126, 92, 57, 60, 43, 58, 17, 50, 26, 38, 120, 47, 125, 44, 45, 51, 54, 89, 62, 105, 111, 94, 21, 52, 46, 87, 41, 36, 108, 32, 106, 91, 49, 63, 112, 86, 118, 40, 23, 31, 30, 76, 79, 99, 22, 34, 103, 84, 96, 100, 72, 97, 98, 28, 78, 82, 25, 24, 95, 93, 16, 75, 0, 29, 35, 15, 13, 81, 10, 66, 6, 5, 73, 2, 11, 14, 3, 85, 19, 64, 8, 12, 74, 4, 68, 90, 69, 20, 65, 70, 88, 67, 7, 80, 71, 77, 9, 1, 18], [42, 101, 117, 114, 123, 56, 122, 102, 33, 53, 48, 124, 107, 59, 111, 39, 89, 94, 91, 27, 86, 121, 54, 57, 21, 37, 104, 28, 125, 112, 49, 79, 116, 40, 109, 36, 113, 43, 44, 41, 60, 118, 99, 26, 47, 38, 84, 82, 58, 46, 87, 25, 32, 17, 115, 52, 110, 61, 95, 92, 108, 62, 119, 24, 127, 45, 50, 120, 55, 51, 85, 29, 63, 90, 126, 30, 15, 93, 105, 106, 81, 34, 83, 22, 12, 96, 76, 16, 31, 98, 78, 75, 19, 35, 23, 74, 97, 13, 88, 8, 73, 20, 14, 100, 103, 77, 68, 72, 71, 6, 7, 69, 5, 11, 9, 66, 10, 4, 1, 65, 70, 3, 64, 67, 2, 0, 18, 80], [101, 42, 117, 33, 123, 122, 56, 53, 114, 87, 84, 82, 124, 91, 94, 39, 61, 79, 78, 75, 16, 73, 13, 6, 48, 86, 27, 72, 46, 111, 107, 2, 57, 89, 32, 95, 30, 21, 68, 22, 104, 20, 19, 74, 54, 71, 58, 102, 23, 109, 67, 44, 14, 85, 25, 8, 11, 88, 63, 55, 90, 10, 24, 106, 65, 41, 110, 36, 98, 64, 120, 35, 121, 92, 105, 7, 81, 99, 17, 49, 113, 28, 15, 60, 3, 77, 118, 108, 40, 116, 93, 76, 80, 18, 29, 52, 12, 38, 83, 26, 96, 4, 69, 119, 34, 70, 43, 31, 1, 9, 62, 47, 103, 115, 125, 51, 112, 127, 126, 59, 50, 97, 37, 5, 100, 0, 45, 66], [101, 117, 56, 122, 123, 33, 53, 42, 87, 82, 91, 84, 124, 54, 73, 15, 16, 103, 76, 78, 75, 24, 71, 27, 121, 13, 88, 68, 6, 37, 72, 114, 107, 86, 85, 104, 58, 59, 67, 66, 10, 3, 48, 65, 1, 126, 116, 25, 105, 95, 12, 109, 61, 118, 44, 89, 64, 45, 14, 92, 7, 120, 79, 115, 9, 127, 18, 29, 80, 43, 26, 98, 90, 77, 4, 94, 102, 70, 49, 74, 93, 0, 110, 11, 28, 23, 55, 19, 8, 81, 34, 99, 35, 20, 36, 100, 41, 125, 111, 52, 22, 32, 31, 17, 113, 46, 47, 83, 39, 51, 96, 21, 69, 5, 2, 112, 30, 62, 50, 63, 40, 60, 57, 38, 106, 119, 108, 97], [39, 52, 118, 33, 31, 79, 13, 87, 8, 11, 53, 70, 20, 117, 54, 82, 81, 9, 68, 63, 50, 66, 113, 26, 120, 121, 111, 125, 90, 10, 61, 114, 49, 119, 46, 23, 74, 3, 51, 112, 25, 47, 116, 84, 55, 21, 6, 29, 95, 38, 67, 69, 12, 73, 18, 5, 19, 60, 77, 17, 107, 0, 28, 93, 85, 97, 57, 44, 106, 62, 15, 59, 22, 100, 72, 2, 24, 83, 92, 91, 7, 94, 86, 80, 98, 36, 14, 115, 71, 102, 78, 75, 105, 34, 1, 16, 123, 65, 58, 110, 30, 42, 76, 89, 43, 32, 4, 27, 45, 124, 104, 40, 88, 48, 96, 56, 99, 41, 109, 127, 37, 101, 64, 108, 35, 122, 126, 103], [39, 118, 52, 33, 31, 102, 20, 87, 63, 25, 116, 11, 54, 47, 82, 81, 90, 117, 13, 85, 79, 49, 80, 123, 111, 121, 56, 86, 76, 104, 26, 50, 28, 53, 23, 91, 9, 7, 105, 120, 41, 112, 51, 71, 61, 44, 70, 83, 55, 114, 27, 57, 8, 60, 125, 107, 46, 110, 115, 106, 101, 124, 40, 97, 42, 10, 14, 59, 127, 29, 43, 78, 126, 113, 35, 45, 16, 69, 122, 74, 62, 119, 65, 22, 98, 108, 58, 36, 48, 3, 93, 17, 84, 37, 30, 12, 68, 99, 109, 88, 92, 64, 18, 67, 38, 100, 34, 66, 96, 19, 89, 21, 24, 32, 75, 72, 77, 94, 5, 95, 15, 73, 1, 6, 4, 103, 0, 2], [53, 39, 33, 52, 118, 31, 90, 50, 102, 81, 87, 113, 59, 63, 54, 125, 20, 114, 98, 61, 55, 97, 60, 117, 108, 47, 107, 26, 124, 121, 92, 49, 24, 44, 86, 112, 42, 79, 126, 110, 28, 116, 103, 57, 111, 82, 14, 43, 83, 120, 45, 119, 10, 127, 46, 29, 122, 85, 106, 19, 12, 51, 56, 41, 37, 32, 115, 96, 48, 62, 109, 22, 100, 105, 34, 88, 13, 40, 89, 58, 123, 101, 11, 94, 91, 25, 104, 93, 36, 5, 78, 27, 38, 99, 35, 17, 71, 21, 80, 23, 76, 74, 84, 30, 16, 95, 77, 75, 8, 18, 15, 72, 67, 6, 9, 7, 73, 69, 3, 4, 65, 1, 66, 70, 0, 68, 64, 2], [39, 118, 52, 33, 53, 31, 8, 20, 82, 87, 13, 79, 11, 54, 70, 63, 117, 102, 81, 74, 9, 50, 125, 68, 121, 46, 90, 86, 23, 61, 16, 49, 66, 10, 55, 59, 60, 15, 47, 42, 51, 69, 115, 67, 95, 12, 26, 111, 119, 97, 22, 72, 112, 114, 77, 120, 17, 80, 124, 21, 18, 94, 89, 116, 3, 25, 2, 45, 56, 92, 75, 84, 78, 85, 28, 0, 19, 14, 38, 29, 57, 27, 83, 30, 71, 24, 113, 96, 107, 76, 40, 7, 123, 99, 62, 122, 88, 44, 48, 36, 6, 100, 41, 5, 73, 98, 106, 43, 34, 127, 91, 64, 1, 93, 32, 105, 37, 126, 4, 108, 110, 101, 104, 35, 58, 109, 65, 103], [41, 39, 99, 116, 124, 118, 52, 95, 54, 28, 86, 92, 62, 75, 105, 81, 26, 31, 24, 21, 122, 121, 58, 17, 83, 55, 87, 127, 22, 61, 79, 56, 123, 117, 93, 50, 12, 53, 14, 115, 35, 88, 110, 106, 13, 82, 112, 113, 94, 47, 18, 34, 125, 57, 42, 120, 101, 108, 30, 48, 20, 126, 98, 32, 111, 46, 89, 43, 51, 78, 114, 96, 49, 104, 90, 15, 72, 40, 25, 109, 9, 84, 59, 38, 37, 107, 100, 63, 60, 44, 45, 33, 23, 16, 11, 119, 29, 85, 102, 36, 97, 27, 91, 74, 19, 8, 65, 68, 1, 69, 73, 76, 5, 4, 70, 77, 71, 6, 3, 10, 80, 7, 66, 64, 2, 0, 67, 103], [39, 99, 28, 116, 124, 118, 16, 52, 41, 83, 74, 13, 86, 95, 72, 54, 70, 49, 75, 71, 79, 92, 4, 26, 22, 3, 21, 35, 24, 12, 81, 1, 62, 2, 122, 23, 61, 31, 18, 55, 32, 85, 123, 87, 30, 50, 11, 73, 77, 127, 58, 89, 7, 112, 19, 48, 27, 117, 78, 126, 64, 111, 25, 88, 80, 42, 47, 66, 82, 53, 14, 121, 8, 59, 67, 100, 10, 101, 51, 5, 17, 6, 106, 94, 110, 113, 76, 0, 15, 90, 105, 34, 65, 107, 91, 104, 60, 63, 119, 69, 20, 46, 93, 120, 102, 125, 84, 43, 33, 97, 37, 96, 29, 36, 115, 40, 108, 68, 98, 103, 56, 45, 9, 57, 109, 38, 44, 114], [39, 116, 99, 124, 118, 52, 41, 95, 92, 86, 26, 31, 28, 54, 49, 83, 98, 87, 81, 112, 24, 62, 79, 75, 105, 22, 30, 35, 61, 84, 16, 34, 93, 13, 117, 88, 47, 121, 120, 58, 106, 21, 123, 114, 44, 110, 46, 104, 45, 60, 122, 14, 42, 108, 53, 96, 89, 125, 113, 25, 33, 27, 50, 127, 90, 48, 40, 109, 59, 20, 38, 111, 63, 37, 97, 126, 17, 55, 51, 91, 43, 107, 32, 119, 102, 101, 36, 100, 12, 57, 94, 72, 29, 56, 115, 74, 23, 78, 85, 103, 82, 11, 18, 15, 19, 4, 70, 1, 68, 76, 71, 73, 9, 5, 69, 8, 6, 65, 2, 64, 80, 77, 66, 7, 0, 10, 3, 67], [39, 99, 118, 124, 52, 116, 28, 95, 72, 13, 83, 4, 79, 86, 16, 22, 70, 75, 74, 66, 54, 105, 2, 68, 65, 64, 81, 35, 84, 87, 92, 61, 1, 21, 0, 26, 24, 3, 17, 115, 106, 77, 98, 55, 11, 89, 31, 30, 8, 14, 19, 71, 67, 88, 62, 110, 7, 123, 23, 73, 58, 82, 6, 94, 42, 33, 15, 117, 10, 41, 109, 12, 5, 103, 49, 122, 121, 76, 80, 127, 25, 57, 125, 56, 51, 40, 93, 48, 53, 34, 113, 78, 85, 18, 96, 69, 47, 59, 114, 46, 63, 27, 20, 90, 45, 44, 100, 29, 32, 60, 120, 111, 126, 43, 101, 91, 36, 119, 50, 37, 97, 108, 38, 112, 9, 102, 107, 104], [45, 102, 0, 97, 64, 31, 109, 25, 4, 66, 1, 65, 6, 76, 10, 124, 79, 69, 9, 68, 22, 17, 2, 11, 48, 38, 105, 126, 95, 125, 81, 51, 82, 59, 20, 56, 52, 5, 78, 104, 90, 18, 127, 84, 67, 118, 61, 7, 41, 63, 60, 122, 53, 71, 113, 74, 70, 73, 57, 75, 100, 15, 50, 120, 42, 14, 77, 13, 72, 28, 110, 12, 47, 114, 24, 49, 87, 107, 93, 19, 88, 121, 23, 33, 26, 16, 83, 36, 3, 46, 80, 112, 55, 39, 40, 85, 92, 29, 58, 115, 8, 30, 43, 101, 86, 96, 108, 106, 34, 111, 37, 91, 35, 99, 89, 21, 98, 32, 117, 119, 27, 123, 94, 103, 54, 62, 116, 44], [102, 45, 97, 31, 124, 79, 10, 17, 11, 5, 109, 25, 76, 89, 22, 70, 9, 4, 69, 6, 84, 48, 52, 51, 66, 82, 1, 78, 115, 75, 56, 65, 59, 104, 126, 53, 19, 90, 61, 72, 2, 105, 14, 87, 125, 20, 41, 77, 81, 42, 57, 71, 122, 38, 114, 15, 86, 28, 16, 120, 127, 118, 3, 60, 73, 63, 23, 74, 7, 83, 67, 0, 18, 13, 50, 123, 92, 12, 121, 95, 99, 24, 35, 68, 111, 88, 43, 39, 80, 113, 21, 34, 106, 30, 27, 29, 46, 93, 107, 117, 85, 101, 8, 110, 100, 94, 108, 26, 47, 96, 116, 98, 91, 36, 119, 58, 37, 62, 33, 112, 49, 32, 40, 44, 54, 55, 103, 64], [45, 102, 97, 31, 65, 25, 124, 109, 6, 11, 4, 1, 10, 76, 9, 79, 17, 0, 67, 69, 3, 48, 81, 66, 84, 51, 20, 105, 5, 52, 71, 126, 61, 90, 56, 15, 122, 125, 70, 72, 78, 118, 22, 95, 59, 120, 41, 23, 53, 57, 63, 38, 8, 7, 16, 82, 60, 113, 18, 73, 12, 64, 24, 30, 74, 127, 2, 14, 75, 47, 83, 80, 85, 68, 111, 86, 28, 19, 39, 21, 46, 87, 50, 49, 13, 29, 77, 104, 121, 26, 100, 88, 91, 93, 42, 27, 110, 58, 107, 117, 94, 101, 119, 114, 40, 108, 115, 96, 36, 103, 89, 116, 123, 37, 62, 92, 44, 34, 43, 55, 106, 112, 35, 98, 99, 54, 32, 33], [124, 102, 97, 45, 31, 90, 22, 52, 86, 120, 84, 109, 89, 78, 111, 104, 41, 53, 121, 13, 122, 18, 36, 33, 38, 115, 51, 114, 59, 113, 107, 63, 25, 34, 117, 57, 79, 127, 106, 46, 42, 26, 48, 44, 62, 17, 123, 126, 61, 100, 39, 54, 96, 56, 116, 21, 60, 49, 108, 103, 58, 105, 29, 43, 112, 110, 28, 99, 32, 101, 27, 24, 118, 35, 119, 50, 37, 91, 55, 88, 23, 76, 98, 47, 30, 40, 94, 125, 93, 82, 20, 11, 92, 14, 19, 83, 85, 77, 80, 16, 87, 10, 72, 9, 8, 12, 81, 15, 73, 74, 95, 5, 75, 7, 71, 70, 69, 67, 68, 1, 3, 6, 2, 65, 4, 66, 64, 0], [63, 59, 52, 104, 119, 125, 56, 122, 60, 113, 62, 123, 114, 124, 50, 61, 127, 53, 54, 57, 58, 46, 109, 118, 111, 49, 48, 126, 116, 51, 47, 55, 117, 82, 120, 44, 115, 108, 45, 121, 112, 43, 110, 79, 88, 107, 84, 106, 42, 39, 41, 80, 87, 102, 28, 90, 105, 91, 76, 85, 86, 96, 103, 19, 30, 93, 9, 38, 100, 12, 81, 14, 101, 16, 13, 37, 36, 7, 89, 73, 4, 5, 33, 40, 74, 32, 99, 83, 35, 21, 97, 8, 94, 11, 25, 10, 15, 27, 65, 77, 72, 70, 92, 69, 34, 22, 2, 78, 23, 75, 17, 98, 31, 95, 1, 67, 24, 18, 29, 71, 0, 26, 20, 3, 6, 66, 68, 64], [104, 34, 52, 63, 59, 77, 17, 90, 10, 7, 78, 87, 68, 75, 27, 20, 31, 21, 13, 118, 4, 119, 66, 79, 89, 93, 124, 71, 46, 70, 125, 14, 100, 26, 23, 82, 6, 83, 73, 84, 22, 19, 122, 86, 95, 64, 74, 85, 11, 1, 76, 113, 8, 81, 88, 15, 18, 80, 25, 111, 29, 56, 28, 60, 2, 65, 94, 67, 108, 123, 50, 72, 120, 36, 0, 69, 12, 5, 3, 91, 40, 102, 41, 30, 32, 61, 116, 39, 92, 127, 48, 9, 98, 16, 115, 62, 24, 126, 47, 99, 121, 38, 42, 110, 45, 109, 37, 49, 57, 101, 96, 97, 117, 33, 54, 103, 106, 105, 58, 35, 55, 51, 44, 43, 112, 107, 53, 114], [104, 34, 52, 63, 59, 7, 92, 71, 87, 17, 78, 119, 28, 68, 10, 91, 31, 75, 83, 125, 90, 77, 85, 94, 60, 1, 22, 3, 122, 100, 18, 4, 16, 8, 124, 26, 65, 113, 118, 23, 84, 46, 74, 56, 89, 93, 72, 123, 14, 25, 5, 13, 76, 79, 9, 81, 19, 12, 21, 6, 82, 62, 30, 116, 11, 88, 86, 70, 27, 66, 98, 102, 80, 24, 67, 120, 36, 111, 69, 40, 108, 20, 2, 64, 61, 48, 39, 73, 15, 96, 95, 117, 43, 101, 0, 38, 29, 37, 33, 50, 121, 32, 55, 35, 97, 109, 47, 103, 45, 99, 105, 106, 127, 54, 110, 57, 44, 41, 42, 49, 115, 51, 107, 112, 58, 126, 114, 53], [104, 52, 63, 59, 119, 125, 124, 60, 46, 62, 113, 118, 27, 123, 48, 39, 116, 50, 56, 40, 122, 88, 34, 32, 108, 57, 111, 109, 106, 61, 115, 47, 110, 30, 28, 51, 54, 120, 55, 43, 35, 117, 49, 114, 107, 121, 42, 41, 98, 58, 103, 105, 36, 102, 100, 79, 33, 45, 53, 37, 97, 76, 44, 38, 126, 80, 86, 101, 24, 82, 112, 127, 85, 26, 99, 94, 95, 9, 96, 84, 19, 73, 20, 91, 22, 31, 7, 25, 93, 81, 14, 13, 5, 74, 92, 4, 87, 90, 8, 89, 11, 16, 1, 12, 70, 17, 69, 23, 18, 72, 10, 2, 29, 21, 83, 65, 77, 15, 78, 75, 71, 67, 3, 0, 64, 6, 66, 68], [105, 112, 39, 34, 101, 48, 50, 111, 118, 85, 124, 53, 23, 59, 21, 125, 90, 31, 82, 119, 30, 108, 56, 54, 117, 25, 93, 87, 60, 18, 96, 63, 127, 41, 19, 22, 52, 61, 122, 28, 123, 92, 88, 51, 103, 94, 89, 47, 95, 26, 37, 86, 16, 80, 57, 110, 121, 81, 38, 102, 84, 55, 29, 49, 91, 17, 97, 32, 45, 33, 11, 106, 44, 40, 107, 27, 99, 109, 62, 58, 114, 20, 78, 42, 120, 79, 14, 100, 104, 13, 116, 126, 46, 15, 24, 115, 36, 83, 12, 76, 35, 43, 75, 113, 71, 77, 9, 6, 5, 74, 98, 10, 73, 72, 8, 69, 7, 70, 2, 66, 68, 3, 65, 67, 0, 1, 64, 4], [50, 48, 105, 124, 34, 111, 38, 112, 117, 108, 119, 90, 57, 54, 39, 106, 116, 94, 114, 120, 122, 125, 118, 109, 127, 42, 63, 110, 103, 126, 52, 51, 93, 49, 92, 45, 85, 35, 82, 23, 59, 36, 31, 60, 107, 113, 55, 46, 58, 29, 104, 87, 9, 101, 47, 61, 83, 62, 40, 121, 89, 56, 53, 99, 97, 123, 44, 80, 96, 43, 115, 28, 91, 76, 37, 100, 24, 19, 88, 102, 78, 71, 16, 14, 33, 32, 86, 77, 22, 30, 95, 26, 41, 27, 84, 25, 74, 98, 20, 79, 72, 17, 13, 21, 15, 18, 7, 68, 73, 75, 11, 6, 10, 2, 67, 8, 12, 4, 81, 69, 70, 66, 64, 3, 5, 0, 1, 65], [105, 101, 34, 112, 48, 50, 89, 31, 39, 85, 83, 124, 23, 111, 87, 80, 54, 14, 82, 76, 118, 123, 59, 25, 41, 93, 21, 119, 16, 29, 63, 125, 108, 53, 19, 95, 56, 9, 117, 103, 94, 49, 116, 84, 90, 22, 20, 60, 15, 75, 52, 122, 47, 71, 26, 24, 109, 46, 74, 127, 17, 28, 32, 35, 11, 33, 57, 12, 38, 96, 62, 102, 42, 120, 92, 91, 113, 30, 114, 77, 13, 79, 107, 10, 88, 73, 104, 70, 121, 43, 78, 97, 100, 61, 81, 18, 99, 44, 27, 72, 51, 6, 106, 58, 8, 110, 55, 5, 115, 126, 7, 86, 40, 36, 69, 98, 45, 37, 68, 67, 3, 66, 4, 2, 65, 1, 0, 64], [50, 101, 38, 105, 34, 112, 118, 59, 119, 83, 31, 85, 89, 37, 23, 111, 90, 78, 49, 61, 103, 124, 21, 20, 74, 71, 125, 52, 88, 113, 53, 57, 19, 122, 104, 92, 11, 44, 106, 82, 54, 14, 109, 48, 63, 29, 22, 123, 40, 51, 28, 96, 76, 79, 108, 8, 41, 17, 9, 87, 60, 18, 68, 102, 16, 127, 56, 107, 126, 25, 114, 42, 72, 121, 24, 84, 30, 110, 93, 13, 27, 10, 117, 100, 62, 46, 39, 116, 4, 47, 35, 15, 81, 58, 91, 2, 115, 67, 45, 97, 120, 32, 94, 69, 33, 43, 7, 86, 26, 99, 55, 80, 77, 6, 1, 64, 36, 95, 70, 66, 5, 65, 0, 3, 75, 98, 12, 73], [110, 39, 33, 31, 46, 85, 70, 16, 53, 10, 14, 72, 94, 82, 12, 75, 68, 3, 0, 2, 1, 9, 65, 66, 6, 50, 67, 8, 104, 21, 86, 13, 120, 124, 17, 22, 4, 79, 18, 83, 57, 54, 118, 63, 90, 11, 69, 58, 123, 73, 119, 51, 126, 103, 78, 7, 127, 41, 24, 20, 99, 38, 115, 87, 80, 88, 29, 121, 56, 49, 101, 71, 102, 114, 35, 76, 98, 28, 74, 81, 36, 23, 84, 108, 125, 89, 47, 19, 96, 77, 113, 107, 45, 111, 100, 64, 5, 40, 61, 48, 117, 95, 34, 92, 26, 25, 91, 42, 52, 55, 37, 122, 93, 116, 112, 27, 105, 15, 43, 62, 106, 60, 59, 30, 109, 44, 32, 97], [110, 39, 33, 31, 46, 12, 94, 10, 6, 16, 82, 4, 72, 14, 85, 50, 74, 2, 79, 53, 70, 17, 86, 67, 115, 124, 51, 69, 21, 23, 28, 24, 76, 80, 120, 22, 57, 78, 8, 63, 117, 75, 18, 123, 66, 73, 127, 65, 126, 104, 49, 84, 118, 103, 88, 56, 58, 96, 64, 48, 77, 90, 13, 54, 20, 32, 9, 68, 121, 0, 38, 122, 108, 41, 62, 7, 11, 101, 81, 83, 59, 113, 45, 43, 93, 87, 44, 55, 47, 15, 89, 112, 27, 109, 92, 119, 114, 107, 25, 116, 5, 98, 125, 19, 95, 105, 99, 60, 91, 40, 34, 36, 26, 61, 37, 1, 111, 29, 52, 71, 35, 100, 102, 106, 42, 3, 30, 97], [39, 33, 110, 53, 31, 23, 85, 46, 82, 79, 94, 16, 12, 14, 88, 19, 104, 10, 126, 57, 17, 112, 50, 70, 75, 51, 5, 63, 72, 124, 66, 115, 68, 120, 106, 64, 65, 71, 86, 81, 60, 48, 84, 55, 8, 123, 9, 125, 47, 11, 108, 56, 21, 40, 42, 3, 121, 41, 59, 24, 116, 95, 25, 49, 91, 119, 58, 38, 101, 96, 54, 22, 111, 117, 99, 122, 87, 113, 35, 18, 26, 105, 32, 34, 73, 118, 90, 127, 78, 29, 37, 100, 107, 114, 83, 27, 15, 44, 13, 45, 61, 102, 28, 109, 52, 89, 7, 80, 98, 67, 20, 93, 43, 36, 62, 74, 103, 92, 4, 2, 6, 76, 30, 0, 69, 77, 1, 97], [110, 33, 39, 31, 85, 46, 94, 24, 74, 82, 16, 14, 72, 10, 12, 86, 88, 83, 22, 53, 79, 23, 4, 13, 124, 51, 73, 90, 50, 80, 126, 5, 115, 123, 11, 120, 57, 71, 6, 78, 18, 63, 2, 70, 8, 48, 17, 75, 9, 103, 21, 67, 58, 119, 65, 118, 91, 106, 68, 56, 30, 104, 15, 69, 84, 92, 121, 87, 49, 19, 54, 40, 28, 41, 117, 77, 61, 59, 81, 125, 76, 36, 114, 20, 26, 27, 111, 99, 127, 7, 25, 62, 3, 32, 55, 60, 35, 42, 96, 112, 101, 89, 45, 43, 47, 113, 44, 93, 34, 38, 100, 102, 52, 105, 98, 108, 109, 66, 37, 116, 122, 1, 64, 95, 97, 29, 107, 0], [112, 40, 34, 53, 23, 27, 122, 29, 19, 81, 76, 21, 25, 87, 63, 31, 56, 79, 118, 119, 77, 59, 15, 93, 62, 57, 30, 16, 72, 90, 47, 9, 82, 18, 26, 11, 92, 97, 109, 125, 33, 95, 78, 101, 123, 32, 44, 96, 49, 83, 35, 105, 73, 120, 24, 51, 86, 80, 106, 121, 103, 88, 98, 55, 36, 14, 22, 52, 38, 10, 71, 46, 67, 12, 28, 89, 91, 110, 127, 111, 61, 6, 54, 37, 107, 13, 94, 60, 42, 85, 84, 17, 102, 58, 5, 41, 99, 43, 113, 126, 74, 20, 115, 124, 108, 48, 45, 100, 8, 69, 116, 7, 39, 114, 50, 70, 4, 104, 117, 68, 2, 3, 1, 75, 65, 66, 0, 64], [112, 53, 40, 38, 34, 56, 47, 119, 44, 43, 52, 63, 62, 49, 122, 117, 114, 125, 61, 116, 120, 111, 109, 45, 123, 55, 31, 23, 25, 95, 59, 60, 126, 51, 118, 29, 106, 42, 127, 57, 107, 50, 54, 121, 27, 92, 41, 58, 24, 115, 46, 16, 113, 105, 48, 30, 108, 39, 110, 124, 84, 21, 93, 82, 96, 103, 88, 101, 19, 35, 22, 100, 90, 89, 102, 37, 36, 33, 32, 18, 87, 26, 17, 97, 99, 81, 86, 28, 20, 94, 85, 80, 77, 11, 14, 98, 78, 83, 13, 15, 12, 91, 76, 3, 74, 68, 8, 9, 10, 65, 7, 6, 79, 71, 4, 72, 75, 66, 73, 70, 2, 104, 5, 0, 69, 1, 67, 64], [53, 40, 112, 34, 19, 23, 81, 27, 56, 125, 21, 77, 93, 47, 82, 57, 122, 88, 120, 58, 25, 76, 59, 11, 9, 28, 14, 118, 61, 95, 62, 67, 79, 70, 104, 30, 91, 114, 92, 31, 26, 36, 44, 80, 16, 73, 119, 107, 52, 15, 7, 72, 90, 121, 20, 29, 63, 83, 126, 6, 85, 99, 12, 116, 105, 100, 18, 86, 106, 48, 60, 38, 22, 43, 78, 110, 115, 127, 69, 24, 50, 97, 87, 46, 103, 1, 84, 35, 123, 13, 41, 17, 4, 5, 89, 94, 32, 55, 10, 54, 102, 37, 98, 96, 74, 109, 45, 42, 108, 49, 111, 124, 75, 33, 51, 39, 117, 68, 113, 101, 3, 65, 71, 8, 0, 2, 64, 66], [40, 53, 34, 112, 91, 19, 21, 29, 81, 122, 79, 73, 93, 76, 6, 56, 16, 11, 47, 125, 52, 8, 63, 70, 67, 59, 62, 65, 95, 72, 17, 119, 9, 118, 27, 126, 57, 1, 12, 38, 104, 10, 120, 74, 77, 71, 61, 7, 121, 58, 86, 85, 24, 44, 30, 75, 23, 88, 84, 4, 3, 31, 68, 15, 28, 13, 78, 32, 83, 114, 14, 80, 46, 89, 20, 82, 25, 90, 123, 92, 87, 0, 2, 94, 117, 69, 26, 66, 36, 96, 18, 22, 5, 100, 35, 99, 60, 45, 37, 54, 51, 124, 108, 97, 33, 55, 101, 105, 102, 42, 103, 41, 107, 43, 49, 127, 64, 39, 106, 109, 111, 115, 110, 50, 113, 116, 48, 98]], "model.layers.4.self_attn.k_proj": [[37, 53, 56, 123, 122, 97, 117, 106, 30, 84, 16, 27, 87, 82, 13, 26, 75, 100, 50, 6, 40, 77, 73, 83, 78, 31, 42, 72, 17, 28, 64, 86, 103, 65, 38, 108, 21, 94, 35, 120, 5, 68, 118, 60, 45, 25, 47, 3, 74, 61, 127, 101, 2, 0, 89, 126, 24, 88, 43, 29, 58, 79, 52, 96, 113, 110, 55, 51, 4, 46, 49, 125, 119, 54, 66, 33, 109, 62, 105, 104, 57, 59, 121, 115, 63, 116, 41, 91, 111, 8, 10, 12, 112, 85, 44, 107, 23, 98, 93, 71, 34, 114, 22, 48, 124, 67, 32, 99, 39, 70, 92, 76, 14, 1, 11, 7, 90, 36, 102, 18, 95, 69, 19, 81, 80, 15, 20, 9], [103, 118, 52, 87, 97, 95, 20, 0, 13, 82, 81, 79, 70, 11, 9, 10, 68, 90, 3, 8, 66, 1, 47, 28, 110, 42, 53, 14, 63, 5, 71, 121, 86, 43, 22, 127, 55, 26, 65, 113, 92, 123, 2, 120, 69, 58, 46, 74, 122, 44, 88, 109, 48, 102, 39, 56, 61, 107, 125, 115, 16, 36, 89, 67, 57, 117, 126, 54, 50, 62, 73, 64, 45, 59, 29, 80, 93, 124, 114, 106, 116, 7, 38, 78, 6, 104, 100, 25, 96, 40, 27, 60, 108, 51, 83, 77, 37, 15, 119, 112, 101, 19, 99, 85, 105, 32, 21, 91, 30, 12, 76, 72, 98, 111, 4, 94, 41, 34, 35, 18, 49, 33, 24, 23, 17, 84, 75, 31], [103, 52, 118, 124, 35, 86, 92, 83, 79, 13, 1, 74, 72, 75, 31, 0, 16, 4, 70, 81, 105, 71, 26, 3, 21, 69, 2, 117, 116, 24, 53, 28, 14, 94, 5, 39, 12, 61, 112, 123, 41, 46, 95, 121, 82, 22, 56, 113, 54, 47, 55, 106, 84, 58, 59, 99, 48, 122, 9, 89, 98, 49, 126, 115, 50, 30, 40, 96, 34, 114, 63, 45, 23, 119, 111, 60, 51, 108, 100, 125, 17, 101, 7, 42, 107, 127, 87, 36, 93, 104, 67, 62, 43, 88, 20, 102, 85, 120, 33, 109, 29, 57, 91, 37, 44, 76, 32, 97, 38, 110, 90, 27, 25, 18, 73, 77, 80, 66, 19, 78, 68, 6, 64, 15, 65, 10, 11, 8], [109, 38, 0, 95, 33, 45, 89, 124, 66, 65, 17, 6, 79, 4, 10, 76, 2, 112, 64, 9, 69, 11, 126, 60, 56, 84, 59, 115, 61, 3, 47, 50, 18, 125, 52, 7, 105, 63, 118, 68, 78, 39, 116, 111, 46, 67, 40, 113, 110, 122, 121, 57, 23, 8, 127, 58, 51, 22, 44, 86, 53, 83, 88, 13, 70, 16, 108, 104, 117, 41, 92, 5, 37, 43, 119, 99, 48, 103, 101, 42, 90, 96, 80, 54, 29, 30, 55, 71, 77, 91, 123, 120, 35, 85, 94, 32, 93, 98, 62, 100, 114, 107, 26, 75, 102, 27, 36, 106, 87, 24, 28, 73, 19, 49, 34, 82, 1, 21, 74, 72, 97, 12, 15, 14, 81, 20, 25, 31], [40, 52, 59, 63, 98, 75, 26, 95, 17, 119, 78, 116, 22, 10, 77, 20, 0, 2, 6, 4, 83, 7, 118, 29, 113, 72, 123, 124, 110, 67, 18, 60, 125, 70, 111, 3, 23, 28, 50, 122, 15, 115, 55, 56, 54, 46, 121, 57, 48, 53, 47, 49, 120, 87, 62, 51, 117, 126, 114, 58, 127, 16, 61, 89, 106, 24, 9, 109, 112, 12, 43, 42, 45, 44, 21, 5, 103, 91, 108, 65, 107, 105, 8, 64, 41, 36, 39, 11, 102, 37, 27, 92, 99, 101, 93, 84, 38, 104, 35, 33, 31, 94, 32, 14, 90, 30, 96, 86, 97, 100, 69, 85, 66, 1, 25, 19, 74, 88, 80, 82, 71, 13, 79, 34, 81, 73, 76, 68], [41, 48, 98, 50, 37, 95, 23, 89, 82, 93, 90, 118, 114, 76, 80, 85, 74, 65, 44, 83, 64, 14, 59, 6, 77, 47, 53, 19, 68, 124, 86, 16, 27, 81, 20, 8, 111, 102, 9, 78, 66, 121, 125, 17, 71, 7, 112, 67, 119, 84, 106, 39, 94, 12, 32, 54, 63, 116, 117, 88, 72, 113, 96, 26, 62, 42, 69, 110, 115, 127, 3, 87, 22, 43, 107, 104, 4, 56, 51, 122, 60, 99, 58, 52, 123, 30, 15, 49, 92, 46, 108, 11, 38, 36, 61, 79, 35, 97, 33, 120, 45, 126, 91, 18, 28, 13, 109, 103, 70, 29, 40, 5, 75, 21, 2, 100, 57, 55, 10, 24, 0, 31, 25, 73, 1, 105, 101, 34], [46, 103, 110, 97, 53, 14, 0, 85, 95, 82, 16, 75, 12, 72, 17, 10, 66, 70, 71, 68, 79, 3, 51, 1, 118, 5, 124, 63, 30, 126, 8, 114, 11, 23, 40, 9, 50, 123, 90, 120, 56, 65, 86, 117, 77, 119, 57, 81, 113, 125, 122, 58, 112, 88, 67, 18, 121, 83, 84, 19, 116, 87, 62, 47, 32, 20, 127, 29, 78, 91, 44, 89, 28, 25, 93, 6, 24, 54, 2, 69, 92, 107, 27, 37, 42, 43, 76, 105, 100, 101, 34, 15, 22, 35, 59, 45, 7, 109, 99, 102, 36, 60, 55, 98, 108, 61, 115, 26, 96, 48, 111, 49, 80, 106, 41, 33, 38, 73, 52, 13, 94, 104, 74, 64, 4, 21, 31, 39], [104, 53, 112, 98, 48, 21, 11, 81, 19, 16, 0, 27, 93, 5, 73, 77, 117, 76, 2, 79, 8, 6, 65, 59, 122, 111, 25, 62, 102, 68, 110, 69, 57, 67, 72, 30, 50, 71, 56, 3, 84, 116, 15, 52, 94, 92, 74, 125, 91, 45, 121, 119, 31, 39, 58, 63, 13, 108, 70, 44, 55, 54, 7, 46, 61, 107, 90, 51, 32, 105, 118, 126, 28, 88, 109, 66, 41, 22, 96, 10, 114, 82, 113, 106, 86, 100, 60, 115, 120, 26, 124, 99, 24, 64, 127, 97, 23, 33, 29, 38, 103, 36, 14, 78, 42, 1, 95, 47, 43, 49, 35, 9, 18, 37, 101, 123, 87, 89, 75, 12, 85, 20, 83, 17, 4, 80, 40, 34]], "model.layers.4.self_attn.qk_proj": [[52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 104, 31, 87, 17, 23, 11, 75, 81, 15, 38, 79, 85, 21, 28, 116, 102, 119, 40, 74, 18, 82, 42, 13, 80, 22, 10, 72, 97, 25, 64, 77, 20, 16, 6, 19, 83, 0, 12, 84, 70, 89, 91, 26, 86, 90, 78, 14, 76, 68, 125, 4, 37, 99, 61, 8, 94, 73, 9, 101, 126, 47, 65, 1, 35, 2, 98, 111, 54, 113, 66, 60, 114, 105, 51, 120, 29, 121, 92, 71, 7, 57, 108, 62, 67, 115, 127, 30, 44, 93, 5, 88, 27, 69, 3, 34, 106, 49, 24, 55, 107, 58, 43, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 48, 56, 117, 122, 123, 39, 95, 103, 33, 41, 31, 87, 104, 11, 23, 17, 81, 15, 38, 116, 28, 0, 79, 40, 85, 75, 64, 21, 74, 18, 72, 10, 97, 102, 25, 89, 42, 22, 6, 70, 19, 82, 26, 20, 83, 91, 68, 77, 16, 80, 12, 13, 125, 78, 119, 1, 90, 14, 84, 37, 4, 76, 86, 98, 101, 54, 105, 94, 65, 66, 113, 2, 47, 9, 111, 121, 73, 126, 35, 8, 114, 61, 120, 108, 99, 57, 51, 62, 3, 115, 60, 29, 69, 71, 5, 67, 92, 106, 7, 43, 58, 34, 24, 93, 127, 27, 30, 49, 44, 55, 88, 107, 96, 32, 100, 36], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 50, 56, 122, 117, 48, 123, 39, 103, 95, 33, 41, 87, 104, 31, 23, 11, 15, 81, 0, 38, 72, 70, 75, 85, 17, 28, 64, 18, 116, 74, 82, 10, 79, 21, 13, 80, 40, 16, 22, 6, 97, 4, 83, 25, 42, 102, 19, 77, 20, 14, 12, 91, 26, 89, 84, 65, 68, 78, 76, 1, 90, 119, 125, 54, 86, 37, 66, 105, 2, 98, 94, 99, 121, 9, 8, 101, 126, 47, 114, 73, 3, 61, 120, 35, 111, 67, 5, 113, 7, 71, 69, 93, 57, 60, 108, 92, 51, 24, 29, 62, 30, 115, 44, 34, 88, 127, 58, 55, 27, 106, 49, 107, 43, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 122, 48, 117, 39, 123, 95, 103, 33, 41, 31, 104, 87, 79, 23, 17, 11, 64, 81, 75, 72, 18, 38, 0, 70, 28, 85, 15, 21, 102, 74, 80, 97, 25, 116, 13, 40, 19, 91, 10, 83, 82, 16, 42, 6, 89, 4, 14, 22, 84, 77, 12, 86, 20, 76, 119, 78, 26, 94, 90, 98, 68, 2, 121, 65, 125, 66, 37, 101, 1, 47, 99, 105, 54, 9, 8, 114, 67, 113, 73, 3, 69, 60, 111, 120, 115, 35, 126, 61, 71, 58, 34, 29, 92, 93, 5, 88, 57, 108, 30, 44, 7, 62, 51, 24, 127, 43, 27, 106, 49, 107, 55, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 56, 117, 50, 122, 48, 123, 95, 103, 39, 41, 33, 104, 87, 79, 64, 11, 75, 81, 17, 31, 70, 72, 23, 0, 38, 40, 85, 6, 15, 116, 28, 18, 21, 80, 82, 10, 74, 13, 97, 16, 22, 83, 84, 4, 42, 77, 12, 19, 25, 78, 76, 102, 26, 20, 91, 2, 89, 68, 86, 8, 65, 119, 14, 66, 120, 1, 90, 98, 9, 94, 61, 125, 113, 73, 37, 126, 47, 111, 101, 99, 121, 35, 114, 7, 67, 115, 105, 51, 57, 69, 29, 5, 60, 71, 3, 93, 92, 54, 30, 34, 108, 62, 44, 58, 106, 127, 88, 27, 24, 49, 107, 55, 43, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 48, 56, 122, 117, 123, 95, 39, 103, 33, 41, 104, 81, 31, 87, 79, 11, 17, 23, 75, 0, 38, 85, 64, 15, 28, 82, 21, 40, 70, 10, 6, 72, 4, 102, 80, 18, 116, 13, 74, 97, 25, 12, 16, 77, 78, 76, 83, 20, 22, 8, 91, 84, 86, 68, 19, 89, 42, 119, 14, 90, 26, 66, 125, 47, 126, 2, 1, 73, 101, 61, 94, 65, 98, 37, 54, 99, 105, 9, 113, 35, 5, 111, 29, 121, 114, 3, 71, 120, 7, 51, 115, 34, 49, 62, 60, 67, 57, 92, 108, 69, 58, 127, 93, 107, 24, 106, 27, 30, 44, 88, 43, 55, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 103, 95, 33, 41, 104, 17, 31, 87, 79, 23, 81, 38, 11, 40, 85, 75, 28, 25, 82, 116, 18, 15, 21, 6, 74, 0, 42, 102, 13, 22, 84, 97, 76, 26, 78, 10, 20, 64, 14, 83, 16, 80, 70, 72, 19, 12, 89, 86, 119, 8, 68, 91, 4, 77, 90, 47, 61, 37, 101, 126, 1, 94, 125, 120, 9, 66, 2, 73, 98, 65, 121, 54, 113, 35, 29, 99, 111, 114, 57, 60, 71, 105, 7, 34, 93, 127, 51, 67, 62, 3, 58, 92, 108, 69, 115, 27, 5, 44, 30, 88, 24, 106, 55, 49, 43, 107, 96, 36, 100, 32], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 117, 123, 39, 95, 103, 33, 41, 104, 81, 87, 23, 17, 79, 75, 31, 85, 28, 38, 6, 15, 11, 21, 40, 64, 82, 116, 74, 10, 25, 18, 97, 76, 0, 42, 16, 77, 70, 22, 19, 84, 80, 78, 8, 119, 13, 83, 89, 20, 26, 91, 102, 14, 12, 86, 68, 4, 101, 37, 125, 66, 72, 1, 90, 65, 9, 54, 111, 98, 94, 60, 35, 126, 44, 105, 73, 120, 113, 57, 2, 93, 47, 121, 61, 7, 99, 29, 51, 24, 69, 58, 92, 67, 114, 3, 27, 71, 115, 108, 34, 5, 30, 127, 62, 106, 43, 88, 49, 55, 107, 96, 36, 32, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 87, 104, 23, 81, 31, 17, 11, 79, 85, 21, 75, 38, 82, 6, 15, 40, 97, 10, 64, 74, 83, 0, 28, 18, 80, 25, 8, 116, 22, 20, 16, 13, 42, 102, 70, 91, 76, 89, 78, 19, 84, 86, 77, 90, 12, 68, 54, 14, 119, 26, 125, 105, 4, 37, 98, 101, 66, 47, 72, 94, 111, 65, 1, 2, 61, 121, 99, 9, 126, 120, 73, 58, 113, 35, 34, 92, 93, 29, 57, 114, 51, 115, 7, 67, 108, 69, 62, 60, 3, 30, 71, 24, 44, 27, 106, 5, 43, 55, 88, 127, 49, 32, 107, 96, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 48, 56, 122, 117, 123, 39, 103, 95, 33, 87, 41, 81, 23, 31, 104, 17, 75, 15, 21, 38, 11, 28, 85, 116, 79, 42, 18, 25, 82, 74, 40, 97, 83, 102, 0, 78, 8, 84, 16, 13, 19, 10, 90, 6, 22, 64, 12, 80, 89, 86, 26, 91, 20, 77, 70, 68, 14, 76, 119, 37, 4, 125, 101, 105, 94, 99, 54, 1, 111, 65, 47, 121, 98, 120, 72, 60, 73, 114, 126, 9, 92, 35, 66, 29, 2, 71, 61, 24, 34, 69, 7, 3, 115, 27, 113, 67, 51, 106, 57, 93, 44, 108, 5, 127, 49, 88, 58, 30, 62, 55, 107, 43, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 56, 48, 50, 122, 117, 39, 123, 95, 103, 33, 41, 104, 31, 87, 75, 11, 64, 17, 79, 8, 81, 15, 23, 6, 82, 38, 28, 10, 40, 0, 21, 116, 85, 42, 70, 80, 18, 102, 97, 13, 74, 16, 83, 68, 22, 19, 77, 12, 119, 25, 20, 84, 91, 78, 26, 4, 14, 76, 86, 89, 47, 1, 66, 2, 65, 72, 101, 99, 90, 94, 9, 73, 98, 125, 37, 126, 121, 114, 105, 35, 67, 115, 127, 61, 111, 5, 60, 120, 69, 54, 113, 71, 92, 7, 3, 57, 108, 106, 29, 58, 51, 93, 30, 62, 24, 34, 107, 43, 49, 44, 27, 88, 55, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 56, 50, 122, 48, 117, 123, 95, 39, 33, 103, 41, 104, 31, 15, 87, 23, 81, 75, 8, 17, 116, 64, 28, 70, 11, 40, 38, 85, 102, 18, 10, 6, 21, 0, 82, 16, 97, 79, 83, 42, 25, 77, 68, 13, 91, 19, 12, 20, 74, 22, 84, 80, 119, 89, 26, 125, 4, 14, 66, 76, 86, 78, 113, 2, 90, 101, 9, 61, 72, 65, 94, 47, 121, 51, 98, 73, 126, 111, 1, 37, 120, 105, 99, 54, 67, 35, 57, 7, 34, 69, 115, 60, 5, 93, 114, 92, 3, 62, 108, 55, 29, 71, 44, 30, 106, 127, 24, 58, 27, 88, 43, 49, 107, 32, 96, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 50, 59, 63, 48, 56, 122, 117, 123, 39, 103, 95, 33, 41, 104, 87, 17, 81, 31, 23, 38, 40, 28, 85, 75, 15, 11, 79, 25, 116, 21, 82, 83, 97, 16, 10, 22, 0, 8, 70, 18, 102, 42, 86, 26, 74, 6, 20, 89, 12, 84, 14, 80, 13, 76, 91, 19, 119, 90, 77, 78, 64, 37, 4, 101, 94, 68, 9, 72, 125, 98, 113, 1, 65, 29, 35, 105, 111, 61, 126, 2, 47, 99, 54, 60, 66, 51, 93, 121, 120, 7, 114, 73, 34, 57, 3, 27, 92, 58, 24, 71, 44, 108, 55, 5, 115, 88, 62, 49, 69, 127, 106, 67, 30, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 117, 122, 123, 95, 39, 33, 103, 41, 87, 31, 104, 81, 17, 11, 23, 38, 75, 28, 79, 70, 15, 0, 85, 40, 64, 18, 10, 74, 8, 25, 97, 16, 22, 42, 102, 21, 82, 83, 116, 80, 86, 6, 84, 77, 78, 20, 12, 13, 14, 19, 91, 76, 90, 68, 72, 89, 119, 4, 9, 26, 1, 37, 125, 98, 47, 65, 2, 66, 101, 99, 105, 94, 51, 7, 61, 121, 54, 111, 35, 126, 73, 29, 120, 92, 113, 5, 60, 57, 108, 114, 67, 34, 115, 44, 127, 27, 71, 88, 3, 62, 93, 69, 58, 55, 24, 49, 30, 106, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 104, 87, 31, 75, 11, 23, 17, 38, 116, 70, 15, 74, 0, 81, 79, 21, 85, 42, 64, 28, 40, 119, 82, 6, 18, 16, 97, 102, 72, 19, 8, 25, 10, 13, 83, 22, 20, 80, 89, 84, 14, 4, 68, 78, 91, 77, 12, 76, 86, 26, 2, 90, 125, 1, 94, 9, 98, 37, 66, 65, 101, 120, 47, 73, 121, 126, 54, 3, 67, 7, 111, 105, 44, 99, 35, 92, 113, 57, 5, 115, 51, 61, 60, 114, 93, 27, 69, 29, 34, 71, 30, 49, 127, 108, 62, 88, 24, 55, 58, 106, 43, 107, 36, 96, 100, 32], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 95, 39, 103, 41, 33, 104, 87, 31, 11, 75, 23, 81, 38, 79, 17, 74, 28, 15, 70, 85, 64, 40, 18, 20, 116, 97, 102, 21, 16, 0, 72, 25, 6, 13, 10, 82, 91, 19, 80, 83, 84, 89, 22, 42, 86, 12, 76, 119, 125, 77, 4, 90, 68, 78, 8, 14, 94, 101, 66, 120, 26, 54, 73, 2, 98, 37, 111, 9, 1, 35, 126, 47, 105, 65, 34, 113, 99, 61, 51, 108, 29, 57, 121, 114, 3, 115, 60, 71, 7, 92, 67, 62, 93, 69, 5, 106, 30, 127, 58, 107, 44, 24, 43, 88, 49, 55, 27, 96, 100, 36, 32], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 122, 56, 48, 117, 123, 39, 103, 95, 33, 41, 87, 23, 17, 31, 81, 15, 38, 11, 85, 79, 28, 75, 104, 72, 82, 18, 21, 74, 16, 25, 40, 14, 20, 22, 97, 6, 86, 89, 19, 10, 0, 102, 70, 84, 76, 77, 116, 91, 83, 64, 68, 13, 42, 12, 78, 80, 90, 4, 26, 125, 8, 47, 1, 9, 101, 94, 119, 105, 61, 37, 65, 98, 73, 99, 126, 113, 29, 2, 121, 66, 120, 114, 35, 111, 7, 71, 60, 34, 51, 54, 5, 58, 92, 3, 57, 24, 69, 93, 67, 88, 127, 108, 27, 115, 55, 62, 44, 30, 49, 106, 43, 96, 107, 36, 32, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 122, 48, 39, 123, 117, 103, 95, 33, 41, 87, 31, 17, 104, 15, 75, 81, 21, 85, 23, 72, 82, 11, 28, 38, 79, 6, 18, 16, 64, 42, 10, 40, 74, 116, 89, 83, 0, 97, 70, 20, 25, 22, 12, 19, 14, 13, 102, 125, 84, 78, 80, 76, 77, 91, 86, 68, 90, 119, 26, 4, 105, 9, 126, 37, 99, 1, 98, 94, 66, 65, 8, 101, 73, 121, 2, 51, 111, 113, 3, 54, 47, 60, 7, 120, 67, 29, 127, 114, 61, 35, 71, 93, 5, 92, 57, 115, 34, 69, 44, 49, 108, 24, 62, 27, 88, 30, 43, 58, 55, 106, 107, 32, 96, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 117, 123, 39, 95, 103, 33, 41, 31, 104, 17, 87, 75, 38, 23, 15, 0, 72, 6, 81, 79, 21, 11, 85, 116, 82, 10, 40, 97, 16, 28, 64, 89, 18, 70, 83, 19, 42, 20, 74, 102, 25, 13, 76, 14, 84, 80, 119, 22, 77, 90, 78, 68, 91, 26, 12, 4, 65, 37, 86, 101, 66, 98, 1, 8, 111, 9, 105, 73, 125, 94, 114, 2, 120, 3, 99, 35, 47, 67, 126, 121, 29, 60, 113, 51, 54, 61, 62, 34, 69, 93, 7, 30, 57, 92, 5, 71, 44, 115, 108, 88, 127, 24, 49, 27, 43, 58, 55, 106, 107, 96, 32, 36, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 17, 31, 87, 81, 23, 75, 11, 116, 104, 38, 15, 79, 28, 72, 21, 85, 74, 10, 0, 6, 40, 64, 70, 20, 18, 97, 16, 83, 19, 77, 82, 102, 90, 78, 84, 68, 25, 89, 13, 91, 76, 22, 4, 80, 42, 119, 125, 86, 12, 26, 14, 94, 65, 8, 37, 1, 98, 9, 101, 73, 66, 54, 105, 99, 120, 113, 47, 2, 121, 111, 61, 35, 71, 29, 3, 126, 7, 58, 62, 5, 69, 60, 92, 34, 51, 115, 114, 57, 93, 67, 106, 108, 24, 43, 127, 88, 44, 49, 27, 55, 30, 96, 36, 32, 107, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 56, 50, 122, 117, 48, 39, 123, 95, 33, 103, 41, 31, 87, 104, 11, 81, 79, 23, 75, 17, 38, 18, 15, 85, 6, 72, 21, 28, 97, 40, 102, 10, 83, 0, 74, 64, 119, 82, 16, 116, 91, 25, 70, 77, 89, 76, 22, 80, 4, 19, 84, 42, 13, 68, 20, 12, 121, 26, 86, 78, 90, 14, 8, 47, 99, 105, 125, 94, 65, 66, 73, 120, 9, 37, 98, 114, 61, 1, 2, 101, 35, 34, 60, 126, 113, 54, 111, 29, 51, 5, 3, 69, 57, 24, 71, 108, 115, 92, 62, 7, 106, 30, 67, 93, 127, 58, 49, 43, 44, 88, 27, 55, 107, 96, 32, 36, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 31, 41, 104, 87, 17, 38, 79, 75, 81, 85, 23, 15, 64, 116, 28, 11, 21, 0, 70, 18, 102, 83, 97, 89, 72, 40, 19, 82, 42, 26, 10, 80, 74, 91, 16, 6, 25, 77, 14, 84, 13, 20, 78, 4, 8, 22, 76, 119, 12, 68, 90, 1, 86, 37, 65, 66, 98, 47, 125, 73, 105, 94, 101, 9, 99, 60, 113, 3, 2, 126, 54, 111, 115, 57, 114, 61, 121, 35, 92, 7, 24, 69, 29, 120, 5, 67, 62, 51, 34, 49, 27, 108, 127, 93, 43, 71, 30, 58, 44, 106, 107, 88, 55, 32, 96, 100, 36], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 123, 117, 39, 95, 103, 33, 41, 87, 104, 31, 81, 11, 38, 23, 15, 17, 21, 28, 40, 85, 79, 0, 116, 75, 70, 74, 10, 16, 83, 97, 18, 22, 19, 64, 89, 82, 8, 84, 25, 77, 80, 6, 20, 42, 86, 119, 12, 91, 26, 72, 76, 13, 102, 4, 90, 78, 68, 37, 14, 66, 94, 65, 98, 1, 73, 125, 105, 101, 2, 47, 9, 35, 121, 111, 61, 120, 29, 113, 99, 114, 3, 54, 126, 51, 92, 60, 69, 34, 7, 24, 93, 30, 71, 58, 115, 27, 67, 57, 106, 5, 127, 62, 49, 108, 44, 43, 88, 107, 55, 96, 32, 100, 36], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 48, 56, 122, 117, 123, 39, 95, 33, 103, 41, 31, 87, 104, 11, 81, 23, 17, 79, 38, 85, 75, 74, 40, 15, 116, 64, 28, 97, 18, 70, 8, 83, 25, 22, 21, 102, 42, 91, 80, 16, 6, 82, 90, 119, 68, 76, 13, 77, 89, 10, 19, 12, 86, 84, 26, 20, 78, 72, 14, 4, 0, 98, 37, 101, 9, 111, 65, 66, 105, 94, 54, 126, 125, 120, 99, 47, 2, 1, 73, 61, 113, 35, 121, 51, 115, 127, 29, 60, 71, 34, 57, 55, 93, 114, 69, 3, 7, 62, 92, 5, 67, 108, 106, 24, 30, 27, 88, 58, 49, 44, 107, 43, 96, 100, 36, 32], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 122, 48, 117, 123, 39, 103, 95, 33, 41, 104, 81, 87, 31, 17, 11, 23, 116, 15, 38, 75, 70, 85, 21, 18, 40, 79, 28, 82, 74, 83, 22, 84, 8, 77, 119, 42, 16, 19, 25, 20, 97, 80, 0, 6, 89, 10, 26, 76, 64, 12, 91, 102, 14, 78, 90, 86, 68, 47, 13, 4, 72, 65, 1, 125, 101, 37, 94, 113, 126, 73, 98, 54, 111, 9, 35, 2, 61, 114, 66, 71, 29, 60, 57, 120, 127, 121, 105, 93, 99, 34, 108, 51, 3, 92, 7, 24, 62, 69, 67, 55, 30, 106, 115, 88, 5, 44, 58, 27, 49, 43, 107, 96, 32, 100, 36], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 50, 59, 56, 48, 123, 117, 122, 39, 95, 103, 33, 41, 87, 31, 81, 23, 104, 75, 15, 85, 11, 79, 17, 38, 28, 8, 40, 70, 74, 116, 82, 21, 97, 25, 22, 64, 18, 83, 42, 10, 89, 16, 119, 0, 102, 19, 20, 6, 14, 91, 13, 77, 80, 84, 76, 12, 90, 78, 26, 4, 68, 101, 86, 37, 111, 125, 1, 9, 98, 61, 66, 120, 72, 73, 105, 47, 65, 2, 94, 60, 57, 35, 113, 121, 51, 99, 126, 58, 92, 54, 7, 29, 106, 34, 115, 93, 62, 127, 67, 108, 114, 24, 5, 3, 49, 69, 27, 43, 55, 44, 30, 88, 71, 107, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 48, 117, 122, 123, 39, 103, 95, 33, 41, 87, 104, 31, 11, 81, 85, 23, 15, 97, 0, 17, 28, 75, 79, 21, 8, 40, 38, 82, 102, 70, 18, 25, 119, 74, 22, 116, 16, 83, 19, 20, 80, 13, 77, 91, 10, 6, 64, 12, 14, 42, 89, 4, 26, 84, 76, 68, 90, 78, 47, 86, 99, 94, 125, 101, 37, 105, 66, 65, 72, 1, 126, 98, 73, 60, 9, 57, 35, 113, 2, 111, 61, 114, 29, 54, 121, 92, 120, 3, 67, 7, 51, 127, 69, 93, 30, 24, 71, 5, 106, 34, 27, 62, 115, 49, 44, 108, 58, 88, 107, 43, 55, 96, 36, 32, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 104, 31, 11, 17, 87, 75, 81, 0, 8, 15, 23, 79, 64, 116, 74, 40, 85, 28, 6, 38, 42, 21, 18, 20, 10, 82, 83, 97, 70, 4, 80, 76, 77, 22, 25, 102, 16, 91, 13, 14, 78, 86, 19, 12, 89, 68, 1, 119, 26, 66, 84, 113, 125, 65, 90, 47, 98, 72, 120, 94, 9, 2, 101, 37, 105, 60, 111, 73, 54, 99, 121, 35, 126, 61, 51, 67, 114, 7, 71, 34, 57, 3, 29, 69, 5, 115, 92, 127, 44, 62, 93, 58, 88, 108, 49, 55, 30, 24, 27, 106, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 11, 31, 104, 87, 75, 81, 116, 38, 79, 8, 23, 28, 17, 6, 15, 21, 74, 0, 85, 40, 82, 80, 64, 97, 25, 10, 42, 83, 102, 18, 19, 70, 89, 76, 91, 22, 77, 14, 119, 68, 13, 84, 16, 20, 4, 86, 12, 78, 72, 26, 90, 101, 121, 2, 125, 1, 37, 73, 9, 105, 65, 47, 66, 114, 94, 113, 98, 120, 111, 35, 99, 51, 108, 54, 34, 115, 61, 71, 3, 60, 67, 7, 5, 92, 29, 69, 126, 57, 62, 93, 58, 24, 30, 127, 106, 55, 49, 88, 44, 27, 43, 107, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 122, 48, 56, 117, 123, 39, 95, 33, 103, 41, 104, 31, 15, 87, 11, 75, 23, 17, 81, 6, 79, 38, 0, 28, 8, 85, 10, 18, 64, 22, 21, 82, 40, 97, 116, 16, 83, 13, 74, 25, 102, 77, 80, 70, 19, 86, 42, 76, 4, 91, 68, 78, 20, 12, 84, 90, 14, 89, 72, 26, 47, 66, 60, 1, 119, 125, 101, 73, 94, 37, 105, 2, 65, 98, 9, 126, 35, 113, 114, 61, 111, 120, 99, 121, 69, 7, 51, 62, 3, 54, 57, 29, 71, 108, 67, 5, 34, 115, 58, 93, 92, 106, 127, 30, 49, 24, 27, 44, 107, 88, 43, 55, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 63, 50, 59, 56, 122, 48, 117, 123, 103, 39, 95, 33, 41, 87, 31, 38, 81, 17, 23, 104, 85, 75, 11, 15, 6, 28, 0, 18, 79, 40, 42, 97, 89, 21, 10, 82, 25, 64, 70, 80, 102, 86, 74, 22, 8, 14, 20, 19, 26, 90, 68, 76, 77, 83, 116, 78, 72, 16, 12, 4, 13, 91, 37, 101, 119, 65, 84, 9, 1, 125, 66, 98, 111, 113, 126, 105, 73, 94, 61, 35, 47, 120, 2, 54, 99, 121, 69, 67, 51, 7, 29, 34, 114, 115, 60, 71, 62, 57, 93, 49, 58, 92, 27, 5, 44, 3, 127, 108, 24, 30, 106, 88, 55, 43, 107, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 122, 123, 117, 39, 95, 103, 41, 33, 104, 31, 81, 87, 23, 11, 75, 15, 17, 85, 40, 79, 38, 6, 10, 42, 82, 74, 97, 25, 72, 0, 28, 20, 89, 64, 21, 102, 70, 116, 77, 18, 16, 19, 91, 80, 119, 76, 14, 125, 101, 83, 8, 13, 78, 26, 68, 22, 12, 84, 86, 47, 4, 90, 37, 111, 98, 54, 105, 126, 9, 66, 2, 121, 1, 94, 61, 65, 73, 120, 114, 99, 60, 35, 115, 29, 71, 113, 57, 62, 51, 108, 58, 67, 3, 93, 5, 7, 34, 92, 24, 69, 44, 106, 127, 88, 30, 27, 49, 43, 96, 55, 107, 32, 36, 100]], "model.layers.5.self_attn.q_proj": [[39, 106, 98, 50, 20, 80, 42, 23, 127, 116, 14, 49, 18, 10, 124, 51, 27, 47, 74, 61, 120, 103, 54, 12, 24, 112, 87, 57, 84, 71, 25, 115, 68, 48, 63, 29, 111, 123, 16, 96, 15, 26, 76, 113, 85, 82, 83, 9, 79, 88, 125, 52, 118, 92, 114, 28, 121, 91, 122, 21, 78, 93, 41, 62, 94, 99, 8, 59, 75, 86, 58, 6, 89, 81, 126, 30, 44, 22, 90, 97, 35, 11, 95, 60, 31, 2, 45, 104, 109, 110, 119, 36, 108, 70, 117, 102, 107, 19, 33, 40, 105, 56, 38, 32, 46, 13, 53, 3, 17, 77, 37, 43, 100, 101, 55, 7, 69, 34, 73, 4, 72, 5, 1, 65, 0, 66, 67, 64], [39, 98, 106, 49, 47, 51, 127, 57, 54, 23, 48, 27, 120, 116, 44, 63, 124, 50, 111, 92, 59, 123, 126, 60, 28, 46, 121, 105, 61, 41, 42, 52, 55, 112, 115, 43, 107, 118, 58, 125, 108, 53, 36, 119, 22, 110, 62, 117, 18, 102, 45, 113, 32, 40, 82, 104, 109, 96, 100, 21, 38, 35, 114, 56, 101, 122, 99, 30, 37, 25, 80, 97, 93, 33, 95, 31, 85, 89, 29, 87, 83, 78, 94, 91, 90, 14, 26, 34, 10, 79, 84, 20, 24, 19, 88, 16, 17, 103, 86, 12, 76, 81, 4, 74, 75, 77, 73, 15, 7, 11, 67, 13, 64, 70, 69, 72, 8, 71, 9, 3, 66, 68, 5, 6, 0, 65, 1, 2], [39, 106, 98, 50, 18, 74, 20, 88, 80, 14, 12, 23, 42, 68, 103, 116, 49, 6, 47, 51, 10, 9, 54, 4, 2, 7, 91, 21, 44, 83, 61, 121, 75, 64, 124, 127, 57, 70, 120, 71, 115, 87, 72, 92, 125, 16, 73, 13, 15, 11, 48, 28, 3, 24, 76, 17, 52, 22, 78, 67, 84, 89, 111, 63, 85, 123, 82, 112, 77, 59, 126, 86, 5, 8, 58, 60, 30, 81, 79, 66, 27, 113, 25, 19, 0, 1, 26, 118, 69, 110, 33, 114, 38, 94, 105, 122, 65, 95, 32, 31, 29, 41, 97, 55, 36, 102, 109, 93, 96, 101, 119, 46, 90, 53, 99, 117, 100, 108, 43, 45, 35, 34, 37, 40, 104, 62, 107, 56], [106, 39, 50, 98, 18, 12, 80, 20, 6, 23, 14, 9, 2, 74, 42, 47, 49, 116, 103, 64, 70, 28, 127, 87, 54, 51, 57, 91, 7, 71, 124, 68, 120, 61, 75, 115, 60, 3, 10, 4, 76, 24, 78, 67, 121, 27, 48, 26, 69, 13, 1, 11, 118, 126, 63, 89, 125, 84, 19, 17, 44, 123, 82, 16, 21, 66, 111, 15, 59, 83, 112, 119, 73, 65, 22, 8, 55, 52, 5, 77, 58, 41, 0, 62, 72, 105, 46, 81, 88, 85, 110, 108, 53, 30, 107, 33, 113, 109, 40, 114, 79, 104, 101, 29, 38, 43, 35, 99, 45, 117, 96, 122, 25, 36, 100, 102, 97, 95, 93, 56, 31, 86, 94, 90, 92, 32, 37, 34], [38, 110, 126, 125, 115, 48, 72, 112, 7, 23, 5, 4, 74, 6, 76, 70, 67, 29, 20, 11, 16, 13, 46, 51, 90, 81, 75, 12, 93, 9, 78, 26, 83, 18, 73, 66, 14, 80, 68, 65, 10, 84, 2, 69, 62, 59, 117, 17, 21, 19, 79, 122, 123, 15, 121, 56, 113, 116, 120, 55, 50, 8, 105, 91, 31, 64, 63, 124, 85, 114, 95, 87, 118, 49, 61, 53, 47, 111, 32, 58, 104, 89, 109, 27, 54, 88, 119, 71, 40, 127, 41, 44, 77, 82, 96, 45, 52, 60, 35, 108, 106, 107, 43, 42, 37, 57, 94, 28, 101, 103, 24, 1, 100, 36, 22, 86, 98, 39, 34, 3, 99, 97, 33, 25, 92, 30, 102, 0], [110, 38, 126, 112, 48, 46, 115, 125, 51, 62, 17, 116, 122, 59, 123, 56, 16, 105, 50, 121, 117, 14, 124, 84, 120, 104, 85, 32, 113, 49, 26, 55, 114, 61, 13, 119, 63, 53, 102, 118, 47, 31, 111, 41, 58, 88, 109, 27, 107, 45, 29, 127, 54, 44, 60, 40, 15, 108, 93, 52, 11, 42, 74, 106, 100, 24, 43, 57, 37, 91, 96, 103, 87, 19, 36, 90, 99, 101, 39, 35, 76, 33, 98, 34, 82, 97, 28, 95, 81, 94, 9, 89, 22, 30, 21, 92, 20, 25, 23, 72, 18, 8, 86, 71, 83, 75, 80, 6, 77, 10, 79, 78, 7, 0, 68, 5, 69, 66, 3, 12, 1, 65, 73, 70, 64, 2, 4, 67], [38, 110, 48, 112, 125, 115, 126, 46, 93, 51, 78, 83, 26, 21, 11, 72, 88, 122, 32, 123, 10, 31, 7, 124, 62, 116, 79, 77, 81, 50, 95, 15, 74, 56, 5, 29, 41, 85, 75, 61, 84, 59, 121, 18, 55, 19, 86, 90, 73, 117, 17, 106, 53, 113, 27, 120, 63, 91, 40, 80, 70, 111, 127, 47, 58, 14, 6, 92, 96, 49, 109, 104, 60, 105, 114, 4, 25, 44, 23, 45, 119, 24, 118, 97, 9, 30, 107, 28, 99, 89, 37, 108, 13, 67, 35, 43, 94, 22, 39, 68, 52, 34, 100, 101, 20, 54, 87, 103, 33, 66, 98, 16, 36, 42, 76, 57, 65, 12, 82, 8, 102, 64, 71, 3, 69, 2, 0, 1], [38, 110, 112, 115, 48, 46, 125, 51, 26, 31, 27, 122, 62, 88, 50, 102, 93, 21, 116, 117, 59, 56, 123, 23, 124, 126, 120, 104, 105, 106, 32, 41, 83, 53, 40, 61, 58, 86, 101, 111, 17, 29, 100, 113, 121, 47, 60, 49, 63, 95, 78, 33, 45, 119, 35, 109, 114, 16, 22, 96, 91, 74, 54, 99, 55, 84, 36, 37, 85, 30, 52, 28, 89, 94, 43, 107, 11, 42, 108, 118, 87, 44, 34, 14, 103, 18, 72, 98, 90, 80, 127, 57, 39, 97, 92, 13, 20, 7, 6, 79, 24, 25, 9, 19, 5, 76, 75, 15, 66, 68, 10, 82, 65, 77, 12, 81, 3, 73, 1, 0, 71, 67, 70, 64, 8, 4, 2, 69], [113, 121, 86, 61, 125, 49, 57, 59, 85, 116, 52, 60, 56, 122, 22, 62, 55, 117, 53, 124, 114, 63, 127, 54, 115, 119, 51, 123, 18, 58, 118, 50, 112, 120, 110, 111, 126, 109, 48, 46, 45, 47, 44, 108, 107, 43, 106, 23, 42, 105, 41, 40, 90, 104, 39, 35, 36, 38, 94, 95, 102, 103, 87, 34, 37, 15, 96, 100, 82, 97, 98, 64, 99, 1, 30, 31, 101, 21, 91, 66, 33, 16, 28, 3, 84, 25, 93, 65, 88, 12, 92, 80, 0, 77, 14, 69, 26, 78, 13, 89, 74, 4, 75, 24, 73, 71, 29, 2, 67, 20, 83, 8, 6, 19, 17, 68, 27, 79, 9, 10, 32, 11, 5, 7, 70, 76, 81, 72], [121, 113, 61, 125, 49, 22, 57, 59, 116, 52, 56, 62, 122, 55, 60, 117, 63, 114, 53, 127, 124, 119, 115, 51, 58, 54, 123, 118, 50, 112, 120, 110, 111, 126, 25, 48, 109, 46, 90, 47, 45, 44, 64, 19, 108, 107, 106, 30, 43, 66, 3, 69, 65, 96, 105, 42, 95, 38, 41, 26, 92, 40, 39, 104, 34, 27, 4, 94, 102, 1, 100, 36, 103, 71, 35, 97, 23, 98, 9, 67, 29, 87, 99, 33, 70, 86, 32, 24, 6, 15, 2, 31, 37, 0, 101, 84, 28, 18, 93, 85, 68, 13, 8, 88, 10, 80, 91, 83, 5, 77, 76, 12, 89, 82, 11, 78, 17, 16, 79, 75, 81, 7, 20, 14, 72, 21, 73, 74], [113, 121, 61, 125, 49, 57, 86, 22, 59, 116, 52, 60, 56, 122, 117, 62, 55, 53, 63, 114, 124, 54, 119, 51, 127, 123, 58, 115, 118, 120, 21, 50, 110, 112, 111, 126, 109, 46, 48, 45, 47, 44, 83, 108, 107, 106, 87, 43, 92, 42, 105, 30, 25, 41, 23, 89, 96, 40, 39, 104, 38, 90, 20, 34, 88, 94, 36, 103, 102, 64, 35, 100, 99, 18, 24, 97, 66, 26, 19, 1, 27, 93, 95, 65, 3, 98, 69, 33, 31, 37, 28, 0, 101, 77, 4, 10, 32, 11, 29, 85, 71, 76, 67, 13, 9, 2, 81, 8, 12, 6, 68, 91, 14, 82, 80, 84, 72, 15, 70, 78, 79, 5, 75, 16, 17, 73, 74, 7], [37, 61, 125, 49, 121, 57, 113, 59, 22, 96, 62, 55, 122, 32, 52, 116, 20, 60, 127, 56, 91, 115, 88, 124, 83, 117, 30, 77, 28, 54, 119, 53, 123, 110, 51, 63, 29, 114, 58, 112, 45, 92, 94, 50, 118, 9, 46, 120, 44, 111, 47, 109, 48, 79, 126, 69, 11, 71, 89, 76, 18, 26, 108, 93, 105, 43, 66, 87, 107, 33, 41, 36, 106, 81, 64, 39, 8, 3, 82, 42, 6, 104, 34, 99, 40, 10, 65, 23, 90, 4, 27, 17, 78, 74, 24, 98, 97, 103, 102, 19, 101, 100, 84, 31, 1, 25, 38, 73, 14, 80, 35, 75, 95, 12, 68, 13, 16, 7, 15, 85, 70, 67, 21, 2, 72, 86, 0, 5], [38, 44, 115, 90, 51, 22, 84, 108, 17, 72, 79, 81, 74, 18, 29, 126, 8, 12, 127, 16, 10, 78, 33, 82, 28, 19, 89, 112, 123, 120, 36, 4, 116, 5, 124, 122, 13, 125, 71, 60, 76, 98, 103, 35, 39, 110, 49, 107, 50, 91, 62, 47, 52, 15, 67, 97, 111, 80, 42, 45, 121, 73, 118, 113, 119, 114, 105, 20, 25, 109, 63, 55, 59, 53, 96, 86, 58, 23, 41, 30, 54, 87, 61, 106, 6, 46, 40, 37, 43, 101, 117, 99, 48, 26, 31, 34, 70, 14, 100, 94, 57, 95, 32, 24, 104, 68, 21, 92, 9, 77, 65, 56, 85, 27, 88, 93, 2, 75, 11, 0, 102, 83, 7, 1, 3, 69, 66, 64], [38, 44, 115, 69, 71, 51, 79, 12, 17, 22, 18, 108, 84, 74, 81, 8, 78, 90, 2, 64, 3, 33, 5, 66, 102, 7, 68, 65, 120, 126, 1, 10, 49, 15, 67, 14, 57, 112, 70, 76, 95, 59, 125, 82, 20, 52, 23, 88, 9, 73, 19, 75, 26, 83, 56, 4, 16, 0, 13, 11, 87, 127, 119, 86, 92, 29, 98, 89, 116, 114, 124, 80, 72, 37, 6, 93, 58, 25, 61, 28, 91, 36, 110, 77, 104, 99, 41, 45, 27, 117, 35, 122, 106, 63, 21, 34, 96, 30, 85, 118, 121, 31, 62, 123, 100, 94, 24, 111, 48, 50, 60, 101, 32, 42, 39, 105, 97, 46, 40, 103, 109, 47, 55, 54, 53, 43, 113, 107], [38, 44, 115, 74, 18, 84, 79, 12, 51, 108, 69, 22, 71, 3, 8, 78, 2, 64, 70, 102, 33, 1, 17, 67, 90, 68, 76, 81, 6, 7, 15, 66, 120, 16, 28, 4, 112, 23, 82, 10, 52, 59, 126, 91, 65, 86, 88, 49, 98, 26, 73, 127, 125, 72, 92, 95, 5, 57, 56, 89, 41, 85, 83, 9, 24, 118, 114, 117, 35, 0, 13, 20, 14, 58, 46, 29, 93, 19, 25, 116, 45, 75, 123, 80, 105, 50, 21, 111, 62, 11, 31, 110, 37, 124, 40, 63, 30, 32, 42, 60, 107, 119, 104, 113, 36, 47, 94, 109, 101, 77, 48, 39, 99, 122, 61, 87, 103, 53, 121, 34, 106, 55, 100, 43, 96, 97, 54, 27], [38, 44, 115, 84, 51, 22, 18, 108, 12, 17, 8, 79, 78, 74, 16, 72, 33, 19, 70, 89, 67, 5, 68, 10, 4, 81, 24, 126, 120, 90, 71, 65, 127, 2, 92, 112, 76, 49, 82, 6, 73, 69, 125, 28, 7, 75, 13, 60, 14, 3, 80, 29, 116, 23, 25, 114, 86, 98, 35, 56, 9, 50, 57, 117, 64, 107, 20, 123, 0, 58, 52, 32, 36, 124, 103, 41, 87, 93, 88, 91, 21, 26, 15, 1, 102, 39, 96, 37, 118, 43, 11, 95, 53, 83, 30, 63, 111, 100, 110, 31, 61, 62, 101, 106, 85, 40, 104, 47, 94, 113, 55, 77, 27, 109, 45, 97, 121, 59, 46, 34, 119, 99, 48, 122, 105, 54, 42, 66], [38, 44, 34, 119, 118, 53, 56, 81, 30, 78, 23, 84, 126, 47, 11, 25, 71, 122, 5, 8, 12, 52, 79, 9, 82, 87, 93, 54, 27, 29, 61, 116, 20, 50, 19, 94, 90, 125, 86, 13, 14, 66, 55, 17, 67, 7, 75, 15, 18, 127, 28, 10, 106, 16, 51, 70, 89, 101, 4, 22, 62, 46, 77, 88, 21, 121, 26, 68, 85, 69, 102, 80, 48, 73, 3, 74, 2, 96, 49, 97, 63, 105, 1, 58, 98, 42, 0, 91, 114, 24, 83, 110, 35, 59, 72, 41, 92, 113, 76, 64, 32, 117, 33, 40, 65, 99, 36, 95, 39, 43, 123, 109, 37, 31, 103, 6, 100, 108, 120, 124, 104, 115, 60, 107, 57, 112, 111, 45], [44, 53, 38, 118, 47, 34, 27, 101, 119, 126, 116, 22, 122, 54, 58, 94, 102, 30, 79, 56, 82, 46, 114, 84, 108, 59, 41, 125, 23, 106, 57, 62, 55, 93, 103, 52, 110, 63, 26, 113, 90, 50, 109, 29, 123, 61, 115, 60, 42, 124, 51, 48, 117, 31, 40, 105, 49, 121, 43, 86, 104, 127, 112, 33, 45, 9, 120, 107, 100, 80, 98, 39, 18, 37, 24, 99, 36, 92, 111, 25, 91, 73, 96, 88, 83, 28, 3, 13, 32, 35, 76, 21, 95, 85, 97, 69, 81, 72, 71, 89, 11, 20, 14, 74, 16, 75, 19, 15, 70, 7, 12, 10, 87, 6, 77, 68, 67, 0, 65, 17, 8, 2, 4, 78, 66, 64, 5, 1], [44, 38, 119, 56, 118, 53, 34, 58, 27, 84, 30, 94, 42, 79, 101, 25, 122, 125, 21, 116, 114, 82, 41, 23, 26, 50, 9, 127, 18, 62, 117, 47, 24, 54, 93, 113, 59, 52, 92, 51, 46, 105, 99, 57, 108, 55, 86, 96, 126, 48, 61, 60, 81, 33, 29, 28, 123, 103, 39, 90, 124, 31, 110, 22, 89, 120, 88, 112, 87, 12, 106, 49, 15, 91, 102, 121, 63, 109, 13, 71, 45, 104, 40, 11, 35, 115, 83, 97, 19, 85, 100, 95, 77, 75, 36, 37, 20, 16, 111, 107, 43, 32, 80, 78, 73, 98, 3, 17, 7, 70, 10, 67, 74, 76, 0, 5, 6, 14, 2, 66, 4, 64, 69, 8, 68, 72, 65, 1], [38, 53, 118, 119, 56, 34, 12, 44, 88, 23, 25, 84, 30, 47, 27, 81, 108, 6, 8, 126, 94, 1, 18, 89, 9, 86, 87, 79, 19, 33, 67, 21, 112, 54, 15, 58, 20, 52, 105, 65, 82, 41, 42, 55, 125, 46, 16, 117, 114, 101, 102, 77, 123, 116, 11, 48, 92, 122, 31, 26, 72, 62, 80, 57, 120, 90, 39, 70, 76, 115, 83, 110, 63, 61, 50, 32, 78, 97, 37, 106, 75, 85, 29, 124, 113, 45, 95, 59, 51, 5, 103, 100, 35, 2, 104, 0, 127, 17, 43, 74, 14, 22, 28, 107, 121, 64, 60, 7, 96, 109, 40, 13, 69, 111, 49, 93, 99, 36, 68, 10, 4, 91, 24, 73, 71, 66, 3, 98], [39, 51, 50, 114, 97, 115, 113, 54, 117, 85, 124, 87, 120, 121, 58, 29, 63, 27, 24, 55, 61, 122, 126, 91, 111, 75, 92, 108, 25, 32, 112, 83, 57, 33, 26, 93, 53, 22, 56, 20, 59, 116, 79, 82, 73, 18, 17, 60, 88, 23, 49, 69, 21, 81, 14, 99, 48, 119, 77, 95, 34, 90, 80, 109, 16, 38, 40, 72, 41, 28, 30, 127, 107, 84, 7, 94, 78, 71, 52, 123, 15, 43, 76, 89, 100, 118, 110, 45, 125, 47, 44, 42, 86, 96, 104, 36, 98, 37, 106, 46, 102, 101, 31, 35, 105, 9, 12, 62, 11, 13, 74, 3, 5, 67, 70, 19, 8, 6, 66, 2, 4, 1, 0, 68, 10, 103, 64, 65], [51, 39, 114, 113, 50, 121, 97, 116, 61, 53, 59, 124, 60, 54, 122, 29, 120, 118, 87, 52, 105, 45, 125, 24, 56, 115, 112, 86, 63, 123, 103, 58, 26, 88, 27, 119, 109, 83, 25, 110, 107, 95, 62, 46, 85, 117, 40, 111, 92, 55, 33, 57, 44, 18, 82, 43, 127, 94, 126, 49, 98, 108, 47, 93, 106, 20, 42, 90, 41, 34, 35, 75, 23, 102, 101, 96, 38, 48, 15, 100, 89, 36, 14, 17, 37, 99, 104, 91, 19, 80, 73, 31, 32, 77, 28, 84, 12, 21, 22, 81, 30, 16, 72, 74, 70, 69, 78, 11, 0, 13, 68, 67, 64, 66, 79, 76, 1, 3, 71, 9, 6, 2, 5, 65, 8, 7, 10, 4], [39, 114, 51, 50, 97, 121, 122, 11, 9, 21, 87, 29, 36, 116, 83, 34, 54, 12, 30, 98, 14, 92, 6, 48, 19, 109, 110, 8, 119, 10, 62, 94, 53, 5, 113, 126, 35, 63, 15, 42, 13, 28, 55, 71, 47, 59, 67, 127, 57, 76, 111, 108, 58, 117, 70, 101, 43, 2, 16, 91, 79, 118, 93, 115, 68, 40, 125, 100, 4, 120, 52, 32, 60, 45, 106, 73, 102, 74, 112, 123, 56, 23, 105, 41, 46, 49, 95, 38, 89, 72, 96, 81, 44, 65, 90, 104, 99, 37, 22, 61, 107, 82, 0, 84, 18, 124, 86, 26, 85, 20, 75, 27, 78, 31, 17, 25, 88, 103, 33, 24, 80, 77, 1, 7, 69, 3, 64, 66], [39, 51, 114, 50, 97, 113, 54, 122, 121, 85, 24, 61, 87, 63, 92, 29, 48, 116, 124, 83, 95, 25, 117, 57, 27, 34, 26, 80, 126, 40, 82, 28, 111, 120, 20, 119, 33, 53, 110, 58, 91, 60, 77, 14, 17, 75, 108, 32, 15, 55, 47, 56, 73, 42, 21, 115, 89, 59, 23, 62, 88, 125, 107, 127, 19, 36, 98, 86, 74, 104, 49, 94, 81, 90, 52, 118, 18, 41, 38, 78, 101, 79, 105, 106, 22, 84, 72, 30, 43, 123, 35, 112, 13, 12, 44, 37, 76, 11, 102, 109, 93, 100, 70, 45, 99, 69, 16, 96, 31, 46, 67, 7, 68, 71, 9, 0, 10, 8, 1, 6, 2, 65, 3, 4, 5, 66, 64, 103], [108, 124, 102, 122, 62, 34, 107, 23, 30, 60, 50, 35, 119, 104, 25, 120, 45, 31, 123, 114, 121, 49, 117, 61, 103, 87, 40, 106, 47, 29, 55, 44, 110, 22, 20, 28, 32, 86, 118, 52, 38, 58, 59, 41, 33, 63, 53, 92, 115, 113, 48, 88, 43, 54, 126, 96, 105, 89, 46, 99, 56, 26, 57, 111, 51, 42, 91, 116, 112, 125, 127, 109, 101, 97, 94, 37, 79, 100, 77, 21, 39, 93, 84, 95, 27, 36, 90, 24, 11, 16, 17, 13, 83, 69, 81, 18, 72, 19, 75, 8, 15, 98, 78, 5, 10, 14, 74, 3, 65, 7, 64, 0, 67, 66, 68, 85, 1, 71, 4, 12, 80, 82, 2, 73, 9, 6, 76, 70], [102, 124, 34, 108, 62, 122, 107, 23, 17, 93, 25, 21, 11, 79, 104, 77, 18, 16, 120, 30, 69, 15, 114, 96, 29, 22, 32, 10, 83, 19, 85, 126, 89, 54, 5, 20, 117, 26, 49, 40, 31, 50, 92, 87, 127, 123, 110, 73, 2, 28, 90, 86, 119, 60, 1, 81, 51, 118, 103, 24, 91, 43, 109, 35, 41, 27, 47, 44, 84, 45, 38, 36, 116, 8, 57, 74, 78, 37, 125, 100, 13, 82, 112, 63, 94, 71, 68, 53, 67, 115, 97, 106, 0, 88, 75, 14, 61, 99, 55, 101, 105, 64, 111, 58, 42, 95, 33, 39, 46, 113, 121, 56, 72, 59, 52, 65, 4, 3, 7, 80, 48, 66, 12, 6, 9, 76, 98, 70], [102, 62, 122, 34, 124, 21, 93, 18, 108, 16, 40, 23, 12, 29, 83, 90, 77, 25, 73, 85, 11, 92, 61, 76, 126, 96, 100, 8, 24, 6, 26, 44, 27, 28, 116, 49, 82, 70, 107, 22, 119, 20, 32, 120, 84, 112, 78, 5, 42, 19, 57, 53, 58, 7, 46, 71, 17, 48, 36, 79, 106, 69, 80, 95, 123, 127, 67, 88, 4, 59, 114, 75, 109, 65, 103, 2, 43, 33, 63, 50, 0, 104, 13, 99, 9, 68, 86, 110, 14, 15, 45, 97, 66, 47, 41, 113, 117, 87, 105, 10, 89, 3, 55, 72, 37, 54, 98, 51, 91, 52, 111, 1, 64, 74, 30, 94, 81, 56, 31, 121, 115, 101, 118, 35, 60, 38, 39, 125], [62, 102, 122, 108, 34, 124, 26, 120, 23, 51, 35, 123, 30, 90, 40, 61, 21, 95, 24, 29, 96, 75, 110, 48, 93, 60, 16, 54, 5, 127, 37, 45, 114, 22, 20, 39, 50, 72, 31, 18, 94, 58, 79, 107, 49, 104, 8, 36, 32, 89, 118, 69, 44, 77, 117, 86, 59, 112, 125, 25, 103, 116, 11, 57, 106, 121, 111, 87, 56, 92, 52, 97, 55, 126, 53, 83, 76, 27, 119, 101, 91, 47, 46, 28, 41, 84, 113, 63, 109, 33, 100, 88, 74, 115, 82, 105, 3, 17, 15, 42, 98, 85, 13, 81, 73, 99, 10, 43, 38, 70, 14, 12, 71, 19, 80, 9, 0, 78, 2, 1, 68, 7, 4, 66, 67, 65, 64, 6], [38, 97, 53, 117, 21, 81, 80, 75, 14, 71, 4, 87, 76, 9, 6, 66, 13, 1, 83, 24, 123, 70, 74, 68, 118, 2, 85, 19, 12, 16, 90, 72, 0, 65, 10, 73, 94, 113, 11, 46, 18, 17, 88, 3, 92, 112, 79, 106, 64, 103, 124, 119, 86, 77, 84, 82, 25, 122, 78, 69, 7, 55, 93, 27, 111, 23, 5, 115, 42, 96, 91, 89, 15, 51, 28, 41, 62, 29, 35, 30, 120, 39, 109, 22, 56, 57, 98, 26, 54, 61, 99, 121, 50, 47, 67, 48, 36, 100, 31, 59, 52, 107, 45, 125, 20, 108, 40, 58, 104, 60, 110, 101, 44, 95, 116, 127, 114, 49, 105, 43, 32, 34, 37, 63, 126, 8, 33, 102], [38, 53, 97, 117, 87, 80, 21, 75, 76, 81, 6, 14, 70, 74, 4, 71, 9, 83, 68, 123, 24, 13, 118, 2, 1, 113, 19, 94, 66, 46, 72, 73, 55, 11, 103, 92, 119, 106, 17, 85, 16, 89, 93, 23, 12, 45, 0, 88, 96, 122, 79, 77, 112, 10, 78, 29, 20, 42, 51, 91, 111, 82, 69, 64, 84, 62, 52, 41, 86, 124, 35, 15, 28, 33, 8, 65, 7, 25, 40, 60, 18, 26, 3, 36, 105, 104, 50, 90, 120, 27, 48, 121, 107, 101, 109, 30, 116, 47, 56, 5, 108, 54, 110, 63, 99, 95, 57, 125, 100, 115, 39, 59, 98, 127, 58, 67, 114, 61, 34, 31, 22, 49, 44, 37, 43, 32, 126, 102], [38, 53, 97, 117, 21, 87, 81, 76, 75, 80, 123, 14, 4, 66, 70, 6, 118, 9, 74, 24, 19, 90, 12, 7, 69, 16, 46, 92, 71, 51, 65, 82, 112, 78, 64, 103, 56, 84, 30, 106, 23, 122, 25, 110, 15, 11, 98, 119, 121, 94, 17, 113, 47, 3, 85, 83, 1, 26, 31, 125, 35, 41, 39, 111, 88, 57, 91, 115, 73, 60, 49, 42, 93, 86, 52, 37, 18, 54, 27, 50, 29, 107, 34, 124, 58, 101, 22, 44, 59, 13, 10, 20, 126, 77, 61, 96, 36, 109, 120, 105, 79, 89, 40, 116, 63, 55, 99, 95, 28, 43, 127, 108, 104, 62, 48, 2, 45, 114, 100, 32, 67, 0, 5, 72, 8, 102, 68, 33], [38, 53, 97, 117, 87, 80, 6, 76, 21, 81, 14, 75, 74, 9, 1, 19, 3, 113, 68, 8, 71, 123, 7, 5, 82, 118, 2, 72, 85, 65, 46, 18, 70, 24, 23, 122, 91, 55, 41, 83, 35, 119, 78, 79, 10, 88, 67, 105, 12, 40, 112, 33, 106, 94, 111, 28, 15, 16, 66, 69, 84, 109, 27, 64, 45, 107, 36, 50, 86, 124, 61, 13, 90, 121, 103, 93, 26, 56, 98, 120, 77, 47, 30, 60, 115, 114, 116, 96, 25, 92, 59, 95, 42, 51, 73, 20, 104, 48, 58, 89, 0, 126, 29, 52, 17, 108, 57, 44, 110, 49, 125, 37, 22, 99, 34, 43, 62, 31, 54, 63, 127, 11, 101, 32, 39, 100, 102, 4]], "model.layers.5.self_attn.k_proj": [[42, 103, 50, 34, 12, 9, 80, 23, 20, 14, 18, 114, 7, 51, 66, 111, 6, 4, 74, 52, 64, 113, 79, 49, 126, 54, 8, 61, 127, 57, 67, 112, 116, 92, 27, 65, 108, 73, 0, 88, 121, 58, 1, 21, 124, 5, 77, 120, 59, 28, 125, 26, 86, 123, 69, 46, 62, 98, 106, 38, 81, 60, 94, 105, 91, 99, 55, 47, 44, 85, 93, 22, 45, 71, 24, 70, 41, 102, 122, 109, 37, 48, 117, 119, 90, 40, 63, 118, 11, 97, 53, 43, 25, 17, 95, 115, 104, 89, 56, 32, 3, 96, 110, 30, 101, 19, 100, 35, 107, 75, 16, 72, 29, 33, 83, 13, 31, 36, 10, 84, 15, 76, 82, 87, 78, 2, 68, 39], [46, 102, 115, 112, 125, 93, 126, 23, 26, 83, 9, 79, 80, 21, 76, 81, 20, 6, 68, 24, 3, 12, 122, 15, 13, 71, 124, 50, 77, 8, 41, 66, 116, 5, 106, 91, 123, 117, 113, 31, 16, 56, 55, 62, 58, 82, 69, 53, 78, 105, 63, 104, 95, 127, 74, 108, 111, 44, 45, 109, 118, 0, 47, 51, 96, 121, 107, 52, 40, 29, 59, 48, 60, 119, 114, 75, 33, 1, 120, 72, 49, 61, 43, 37, 19, 11, 54, 42, 36, 17, 10, 22, 18, 85, 39, 35, 110, 101, 34, 97, 99, 57, 32, 100, 88, 103, 98, 65, 84, 30, 94, 14, 25, 86, 27, 89, 28, 7, 92, 73, 87, 90, 2, 70, 38, 64, 67, 4], [61, 121, 125, 101, 57, 49, 113, 22, 32, 127, 62, 115, 55, 56, 122, 117, 17, 123, 52, 84, 51, 60, 58, 99, 59, 124, 63, 119, 54, 120, 116, 93, 114, 126, 50, 112, 88, 118, 53, 30, 48, 91, 110, 109, 47, 111, 46, 45, 43, 44, 92, 24, 95, 14, 12, 102, 35, 108, 79, 42, 82, 18, 107, 39, 10, 106, 83, 40, 41, 77, 100, 38, 37, 94, 21, 20, 97, 105, 16, 87, 104, 103, 11, 85, 36, 78, 72, 89, 80, 25, 98, 9, 33, 96, 29, 34, 31, 75, 86, 15, 90, 71, 13, 28, 74, 70, 19, 4, 23, 8, 26, 81, 66, 64, 27, 69, 3, 65, 73, 5, 6, 7, 1, 0, 76, 67, 68, 2], [108, 115, 102, 12, 22, 78, 84, 74, 18, 17, 79, 44, 69, 64, 71, 8, 68, 66, 90, 3, 75, 97, 120, 73, 2, 65, 48, 116, 1, 6, 16, 126, 24, 49, 70, 29, 67, 125, 113, 51, 93, 57, 59, 114, 19, 89, 88, 9, 21, 117, 11, 85, 124, 14, 23, 92, 58, 7, 91, 83, 0, 80, 25, 81, 62, 107, 39, 118, 30, 63, 77, 27, 55, 87, 95, 53, 10, 36, 127, 56, 99, 26, 32, 34, 31, 4, 119, 112, 41, 105, 28, 103, 43, 94, 60, 45, 52, 61, 13, 121, 104, 106, 40, 37, 100, 42, 54, 96, 123, 122, 82, 98, 110, 47, 35, 46, 101, 111, 109, 50, 33, 76, 86, 72, 38, 20, 15, 5], [102, 119, 118, 56, 53, 98, 94, 27, 108, 23, 84, 81, 25, 79, 64, 12, 9, 66, 117, 58, 5, 78, 57, 54, 37, 109, 125, 42, 67, 113, 8, 82, 114, 104, 55, 52, 105, 11, 39, 126, 24, 115, 85, 71, 62, 124, 123, 46, 110, 45, 50, 61, 19, 122, 48, 63, 121, 38, 1, 107, 6, 13, 22, 127, 112, 120, 41, 80, 4, 51, 43, 49, 60, 35, 59, 47, 74, 33, 111, 40, 18, 69, 65, 99, 90, 103, 31, 10, 116, 3, 29, 36, 97, 83, 21, 44, 68, 77, 100, 106, 96, 92, 70, 86, 32, 75, 93, 0, 30, 91, 7, 95, 88, 87, 16, 28, 14, 26, 89, 76, 72, 2, 101, 17, 20, 15, 34, 73], [103, 114, 51, 33, 31, 87, 85, 79, 49, 115, 83, 77, 27, 46, 17, 24, 7, 122, 76, 78, 25, 18, 108, 29, 4, 10, 84, 98, 121, 75, 57, 6, 3, 8, 74, 86, 30, 62, 82, 55, 48, 42, 65, 113, 69, 54, 119, 111, 5, 110, 99, 109, 127, 64, 101, 45, 89, 104, 63, 9, 112, 126, 56, 91, 123, 2, 38, 93, 58, 47, 92, 100, 52, 44, 13, 66, 125, 43, 118, 72, 94, 73, 41, 102, 40, 61, 35, 106, 23, 50, 96, 105, 116, 117, 26, 59, 12, 32, 81, 120, 36, 53, 67, 15, 22, 97, 60, 37, 107, 21, 95, 34, 88, 124, 28, 16, 80, 19, 14, 20, 90, 71, 70, 11, 39, 1, 0, 68], [38, 122, 62, 29, 98, 124, 44, 16, 18, 104, 78, 21, 22, 20, 25, 79, 73, 12, 85, 19, 11, 71, 23, 24, 6, 32, 108, 90, 77, 66, 14, 81, 82, 43, 64, 92, 17, 126, 26, 46, 68, 58, 30, 48, 40, 8, 27, 9, 117, 116, 49, 65, 57, 109, 114, 119, 102, 76, 34, 61, 60, 7, 74, 51, 103, 113, 41, 110, 91, 121, 55, 93, 101, 56, 39, 63, 10, 36, 33, 50, 59, 97, 28, 4, 105, 112, 45, 15, 99, 106, 13, 100, 125, 95, 87, 84, 75, 42, 52, 54, 96, 31, 86, 3, 53, 83, 118, 88, 120, 111, 5, 72, 115, 107, 94, 47, 123, 37, 127, 80, 35, 0, 67, 89, 1, 69, 2, 70], [117, 53, 102, 33, 14, 21, 80, 87, 81, 74, 76, 0, 75, 9, 68, 6, 8, 71, 2, 83, 65, 3, 5, 55, 24, 78, 7, 82, 1, 10, 73, 67, 42, 77, 38, 118, 123, 19, 113, 27, 84, 111, 66, 72, 110, 11, 69, 62, 91, 70, 105, 86, 94, 93, 90, 35, 26, 25, 89, 103, 112, 88, 79, 92, 15, 115, 46, 124, 57, 20, 119, 122, 48, 120, 54, 31, 59, 41, 106, 56, 17, 44, 60, 4, 30, 29, 116, 61, 37, 18, 43, 22, 114, 85, 49, 98, 127, 104, 126, 40, 100, 47, 125, 45, 51, 32, 109, 12, 28, 95, 52, 96, 58, 34, 99, 107, 36, 121, 16, 63, 108, 50, 13, 101, 39, 23, 64, 97]], "model.layers.5.self_attn.qk_proj": [[53, 115, 117, 108, 51, 125, 122, 124, 114, 62, 38, 61, 119, 112, 118, 44, 102, 50, 121, 42, 46, 56, 113, 126, 49, 23, 87, 57, 103, 76, 84, 85, 78, 82, 21, 17, 12, 81, 20, 18, 86, 80, 93, 14, 16, 110, 55, 91, 116, 10, 29, 22, 127, 74, 98, 97, 120, 15, 111, 79, 54, 26, 48, 106, 59, 63, 30, 52, 60, 9, 7, 73, 11, 47, 39, 34, 123, 8, 90, 71, 104, 75, 89, 45, 83, 24, 72, 27, 19, 5, 33, 28, 88, 109, 6, 58, 107, 70, 25, 13, 69, 77, 105, 68, 94, 0, 4, 32, 40, 35, 3, 43, 67, 37, 66, 31, 41, 64, 101, 2, 95, 1, 99, 92, 36, 65, 96, 100], [53, 115, 117, 108, 51, 38, 122, 125, 124, 62, 61, 114, 112, 118, 44, 102, 119, 46, 121, 50, 42, 113, 49, 56, 126, 23, 87, 103, 57, 82, 76, 48, 84, 86, 21, 85, 93, 81, 20, 12, 116, 78, 29, 18, 80, 17, 97, 127, 14, 91, 120, 16, 22, 55, 60, 15, 98, 110, 59, 106, 10, 30, 54, 47, 26, 63, 74, 34, 39, 58, 111, 79, 90, 123, 9, 8, 73, 6, 52, 75, 33, 11, 89, 7, 107, 71, 27, 109, 24, 28, 45, 83, 104, 5, 19, 88, 105, 25, 40, 64, 94, 72, 4, 3, 68, 69, 0, 37, 13, 35, 77, 95, 41, 2, 66, 32, 96, 70, 67, 1, 92, 43, 65, 99, 31, 101, 100, 36], [53, 115, 117, 108, 51, 124, 122, 125, 62, 38, 114, 61, 44, 112, 121, 102, 118, 46, 119, 42, 50, 56, 113, 49, 23, 103, 87, 126, 82, 57, 76, 21, 84, 12, 85, 93, 20, 86, 78, 81, 14, 116, 29, 18, 17, 16, 80, 22, 97, 98, 48, 8, 91, 120, 60, 15, 10, 110, 79, 74, 106, 26, 30, 39, 127, 55, 54, 6, 90, 9, 63, 71, 11, 7, 34, 47, 59, 75, 52, 109, 73, 123, 27, 89, 88, 19, 33, 28, 5, 58, 24, 107, 111, 104, 0, 83, 77, 64, 69, 40, 68, 25, 45, 67, 4, 13, 41, 94, 66, 105, 2, 3, 37, 35, 72, 32, 1, 43, 92, 95, 70, 65, 101, 31, 99, 96, 36, 100], [53, 115, 117, 108, 51, 114, 38, 122, 124, 125, 62, 61, 118, 102, 44, 46, 121, 112, 50, 119, 42, 113, 56, 49, 126, 103, 23, 87, 21, 85, 84, 86, 82, 17, 76, 57, 14, 20, 18, 93, 12, 80, 81, 78, 16, 22, 29, 98, 79, 91, 116, 97, 127, 55, 8, 30, 26, 54, 39, 48, 34, 106, 58, 63, 10, 59, 15, 110, 120, 90, 52, 123, 74, 47, 9, 6, 60, 75, 89, 71, 83, 73, 11, 7, 64, 24, 69, 28, 33, 88, 27, 104, 111, 19, 0, 77, 109, 25, 13, 107, 45, 5, 67, 4, 72, 3, 105, 94, 40, 66, 65, 37, 68, 1, 92, 2, 41, 96, 70, 32, 95, 31, 101, 35, 43, 99, 100, 36], [53, 115, 117, 108, 51, 122, 114, 124, 62, 125, 61, 38, 118, 121, 46, 102, 44, 112, 119, 50, 42, 113, 56, 49, 23, 103, 126, 87, 57, 82, 76, 12, 78, 85, 21, 20, 17, 86, 16, 84, 93, 81, 14, 18, 120, 98, 8, 80, 29, 22, 34, 52, 10, 110, 79, 106, 116, 15, 58, 39, 91, 123, 74, 63, 127, 6, 55, 48, 59, 7, 9, 71, 30, 54, 11, 75, 90, 111, 97, 64, 47, 109, 73, 33, 60, 26, 0, 5, 89, 83, 69, 104, 19, 27, 68, 88, 4, 3, 45, 77, 13, 67, 28, 24, 1, 40, 2, 107, 72, 25, 94, 41, 70, 66, 37, 35, 65, 32, 43, 31, 105, 99, 92, 95, 101, 36, 100, 96], [53, 115, 117, 51, 122, 124, 108, 125, 114, 61, 62, 38, 118, 119, 112, 121, 50, 102, 44, 46, 42, 56, 113, 49, 126, 87, 103, 57, 23, 76, 12, 18, 21, 78, 20, 84, 86, 85, 82, 17, 81, 93, 16, 14, 80, 55, 48, 110, 22, 120, 98, 29, 91, 74, 10, 15, 34, 47, 8, 54, 30, 59, 52, 127, 79, 123, 58, 97, 106, 116, 60, 63, 7, 9, 39, 11, 73, 71, 90, 75, 26, 111, 104, 6, 83, 33, 89, 5, 88, 68, 109, 24, 4, 45, 27, 19, 69, 72, 28, 94, 2, 70, 13, 25, 0, 64, 77, 40, 107, 3, 67, 105, 43, 41, 31, 1, 95, 32, 35, 37, 66, 101, 65, 96, 92, 100, 99, 36], [53, 115, 117, 51, 124, 125, 108, 122, 114, 38, 62, 119, 118, 61, 44, 112, 102, 121, 46, 50, 49, 42, 56, 113, 126, 87, 21, 23, 57, 103, 85, 76, 17, 82, 12, 78, 14, 110, 18, 20, 84, 81, 16, 93, 86, 80, 106, 54, 120, 48, 22, 29, 55, 60, 98, 79, 97, 127, 59, 11, 10, 111, 91, 63, 74, 39, 15, 30, 58, 34, 47, 116, 45, 8, 52, 9, 90, 123, 7, 26, 73, 88, 71, 75, 33, 19, 83, 89, 104, 27, 24, 70, 109, 40, 28, 94, 25, 41, 69, 43, 68, 6, 32, 5, 107, 72, 77, 4, 13, 3, 37, 35, 67, 105, 2, 95, 64, 31, 92, 0, 66, 101, 96, 65, 1, 36, 99, 100], [53, 115, 117, 51, 108, 125, 122, 114, 124, 62, 38, 61, 118, 119, 112, 102, 44, 46, 121, 42, 50, 49, 56, 113, 23, 87, 21, 126, 57, 103, 12, 20, 84, 85, 80, 86, 76, 17, 82, 14, 78, 18, 81, 93, 16, 54, 29, 127, 52, 79, 97, 106, 91, 15, 111, 120, 22, 110, 58, 116, 98, 55, 10, 48, 74, 39, 60, 70, 30, 34, 47, 9, 11, 63, 88, 26, 104, 45, 27, 33, 90, 75, 19, 7, 123, 8, 71, 73, 83, 24, 109, 89, 59, 28, 43, 25, 107, 5, 37, 41, 40, 72, 69, 77, 32, 13, 94, 66, 0, 35, 4, 67, 64, 3, 105, 68, 2, 92, 31, 65, 101, 1, 6, 99, 95, 36, 100, 96], [53, 115, 117, 108, 51, 122, 38, 125, 124, 62, 114, 61, 44, 118, 112, 102, 50, 119, 121, 46, 42, 56, 126, 113, 49, 23, 87, 103, 21, 57, 20, 17, 84, 85, 14, 18, 81, 76, 12, 86, 82, 93, 80, 16, 29, 22, 127, 78, 54, 120, 110, 98, 116, 55, 97, 106, 91, 48, 10, 34, 30, 111, 79, 15, 26, 70, 52, 47, 74, 39, 90, 123, 58, 59, 33, 27, 11, 75, 24, 104, 25, 88, 71, 7, 89, 72, 9, 83, 73, 8, 19, 60, 5, 45, 94, 109, 105, 63, 32, 28, 35, 13, 0, 69, 4, 64, 41, 40, 66, 77, 67, 68, 37, 107, 43, 95, 3, 31, 92, 101, 99, 2, 1, 96, 6, 65, 36, 100], [53, 115, 117, 108, 51, 122, 38, 114, 125, 62, 124, 61, 44, 118, 112, 119, 121, 102, 46, 42, 50, 113, 56, 49, 87, 23, 103, 85, 82, 17, 76, 12, 126, 21, 57, 86, 93, 18, 84, 20, 81, 14, 78, 80, 16, 29, 127, 22, 48, 15, 97, 55, 98, 79, 91, 110, 59, 52, 74, 54, 26, 30, 10, 106, 39, 60, 90, 120, 34, 47, 111, 109, 70, 9, 58, 11, 72, 89, 75, 71, 33, 116, 63, 7, 73, 83, 27, 88, 24, 104, 123, 19, 77, 94, 69, 45, 25, 105, 4, 68, 28, 40, 13, 8, 5, 43, 37, 3, 66, 0, 107, 32, 2, 35, 6, 92, 95, 67, 31, 96, 36, 64, 41, 1, 101, 100, 99, 65], [53, 115, 117, 108, 122, 125, 51, 114, 62, 38, 124, 61, 102, 118, 44, 121, 50, 112, 119, 42, 46, 56, 113, 49, 126, 23, 87, 103, 17, 18, 12, 76, 85, 78, 57, 93, 20, 21, 81, 84, 82, 14, 86, 74, 80, 110, 16, 98, 29, 22, 79, 15, 72, 91, 48, 97, 10, 54, 116, 58, 52, 30, 120, 34, 111, 47, 55, 106, 39, 127, 90, 109, 9, 11, 73, 26, 7, 75, 70, 33, 60, 63, 27, 71, 83, 59, 69, 88, 89, 5, 107, 104, 64, 4, 6, 68, 77, 0, 3, 13, 19, 123, 8, 28, 2, 65, 45, 24, 31, 94, 66, 105, 37, 25, 67, 1, 40, 95, 41, 32, 101, 43, 35, 96, 92, 100, 99, 36], [53, 115, 117, 51, 108, 122, 125, 114, 62, 124, 38, 61, 121, 102, 118, 50, 44, 112, 46, 119, 42, 56, 113, 49, 126, 87, 23, 103, 57, 76, 84, 21, 17, 85, 18, 20, 12, 14, 78, 86, 82, 93, 58, 48, 80, 81, 72, 16, 29, 98, 22, 91, 74, 34, 116, 79, 54, 30, 10, 55, 52, 120, 106, 63, 59, 15, 127, 97, 9, 111, 110, 11, 39, 109, 47, 90, 7, 89, 75, 104, 6, 71, 107, 26, 123, 68, 27, 73, 45, 70, 5, 0, 33, 19, 28, 88, 69, 60, 24, 83, 8, 64, 67, 4, 37, 77, 3, 13, 25, 66, 94, 105, 40, 1, 2, 43, 65, 35, 41, 32, 31, 95, 92, 101, 99, 100, 36, 96], [53, 115, 117, 108, 51, 122, 62, 125, 114, 38, 124, 61, 112, 44, 119, 118, 102, 46, 50, 121, 42, 49, 113, 56, 126, 87, 23, 103, 57, 84, 76, 21, 20, 85, 81, 18, 17, 86, 93, 82, 12, 16, 78, 14, 80, 58, 48, 29, 22, 59, 15, 52, 127, 54, 91, 63, 97, 106, 110, 98, 120, 111, 26, 34, 10, 39, 9, 55, 116, 79, 74, 30, 72, 47, 90, 7, 109, 6, 19, 11, 33, 75, 27, 71, 24, 89, 73, 88, 83, 104, 123, 28, 107, 25, 60, 40, 68, 5, 45, 94, 77, 4, 43, 37, 13, 32, 41, 69, 3, 35, 105, 67, 101, 66, 95, 70, 2, 0, 31, 64, 92, 36, 8, 65, 99, 100, 96, 1], [53, 115, 117, 108, 51, 122, 125, 38, 114, 62, 124, 61, 112, 118, 44, 119, 102, 121, 46, 42, 50, 49, 56, 113, 126, 23, 103, 87, 21, 76, 78, 20, 18, 12, 57, 85, 84, 17, 48, 82, 81, 80, 86, 55, 93, 110, 14, 22, 16, 54, 6, 127, 72, 79, 29, 91, 10, 15, 58, 26, 98, 74, 30, 75, 116, 9, 39, 106, 34, 123, 120, 7, 97, 90, 71, 59, 111, 52, 11, 89, 88, 63, 73, 19, 47, 60, 33, 24, 104, 83, 68, 27, 40, 28, 5, 25, 77, 69, 0, 4, 94, 105, 2, 64, 37, 66, 8, 43, 109, 45, 100, 107, 13, 32, 3, 92, 41, 101, 35, 67, 95, 36, 65, 1, 31, 96, 99, 70], [53, 115, 117, 108, 51, 122, 125, 62, 38, 124, 61, 114, 102, 118, 44, 121, 112, 119, 46, 42, 50, 56, 126, 49, 87, 23, 113, 103, 84, 57, 81, 48, 85, 18, 76, 20, 21, 12, 78, 93, 82, 17, 110, 80, 14, 22, 98, 16, 86, 10, 29, 91, 79, 6, 72, 34, 63, 15, 74, 120, 39, 55, 97, 106, 54, 30, 116, 90, 127, 71, 104, 26, 58, 7, 52, 60, 89, 27, 59, 109, 11, 75, 9, 73, 33, 123, 19, 64, 88, 83, 24, 47, 111, 37, 5, 8, 69, 68, 107, 28, 94, 25, 40, 13, 66, 77, 4, 45, 2, 0, 105, 3, 32, 43, 67, 1, 92, 41, 70, 65, 35, 96, 101, 100, 31, 95, 99, 36], [53, 115, 117, 108, 51, 125, 122, 62, 124, 114, 38, 61, 44, 118, 112, 102, 121, 119, 50, 46, 42, 113, 56, 126, 49, 87, 23, 103, 57, 20, 14, 82, 85, 21, 12, 84, 76, 17, 18, 81, 86, 63, 80, 48, 78, 93, 22, 91, 16, 98, 110, 58, 34, 15, 74, 10, 29, 106, 60, 116, 127, 79, 54, 120, 6, 55, 59, 52, 39, 30, 72, 97, 11, 104, 90, 8, 7, 9, 89, 33, 47, 73, 26, 19, 111, 109, 107, 37, 75, 71, 69, 5, 83, 28, 123, 24, 43, 88, 25, 27, 64, 67, 13, 0, 68, 4, 70, 3, 77, 40, 94, 35, 45, 105, 92, 41, 65, 31, 100, 32, 95, 2, 101, 1, 66, 36, 96, 99], [53, 115, 117, 108, 51, 125, 124, 114, 122, 38, 62, 61, 112, 119, 118, 44, 121, 46, 102, 42, 50, 49, 113, 56, 126, 23, 87, 103, 84, 85, 81, 20, 14, 21, 12, 76, 18, 82, 80, 93, 57, 86, 78, 17, 15, 22, 16, 97, 10, 116, 52, 110, 29, 98, 48, 59, 54, 55, 60, 127, 91, 106, 58, 79, 30, 9, 120, 74, 7, 90, 8, 63, 26, 33, 73, 11, 71, 34, 39, 19, 24, 111, 75, 83, 89, 109, 72, 27, 28, 88, 104, 107, 47, 105, 6, 68, 123, 5, 25, 13, 40, 77, 70, 69, 4, 37, 32, 94, 43, 45, 92, 0, 64, 2, 31, 66, 35, 101, 41, 3, 67, 95, 65, 96, 100, 99, 36, 1], [53, 115, 117, 108, 51, 122, 62, 125, 38, 124, 61, 114, 44, 118, 112, 121, 102, 119, 50, 42, 46, 23, 49, 56, 113, 126, 87, 103, 84, 21, 12, 20, 82, 81, 57, 76, 14, 78, 85, 93, 18, 86, 17, 80, 16, 48, 110, 29, 97, 22, 91, 15, 106, 98, 10, 26, 8, 79, 116, 120, 74, 59, 55, 39, 30, 90, 58, 127, 54, 34, 9, 60, 47, 71, 11, 75, 52, 70, 33, 89, 7, 27, 73, 19, 28, 123, 111, 88, 5, 63, 24, 105, 83, 25, 43, 45, 104, 77, 72, 94, 69, 107, 13, 4, 40, 64, 109, 68, 32, 6, 67, 92, 66, 37, 0, 2, 95, 96, 41, 3, 101, 31, 99, 65, 35, 36, 100, 1], [53, 115, 117, 108, 51, 125, 38, 122, 62, 114, 124, 61, 118, 44, 102, 112, 121, 119, 50, 42, 46, 113, 56, 49, 23, 126, 103, 87, 20, 21, 12, 84, 14, 76, 85, 93, 82, 18, 16, 78, 110, 80, 17, 81, 22, 57, 48, 86, 29, 98, 39, 8, 55, 58, 52, 74, 34, 79, 106, 10, 70, 63, 97, 54, 91, 15, 30, 26, 90, 127, 116, 120, 109, 9, 89, 59, 60, 11, 75, 47, 71, 104, 7, 73, 33, 111, 27, 28, 19, 83, 24, 69, 0, 88, 64, 5, 25, 68, 123, 40, 94, 67, 66, 3, 65, 32, 45, 77, 13, 37, 43, 4, 41, 31, 2, 92, 107, 105, 6, 72, 95, 100, 1, 36, 35, 99, 96, 101], [53, 115, 117, 51, 108, 125, 122, 114, 38, 124, 62, 61, 121, 118, 44, 119, 112, 102, 46, 50, 113, 42, 49, 56, 87, 23, 126, 21, 103, 84, 57, 12, 76, 18, 82, 16, 20, 81, 86, 14, 78, 17, 85, 80, 93, 22, 48, 8, 29, 91, 10, 79, 98, 74, 15, 70, 47, 39, 55, 54, 109, 110, 63, 97, 9, 11, 59, 30, 116, 58, 127, 106, 34, 90, 60, 7, 73, 71, 111, 120, 89, 75, 52, 26, 83, 107, 33, 27, 104, 24, 4, 5, 19, 69, 77, 28, 68, 88, 123, 25, 94, 45, 43, 40, 13, 0, 105, 64, 72, 2, 37, 32, 67, 66, 3, 35, 99, 41, 92, 31, 65, 1, 95, 6, 36, 100, 96, 101], [53, 115, 117, 51, 108, 114, 125, 62, 122, 38, 61, 124, 44, 102, 119, 121, 118, 50, 112, 46, 42, 113, 126, 56, 49, 87, 103, 23, 82, 84, 21, 76, 78, 12, 86, 57, 93, 85, 20, 17, 18, 14, 81, 16, 110, 80, 8, 91, 97, 120, 22, 29, 54, 116, 15, 98, 30, 79, 63, 60, 48, 74, 10, 106, 34, 39, 58, 55, 70, 90, 127, 47, 52, 9, 11, 71, 26, 73, 107, 7, 111, 28, 75, 33, 19, 24, 59, 89, 104, 69, 27, 88, 83, 123, 109, 25, 37, 5, 77, 105, 68, 43, 40, 94, 4, 13, 64, 3, 72, 0, 35, 45, 31, 32, 66, 65, 67, 92, 95, 101, 6, 41, 2, 99, 96, 36, 1, 100], [53, 115, 117, 51, 108, 114, 125, 62, 122, 38, 124, 61, 44, 118, 121, 102, 112, 119, 46, 42, 50, 113, 49, 126, 87, 56, 103, 57, 21, 23, 76, 82, 84, 93, 20, 12, 86, 80, 14, 78, 85, 18, 81, 17, 120, 110, 16, 22, 29, 60, 55, 79, 48, 98, 97, 91, 63, 8, 54, 106, 15, 127, 39, 59, 116, 30, 74, 34, 10, 71, 26, 90, 52, 33, 47, 11, 9, 58, 75, 104, 19, 7, 27, 111, 89, 123, 83, 24, 45, 25, 70, 107, 73, 69, 5, 88, 13, 77, 109, 28, 94, 64, 40, 0, 68, 37, 105, 72, 6, 3, 66, 43, 67, 4, 101, 92, 65, 1, 31, 2, 32, 41, 35, 99, 96, 95, 100, 36], [53, 115, 117, 108, 51, 114, 125, 62, 38, 124, 122, 61, 112, 121, 44, 118, 46, 102, 119, 42, 50, 113, 56, 49, 87, 126, 103, 23, 57, 21, 93, 76, 20, 18, 84, 82, 86, 14, 85, 81, 17, 12, 22, 16, 80, 78, 39, 106, 98, 97, 127, 29, 15, 120, 34, 79, 91, 48, 116, 10, 30, 55, 54, 111, 74, 110, 8, 52, 58, 59, 26, 9, 11, 71, 7, 90, 33, 60, 109, 104, 63, 6, 27, 73, 25, 75, 47, 83, 89, 19, 28, 5, 24, 88, 69, 77, 107, 13, 70, 45, 72, 123, 68, 94, 64, 37, 40, 0, 32, 43, 67, 4, 35, 3, 31, 65, 101, 1, 66, 99, 2, 105, 92, 41, 95, 36, 100, 96], [53, 115, 117, 51, 108, 122, 125, 62, 38, 114, 124, 61, 102, 118, 112, 44, 119, 50, 121, 46, 42, 56, 113, 87, 49, 126, 103, 23, 21, 76, 86, 82, 18, 85, 84, 20, 93, 17, 57, 12, 14, 59, 81, 78, 16, 48, 80, 29, 91, 22, 116, 97, 110, 98, 127, 15, 30, 10, 34, 26, 106, 111, 74, 55, 6, 60, 11, 39, 120, 52, 90, 54, 79, 63, 47, 75, 58, 8, 73, 72, 7, 89, 109, 9, 24, 33, 27, 19, 71, 83, 123, 104, 25, 68, 28, 88, 40, 13, 107, 69, 5, 94, 3, 37, 43, 64, 77, 105, 4, 101, 41, 95, 45, 0, 31, 70, 35, 67, 96, 32, 100, 66, 92, 36, 2, 99, 65, 1], [53, 115, 117, 51, 108, 122, 125, 114, 62, 124, 38, 118, 61, 44, 119, 112, 102, 121, 42, 46, 50, 113, 49, 56, 126, 87, 23, 103, 21, 57, 76, 20, 85, 18, 14, 12, 17, 86, 81, 84, 82, 78, 80, 93, 59, 54, 16, 110, 22, 116, 98, 91, 120, 6, 29, 15, 63, 10, 48, 97, 106, 39, 55, 60, 79, 74, 30, 75, 127, 111, 52, 71, 58, 9, 11, 90, 26, 34, 33, 19, 72, 73, 47, 109, 27, 7, 104, 89, 88, 28, 83, 8, 24, 37, 107, 69, 25, 68, 123, 5, 43, 13, 94, 105, 40, 77, 45, 32, 67, 41, 4, 70, 2, 3, 64, 0, 35, 31, 66, 101, 95, 65, 92, 1, 100, 99, 96, 36], [53, 115, 117, 51, 108, 122, 125, 38, 114, 124, 62, 61, 118, 44, 121, 119, 102, 112, 46, 42, 50, 113, 49, 126, 56, 87, 57, 103, 23, 21, 76, 20, 82, 81, 78, 84, 18, 85, 12, 86, 14, 80, 93, 17, 54, 16, 120, 97, 29, 48, 91, 59, 98, 22, 15, 116, 10, 110, 52, 79, 127, 106, 90, 26, 74, 30, 47, 72, 60, 11, 6, 39, 55, 34, 9, 63, 58, 75, 73, 33, 111, 104, 19, 24, 7, 27, 71, 88, 89, 5, 28, 37, 123, 13, 69, 25, 107, 83, 94, 41, 40, 45, 105, 0, 77, 109, 43, 4, 2, 8, 101, 70, 32, 68, 64, 31, 3, 35, 66, 96, 65, 67, 1, 95, 92, 99, 100, 36], [53, 115, 117, 108, 51, 122, 38, 125, 62, 124, 61, 114, 44, 118, 112, 102, 50, 121, 119, 42, 46, 126, 56, 113, 23, 49, 103, 87, 21, 57, 17, 85, 20, 18, 84, 12, 82, 93, 76, 81, 86, 80, 78, 14, 22, 110, 29, 97, 55, 48, 16, 98, 106, 10, 91, 72, 26, 127, 47, 79, 30, 116, 39, 15, 59, 34, 74, 58, 90, 120, 111, 54, 123, 73, 33, 60, 52, 7, 89, 75, 9, 11, 19, 63, 104, 6, 27, 25, 69, 83, 24, 88, 71, 94, 28, 109, 5, 45, 77, 4, 105, 64, 32, 68, 40, 35, 70, 13, 67, 8, 107, 31, 37, 0, 3, 66, 41, 101, 96, 95, 92, 2, 65, 43, 99, 1, 36, 100], [53, 115, 117, 108, 51, 122, 125, 124, 62, 38, 114, 61, 118, 121, 44, 119, 112, 102, 46, 42, 50, 126, 49, 56, 113, 87, 23, 103, 12, 21, 81, 14, 76, 18, 110, 84, 78, 57, 82, 17, 20, 86, 85, 93, 16, 22, 80, 72, 10, 98, 127, 48, 74, 58, 106, 29, 120, 116, 63, 91, 7, 15, 47, 9, 26, 30, 79, 75, 71, 55, 97, 34, 11, 39, 59, 73, 90, 54, 89, 52, 109, 19, 33, 70, 123, 45, 111, 24, 6, 69, 5, 60, 28, 27, 83, 68, 77, 4, 0, 64, 104, 40, 88, 3, 67, 13, 107, 66, 94, 2, 8, 25, 65, 37, 105, 35, 32, 43, 41, 1, 31, 92, 96, 95, 101, 99, 100, 36], [53, 115, 117, 51, 108, 122, 125, 114, 124, 62, 38, 61, 118, 44, 121, 119, 102, 112, 50, 42, 46, 56, 113, 126, 49, 87, 23, 103, 12, 57, 21, 18, 84, 78, 76, 20, 86, 82, 14, 110, 85, 93, 81, 48, 16, 80, 17, 22, 72, 58, 98, 60, 106, 91, 116, 54, 10, 29, 79, 55, 127, 74, 120, 97, 34, 30, 39, 15, 63, 59, 70, 52, 33, 47, 90, 73, 75, 26, 9, 11, 104, 111, 71, 7, 89, 109, 123, 5, 19, 37, 88, 69, 24, 28, 45, 27, 35, 68, 13, 83, 40, 4, 25, 0, 64, 105, 8, 6, 2, 77, 43, 94, 66, 107, 3, 67, 1, 95, 65, 32, 31, 100, 96, 41, 99, 36, 101, 92], [53, 115, 117, 51, 108, 125, 122, 114, 124, 62, 38, 61, 119, 118, 44, 112, 50, 121, 102, 46, 126, 42, 113, 49, 56, 87, 23, 57, 103, 85, 12, 21, 76, 20, 82, 18, 78, 14, 17, 86, 93, 84, 16, 59, 80, 81, 116, 54, 98, 22, 29, 55, 58, 47, 72, 70, 91, 120, 15, 110, 10, 74, 63, 79, 48, 106, 127, 34, 9, 7, 30, 97, 52, 39, 75, 11, 60, 71, 111, 123, 73, 90, 26, 33, 69, 89, 24, 28, 5, 109, 19, 4, 40, 67, 27, 83, 64, 88, 37, 104, 68, 45, 25, 94, 0, 3, 8, 105, 77, 13, 107, 66, 2, 43, 1, 32, 65, 35, 41, 6, 31, 100, 95, 101, 96, 92, 99, 36], [53, 115, 117, 108, 51, 125, 122, 124, 38, 62, 114, 119, 61, 44, 118, 112, 121, 46, 102, 50, 42, 49, 113, 126, 56, 87, 103, 23, 21, 57, 84, 85, 12, 18, 82, 76, 20, 93, 81, 78, 17, 14, 59, 86, 80, 22, 16, 98, 116, 106, 74, 79, 47, 54, 55, 127, 29, 110, 48, 120, 39, 15, 70, 63, 72, 60, 30, 26, 97, 11, 91, 10, 73, 9, 7, 34, 75, 52, 58, 19, 111, 71, 90, 33, 45, 69, 4, 109, 68, 123, 104, 27, 89, 28, 83, 24, 37, 8, 40, 13, 88, 25, 5, 66, 0, 107, 105, 67, 3, 64, 94, 77, 6, 2, 43, 41, 32, 31, 1, 35, 65, 95, 92, 101, 96, 36, 100, 99], [53, 115, 117, 108, 51, 122, 125, 124, 38, 114, 62, 61, 119, 44, 118, 50, 112, 121, 102, 46, 42, 49, 113, 56, 126, 87, 23, 103, 21, 76, 57, 54, 14, 85, 20, 81, 78, 18, 84, 82, 127, 86, 93, 12, 17, 80, 55, 60, 59, 16, 22, 34, 63, 116, 74, 110, 29, 10, 26, 120, 48, 91, 98, 70, 97, 79, 30, 15, 123, 73, 47, 11, 111, 39, 9, 75, 33, 52, 72, 106, 109, 7, 71, 24, 19, 89, 90, 88, 8, 105, 5, 104, 58, 27, 43, 69, 83, 28, 45, 37, 4, 25, 68, 77, 64, 6, 94, 40, 107, 13, 0, 32, 3, 66, 41, 101, 1, 92, 67, 2, 95, 31, 96, 100, 35, 65, 36, 99]], "model.layers.6.self_attn.q_proj": [[108, 55, 36, 96, 91, 23, 44, 84, 77, 81, 15, 86, 10, 16, 7, 114, 57, 75, 78, 32, 30, 28, 111, 89, 6, 25, 87, 29, 66, 70, 11, 69, 68, 22, 101, 13, 74, 72, 85, 19, 95, 17, 92, 26, 90, 51, 88, 79, 121, 14, 40, 18, 3, 9, 20, 61, 27, 76, 126, 4, 1, 54, 123, 80, 71, 107, 97, 103, 12, 64, 94, 118, 99, 102, 83, 33, 21, 39, 93, 73, 49, 58, 60, 24, 82, 125, 41, 31, 34, 98, 112, 35, 110, 48, 37, 104, 56, 5, 115, 109, 53, 38, 120, 50, 124, 2, 106, 59, 113, 47, 105, 127, 8, 46, 52, 122, 63, 42, 65, 43, 119, 116, 117, 62, 45, 67, 100, 0], [108, 55, 36, 23, 44, 84, 15, 75, 91, 81, 77, 10, 8, 96, 29, 114, 3, 72, 70, 74, 6, 64, 111, 11, 67, 86, 69, 30, 57, 2, 79, 89, 83, 27, 4, 26, 25, 65, 94, 28, 73, 18, 103, 0, 14, 5, 82, 17, 9, 87, 13, 31, 68, 85, 12, 93, 21, 1, 88, 20, 16, 80, 112, 78, 51, 7, 22, 95, 39, 32, 24, 19, 66, 97, 76, 90, 71, 33, 118, 121, 56, 123, 35, 92, 54, 98, 34, 107, 102, 48, 40, 52, 127, 41, 99, 37, 61, 59, 119, 120, 110, 49, 115, 125, 101, 104, 122, 60, 58, 100, 38, 126, 46, 109, 47, 43, 105, 124, 63, 53, 62, 106, 45, 113, 116, 42, 50, 117], [55, 108, 112, 39, 127, 36, 111, 61, 44, 54, 51, 124, 113, 118, 60, 121, 50, 59, 122, 63, 56, 58, 119, 115, 101, 116, 123, 120, 97, 62, 53, 42, 92, 26, 117, 19, 47, 43, 48, 40, 49, 94, 98, 52, 32, 90, 46, 109, 106, 110, 105, 96, 83, 45, 114, 104, 107, 41, 38, 35, 95, 88, 125, 102, 99, 33, 86, 30, 34, 23, 37, 103, 126, 29, 91, 100, 57, 78, 25, 89, 16, 93, 27, 28, 24, 31, 73, 21, 85, 81, 22, 18, 7, 14, 82, 15, 84, 75, 87, 65, 2, 77, 9, 12, 66, 11, 17, 10, 6, 4, 68, 67, 80, 76, 0, 64, 71, 5, 70, 8, 72, 79, 13, 1, 20, 69, 74, 3], [108, 55, 39, 44, 36, 94, 103, 126, 111, 90, 102, 114, 96, 57, 34, 121, 127, 29, 32, 83, 51, 54, 123, 91, 86, 60, 58, 63, 0, 19, 61, 59, 2, 110, 115, 49, 23, 125, 112, 52, 113, 98, 104, 56, 50, 38, 37, 107, 31, 42, 124, 47, 101, 106, 40, 78, 82, 117, 97, 45, 120, 109, 16, 89, 122, 92, 105, 119, 116, 33, 48, 53, 15, 10, 77, 118, 62, 46, 35, 84, 28, 43, 81, 70, 7, 41, 67, 88, 4, 68, 99, 65, 26, 95, 93, 100, 30, 5, 71, 64, 85, 25, 76, 21, 11, 75, 24, 72, 8, 12, 18, 66, 9, 73, 27, 80, 1, 22, 14, 6, 87, 69, 17, 79, 13, 3, 74, 20], [42, 102, 46, 32, 103, 89, 56, 20, 106, 76, 22, 17, 14, 19, 71, 127, 10, 110, 28, 93, 8, 120, 26, 51, 80, 60, 116, 29, 66, 50, 11, 27, 96, 38, 82, 25, 47, 123, 69, 36, 109, 63, 3, 59, 34, 118, 13, 113, 105, 79, 6, 88, 54, 33, 21, 41, 67, 52, 83, 30, 75, 81, 119, 15, 35, 77, 12, 7, 91, 86, 2, 114, 58, 112, 78, 61, 94, 84, 90, 5, 23, 16, 125, 85, 98, 101, 18, 62, 87, 70, 95, 97, 92, 74, 122, 49, 111, 4, 126, 24, 31, 9, 64, 99, 73, 72, 55, 43, 115, 57, 107, 108, 121, 44, 45, 37, 40, 124, 68, 0, 117, 53, 100, 104, 48, 39, 65, 1], [42, 41, 46, 103, 23, 32, 106, 102, 80, 92, 56, 8, 20, 26, 24, 89, 123, 29, 17, 110, 105, 96, 127, 19, 22, 116, 120, 25, 93, 28, 51, 52, 14, 109, 121, 60, 76, 27, 83, 54, 10, 69, 107, 47, 44, 49, 119, 86, 100, 61, 111, 35, 34, 58, 97, 5, 85, 91, 112, 122, 50, 63, 18, 104, 31, 39, 43, 59, 4, 9, 95, 66, 65, 117, 115, 15, 72, 124, 90, 68, 3, 126, 11, 62, 118, 48, 55, 101, 30, 98, 33, 21, 73, 108, 87, 114, 57, 53, 36, 45, 38, 94, 125, 99, 79, 37, 2, 40, 1, 77, 113, 82, 88, 71, 16, 78, 75, 81, 0, 70, 84, 6, 64, 13, 67, 12, 74, 7], [102, 46, 42, 56, 51, 116, 127, 110, 26, 123, 41, 47, 60, 50, 32, 59, 122, 120, 103, 63, 22, 111, 61, 90, 52, 19, 109, 33, 112, 27, 121, 115, 55, 49, 96, 34, 54, 118, 93, 119, 117, 43, 113, 53, 89, 57, 62, 48, 124, 16, 58, 39, 107, 126, 125, 114, 44, 29, 105, 31, 108, 101, 97, 45, 85, 79, 104, 36, 35, 98, 99, 38, 40, 24, 11, 25, 100, 37, 95, 94, 92, 106, 87, 28, 88, 20, 23, 91, 30, 83, 13, 80, 72, 86, 18, 15, 84, 17, 21, 14, 75, 73, 12, 8, 82, 69, 77, 9, 66, 5, 76, 78, 2, 81, 6, 70, 4, 74, 1, 68, 71, 10, 7, 65, 3, 0, 64, 67], [42, 46, 102, 32, 20, 17, 89, 76, 103, 22, 106, 10, 14, 71, 96, 29, 110, 69, 127, 56, 26, 51, 25, 66, 38, 11, 120, 93, 19, 5, 60, 24, 116, 79, 41, 61, 63, 119, 81, 78, 6, 109, 8, 118, 87, 82, 12, 58, 13, 75, 83, 2, 80, 23, 3, 86, 55, 15, 52, 30, 97, 54, 27, 113, 88, 59, 21, 16, 47, 28, 84, 77, 68, 85, 91, 90, 122, 35, 125, 0, 18, 72, 105, 111, 115, 50, 95, 74, 101, 70, 114, 7, 65, 112, 92, 73, 126, 99, 94, 67, 36, 43, 34, 49, 33, 98, 31, 107, 4, 37, 64, 1, 9, 121, 48, 100, 45, 123, 108, 104, 62, 44, 40, 117, 124, 39, 53, 57], [109, 103, 34, 45, 4, 20, 74, 71, 1, 18, 9, 14, 12, 0, 80, 88, 3, 66, 112, 70, 68, 79, 72, 39, 75, 67, 85, 10, 21, 7, 19, 5, 69, 13, 124, 77, 6, 16, 65, 23, 84, 11, 64, 76, 60, 8, 24, 73, 15, 2, 98, 87, 51, 22, 48, 82, 78, 83, 52, 81, 25, 86, 17, 108, 59, 57, 27, 63, 49, 55, 113, 89, 91, 121, 126, 123, 96, 110, 120, 92, 61, 114, 111, 30, 62, 90, 94, 119, 97, 28, 127, 54, 29, 41, 115, 95, 125, 93, 116, 44, 31, 58, 33, 38, 42, 26, 105, 106, 46, 107, 32, 40, 99, 122, 36, 35, 43, 56, 104, 50, 101, 100, 102, 117, 37, 118, 47, 53], [103, 109, 34, 113, 45, 88, 84, 24, 16, 76, 92, 11, 124, 8, 15, 82, 83, 73, 10, 60, 22, 127, 112, 7, 78, 18, 119, 63, 48, 57, 51, 52, 49, 62, 19, 123, 91, 114, 89, 56, 6, 115, 108, 53, 90, 120, 104, 94, 28, 9, 95, 110, 86, 26, 29, 111, 93, 97, 35, 126, 59, 122, 30, 54, 121, 117, 27, 50, 58, 96, 125, 118, 100, 31, 61, 101, 116, 32, 55, 20, 105, 33, 44, 46, 43, 42, 37, 38, 40, 99, 36, 41, 106, 25, 12, 107, 102, 47, 87, 77, 21, 13, 23, 69, 79, 75, 81, 17, 74, 72, 14, 85, 68, 98, 80, 71, 70, 66, 4, 5, 39, 67, 2, 65, 3, 1, 0, 64], [109, 103, 34, 45, 20, 74, 22, 18, 19, 4, 12, 14, 39, 67, 71, 79, 10, 72, 80, 68, 73, 1, 5, 3, 9, 13, 112, 88, 7, 66, 2, 77, 75, 0, 23, 85, 64, 70, 76, 21, 16, 15, 27, 69, 60, 92, 82, 8, 78, 65, 84, 11, 83, 81, 48, 6, 120, 123, 24, 124, 52, 63, 58, 91, 113, 51, 86, 55, 87, 17, 114, 41, 57, 25, 98, 89, 119, 90, 31, 49, 59, 126, 97, 127, 108, 94, 42, 29, 62, 28, 30, 56, 115, 96, 36, 61, 53, 105, 33, 106, 95, 32, 40, 46, 125, 93, 111, 122, 110, 47, 99, 118, 26, 107, 121, 102, 38, 100, 35, 104, 101, 54, 37, 50, 117, 44, 43, 116], [109, 103, 34, 45, 22, 80, 14, 20, 4, 18, 24, 74, 70, 75, 88, 19, 72, 112, 13, 12, 124, 89, 68, 10, 9, 60, 71, 63, 51, 73, 15, 66, 39, 76, 2, 8, 48, 52, 82, 78, 113, 77, 16, 83, 84, 49, 87, 6, 62, 127, 23, 57, 81, 11, 121, 5, 59, 79, 1, 17, 91, 0, 64, 25, 85, 92, 86, 94, 106, 7, 97, 21, 123, 27, 31, 90, 38, 28, 61, 110, 67, 69, 119, 36, 104, 126, 114, 99, 96, 3, 101, 111, 108, 95, 32, 41, 116, 40, 102, 55, 26, 120, 118, 98, 56, 50, 54, 100, 53, 30, 58, 35, 47, 93, 29, 115, 33, 125, 44, 107, 37, 117, 43, 46, 122, 65, 105, 42], [38, 124, 49, 30, 56, 86, 113, 15, 82, 84, 81, 26, 76, 77, 8, 5, 74, 71, 66, 122, 94, 67, 24, 11, 25, 1, 60, 22, 102, 72, 32, 12, 39, 97, 2, 120, 75, 21, 9, 17, 64, 10, 79, 14, 16, 89, 78, 91, 20, 6, 23, 80, 83, 87, 29, 4, 51, 73, 13, 68, 90, 27, 70, 112, 69, 33, 92, 93, 59, 85, 18, 96, 99, 117, 125, 31, 7, 114, 0, 19, 3, 107, 28, 105, 88, 118, 111, 110, 109, 44, 63, 104, 50, 108, 123, 95, 106, 52, 55, 36, 126, 101, 34, 65, 58, 116, 119, 47, 103, 62, 40, 45, 100, 42, 37, 121, 57, 54, 61, 48, 35, 41, 127, 115, 98, 43, 46, 53], [38, 49, 56, 124, 113, 30, 7, 84, 86, 82, 3, 81, 77, 15, 10, 13, 65, 89, 5, 76, 1, 67, 69, 8, 2, 64, 68, 94, 74, 73, 26, 72, 91, 122, 120, 66, 0, 27, 22, 97, 20, 79, 6, 18, 12, 9, 11, 14, 25, 87, 17, 23, 32, 24, 29, 51, 90, 75, 80, 60, 16, 99, 92, 78, 102, 83, 71, 59, 4, 88, 19, 39, 70, 33, 21, 31, 28, 103, 43, 114, 45, 125, 85, 112, 98, 93, 46, 111, 107, 118, 40, 95, 58, 37, 101, 126, 109, 104, 108, 44, 55, 105, 63, 41, 57, 61, 127, 96, 42, 115, 53, 116, 35, 54, 48, 100, 36, 47, 121, 62, 50, 117, 106, 119, 110, 34, 52, 123], [38, 49, 56, 124, 30, 86, 94, 89, 113, 81, 84, 91, 102, 122, 39, 15, 25, 82, 18, 51, 22, 26, 23, 76, 59, 97, 32, 120, 17, 33, 87, 105, 125, 29, 114, 99, 126, 72, 60, 103, 90, 62, 24, 31, 116, 77, 58, 20, 109, 118, 110, 52, 79, 45, 127, 112, 121, 63, 41, 117, 104, 96, 13, 55, 57, 50, 43, 44, 119, 111, 12, 115, 42, 48, 93, 95, 54, 101, 78, 40, 123, 75, 108, 37, 27, 53, 47, 46, 35, 100, 92, 61, 106, 36, 34, 107, 83, 28, 98, 21, 10, 19, 88, 9, 85, 80, 69, 14, 16, 73, 8, 7, 74, 11, 6, 68, 70, 3, 5, 71, 66, 4, 2, 65, 67, 1, 64, 0], [38, 49, 30, 124, 56, 113, 84, 86, 15, 77, 82, 74, 81, 5, 73, 76, 89, 8, 67, 71, 1, 20, 22, 72, 10, 122, 78, 66, 14, 64, 17, 9, 75, 79, 26, 94, 120, 16, 87, 11, 2, 7, 12, 6, 65, 99, 13, 91, 80, 69, 25, 27, 4, 3, 24, 39, 90, 97, 18, 32, 70, 93, 51, 85, 23, 92, 68, 29, 19, 21, 118, 31, 83, 60, 28, 88, 0, 114, 107, 33, 45, 46, 42, 43, 95, 103, 44, 117, 98, 125, 101, 100, 109, 63, 104, 62, 126, 54, 58, 111, 112, 40, 115, 57, 37, 59, 116, 50, 96, 55, 61, 52, 53, 34, 105, 41, 48, 35, 36, 119, 106, 121, 47, 110, 127, 123, 108, 102], [40, 98, 116, 32, 20, 78, 9, 81, 11, 89, 86, 7, 68, 69, 16, 47, 104, 4, 3, 53, 2, 49, 73, 51, 13, 117, 111, 126, 58, 121, 64, 8, 46, 56, 0, 67, 60, 94, 52, 88, 10, 127, 34, 42, 90, 71, 62, 72, 80, 74, 84, 113, 92, 50, 45, 79, 70, 63, 77, 75, 5, 27, 120, 6, 57, 44, 65, 1, 87, 14, 19, 106, 54, 17, 41, 101, 24, 125, 100, 21, 114, 99, 29, 122, 85, 105, 108, 33, 66, 15, 43, 124, 28, 119, 23, 115, 30, 12, 91, 18, 110, 25, 38, 39, 26, 22, 107, 118, 96, 61, 59, 103, 36, 83, 35, 97, 112, 123, 48, 37, 109, 102, 93, 95, 82, 31, 55, 76], [40, 98, 116, 32, 86, 20, 94, 88, 89, 28, 81, 62, 50, 121, 58, 78, 117, 52, 11, 47, 115, 45, 21, 104, 46, 126, 53, 111, 49, 87, 13, 30, 101, 41, 51, 96, 42, 16, 22, 92, 119, 90, 12, 122, 113, 24, 31, 91, 63, 26, 18, 29, 44, 37, 23, 109, 83, 93, 95, 107, 85, 38, 36, 125, 120, 110, 99, 103, 76, 79, 43, 114, 105, 102, 108, 8, 59, 25, 84, 124, 19, 9, 15, 7, 112, 54, 48, 60, 123, 82, 100, 127, 56, 27, 97, 106, 118, 35, 55, 80, 61, 57, 33, 17, 39, 14, 74, 34, 77, 75, 69, 10, 72, 2, 71, 3, 6, 68, 70, 73, 5, 67, 0, 66, 64, 4, 65, 1], [40, 98, 32, 86, 104, 116, 81, 20, 13, 47, 2, 66, 7, 8, 78, 49, 58, 51, 11, 9, 121, 60, 94, 5, 111, 45, 53, 52, 62, 68, 73, 46, 4, 16, 122, 92, 126, 70, 88, 89, 50, 0, 69, 56, 115, 34, 113, 10, 87, 64, 117, 125, 79, 3, 103, 82, 18, 28, 119, 77, 127, 63, 37, 57, 1, 108, 100, 27, 72, 25, 43, 114, 106, 44, 75, 120, 71, 22, 84, 65, 17, 30, 23, 42, 93, 14, 41, 19, 59, 54, 35, 48, 21, 31, 101, 76, 110, 12, 118, 29, 99, 90, 112, 80, 6, 85, 109, 15, 61, 124, 83, 36, 26, 91, 102, 55, 107, 24, 97, 39, 123, 95, 33, 38, 105, 74, 67, 96], [40, 98, 116, 86, 81, 32, 89, 78, 20, 88, 11, 7, 104, 47, 117, 111, 13, 49, 16, 28, 53, 74, 10, 51, 9, 121, 94, 58, 52, 126, 62, 46, 21, 45, 75, 60, 69, 25, 92, 96, 17, 8, 115, 79, 30, 50, 63, 77, 113, 90, 73, 71, 80, 56, 14, 67, 127, 95, 38, 2, 23, 42, 120, 122, 87, 72, 84, 99, 18, 44, 41, 29, 101, 105, 114, 125, 3, 22, 61, 19, 83, 91, 100, 6, 5, 54, 43, 97, 24, 27, 37, 119, 112, 108, 118, 15, 68, 85, 70, 102, 57, 107, 59, 34, 124, 55, 33, 12, 48, 110, 76, 66, 106, 26, 123, 103, 109, 31, 39, 93, 35, 65, 36, 82, 0, 1, 4, 64], [119, 38, 118, 42, 33, 25, 84, 31, 88, 52, 94, 103, 114, 99, 57, 55, 102, 28, 83, 78, 96, 54, 50, 62, 22, 106, 60, 45, 87, 93, 112, 29, 23, 53, 97, 27, 71, 61, 34, 74, 32, 85, 109, 111, 17, 24, 120, 46, 122, 110, 41, 80, 104, 21, 40, 18, 116, 125, 35, 121, 39, 105, 10, 123, 79, 56, 36, 92, 98, 47, 113, 107, 89, 127, 115, 126, 26, 20, 101, 16, 30, 43, 12, 49, 48, 81, 58, 91, 117, 37, 14, 13, 90, 63, 82, 100, 73, 11, 4, 7, 75, 44, 59, 51, 68, 124, 15, 108, 77, 95, 19, 72, 86, 1, 76, 8, 5, 6, 3, 0, 70, 69, 9, 65, 2, 67, 66, 64], [119, 118, 38, 33, 103, 42, 25, 94, 90, 54, 88, 109, 84, 31, 112, 120, 52, 102, 62, 111, 22, 45, 93, 53, 50, 57, 55, 83, 125, 47, 114, 87, 110, 106, 122, 74, 35, 18, 60, 101, 48, 58, 116, 85, 127, 61, 23, 78, 28, 59, 91, 100, 124, 121, 46, 63, 99, 49, 79, 126, 56, 123, 51, 113, 108, 30, 115, 105, 107, 117, 29, 43, 98, 26, 41, 39, 96, 0, 44, 24, 71, 34, 97, 40, 104, 27, 72, 89, 8, 11, 21, 75, 16, 68, 82, 12, 5, 37, 13, 69, 36, 64, 1, 4, 32, 95, 80, 67, 7, 2, 92, 65, 17, 14, 66, 73, 9, 3, 15, 70, 77, 10, 20, 81, 76, 6, 86, 19], [118, 38, 119, 42, 33, 25, 31, 109, 83, 55, 22, 94, 114, 84, 78, 90, 88, 106, 99, 52, 53, 54, 12, 74, 103, 120, 48, 111, 121, 102, 125, 17, 110, 50, 93, 62, 87, 107, 127, 34, 112, 60, 43, 97, 89, 115, 113, 122, 61, 57, 105, 58, 47, 91, 116, 73, 104, 35, 63, 46, 123, 30, 96, 28, 29, 72, 41, 59, 124, 126, 23, 44, 70, 37, 56, 117, 49, 101, 100, 27, 40, 85, 51, 79, 71, 95, 7, 45, 108, 80, 92, 68, 19, 39, 98, 81, 86, 26, 24, 75, 10, 36, 32, 67, 15, 76, 16, 21, 2, 18, 14, 11, 77, 82, 3, 66, 8, 1, 6, 20, 13, 0, 69, 4, 64, 5, 65, 9], [38, 118, 119, 33, 25, 83, 31, 79, 84, 89, 94, 12, 81, 78, 90, 88, 28, 85, 73, 22, 29, 54, 42, 24, 19, 112, 45, 26, 87, 93, 70, 17, 6, 102, 97, 48, 9, 55, 27, 116, 106, 35, 59, 127, 121, 114, 74, 110, 32, 125, 2, 103, 82, 20, 124, 52, 98, 96, 34, 21, 71, 50, 15, 80, 67, 18, 40, 16, 95, 109, 92, 0, 75, 91, 76, 62, 117, 72, 8, 39, 120, 69, 77, 65, 43, 30, 108, 36, 100, 99, 5, 58, 115, 57, 101, 14, 13, 23, 51, 60, 3, 61, 64, 107, 37, 86, 113, 104, 4, 66, 11, 126, 68, 41, 47, 122, 46, 111, 53, 1, 10, 123, 105, 49, 63, 7, 56, 44], [39, 57, 58, 52, 56, 116, 117, 55, 118, 51, 37, 59, 120, 86, 97, 31, 50, 126, 28, 98, 110, 99, 53, 54, 63, 112, 40, 122, 41, 20, 29, 47, 43, 92, 111, 109, 127, 124, 38, 125, 114, 95, 115, 108, 49, 123, 119, 105, 88, 89, 19, 61, 106, 62, 30, 60, 107, 113, 45, 34, 44, 104, 100, 121, 48, 36, 42, 46, 102, 33, 25, 32, 23, 6, 81, 101, 35, 80, 93, 10, 91, 24, 73, 96, 18, 87, 15, 78, 69, 27, 13, 14, 94, 22, 26, 90, 74, 66, 84, 67, 76, 71, 16, 12, 7, 68, 103, 64, 8, 21, 1, 2, 17, 72, 75, 5, 85, 83, 82, 11, 77, 0, 9, 3, 65, 79, 4, 70], [56, 39, 58, 52, 19, 59, 120, 34, 28, 90, 26, 31, 95, 96, 55, 37, 79, 53, 12, 21, 97, 88, 98, 86, 50, 116, 127, 118, 57, 24, 92, 8, 99, 25, 122, 49, 123, 111, 89, 14, 121, 18, 83, 27, 108, 29, 74, 63, 62, 51, 110, 44, 43, 87, 105, 109, 115, 91, 104, 38, 70, 126, 114, 1, 42, 10, 80, 41, 119, 68, 46, 33, 102, 72, 35, 60, 32, 45, 124, 125, 15, 93, 100, 106, 4, 48, 107, 85, 103, 112, 94, 47, 6, 78, 76, 23, 113, 81, 61, 75, 40, 101, 54, 11, 117, 0, 84, 16, 82, 36, 9, 2, 30, 13, 17, 22, 67, 71, 20, 64, 3, 69, 65, 73, 66, 77, 5, 7], [56, 39, 52, 117, 58, 57, 116, 55, 37, 118, 51, 53, 86, 54, 120, 10, 31, 73, 127, 126, 110, 111, 63, 97, 123, 122, 6, 119, 38, 76, 99, 121, 16, 29, 49, 43, 109, 13, 105, 112, 80, 42, 124, 125, 44, 78, 83, 106, 61, 7, 108, 93, 89, 28, 115, 50, 72, 48, 40, 62, 107, 45, 60, 46, 114, 77, 47, 113, 68, 88, 81, 11, 98, 41, 66, 104, 87, 82, 15, 67, 102, 69, 92, 36, 79, 30, 101, 34, 27, 59, 19, 96, 75, 22, 100, 33, 95, 20, 35, 1, 103, 71, 17, 32, 5, 84, 24, 94, 64, 26, 23, 85, 14, 4, 91, 8, 25, 0, 65, 9, 2, 12, 74, 21, 90, 70, 18, 3], [39, 52, 56, 34, 117, 57, 58, 51, 88, 116, 120, 50, 118, 95, 18, 98, 80, 20, 86, 55, 122, 21, 92, 30, 22, 90, 28, 53, 114, 97, 13, 74, 115, 19, 29, 32, 109, 60, 126, 62, 77, 61, 125, 99, 26, 89, 27, 25, 121, 37, 68, 11, 78, 49, 82, 14, 8, 54, 23, 48, 79, 24, 31, 71, 96, 112, 40, 105, 123, 127, 35, 46, 36, 94, 33, 124, 63, 12, 108, 119, 100, 110, 45, 10, 43, 44, 111, 47, 9, 107, 93, 85, 42, 16, 41, 38, 106, 15, 59, 113, 104, 102, 6, 101, 75, 91, 70, 81, 83, 2, 69, 84, 17, 1, 87, 76, 73, 0, 4, 3, 67, 5, 66, 72, 103, 7, 65, 64], [57, 48, 50, 38, 58, 54, 118, 56, 61, 100, 53, 63, 124, 115, 41, 116, 117, 119, 52, 102, 59, 84, 120, 46, 51, 121, 47, 126, 122, 125, 62, 113, 114, 111, 127, 97, 20, 44, 123, 49, 109, 60, 55, 45, 110, 112, 108, 43, 103, 106, 40, 107, 74, 42, 89, 105, 30, 27, 34, 31, 7, 33, 13, 101, 39, 24, 95, 78, 104, 32, 36, 98, 35, 21, 37, 88, 25, 99, 92, 80, 77, 87, 91, 96, 29, 23, 85, 93, 16, 71, 9, 69, 94, 28, 10, 66, 12, 90, 18, 73, 4, 26, 2, 5, 6, 65, 64, 3, 14, 19, 68, 72, 79, 83, 11, 70, 17, 0, 75, 82, 86, 1, 76, 15, 67, 22, 81, 8], [50, 38, 57, 48, 97, 112, 114, 7, 54, 115, 20, 88, 111, 77, 124, 90, 81, 100, 29, 95, 18, 79, 11, 28, 4, 27, 58, 24, 53, 68, 35, 23, 52, 59, 107, 94, 86, 119, 1, 101, 25, 63, 10, 56, 120, 99, 61, 42, 72, 104, 41, 121, 16, 103, 40, 125, 26, 43, 118, 117, 39, 108, 32, 123, 109, 110, 51, 37, 8, 46, 102, 116, 47, 105, 87, 106, 14, 93, 44, 6, 45, 85, 71, 49, 62, 126, 113, 55, 3, 76, 83, 65, 96, 75, 60, 34, 127, 89, 91, 30, 21, 122, 36, 31, 98, 78, 15, 84, 73, 12, 82, 17, 66, 5, 19, 80, 2, 74, 33, 0, 64, 92, 22, 9, 69, 13, 67, 70], [38, 57, 50, 48, 97, 23, 79, 72, 81, 76, 100, 18, 114, 28, 112, 6, 10, 3, 68, 14, 21, 65, 8, 70, 86, 77, 11, 82, 80, 87, 0, 67, 17, 54, 12, 75, 90, 88, 24, 56, 15, 2, 20, 95, 59, 73, 78, 5, 83, 9, 13, 85, 53, 84, 16, 29, 69, 58, 93, 102, 19, 92, 115, 91, 26, 25, 74, 27, 52, 66, 22, 117, 71, 94, 30, 121, 111, 61, 89, 32, 107, 7, 64, 124, 101, 36, 1, 55, 31, 35, 96, 63, 99, 4, 98, 119, 46, 34, 41, 60, 118, 125, 43, 110, 120, 106, 37, 126, 62, 123, 104, 40, 47, 42, 39, 103, 44, 127, 51, 116, 49, 105, 33, 108, 122, 109, 45, 113], [48, 50, 57, 38, 122, 115, 126, 51, 46, 54, 118, 56, 47, 62, 58, 109, 114, 124, 116, 49, 125, 55, 52, 112, 63, 113, 119, 61, 127, 60, 110, 59, 53, 45, 106, 43, 117, 44, 123, 121, 120, 108, 41, 107, 105, 40, 42, 103, 34, 102, 111, 101, 39, 89, 104, 20, 97, 37, 95, 32, 29, 90, 86, 100, 36, 31, 30, 35, 93, 99, 88, 77, 92, 96, 94, 98, 16, 22, 83, 33, 81, 27, 25, 91, 23, 80, 84, 14, 28, 24, 19, 21, 73, 26, 17, 82, 13, 18, 79, 78, 87, 85, 76, 11, 66, 10, 6, 12, 74, 9, 15, 0, 75, 2, 71, 69, 7, 1, 72, 8, 64, 68, 3, 4, 70, 5, 65, 67]], "model.layers.6.self_attn.k_proj": [[44, 55, 100, 86, 32, 84, 81, 57, 15, 77, 91, 108, 23, 12, 10, 64, 5, 75, 67, 8, 7, 65, 94, 114, 90, 16, 111, 29, 71, 76, 50, 82, 89, 103, 68, 72, 2, 83, 25, 39, 123, 78, 51, 66, 93, 121, 63, 120, 6, 58, 46, 48, 118, 125, 19, 124, 31, 126, 106, 61, 127, 104, 21, 70, 40, 1, 45, 18, 95, 88, 47, 0, 122, 69, 113, 53, 119, 34, 117, 62, 92, 54, 56, 42, 37, 109, 43, 73, 35, 41, 33, 11, 38, 116, 80, 30, 97, 49, 52, 98, 85, 28, 74, 107, 112, 59, 115, 110, 102, 4, 24, 60, 87, 99, 105, 101, 14, 27, 22, 26, 20, 79, 96, 13, 9, 17, 3, 36], [106, 110, 46, 96, 22, 93, 17, 89, 39, 10, 14, 19, 76, 20, 56, 71, 38, 120, 0, 11, 24, 45, 111, 42, 1, 123, 3, 52, 69, 66, 26, 28, 13, 103, 119, 58, 16, 5, 8, 49, 79, 37, 92, 18, 121, 116, 48, 107, 54, 43, 99, 114, 70, 105, 57, 127, 68, 122, 61, 85, 113, 60, 50, 90, 47, 73, 63, 55, 40, 36, 126, 117, 62, 109, 35, 53, 125, 124, 100, 112, 118, 30, 65, 102, 27, 115, 108, 4, 44, 104, 67, 23, 80, 82, 6, 101, 31, 51, 98, 34, 59, 97, 29, 9, 84, 64, 91, 33, 41, 94, 86, 7, 72, 21, 95, 74, 88, 2, 87, 77, 78, 12, 75, 15, 83, 25, 32, 81], [45, 39, 98, 109, 0, 20, 80, 18, 72, 12, 14, 24, 79, 75, 71, 66, 74, 5, 4, 70, 65, 9, 3, 112, 1, 67, 77, 69, 22, 92, 19, 64, 113, 123, 60, 120, 85, 91, 59, 86, 89, 23, 58, 48, 88, 17, 73, 6, 34, 78, 114, 25, 10, 90, 21, 27, 55, 63, 108, 13, 30, 122, 28, 47, 96, 126, 29, 119, 104, 33, 61, 105, 93, 83, 95, 35, 115, 31, 57, 11, 100, 51, 8, 54, 50, 38, 2, 121, 94, 111, 110, 49, 97, 26, 125, 81, 32, 87, 124, 7, 40, 46, 56, 52, 68, 101, 127, 42, 44, 117, 106, 116, 107, 118, 36, 43, 37, 41, 62, 102, 99, 53, 16, 76, 82, 84, 15, 103], [113, 124, 102, 56, 49, 94, 84, 82, 15, 76, 86, 81, 74, 77, 64, 5, 71, 8, 89, 73, 1, 66, 69, 39, 67, 122, 26, 91, 4, 75, 2, 51, 68, 70, 25, 65, 3, 72, 12, 24, 13, 38, 30, 6, 22, 23, 20, 59, 33, 79, 14, 83, 78, 90, 32, 80, 87, 7, 27, 9, 29, 88, 96, 10, 60, 99, 19, 16, 28, 21, 85, 18, 0, 11, 105, 97, 93, 17, 31, 95, 92, 103, 114, 35, 58, 125, 54, 47, 117, 43, 46, 104, 45, 40, 107, 118, 112, 36, 108, 109, 116, 115, 110, 126, 42, 62, 48, 111, 63, 98, 101, 55, 50, 53, 57, 44, 61, 127, 52, 106, 41, 34, 37, 121, 119, 120, 123, 100], [104, 116, 34, 111, 64, 20, 11, 53, 81, 86, 78, 51, 8, 16, 9, 7, 13, 121, 96, 3, 113, 2, 58, 62, 68, 69, 109, 60, 110, 122, 42, 50, 117, 56, 41, 89, 46, 127, 30, 49, 1, 126, 90, 108, 21, 98, 67, 39, 6, 105, 40, 63, 35, 37, 88, 83, 82, 32, 27, 124, 125, 65, 92, 44, 15, 57, 76, 107, 118, 61, 120, 54, 36, 74, 29, 0, 87, 70, 72, 28, 24, 31, 23, 25, 99, 93, 59, 106, 119, 47, 85, 91, 5, 43, 103, 77, 94, 33, 112, 97, 18, 55, 26, 45, 22, 38, 95, 115, 114, 19, 101, 123, 12, 52, 84, 100, 48, 71, 102, 66, 17, 79, 10, 75, 14, 4, 73, 80], [102, 119, 118, 22, 97, 25, 28, 83, 84, 78, 95, 12, 64, 74, 94, 71, 16, 18, 52, 79, 88, 30, 68, 125, 106, 111, 73, 38, 124, 46, 13, 91, 54, 45, 127, 66, 109, 6, 116, 17, 48, 104, 62, 43, 51, 58, 40, 53, 61, 57, 126, 90, 59, 123, 117, 67, 65, 1, 120, 121, 113, 44, 93, 3, 60, 50, 11, 56, 49, 69, 103, 108, 21, 47, 37, 112, 81, 19, 99, 42, 115, 122, 23, 63, 35, 82, 114, 41, 75, 2, 105, 20, 39, 26, 100, 70, 110, 55, 36, 87, 34, 32, 72, 15, 107, 101, 4, 80, 0, 98, 7, 5, 31, 77, 85, 96, 8, 27, 29, 92, 76, 86, 9, 89, 10, 24, 14, 33], [103, 57, 52, 56, 31, 58, 21, 88, 18, 90, 117, 59, 79, 98, 28, 14, 111, 72, 89, 75, 4, 120, 118, 25, 60, 12, 116, 110, 126, 74, 64, 85, 115, 55, 50, 62, 63, 124, 121, 66, 70, 46, 32, 112, 125, 49, 123, 65, 109, 39, 108, 105, 51, 53, 114, 48, 43, 73, 54, 34, 44, 92, 127, 27, 35, 113, 119, 81, 106, 19, 107, 102, 45, 47, 86, 84, 101, 20, 61, 122, 96, 7, 97, 104, 87, 77, 76, 91, 29, 41, 100, 13, 40, 82, 42, 71, 11, 94, 26, 16, 33, 6, 38, 95, 67, 3, 99, 36, 69, 37, 24, 80, 78, 30, 93, 1, 9, 68, 5, 8, 17, 15, 10, 23, 22, 83, 0, 2], [102, 50, 57, 86, 48, 33, 92, 18, 79, 76, 81, 23, 72, 6, 88, 83, 112, 14, 90, 11, 0, 93, 100, 30, 3, 10, 111, 31, 53, 58, 77, 124, 68, 73, 119, 61, 94, 117, 114, 2, 16, 63, 21, 118, 52, 36, 29, 54, 125, 38, 35, 99, 56, 115, 120, 9, 59, 65, 121, 108, 123, 105, 41, 98, 47, 113, 126, 127, 69, 55, 25, 45, 62, 51, 46, 24, 109, 60, 116, 20, 107, 110, 49, 85, 43, 32, 44, 66, 5, 96, 34, 122, 89, 106, 75, 42, 104, 1, 103, 91, 27, 39, 40, 101, 26, 37, 64, 4, 71, 95, 82, 84, 12, 78, 67, 80, 22, 15, 87, 19, 7, 8, 74, 28, 17, 13, 70, 97]], "model.layers.6.self_attn.qk_proj": [[57, 56, 118, 119, 45, 50, 55, 124, 113, 109, 48, 49, 38, 46, 44, 106, 22, 104, 52, 116, 102, 39, 108, 86, 20, 84, 58, 17, 110, 42, 30, 81, 25, 111, 34, 79, 15, 78, 76, 82, 98, 18, 88, 94, 89, 24, 10, 14, 32, 12, 112, 7, 13, 74, 0, 103, 40, 77, 117, 114, 51, 29, 72, 120, 8, 53, 75, 64, 16, 71, 11, 96, 90, 19, 28, 80, 67, 62, 31, 60, 87, 26, 121, 33, 3, 92, 63, 69, 70, 123, 127, 23, 83, 73, 97, 5, 66, 2, 9, 59, 126, 122, 68, 27, 61, 41, 47, 4, 93, 100, 21, 43, 95, 36, 115, 125, 54, 107, 1, 91, 65, 99, 85, 105, 35, 6, 101, 37], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 49, 48, 38, 46, 44, 106, 104, 22, 116, 52, 102, 39, 84, 86, 108, 20, 58, 110, 17, 42, 111, 81, 30, 25, 103, 34, 89, 94, 79, 82, 78, 98, 32, 15, 18, 76, 112, 14, 117, 24, 88, 12, 51, 10, 40, 74, 29, 7, 53, 75, 77, 120, 16, 28, 71, 72, 60, 11, 8, 64, 13, 114, 90, 80, 0, 96, 19, 121, 26, 127, 59, 97, 83, 87, 66, 31, 69, 2, 23, 62, 92, 4, 5, 33, 9, 122, 73, 123, 126, 27, 68, 67, 63, 1, 70, 100, 54, 61, 93, 125, 95, 3, 47, 43, 65, 21, 41, 91, 107, 6, 115, 99, 85, 36, 105, 35, 37, 101], [57, 56, 119, 118, 45, 113, 50, 55, 109, 48, 124, 49, 38, 46, 44, 102, 22, 106, 104, 39, 52, 116, 86, 84, 108, 20, 58, 110, 42, 17, 30, 25, 81, 89, 111, 98, 79, 94, 15, 34, 18, 82, 88, 14, 103, 32, 78, 76, 74, 0, 120, 13, 77, 75, 40, 112, 10, 24, 51, 71, 117, 12, 8, 7, 53, 60, 64, 114, 96, 11, 28, 72, 16, 90, 5, 29, 67, 31, 80, 26, 19, 83, 92, 69, 66, 2, 62, 3, 123, 97, 23, 68, 87, 73, 121, 9, 59, 127, 4, 122, 27, 93, 100, 63, 33, 54, 126, 6, 61, 1, 115, 70, 95, 43, 107, 21, 85, 41, 65, 91, 47, 125, 99, 105, 35, 36, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 48, 109, 49, 38, 46, 44, 106, 104, 102, 52, 116, 39, 22, 86, 108, 84, 20, 58, 42, 110, 17, 30, 81, 89, 18, 79, 34, 103, 25, 98, 94, 15, 111, 14, 78, 32, 82, 64, 112, 13, 117, 88, 76, 10, 51, 40, 24, 12, 8, 74, 77, 0, 3, 71, 120, 11, 60, 7, 28, 114, 53, 67, 96, 127, 90, 66, 123, 75, 121, 80, 16, 29, 31, 5, 62, 83, 72, 26, 19, 2, 69, 92, 73, 97, 33, 122, 23, 63, 59, 87, 4, 6, 27, 126, 100, 9, 54, 68, 93, 115, 1, 61, 95, 65, 21, 41, 107, 43, 70, 47, 91, 85, 125, 37, 105, 36, 99, 35, 101], [57, 56, 118, 119, 45, 50, 55, 113, 48, 124, 109, 38, 49, 46, 44, 106, 104, 52, 116, 102, 39, 22, 108, 86, 20, 84, 42, 58, 17, 110, 81, 34, 78, 30, 89, 111, 79, 15, 14, 94, 76, 18, 32, 12, 98, 112, 74, 10, 25, 8, 71, 82, 103, 88, 64, 51, 7, 11, 40, 114, 77, 13, 120, 0, 72, 24, 96, 53, 16, 28, 75, 29, 62, 80, 69, 117, 83, 66, 19, 90, 3, 92, 127, 5, 4, 60, 2, 6, 123, 26, 63, 73, 9, 67, 68, 31, 121, 59, 100, 65, 33, 23, 97, 87, 122, 126, 93, 115, 27, 41, 54, 95, 1, 61, 105, 43, 91, 47, 36, 107, 125, 85, 37, 70, 35, 99, 21, 101], [57, 56, 119, 118, 45, 50, 55, 113, 124, 109, 48, 49, 46, 38, 44, 106, 104, 52, 116, 102, 39, 22, 86, 84, 108, 20, 58, 110, 17, 42, 81, 34, 79, 82, 111, 30, 25, 14, 98, 12, 78, 15, 76, 8, 74, 89, 10, 94, 103, 18, 13, 112, 88, 11, 7, 40, 51, 16, 32, 77, 117, 24, 71, 114, 0, 72, 64, 3, 53, 80, 120, 75, 29, 123, 60, 28, 127, 69, 5, 90, 96, 19, 26, 83, 97, 23, 31, 4, 68, 6, 73, 67, 66, 122, 59, 92, 62, 121, 9, 100, 87, 126, 63, 2, 33, 93, 27, 47, 41, 54, 107, 115, 95, 1, 61, 65, 43, 36, 21, 125, 85, 105, 99, 91, 70, 37, 101, 35], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 49, 38, 46, 44, 104, 106, 52, 102, 22, 116, 39, 86, 20, 84, 108, 17, 58, 81, 30, 110, 42, 82, 79, 89, 25, 40, 10, 78, 15, 18, 13, 12, 77, 98, 76, 34, 74, 51, 14, 94, 111, 8, 103, 16, 88, 11, 32, 112, 24, 72, 7, 96, 117, 75, 80, 120, 19, 90, 114, 71, 53, 29, 60, 59, 28, 83, 26, 127, 92, 73, 61, 63, 97, 62, 64, 123, 122, 121, 0, 33, 100, 9, 87, 31, 69, 23, 67, 5, 27, 126, 2, 4, 68, 93, 6, 3, 66, 47, 125, 107, 21, 41, 54, 91, 43, 85, 105, 95, 1, 99, 115, 65, 36, 37, 101, 70, 35], [57, 56, 119, 118, 45, 50, 124, 113, 55, 109, 48, 49, 46, 38, 106, 44, 104, 52, 39, 116, 102, 22, 86, 84, 20, 108, 17, 58, 81, 89, 42, 110, 25, 78, 30, 34, 82, 14, 79, 15, 12, 18, 111, 74, 98, 77, 88, 103, 76, 10, 40, 13, 51, 8, 11, 16, 80, 94, 75, 117, 32, 24, 71, 112, 120, 19, 72, 114, 7, 28, 83, 53, 64, 96, 92, 121, 59, 73, 69, 31, 29, 5, 0, 87, 62, 90, 60, 97, 66, 9, 26, 100, 126, 27, 23, 122, 2, 123, 33, 63, 68, 4, 47, 3, 67, 127, 54, 21, 6, 91, 61, 93, 85, 125, 43, 95, 65, 70, 107, 41, 35, 1, 115, 105, 36, 99, 37, 101], [57, 56, 119, 118, 45, 50, 55, 124, 113, 109, 49, 48, 38, 46, 44, 106, 104, 52, 116, 102, 39, 22, 86, 84, 20, 108, 58, 81, 17, 42, 25, 110, 89, 78, 34, 82, 30, 14, 111, 79, 18, 32, 103, 98, 15, 94, 40, 10, 12, 88, 77, 80, 13, 74, 76, 24, 8, 112, 117, 29, 11, 75, 83, 53, 19, 114, 28, 51, 96, 72, 120, 90, 7, 71, 121, 16, 62, 92, 31, 87, 64, 26, 59, 127, 23, 97, 100, 66, 9, 67, 122, 60, 5, 123, 61, 33, 27, 73, 63, 3, 4, 68, 0, 126, 69, 2, 54, 93, 21, 125, 91, 95, 6, 47, 85, 65, 70, 41, 99, 35, 1, 43, 37, 115, 105, 107, 36, 101], [57, 56, 118, 119, 50, 45, 55, 113, 109, 124, 48, 49, 38, 46, 44, 116, 106, 104, 102, 52, 22, 39, 84, 86, 20, 108, 58, 81, 42, 17, 110, 30, 25, 82, 79, 111, 14, 15, 78, 76, 18, 89, 98, 34, 94, 103, 51, 77, 40, 32, 88, 10, 127, 12, 29, 24, 7, 74, 11, 112, 80, 117, 28, 8, 90, 13, 96, 72, 75, 53, 87, 16, 83, 60, 71, 19, 114, 120, 26, 31, 23, 0, 5, 123, 66, 92, 59, 64, 97, 62, 69, 73, 67, 100, 121, 68, 9, 33, 61, 27, 4, 126, 122, 63, 2, 3, 70, 54, 93, 47, 41, 125, 43, 91, 21, 85, 95, 107, 1, 105, 65, 115, 6, 35, 99, 37, 101, 36], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 44, 106, 46, 52, 104, 102, 116, 39, 22, 86, 108, 84, 20, 58, 42, 110, 17, 81, 34, 111, 30, 15, 14, 78, 25, 76, 98, 79, 103, 82, 18, 89, 94, 10, 0, 32, 88, 12, 64, 112, 7, 72, 71, 11, 13, 24, 74, 40, 114, 51, 117, 96, 53, 77, 8, 19, 28, 2, 70, 75, 5, 80, 66, 69, 31, 120, 127, 16, 62, 26, 123, 3, 92, 63, 90, 29, 83, 67, 60, 73, 68, 23, 9, 97, 121, 41, 4, 87, 122, 59, 93, 100, 61, 65, 47, 54, 33, 126, 27, 1, 95, 115, 107, 85, 125, 43, 21, 91, 105, 6, 36, 35, 99, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 44, 46, 106, 104, 52, 39, 116, 102, 22, 84, 86, 108, 20, 58, 42, 110, 17, 81, 111, 98, 34, 82, 30, 79, 25, 78, 15, 14, 89, 94, 32, 18, 103, 13, 88, 12, 76, 51, 40, 74, 72, 10, 7, 71, 24, 112, 80, 77, 53, 67, 75, 64, 0, 123, 3, 11, 121, 8, 5, 16, 29, 19, 117, 62, 127, 28, 92, 68, 120, 96, 114, 69, 83, 9, 31, 90, 122, 4, 59, 66, 60, 2, 70, 26, 87, 63, 97, 73, 100, 41, 27, 33, 23, 1, 65, 95, 93, 54, 47, 126, 61, 91, 107, 125, 115, 21, 85, 36, 105, 6, 99, 37, 43, 35, 101], [57, 56, 118, 119, 50, 45, 55, 124, 113, 48, 109, 49, 46, 38, 44, 104, 106, 116, 52, 102, 22, 39, 84, 86, 20, 108, 17, 58, 81, 110, 30, 42, 25, 89, 82, 111, 15, 79, 18, 78, 34, 14, 94, 12, 74, 10, 98, 32, 76, 103, 13, 72, 40, 88, 77, 24, 11, 29, 16, 7, 51, 80, 117, 8, 83, 75, 112, 71, 114, 53, 96, 120, 92, 122, 28, 97, 121, 5, 100, 90, 59, 60, 19, 127, 26, 31, 9, 87, 123, 27, 126, 69, 73, 23, 66, 4, 41, 68, 62, 33, 2, 64, 3, 0, 93, 61, 54, 63, 47, 91, 125, 115, 43, 107, 85, 67, 21, 70, 95, 65, 6, 35, 37, 99, 1, 36, 105, 101], [57, 56, 119, 118, 45, 55, 50, 124, 113, 109, 48, 49, 46, 38, 44, 106, 52, 104, 116, 22, 102, 39, 86, 84, 108, 20, 58, 17, 110, 81, 42, 34, 82, 78, 25, 18, 12, 30, 72, 79, 15, 103, 111, 74, 98, 14, 10, 0, 32, 89, 76, 88, 24, 112, 71, 77, 7, 51, 8, 94, 64, 40, 13, 16, 75, 117, 11, 123, 80, 5, 29, 83, 53, 19, 114, 62, 69, 60, 26, 96, 127, 28, 31, 92, 9, 126, 90, 59, 120, 122, 2, 66, 73, 23, 121, 33, 54, 63, 68, 4, 87, 97, 3, 61, 67, 100, 70, 27, 21, 6, 47, 93, 115, 43, 95, 91, 41, 125, 1, 65, 85, 105, 107, 36, 35, 99, 37, 101], [57, 56, 118, 119, 45, 113, 50, 124, 55, 109, 48, 49, 38, 46, 44, 106, 104, 52, 116, 39, 102, 86, 22, 84, 108, 20, 42, 58, 17, 81, 110, 30, 18, 82, 79, 89, 25, 78, 34, 14, 98, 15, 94, 111, 12, 103, 72, 32, 10, 74, 76, 71, 40, 77, 112, 88, 7, 24, 11, 51, 16, 8, 96, 117, 26, 5, 28, 29, 62, 13, 3, 75, 53, 83, 64, 31, 114, 0, 90, 19, 69, 120, 80, 127, 73, 66, 60, 121, 59, 92, 67, 2, 123, 100, 9, 97, 23, 68, 126, 87, 6, 63, 93, 54, 4, 122, 27, 33, 41, 115, 43, 65, 21, 95, 47, 1, 85, 61, 36, 35, 107, 70, 91, 125, 37, 105, 99, 101], [57, 56, 119, 118, 45, 50, 55, 124, 113, 48, 109, 38, 49, 44, 46, 106, 52, 104, 116, 102, 39, 22, 86, 108, 84, 20, 58, 42, 17, 110, 81, 34, 30, 111, 78, 25, 98, 15, 94, 14, 79, 82, 10, 74, 89, 76, 18, 12, 103, 117, 24, 77, 72, 88, 32, 112, 40, 7, 71, 11, 16, 8, 120, 26, 96, 51, 28, 75, 80, 29, 13, 64, 83, 121, 19, 114, 0, 92, 127, 69, 59, 53, 90, 3, 123, 31, 97, 47, 67, 62, 60, 73, 5, 122, 93, 126, 4, 87, 23, 100, 9, 68, 2, 66, 6, 41, 33, 63, 27, 54, 115, 43, 21, 95, 61, 125, 65, 1, 107, 35, 99, 105, 91, 70, 85, 36, 37, 101], [57, 56, 118, 119, 50, 45, 113, 124, 55, 109, 48, 38, 46, 49, 44, 116, 22, 106, 104, 52, 102, 39, 86, 84, 108, 20, 17, 42, 58, 110, 81, 30, 25, 18, 89, 15, 111, 77, 98, 78, 79, 34, 74, 82, 14, 10, 12, 94, 76, 11, 88, 40, 24, 7, 13, 16, 72, 51, 120, 117, 32, 103, 8, 75, 80, 53, 28, 71, 114, 112, 122, 123, 19, 26, 96, 29, 60, 92, 31, 0, 87, 127, 83, 5, 69, 126, 97, 23, 90, 62, 73, 66, 63, 59, 121, 9, 4, 33, 61, 64, 100, 68, 21, 27, 6, 47, 43, 54, 115, 107, 3, 2, 67, 93, 95, 41, 105, 125, 99, 35, 91, 1, 65, 36, 85, 37, 70, 101], [57, 56, 118, 119, 45, 50, 113, 124, 55, 109, 48, 46, 49, 38, 44, 106, 52, 104, 116, 22, 39, 102, 86, 84, 20, 108, 58, 42, 110, 17, 81, 89, 30, 82, 25, 18, 14, 78, 15, 98, 34, 74, 12, 10, 79, 111, 94, 72, 88, 40, 117, 8, 32, 13, 75, 76, 103, 24, 77, 7, 53, 28, 51, 112, 80, 71, 11, 16, 60, 31, 83, 64, 96, 19, 121, 5, 90, 114, 120, 126, 87, 122, 69, 26, 62, 66, 29, 92, 9, 3, 73, 23, 97, 123, 0, 127, 100, 59, 63, 67, 68, 33, 2, 61, 6, 27, 4, 47, 115, 54, 43, 21, 93, 91, 95, 41, 1, 85, 107, 65, 125, 70, 105, 99, 35, 37, 36, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 46, 44, 106, 102, 52, 104, 116, 39, 22, 86, 84, 108, 20, 58, 42, 17, 110, 30, 81, 89, 94, 15, 34, 18, 82, 98, 14, 111, 12, 76, 88, 78, 79, 25, 103, 40, 10, 74, 8, 0, 51, 24, 112, 71, 7, 77, 72, 11, 32, 64, 96, 117, 31, 13, 75, 28, 26, 100, 60, 90, 5, 16, 80, 114, 29, 69, 121, 122, 66, 120, 3, 92, 53, 83, 19, 2, 127, 97, 23, 62, 87, 123, 33, 9, 73, 59, 4, 68, 126, 27, 67, 63, 41, 61, 6, 93, 95, 105, 54, 115, 70, 65, 1, 47, 91, 21, 37, 85, 99, 35, 43, 107, 36, 125, 101], [57, 56, 118, 119, 50, 45, 55, 124, 113, 48, 109, 49, 38, 46, 44, 106, 52, 116, 102, 104, 22, 39, 86, 84, 108, 20, 58, 17, 110, 42, 30, 81, 14, 34, 89, 98, 82, 76, 78, 18, 10, 74, 25, 94, 15, 8, 117, 88, 12, 111, 103, 32, 79, 112, 51, 71, 40, 11, 75, 13, 24, 72, 120, 77, 19, 123, 7, 29, 26, 96, 16, 126, 80, 114, 83, 28, 90, 127, 0, 60, 53, 5, 69, 121, 66, 73, 31, 4, 64, 122, 61, 97, 92, 68, 59, 100, 3, 62, 87, 2, 23, 33, 9, 63, 47, 70, 27, 93, 67, 43, 95, 21, 54, 41, 65, 1, 115, 6, 107, 85, 105, 91, 35, 125, 36, 99, 37, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 48, 109, 49, 44, 38, 46, 106, 52, 104, 116, 22, 102, 39, 86, 108, 84, 20, 58, 17, 42, 110, 81, 30, 14, 98, 89, 18, 111, 79, 25, 88, 34, 78, 8, 15, 32, 82, 12, 94, 76, 10, 74, 103, 77, 112, 24, 40, 71, 117, 13, 120, 67, 16, 75, 114, 64, 51, 80, 11, 7, 28, 62, 3, 96, 72, 123, 60, 19, 92, 90, 31, 83, 29, 87, 26, 53, 127, 126, 5, 0, 121, 69, 97, 73, 66, 61, 63, 4, 47, 70, 115, 122, 33, 41, 9, 59, 93, 100, 27, 68, 107, 2, 23, 95, 54, 91, 125, 43, 21, 36, 85, 1, 6, 105, 65, 99, 37, 35, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 109, 48, 49, 38, 44, 46, 52, 106, 104, 102, 116, 39, 22, 86, 108, 84, 20, 58, 42, 17, 110, 81, 30, 34, 15, 98, 25, 94, 79, 18, 89, 14, 111, 78, 82, 12, 103, 88, 74, 40, 24, 8, 112, 10, 117, 76, 77, 51, 32, 120, 71, 7, 16, 75, 64, 13, 11, 96, 0, 28, 60, 80, 26, 66, 90, 114, 72, 69, 53, 59, 5, 83, 19, 29, 92, 3, 126, 87, 97, 121, 62, 2, 31, 122, 123, 127, 9, 23, 63, 27, 73, 33, 100, 70, 67, 4, 43, 68, 61, 47, 93, 41, 95, 115, 107, 105, 1, 54, 91, 125, 21, 85, 36, 37, 65, 6, 99, 35, 101], [57, 56, 118, 119, 45, 50, 55, 124, 113, 48, 109, 49, 38, 46, 44, 52, 106, 116, 104, 102, 22, 39, 20, 108, 86, 84, 58, 42, 17, 110, 30, 81, 89, 25, 34, 18, 88, 79, 32, 78, 94, 14, 15, 12, 82, 98, 74, 111, 8, 10, 112, 13, 51, 103, 76, 117, 77, 24, 71, 11, 114, 7, 120, 75, 80, 40, 16, 19, 72, 28, 87, 83, 59, 29, 26, 0, 126, 62, 96, 92, 97, 69, 90, 127, 31, 121, 5, 64, 60, 3, 122, 53, 9, 73, 66, 123, 33, 4, 68, 100, 23, 2, 27, 70, 93, 67, 61, 63, 47, 43, 41, 1, 115, 95, 54, 85, 91, 125, 35, 37, 107, 105, 21, 6, 36, 65, 99, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 109, 48, 38, 49, 46, 44, 106, 52, 116, 104, 102, 39, 22, 84, 108, 86, 20, 110, 58, 42, 17, 30, 81, 111, 98, 89, 78, 34, 25, 94, 10, 79, 88, 103, 8, 18, 74, 76, 117, 14, 12, 82, 32, 15, 51, 75, 112, 77, 24, 40, 71, 28, 7, 114, 120, 72, 26, 80, 13, 29, 96, 16, 127, 11, 3, 123, 19, 53, 83, 97, 90, 60, 87, 92, 69, 126, 121, 31, 64, 122, 23, 9, 62, 68, 5, 73, 67, 100, 33, 4, 59, 63, 27, 0, 66, 41, 47, 115, 54, 2, 70, 93, 61, 107, 105, 125, 36, 65, 95, 43, 21, 85, 91, 35, 1, 6, 99, 37, 101], [57, 56, 118, 119, 45, 50, 124, 113, 55, 109, 48, 49, 38, 46, 44, 52, 104, 102, 106, 116, 22, 39, 108, 20, 84, 86, 58, 17, 42, 81, 110, 30, 10, 89, 34, 79, 76, 18, 111, 25, 15, 98, 14, 78, 94, 82, 11, 88, 12, 74, 40, 51, 7, 103, 13, 77, 117, 112, 8, 71, 24, 32, 120, 16, 96, 72, 114, 75, 29, 80, 60, 26, 28, 90, 31, 19, 83, 53, 122, 5, 69, 92, 73, 123, 64, 9, 0, 62, 126, 100, 97, 121, 23, 59, 127, 63, 33, 87, 68, 2, 4, 67, 3, 41, 47, 66, 93, 115, 27, 61, 6, 125, 95, 43, 107, 70, 105, 21, 65, 1, 85, 36, 91, 54, 99, 35, 37, 101], [57, 56, 118, 119, 50, 45, 113, 124, 55, 109, 48, 49, 38, 46, 44, 106, 104, 52, 116, 102, 22, 39, 84, 86, 20, 108, 58, 17, 81, 42, 110, 111, 25, 98, 30, 89, 34, 14, 82, 76, 74, 79, 10, 15, 78, 117, 94, 18, 103, 88, 24, 72, 40, 12, 71, 120, 8, 13, 7, 77, 32, 80, 11, 75, 28, 51, 16, 112, 92, 126, 114, 64, 0, 26, 19, 96, 83, 53, 5, 29, 123, 9, 62, 23, 60, 121, 66, 122, 4, 73, 127, 90, 67, 69, 97, 100, 87, 31, 93, 59, 33, 2, 47, 6, 63, 61, 3, 91, 27, 115, 125, 95, 41, 54, 68, 21, 85, 1, 107, 105, 65, 43, 36, 70, 99, 35, 101, 37], [57, 56, 119, 118, 45, 50, 124, 113, 55, 109, 48, 49, 38, 46, 44, 104, 106, 52, 116, 22, 102, 39, 86, 84, 20, 108, 58, 17, 30, 81, 25, 42, 110, 82, 34, 98, 89, 79, 32, 15, 111, 103, 88, 76, 94, 14, 78, 24, 12, 74, 10, 40, 18, 77, 51, 75, 72, 112, 13, 120, 7, 29, 19, 90, 71, 16, 92, 114, 117, 96, 80, 28, 64, 8, 0, 83, 11, 67, 3, 62, 97, 59, 53, 121, 87, 63, 73, 60, 26, 31, 126, 5, 122, 33, 66, 4, 6, 23, 2, 69, 47, 61, 68, 100, 123, 127, 41, 9, 93, 125, 27, 65, 91, 43, 95, 107, 85, 21, 54, 1, 105, 99, 115, 36, 70, 35, 37, 101], [57, 56, 119, 118, 45, 50, 124, 55, 113, 109, 48, 49, 46, 38, 44, 52, 106, 116, 104, 39, 22, 102, 84, 108, 20, 86, 58, 17, 42, 110, 81, 34, 30, 15, 111, 79, 89, 10, 25, 98, 82, 78, 12, 18, 14, 72, 76, 74, 103, 112, 8, 7, 51, 24, 94, 88, 13, 75, 120, 40, 117, 64, 71, 77, 11, 0, 80, 32, 16, 53, 60, 114, 5, 62, 2, 19, 28, 29, 90, 69, 123, 83, 73, 67, 121, 4, 26, 97, 68, 126, 92, 66, 96, 23, 122, 9, 31, 63, 127, 6, 33, 61, 47, 54, 3, 59, 93, 65, 87, 100, 1, 27, 95, 43, 41, 115, 125, 107, 91, 105, 85, 21, 70, 36, 37, 99, 101, 35], [57, 56, 119, 118, 45, 50, 113, 55, 124, 48, 109, 38, 49, 46, 44, 106, 104, 102, 52, 116, 22, 39, 108, 86, 20, 84, 42, 58, 110, 111, 17, 30, 81, 34, 94, 14, 98, 15, 112, 25, 72, 103, 79, 7, 18, 76, 89, 88, 10, 12, 32, 120, 78, 24, 74, 82, 114, 40, 51, 117, 123, 28, 71, 80, 11, 96, 75, 8, 77, 0, 64, 13, 26, 5, 53, 29, 16, 60, 69, 83, 2, 31, 67, 92, 90, 62, 126, 122, 121, 127, 9, 97, 47, 66, 19, 87, 33, 4, 73, 3, 63, 100, 59, 93, 41, 6, 23, 27, 54, 68, 107, 65, 115, 61, 36, 95, 105, 1, 21, 91, 70, 99, 125, 35, 85, 43, 101, 37], [57, 56, 119, 118, 45, 55, 124, 50, 113, 48, 109, 49, 38, 46, 44, 106, 104, 52, 116, 102, 39, 22, 108, 86, 84, 20, 110, 58, 42, 17, 81, 30, 111, 14, 72, 34, 98, 25, 76, 79, 15, 18, 10, 89, 94, 103, 112, 114, 78, 74, 88, 82, 12, 13, 7, 75, 32, 117, 24, 71, 80, 77, 40, 16, 3, 8, 53, 120, 11, 51, 96, 64, 0, 28, 63, 19, 29, 83, 5, 31, 123, 60, 4, 126, 69, 67, 73, 62, 26, 127, 66, 92, 122, 97, 90, 115, 9, 121, 23, 2, 59, 33, 107, 54, 47, 87, 27, 41, 68, 93, 100, 6, 70, 1, 95, 125, 65, 61, 105, 21, 85, 43, 99, 91, 36, 35, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 49, 38, 46, 44, 106, 104, 116, 52, 39, 102, 22, 84, 108, 86, 20, 110, 58, 81, 17, 42, 34, 30, 25, 15, 98, 111, 18, 82, 103, 72, 14, 79, 94, 78, 12, 76, 74, 51, 89, 88, 13, 71, 112, 10, 7, 24, 120, 40, 32, 11, 77, 80, 60, 0, 117, 114, 64, 75, 66, 16, 83, 123, 5, 8, 69, 96, 26, 68, 53, 63, 62, 29, 28, 87, 19, 90, 31, 127, 122, 9, 2, 4, 92, 97, 73, 67, 59, 47, 23, 33, 100, 61, 115, 70, 121, 3, 27, 41, 126, 93, 21, 107, 54, 6, 65, 125, 95, 91, 36, 1, 85, 43, 105, 37, 99, 35, 101], [57, 56, 118, 119, 45, 50, 113, 55, 124, 109, 49, 48, 38, 46, 44, 106, 104, 52, 116, 39, 102, 22, 20, 86, 108, 84, 58, 81, 17, 42, 110, 30, 25, 34, 111, 82, 14, 18, 79, 98, 12, 72, 76, 94, 10, 15, 89, 88, 117, 78, 51, 74, 32, 40, 103, 112, 75, 7, 8, 71, 80, 16, 24, 13, 114, 77, 60, 11, 120, 83, 28, 19, 53, 31, 96, 90, 69, 9, 0, 26, 5, 87, 62, 59, 73, 29, 97, 92, 23, 123, 66, 126, 121, 64, 3, 127, 70, 122, 68, 4, 27, 33, 2, 115, 47, 67, 61, 63, 100, 41, 93, 54, 65, 43, 125, 21, 91, 95, 1, 105, 6, 85, 107, 36, 99, 37, 101, 35]], "model.layers.7.self_attn.q_proj": [[38, 109, 43, 89, 45, 18, 93, 60, 107, 78, 12, 22, 10, 8, 17, 5, 19, 77, 80, 79, 56, 24, 91, 71, 68, 72, 3, 92, 26, 115, 25, 51, 20, 23, 57, 69, 15, 50, 81, 4, 58, 65, 102, 14, 127, 86, 97, 125, 27, 70, 34, 126, 28, 90, 88, 32, 67, 108, 30, 76, 9, 122, 29, 111, 123, 1, 6, 13, 124, 116, 74, 110, 82, 16, 63, 11, 96, 33, 75, 117, 55, 113, 52, 83, 87, 62, 35, 112, 94, 95, 7, 59, 100, 73, 119, 106, 40, 85, 54, 21, 0, 48, 44, 61, 114, 39, 105, 47, 31, 49, 120, 103, 53, 99, 36, 118, 84, 121, 41, 37, 66, 2, 98, 46, 101, 64, 104, 42], [38, 109, 43, 60, 10, 12, 19, 77, 5, 80, 93, 45, 18, 68, 78, 71, 107, 25, 22, 89, 8, 1, 73, 69, 79, 17, 3, 70, 29, 125, 65, 51, 88, 72, 87, 23, 0, 16, 14, 66, 122, 20, 4, 56, 74, 94, 26, 91, 84, 15, 32, 28, 86, 85, 7, 115, 21, 34, 24, 11, 76, 82, 6, 30, 90, 83, 92, 81, 58, 75, 116, 102, 97, 2, 13, 120, 123, 27, 95, 64, 126, 9, 100, 35, 33, 67, 98, 31, 96, 52, 46, 36, 111, 108, 47, 39, 127, 105, 99, 118, 62, 117, 112, 110, 59, 49, 55, 37, 106, 50, 54, 63, 104, 61, 103, 101, 57, 40, 114, 113, 119, 41, 48, 124, 53, 42, 44, 121], [38, 109, 43, 10, 67, 19, 93, 77, 45, 107, 80, 78, 12, 65, 6, 79, 23, 4, 5, 89, 7, 71, 3, 72, 0, 69, 25, 68, 8, 74, 1, 22, 60, 20, 14, 17, 18, 76, 16, 21, 51, 28, 75, 13, 115, 64, 88, 9, 11, 70, 73, 90, 84, 102, 29, 57, 126, 2, 82, 15, 86, 83, 87, 125, 24, 66, 34, 30, 91, 26, 35, 85, 120, 92, 94, 32, 81, 56, 55, 122, 58, 116, 98, 95, 52, 62, 31, 111, 97, 106, 50, 27, 124, 96, 118, 123, 48, 110, 40, 47, 36, 127, 99, 33, 108, 112, 113, 49, 54, 37, 117, 39, 63, 104, 46, 100, 105, 42, 103, 59, 121, 44, 53, 101, 61, 114, 41, 119], [109, 43, 60, 45, 107, 123, 34, 50, 58, 62, 115, 56, 35, 113, 110, 57, 122, 48, 63, 108, 98, 126, 47, 127, 112, 124, 119, 55, 111, 117, 49, 121, 59, 53, 114, 52, 118, 54, 39, 125, 61, 116, 44, 51, 37, 120, 106, 46, 92, 40, 42, 36, 105, 41, 103, 104, 33, 94, 38, 101, 100, 31, 97, 99, 86, 82, 95, 29, 87, 96, 16, 22, 30, 81, 78, 89, 32, 91, 84, 28, 27, 10, 93, 76, 77, 20, 85, 13, 8, 83, 24, 11, 80, 88, 23, 19, 21, 102, 25, 90, 79, 14, 17, 71, 74, 26, 18, 5, 68, 15, 9, 12, 70, 73, 66, 1, 2, 3, 72, 7, 4, 69, 6, 75, 0, 67, 65, 64], [105, 97, 118, 23, 90, 84, 79, 82, 117, 13, 93, 33, 47, 9, 54, 111, 29, 41, 10, 11, 71, 85, 20, 22, 25, 15, 17, 26, 18, 74, 56, 83, 89, 77, 75, 87, 121, 80, 21, 91, 19, 81, 86, 116, 7, 6, 92, 76, 32, 38, 30, 95, 88, 27, 3, 78, 5, 49, 48, 53, 16, 12, 69, 52, 94, 24, 31, 96, 112, 8, 108, 73, 61, 57, 46, 43, 102, 124, 4, 126, 58, 35, 100, 113, 99, 14, 28, 68, 42, 110, 107, 101, 123, 115, 55, 63, 37, 106, 119, 39, 62, 51, 103, 98, 109, 34, 67, 114, 120, 127, 60, 72, 40, 70, 125, 59, 122, 44, 45, 104, 50, 36, 1, 65, 66, 2, 0, 64], [105, 118, 64, 0, 2, 97, 1, 79, 65, 13, 66, 67, 41, 71, 4, 84, 29, 11, 26, 10, 9, 82, 69, 68, 23, 117, 70, 111, 6, 54, 87, 101, 7, 3, 76, 74, 78, 20, 8, 5, 73, 18, 12, 90, 121, 89, 75, 86, 77, 16, 93, 15, 14, 25, 38, 115, 47, 55, 17, 45, 80, 83, 37, 72, 58, 59, 48, 92, 22, 51, 39, 114, 85, 56, 49, 53, 34, 21, 81, 91, 88, 27, 120, 19, 24, 62, 46, 98, 100, 110, 124, 44, 28, 116, 96, 119, 106, 42, 94, 57, 102, 40, 61, 60, 126, 107, 127, 52, 31, 30, 103, 109, 113, 108, 122, 125, 63, 36, 43, 104, 95, 112, 123, 99, 35, 50, 32, 33], [105, 47, 118, 93, 117, 111, 90, 89, 97, 101, 116, 56, 85, 54, 23, 41, 25, 80, 33, 52, 121, 32, 96, 61, 37, 55, 24, 17, 49, 84, 31, 29, 53, 123, 21, 99, 91, 58, 98, 88, 95, 75, 35, 92, 60, 113, 34, 77, 126, 100, 12, 18, 127, 48, 30, 36, 45, 115, 108, 39, 73, 59, 112, 102, 15, 28, 43, 107, 26, 38, 27, 110, 81, 46, 114, 82, 16, 40, 103, 106, 87, 63, 94, 120, 104, 20, 62, 8, 44, 42, 122, 69, 109, 124, 7, 119, 74, 125, 51, 57, 50, 78, 11, 86, 19, 79, 83, 70, 22, 14, 67, 76, 72, 68, 13, 9, 10, 71, 5, 4, 6, 3, 66, 65, 2, 1, 0, 64], [105, 47, 118, 93, 117, 101, 90, 97, 116, 111, 54, 56, 52, 123, 89, 24, 85, 33, 55, 61, 23, 53, 58, 21, 49, 60, 121, 17, 37, 59, 75, 96, 31, 113, 115, 25, 112, 120, 106, 126, 12, 80, 114, 57, 127, 107, 48, 29, 41, 43, 77, 46, 110, 108, 45, 63, 62, 27, 124, 99, 73, 109, 40, 44, 51, 119, 42, 98, 38, 39, 87, 103, 102, 84, 50, 122, 125, 104, 32, 91, 34, 81, 26, 36, 20, 95, 16, 35, 18, 100, 30, 15, 69, 8, 7, 92, 86, 94, 28, 22, 88, 78, 74, 72, 82, 14, 70, 83, 79, 11, 10, 71, 67, 76, 19, 13, 9, 4, 68, 5, 6, 3, 66, 2, 65, 1, 64, 0], [103, 42, 34, 30, 23, 85, 106, 83, 81, 127, 70, 78, 50, 16, 39, 94, 53, 10, 41, 82, 12, 15, 4, 112, 13, 74, 20, 54, 25, 5, 8, 66, 38, 48, 19, 117, 63, 76, 93, 121, 72, 90, 88, 123, 111, 124, 24, 32, 107, 14, 118, 40, 17, 73, 55, 1, 87, 115, 75, 105, 80, 125, 21, 26, 67, 60, 29, 47, 52, 22, 101, 113, 58, 95, 36, 84, 126, 120, 11, 97, 6, 49, 100, 104, 62, 79, 59, 86, 46, 43, 45, 122, 7, 69, 102, 44, 109, 89, 33, 3, 99, 57, 77, 37, 108, 98, 27, 114, 28, 110, 18, 68, 96, 71, 119, 56, 116, 51, 35, 31, 61, 91, 65, 9, 64, 92, 0, 2], [103, 42, 34, 30, 85, 106, 23, 81, 83, 53, 50, 94, 78, 41, 127, 16, 39, 62, 15, 121, 54, 38, 44, 112, 125, 25, 12, 124, 74, 93, 63, 22, 47, 87, 13, 48, 82, 118, 111, 60, 11, 24, 5, 33, 113, 59, 109, 40, 117, 8, 55, 80, 70, 72, 107, 122, 115, 114, 123, 51, 49, 95, 32, 56, 4, 76, 110, 46, 21, 101, 45, 119, 28, 36, 58, 116, 90, 10, 57, 120, 126, 79, 29, 61, 104, 91, 108, 99, 26, 97, 92, 19, 86, 37, 20, 96, 84, 52, 102, 89, 100, 105, 43, 17, 1, 35, 75, 3, 31, 77, 14, 27, 88, 7, 98, 66, 18, 67, 6, 71, 73, 9, 64, 68, 69, 65, 0, 2], [103, 42, 34, 30, 106, 50, 85, 81, 54, 41, 23, 16, 83, 53, 112, 48, 121, 38, 44, 95, 4, 70, 94, 124, 13, 127, 20, 26, 63, 123, 125, 113, 93, 52, 60, 10, 62, 109, 59, 29, 115, 47, 78, 15, 122, 19, 76, 111, 31, 87, 8, 46, 28, 25, 45, 118, 40, 92, 82, 32, 90, 66, 58, 61, 55, 97, 110, 88, 114, 43, 105, 36, 117, 107, 74, 51, 100, 101, 119, 22, 49, 116, 37, 126, 33, 9, 80, 89, 86, 104, 99, 120, 73, 56, 57, 5, 24, 84, 21, 27, 35, 17, 96, 102, 69, 108, 67, 91, 12, 1, 79, 14, 11, 65, 75, 39, 68, 72, 18, 7, 77, 98, 71, 6, 2, 0, 3, 64], [103, 42, 34, 30, 83, 81, 85, 106, 12, 23, 50, 74, 72, 15, 78, 127, 16, 54, 69, 5, 13, 4, 113, 48, 60, 124, 8, 39, 76, 41, 1, 2, 24, 53, 80, 87, 11, 55, 17, 65, 21, 121, 94, 6, 63, 68, 3, 45, 19, 25, 62, 32, 38, 75, 126, 70, 52, 82, 93, 9, 84, 20, 125, 67, 123, 90, 14, 115, 26, 79, 44, 22, 10, 112, 104, 77, 66, 96, 95, 120, 91, 97, 117, 86, 100, 28, 98, 0, 73, 88, 71, 36, 59, 111, 89, 108, 64, 18, 29, 114, 105, 61, 7, 47, 101, 27, 99, 43, 107, 122, 31, 35, 92, 118, 51, 119, 33, 102, 58, 116, 49, 46, 109, 110, 40, 37, 57, 56], [104, 121, 100, 33, 17, 13, 21, 23, 79, 83, 90, 122, 72, 11, 8, 26, 28, 67, 112, 37, 30, 60, 59, 6, 61, 126, 45, 80, 53, 22, 115, 125, 81, 48, 109, 20, 4, 10, 51, 42, 18, 14, 46, 86, 19, 15, 40, 77, 96, 78, 34, 97, 75, 69, 99, 85, 27, 5, 1, 41, 29, 74, 39, 0, 16, 87, 94, 50, 25, 118, 111, 24, 68, 82, 12, 92, 84, 101, 73, 52, 103, 32, 117, 98, 66, 93, 95, 110, 113, 123, 47, 62, 31, 7, 107, 71, 76, 9, 70, 89, 2, 58, 108, 102, 124, 106, 55, 54, 63, 91, 88, 116, 3, 105, 64, 127, 120, 35, 49, 44, 119, 43, 57, 38, 65, 56, 114, 36], [104, 100, 121, 33, 28, 90, 21, 23, 17, 13, 109, 112, 42, 83, 115, 92, 48, 122, 30, 79, 26, 116, 18, 86, 118, 37, 61, 123, 45, 95, 117, 51, 44, 108, 39, 101, 58, 11, 120, 52, 34, 35, 107, 60, 110, 29, 124, 96, 41, 98, 25, 22, 32, 31, 82, 126, 80, 53, 16, 24, 62, 59, 27, 54, 103, 125, 43, 111, 57, 87, 119, 46, 6, 114, 36, 56, 105, 84, 106, 63, 102, 50, 85, 49, 19, 97, 55, 127, 20, 113, 99, 91, 14, 88, 93, 81, 38, 94, 10, 47, 77, 75, 72, 89, 74, 78, 12, 76, 70, 40, 67, 3, 8, 15, 69, 7, 9, 68, 5, 73, 71, 65, 2, 1, 66, 64, 4, 0], [104, 121, 33, 100, 93, 37, 79, 23, 83, 17, 21, 109, 112, 13, 8, 11, 90, 6, 67, 26, 35, 122, 59, 86, 40, 46, 3, 1, 125, 126, 71, 30, 82, 61, 45, 81, 48, 28, 42, 51, 85, 115, 60, 29, 9, 18, 4, 92, 43, 15, 52, 77, 74, 101, 25, 80, 22, 34, 65, 105, 75, 107, 19, 111, 50, 108, 10, 94, 69, 20, 78, 58, 16, 120, 39, 62, 47, 7, 119, 123, 88, 70, 96, 73, 99, 98, 31, 87, 57, 14, 32, 66, 106, 49, 102, 5, 103, 116, 110, 53, 38, 114, 76, 68, 63, 55, 124, 64, 56, 27, 113, 127, 12, 118, 84, 41, 117, 24, 91, 54, 95, 89, 44, 2, 72, 36, 97, 0], [121, 104, 100, 33, 28, 112, 122, 90, 21, 23, 83, 37, 93, 13, 116, 126, 30, 45, 92, 91, 125, 101, 79, 52, 17, 86, 115, 61, 60, 59, 46, 26, 51, 109, 35, 82, 85, 57, 48, 18, 123, 42, 118, 105, 19, 15, 97, 43, 39, 11, 54, 29, 107, 111, 103, 22, 32, 84, 31, 53, 14, 117, 95, 47, 96, 38, 120, 58, 119, 41, 108, 49, 106, 113, 110, 87, 27, 127, 99, 25, 70, 55, 34, 44, 124, 94, 6, 114, 89, 62, 36, 102, 24, 63, 75, 9, 50, 77, 98, 81, 72, 78, 20, 74, 56, 88, 10, 12, 16, 3, 80, 67, 71, 7, 76, 73, 2, 68, 8, 40, 69, 5, 66, 64, 1, 65, 4, 0], [37, 40, 105, 93, 55, 30, 124, 14, 18, 87, 63, 20, 94, 126, 48, 29, 88, 49, 104, 41, 60, 16, 117, 43, 58, 76, 89, 50, 62, 51, 57, 116, 53, 52, 39, 61, 111, 72, 119, 102, 38, 109, 100, 42, 44, 120, 59, 112, 56, 46, 122, 121, 110, 54, 114, 45, 113, 127, 123, 103, 115, 125, 27, 47, 108, 106, 118, 96, 107, 95, 86, 21, 98, 82, 26, 99, 36, 32, 33, 31, 35, 34, 23, 97, 24, 101, 69, 77, 81, 90, 25, 85, 10, 83, 92, 91, 78, 15, 28, 12, 84, 13, 74, 22, 17, 9, 80, 5, 6, 8, 79, 11, 75, 19, 1, 73, 3, 65, 71, 70, 66, 64, 2, 7, 67, 68, 4, 0], [105, 37, 40, 93, 87, 41, 18, 16, 76, 14, 20, 124, 29, 55, 69, 63, 72, 10, 57, 94, 88, 60, 49, 1, 61, 126, 15, 48, 23, 77, 117, 90, 114, 58, 27, 96, 74, 116, 89, 52, 78, 25, 12, 62, 122, 24, 81, 26, 13, 8, 30, 79, 82, 50, 102, 6, 84, 3, 68, 119, 112, 95, 86, 51, 100, 80, 104, 43, 83, 46, 17, 53, 21, 44, 91, 56, 59, 120, 71, 22, 39, 92, 19, 107, 97, 85, 45, 28, 11, 123, 4, 70, 101, 111, 38, 32, 75, 54, 5, 67, 108, 31, 121, 9, 34, 36, 65, 103, 33, 73, 35, 0, 2, 127, 7, 110, 113, 125, 98, 109, 42, 64, 115, 118, 99, 47, 66, 106], [105, 37, 40, 93, 104, 29, 55, 18, 87, 14, 16, 20, 63, 124, 76, 72, 49, 30, 60, 126, 27, 57, 94, 48, 117, 61, 88, 10, 114, 69, 77, 116, 89, 62, 50, 56, 100, 24, 90, 51, 96, 102, 119, 112, 86, 52, 15, 54, 58, 23, 127, 122, 12, 95, 8, 84, 53, 38, 79, 43, 21, 26, 92, 111, 82, 34, 108, 107, 113, 13, 32, 120, 80, 91, 25, 81, 98, 110, 28, 22, 103, 121, 44, 123, 19, 31, 39, 59, 45, 1, 115, 33, 101, 125, 47, 36, 35, 106, 46, 85, 99, 118, 42, 78, 17, 97, 109, 83, 41, 74, 6, 71, 70, 11, 3, 75, 68, 9, 5, 2, 4, 65, 66, 73, 7, 0, 64, 67], [40, 37, 105, 63, 93, 14, 16, 87, 10, 20, 18, 76, 55, 69, 29, 72, 57, 104, 30, 124, 3, 60, 41, 94, 74, 58, 61, 1, 26, 114, 88, 49, 15, 13, 23, 84, 24, 12, 126, 90, 17, 67, 95, 112, 70, 11, 71, 82, 78, 62, 116, 8, 117, 48, 52, 122, 81, 6, 22, 68, 0, 102, 7, 75, 83, 50, 77, 79, 46, 56, 98, 123, 80, 96, 4, 53, 27, 65, 89, 43, 21, 103, 86, 5, 39, 85, 119, 101, 100, 113, 51, 44, 47, 45, 25, 38, 28, 59, 92, 32, 19, 127, 111, 107, 91, 9, 120, 73, 110, 34, 64, 36, 115, 54, 108, 31, 109, 99, 35, 2, 121, 97, 118, 42, 66, 125, 33, 106], [105, 99, 89, 96, 48, 54, 83, 92, 84, 16, 14, 25, 11, 112, 22, 19, 75, 78, 72, 27, 77, 18, 80, 115, 41, 69, 81, 32, 24, 10, 36, 108, 30, 5, 44, 85, 93, 26, 13, 88, 104, 63, 119, 117, 29, 82, 15, 17, 73, 118, 20, 124, 94, 6, 9, 66, 62, 67, 49, 79, 12, 28, 91, 70, 126, 33, 23, 56, 90, 59, 21, 120, 58, 87, 76, 31, 71, 7, 74, 8, 65, 37, 4, 0, 1, 38, 98, 97, 43, 127, 46, 95, 2, 68, 60, 64, 86, 3, 51, 116, 107, 52, 47, 101, 110, 123, 50, 35, 100, 102, 109, 40, 53, 34, 122, 103, 125, 114, 121, 106, 61, 39, 42, 113, 45, 111, 57, 55], [44, 36, 104, 54, 92, 100, 95, 48, 105, 108, 28, 47, 96, 53, 32, 98, 37, 26, 27, 123, 40, 24, 97, 60, 127, 51, 112, 91, 58, 55, 85, 88, 52, 78, 93, 31, 33, 38, 21, 90, 113, 107, 49, 99, 124, 10, 110, 120, 115, 18, 89, 63, 30, 45, 34, 83, 62, 82, 56, 111, 94, 29, 122, 102, 84, 106, 118, 74, 103, 116, 117, 50, 46, 43, 22, 81, 101, 121, 119, 61, 114, 17, 39, 23, 57, 109, 59, 16, 86, 79, 126, 14, 87, 35, 70, 42, 41, 80, 5, 125, 9, 15, 75, 2, 20, 3, 66, 64, 12, 7, 73, 1, 77, 19, 76, 6, 4, 68, 0, 13, 25, 11, 67, 69, 72, 71, 8, 65], [48, 54, 37, 105, 108, 104, 36, 107, 47, 115, 63, 119, 95, 50, 44, 56, 112, 114, 118, 94, 127, 30, 117, 62, 110, 52, 55, 123, 116, 121, 53, 38, 58, 51, 124, 40, 59, 111, 113, 92, 34, 101, 120, 60, 125, 32, 46, 122, 126, 109, 57, 61, 100, 49, 24, 102, 33, 45, 98, 106, 43, 103, 28, 21, 99, 85, 42, 39, 96, 31, 41, 15, 97, 25, 29, 87, 19, 18, 81, 11, 23, 88, 12, 90, 73, 77, 17, 84, 26, 75, 91, 8, 93, 79, 22, 78, 7, 20, 5, 82, 80, 4, 35, 2, 0, 1, 64, 76, 6, 3, 14, 86, 74, 72, 70, 83, 71, 9, 89, 69, 27, 10, 16, 65, 68, 66, 67, 13], [54, 105, 48, 36, 123, 41, 25, 44, 127, 98, 99, 112, 34, 96, 115, 50, 47, 52, 60, 121, 118, 57, 117, 122, 64, 107, 63, 114, 108, 74, 119, 45, 120, 56, 22, 30, 113, 55, 8, 126, 111, 51, 58, 53, 62, 106, 110, 116, 125, 1, 124, 104, 59, 37, 61, 94, 80, 100, 68, 43, 75, 97, 109, 84, 67, 49, 81, 14, 38, 77, 6, 46, 42, 73, 103, 40, 5, 39, 79, 2, 24, 18, 4, 16, 3, 92, 89, 71, 21, 69, 12, 102, 28, 29, 33, 15, 20, 101, 88, 26, 85, 86, 7, 17, 31, 27, 95, 82, 91, 35, 32, 87, 13, 93, 76, 70, 19, 23, 90, 9, 0, 83, 11, 65, 78, 10, 66, 72], [37, 114, 49, 50, 54, 113, 94, 91, 101, 28, 118, 84, 61, 119, 25, 62, 45, 58, 11, 32, 112, 56, 87, 60, 29, 22, 124, 15, 31, 14, 90, 79, 116, 96, 81, 48, 33, 97, 82, 55, 24, 47, 110, 83, 36, 35, 117, 46, 95, 93, 109, 53, 98, 51, 99, 27, 18, 126, 127, 34, 30, 39, 115, 107, 102, 38, 40, 43, 63, 100, 52, 104, 44, 21, 105, 57, 125, 122, 123, 103, 92, 121, 41, 59, 120, 42, 8, 16, 108, 89, 106, 20, 88, 85, 26, 111, 76, 23, 86, 9, 75, 78, 5, 13, 19, 12, 6, 17, 80, 4, 10, 73, 7, 77, 2, 74, 3, 1, 71, 66, 72, 0, 70, 68, 69, 64, 65, 67], [114, 54, 49, 113, 50, 124, 119, 118, 109, 62, 123, 61, 56, 48, 47, 55, 112, 53, 58, 36, 127, 46, 37, 120, 63, 45, 126, 52, 60, 107, 125, 44, 115, 51, 111, 121, 117, 110, 122, 116, 59, 43, 57, 108, 17, 42, 105, 39, 106, 41, 101, 104, 20, 102, 38, 40, 82, 80, 30, 96, 19, 75, 79, 103, 86, 77, 78, 76, 95, 85, 24, 93, 100, 27, 97, 98, 99, 35, 34, 33, 74, 18, 32, 89, 72, 28, 71, 9, 23, 29, 14, 90, 15, 31, 92, 94, 87, 12, 88, 69, 70, 66, 13, 64, 65, 91, 84, 21, 81, 67, 26, 73, 83, 11, 25, 68, 22, 16, 6, 10, 4, 5, 1, 3, 8, 7, 0, 2], [37, 49, 114, 50, 54, 113, 101, 94, 62, 25, 91, 11, 84, 83, 119, 87, 81, 118, 32, 79, 8, 56, 5, 15, 76, 9, 28, 22, 21, 45, 6, 60, 4, 58, 61, 7, 124, 13, 110, 16, 33, 75, 112, 85, 14, 10, 18, 30, 116, 3, 82, 90, 46, 24, 1, 73, 53, 23, 96, 77, 89, 109, 78, 117, 2, 63, 74, 125, 47, 12, 80, 52, 31, 126, 20, 17, 71, 48, 93, 57, 127, 66, 19, 115, 98, 105, 55, 27, 36, 44, 86, 29, 88, 40, 35, 95, 26, 43, 97, 122, 92, 72, 111, 68, 59, 38, 123, 41, 0, 51, 107, 34, 103, 39, 121, 42, 120, 102, 99, 106, 108, 100, 104, 70, 69, 65, 64, 67], [49, 37, 114, 54, 50, 113, 5, 9, 76, 6, 8, 7, 94, 4, 77, 79, 75, 12, 87, 16, 3, 25, 30, 22, 73, 11, 74, 83, 78, 80, 82, 17, 124, 2, 81, 71, 72, 91, 10, 45, 19, 56, 60, 62, 119, 61, 112, 1, 32, 118, 13, 15, 58, 84, 85, 63, 53, 26, 101, 21, 20, 117, 110, 109, 55, 23, 90, 93, 92, 66, 14, 89, 116, 31, 88, 46, 29, 48, 18, 107, 28, 24, 27, 125, 102, 127, 115, 57, 0, 126, 36, 52, 51, 47, 86, 123, 44, 70, 96, 121, 68, 43, 33, 42, 59, 39, 122, 104, 111, 120, 98, 67, 40, 69, 41, 100, 105, 35, 34, 38, 108, 97, 95, 103, 106, 65, 99, 64], [41, 101, 25, 23, 85, 110, 18, 105, 15, 34, 13, 20, 11, 91, 124, 30, 113, 9, 14, 83, 43, 8, 109, 24, 126, 94, 82, 68, 6, 51, 84, 48, 107, 118, 92, 29, 27, 59, 112, 111, 57, 4, 45, 77, 114, 44, 87, 88, 122, 89, 36, 53, 62, 17, 26, 98, 79, 31, 32, 16, 76, 80, 58, 117, 78, 103, 90, 96, 115, 66, 50, 75, 72, 21, 67, 38, 56, 2, 99, 55, 10, 47, 95, 37, 121, 0, 61, 52, 69, 5, 102, 120, 86, 108, 19, 28, 63, 60, 46, 22, 116, 106, 39, 123, 35, 127, 54, 65, 104, 71, 70, 1, 7, 97, 73, 119, 93, 12, 3, 33, 74, 40, 49, 42, 81, 100, 125, 64], [41, 101, 25, 85, 18, 23, 20, 105, 15, 11, 34, 13, 91, 124, 14, 9, 6, 51, 113, 2, 118, 8, 82, 112, 114, 29, 27, 53, 83, 98, 84, 68, 81, 109, 75, 48, 78, 32, 43, 107, 89, 30, 66, 87, 92, 111, 94, 126, 70, 36, 90, 45, 79, 110, 17, 44, 88, 122, 62, 57, 59, 117, 21, 7, 99, 103, 60, 127, 47, 76, 19, 38, 26, 42, 31, 24, 120, 52, 58, 16, 46, 69, 108, 77, 0, 35, 119, 72, 61, 115, 56, 80, 37, 22, 123, 102, 55, 63, 121, 10, 86, 95, 28, 67, 96, 97, 125, 100, 106, 39, 71, 54, 116, 50, 73, 93, 5, 74, 12, 40, 33, 65, 3, 49, 4, 1, 104, 64], [41, 101, 109, 110, 91, 25, 20, 18, 34, 85, 105, 23, 14, 124, 11, 15, 9, 13, 43, 53, 24, 113, 82, 84, 29, 8, 107, 45, 27, 126, 83, 6, 111, 90, 127, 59, 68, 98, 112, 51, 28, 44, 120, 78, 56, 47, 121, 118, 117, 114, 116, 62, 50, 106, 122, 55, 57, 92, 58, 95, 93, 94, 81, 36, 52, 97, 16, 87, 32, 30, 60, 102, 108, 42, 99, 38, 66, 75, 26, 40, 96, 119, 88, 63, 67, 33, 31, 89, 35, 37, 48, 80, 123, 115, 70, 61, 49, 46, 39, 104, 71, 125, 2, 22, 103, 76, 77, 86, 12, 54, 17, 79, 0, 4, 19, 21, 72, 7, 10, 5, 69, 100, 73, 65, 74, 1, 3, 64], [41, 101, 25, 85, 91, 105, 23, 15, 18, 20, 34, 13, 98, 113, 124, 81, 110, 29, 30, 11, 48, 95, 126, 43, 92, 114, 27, 9, 53, 14, 109, 84, 36, 44, 94, 24, 118, 45, 35, 26, 83, 100, 111, 89, 51, 107, 46, 57, 82, 120, 76, 123, 112, 61, 119, 38, 40, 103, 122, 121, 31, 56, 87, 116, 8, 62, 60, 50, 28, 106, 99, 47, 127, 125, 39, 80, 97, 117, 78, 42, 33, 6, 32, 58, 54, 59, 86, 102, 16, 88, 73, 21, 55, 63, 77, 22, 17, 96, 115, 52, 49, 79, 108, 4, 93, 12, 90, 104, 19, 68, 37, 66, 75, 10, 70, 74, 72, 5, 67, 7, 71, 1, 2, 69, 64, 3, 0, 65]], "model.layers.7.self_attn.k_proj": [[45, 107, 102, 0, 109, 60, 1, 12, 3, 77, 70, 19, 71, 43, 10, 8, 78, 115, 80, 29, 68, 22, 25, 5, 18, 17, 11, 125, 2, 73, 57, 126, 56, 48, 122, 111, 124, 79, 55, 110, 113, 63, 20, 66, 58, 67, 52, 47, 127, 62, 117, 49, 59, 114, 112, 96, 121, 51, 44, 54, 23, 118, 6, 119, 61, 92, 123, 53, 27, 95, 46, 116, 98, 50, 120, 106, 69, 89, 100, 64, 65, 21, 4, 108, 75, 9, 7, 41, 42, 105, 40, 39, 87, 30, 35, 24, 104, 94, 103, 36, 97, 31, 26, 101, 84, 37, 32, 33, 99, 90, 28, 85, 88, 81, 72, 93, 34, 15, 74, 76, 91, 82, 14, 16, 83, 13, 86, 38], [41, 118, 64, 111, 117, 33, 79, 82, 84, 1, 13, 23, 90, 93, 56, 71, 2, 11, 9, 4, 89, 85, 10, 105, 17, 65, 5, 37, 47, 121, 6, 30, 76, 67, 14, 116, 32, 80, 66, 72, 101, 22, 24, 115, 0, 48, 19, 28, 59, 29, 127, 44, 126, 49, 95, 110, 3, 26, 55, 112, 88, 83, 16, 38, 43, 35, 94, 52, 98, 27, 18, 46, 81, 40, 120, 107, 12, 100, 21, 102, 62, 124, 119, 61, 106, 113, 99, 69, 34, 53, 91, 123, 97, 103, 96, 42, 63, 57, 39, 51, 109, 114, 60, 36, 31, 58, 73, 104, 108, 54, 8, 122, 125, 68, 50, 25, 45, 78, 75, 92, 20, 77, 87, 86, 15, 74, 7, 70], [106, 39, 94, 98, 85, 23, 83, 127, 74, 12, 81, 78, 72, 16, 15, 50, 112, 65, 68, 42, 6, 41, 53, 69, 118, 2, 114, 49, 13, 0, 124, 88, 77, 18, 52, 111, 55, 8, 79, 38, 126, 125, 115, 34, 29, 36, 102, 109, 108, 120, 54, 7, 64, 107, 31, 40, 63, 110, 9, 47, 26, 101, 75, 60, 48, 3, 71, 61, 35, 122, 67, 117, 104, 25, 22, 58, 100, 43, 119, 62, 80, 113, 66, 56, 5, 11, 121, 14, 73, 105, 90, 57, 33, 123, 44, 92, 103, 46, 89, 24, 86, 116, 93, 37, 82, 27, 59, 99, 96, 19, 91, 45, 32, 95, 84, 28, 20, 97, 30, 76, 10, 17, 4, 51, 1, 87, 70, 21], [40, 121, 97, 90, 23, 21, 83, 36, 79, 48, 17, 82, 11, 13, 86, 8, 122, 73, 0, 30, 78, 50, 51, 107, 6, 109, 116, 69, 60, 59, 10, 65, 115, 58, 110, 101, 45, 29, 9, 4, 16, 68, 53, 123, 28, 84, 93, 66, 67, 39, 100, 126, 61, 37, 27, 42, 12, 89, 55, 124, 112, 3, 56, 96, 32, 47, 57, 70, 120, 125, 117, 103, 119, 2, 114, 95, 94, 35, 98, 14, 74, 54, 92, 106, 46, 43, 75, 88, 99, 41, 111, 105, 62, 52, 118, 49, 108, 34, 63, 38, 127, 91, 113, 44, 5, 31, 102, 15, 19, 76, 1, 71, 7, 18, 22, 24, 64, 25, 20, 72, 80, 77, 85, 26, 81, 87, 104, 33], [104, 41, 101, 29, 87, 55, 20, 63, 18, 16, 76, 10, 14, 60, 37, 57, 7, 0, 65, 72, 77, 67, 52, 69, 112, 58, 124, 113, 114, 71, 61, 66, 5, 86, 30, 17, 9, 126, 24, 15, 27, 75, 8, 93, 123, 4, 12, 102, 89, 62, 43, 64, 94, 21, 53, 2, 90, 70, 83, 13, 85, 39, 79, 19, 96, 88, 56, 100, 117, 3, 107, 116, 108, 45, 127, 82, 6, 91, 46, 111, 119, 47, 11, 121, 81, 23, 51, 54, 25, 49, 120, 73, 95, 32, 68, 42, 59, 26, 78, 34, 36, 28, 122, 80, 22, 31, 98, 92, 110, 84, 109, 103, 97, 106, 118, 44, 35, 99, 115, 74, 38, 48, 50, 125, 33, 40, 1, 105], [41, 54, 48, 35, 32, 22, 93, 16, 44, 127, 40, 10, 112, 30, 90, 89, 78, 84, 124, 83, 3, 70, 77, 27, 115, 123, 113, 51, 18, 117, 25, 52, 55, 101, 120, 111, 24, 7, 59, 121, 76, 119, 110, 63, 5, 122, 60, 50, 62, 126, 114, 58, 57, 43, 11, 118, 108, 31, 116, 81, 125, 1, 61, 56, 53, 109, 47, 19, 45, 100, 72, 79, 91, 46, 103, 102, 42, 39, 106, 9, 49, 2, 23, 107, 4, 80, 64, 73, 0, 13, 74, 66, 20, 12, 8, 33, 28, 21, 86, 104, 65, 38, 14, 26, 71, 87, 98, 34, 92, 15, 17, 85, 95, 68, 97, 29, 88, 36, 105, 96, 37, 94, 82, 75, 6, 99, 69, 67], [50, 101, 113, 49, 54, 94, 114, 22, 91, 87, 25, 21, 83, 81, 74, 84, 13, 18, 16, 86, 14, 72, 56, 71, 80, 110, 69, 77, 15, 32, 112, 58, 70, 60, 67, 66, 53, 117, 124, 26, 88, 92, 12, 45, 119, 68, 52, 55, 78, 62, 63, 125, 109, 9, 127, 73, 61, 64, 57, 4, 37, 51, 43, 126, 107, 11, 118, 115, 48, 19, 100, 123, 98, 116, 31, 47, 65, 121, 59, 75, 111, 120, 42, 29, 104, 102, 95, 6, 105, 44, 41, 103, 89, 76, 46, 40, 108, 122, 39, 1, 23, 97, 17, 38, 106, 3, 27, 10, 34, 35, 36, 99, 33, 85, 24, 30, 28, 96, 93, 79, 90, 20, 5, 8, 82, 7, 2, 0], [105, 37, 23, 18, 25, 15, 20, 11, 14, 85, 101, 13, 124, 91, 113, 34, 9, 46, 49, 8, 48, 126, 6, 1, 41, 118, 43, 68, 45, 115, 2, 111, 74, 110, 94, 16, 0, 63, 119, 114, 51, 83, 24, 62, 98, 64, 70, 59, 35, 53, 109, 92, 117, 58, 81, 102, 12, 65, 90, 55, 31, 10, 30, 57, 67, 3, 44, 29, 36, 61, 120, 7, 4, 32, 112, 103, 122, 108, 27, 116, 99, 96, 77, 50, 26, 86, 72, 107, 106, 121, 19, 95, 104, 93, 88, 56, 75, 71, 42, 125, 40, 54, 22, 38, 100, 123, 52, 97, 39, 5, 17, 79, 33, 69, 127, 47, 80, 28, 73, 60, 66, 21, 76, 78, 89, 84, 87, 82]], "model.layers.7.self_attn.qk_proj": [[121, 41, 54, 118, 105, 50, 37, 45, 106, 48, 101, 49, 109, 114, 107, 104, 113, 93, 43, 23, 87, 60, 30, 42, 89, 77, 40, 85, 25, 82, 83, 13, 14, 18, 21, 26, 78, 117, 19, 16, 112, 10, 111, 124, 79, 15, 20, 34, 63, 55, 81, 84, 74, 80, 103, 72, 17, 56, 119, 115, 94, 76, 29, 12, 39, 100, 86, 33, 57, 110, 75, 122, 126, 27, 47, 127, 52, 44, 125, 0, 11, 22, 53, 102, 90, 8, 58, 123, 32, 97, 64, 61, 7, 67, 69, 5, 62, 9, 59, 71, 51, 65, 73, 70, 98, 3, 4, 91, 120, 116, 38, 96, 6, 35, 24, 36, 46, 1, 88, 68, 31, 92, 108, 28, 2, 95, 99, 66], [121, 41, 118, 105, 54, 50, 37, 45, 106, 48, 49, 101, 109, 107, 114, 104, 43, 113, 93, 23, 30, 87, 40, 60, 42, 25, 21, 18, 85, 77, 19, 13, 14, 82, 89, 83, 26, 112, 111, 84, 103, 79, 78, 16, 10, 63, 80, 117, 15, 34, 55, 124, 94, 17, 20, 74, 12, 81, 72, 115, 119, 100, 56, 86, 39, 76, 122, 53, 29, 11, 125, 33, 47, 110, 127, 27, 61, 57, 22, 102, 75, 0, 52, 90, 126, 8, 97, 62, 64, 123, 32, 51, 44, 58, 98, 69, 7, 9, 4, 5, 6, 1, 65, 71, 67, 36, 91, 120, 35, 116, 73, 59, 68, 108, 70, 38, 3, 24, 96, 88, 2, 99, 28, 66, 46, 92, 31, 95], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 48, 114, 109, 49, 107, 104, 113, 93, 23, 43, 30, 87, 42, 60, 40, 25, 89, 85, 21, 77, 26, 18, 19, 82, 13, 117, 55, 103, 14, 94, 83, 112, 84, 34, 111, 78, 81, 124, 20, 79, 100, 16, 10, 74, 15, 29, 80, 33, 39, 86, 63, 17, 110, 76, 97, 47, 12, 122, 90, 56, 53, 115, 72, 57, 8, 27, 119, 22, 127, 102, 52, 32, 75, 0, 64, 11, 58, 62, 69, 126, 125, 6, 98, 7, 5, 1, 44, 65, 61, 68, 123, 51, 71, 91, 3, 108, 9, 96, 4, 59, 36, 35, 67, 46, 116, 73, 24, 99, 38, 31, 88, 120, 70, 2, 66, 28, 95, 92], [121, 118, 41, 105, 54, 50, 45, 37, 106, 49, 48, 101, 114, 109, 107, 104, 93, 113, 43, 87, 23, 30, 89, 40, 42, 60, 25, 82, 26, 85, 117, 77, 112, 21, 18, 103, 55, 78, 127, 83, 13, 19, 79, 119, 20, 94, 84, 34, 124, 15, 63, 14, 111, 53, 100, 81, 10, 16, 47, 80, 17, 74, 29, 39, 27, 86, 12, 33, 102, 110, 64, 125, 11, 44, 115, 76, 126, 32, 52, 56, 58, 90, 97, 122, 22, 8, 72, 0, 51, 57, 69, 71, 62, 75, 5, 6, 67, 65, 116, 3, 73, 61, 91, 59, 7, 98, 120, 68, 1, 36, 35, 9, 123, 96, 4, 2, 46, 31, 38, 70, 88, 28, 66, 108, 92, 24, 95, 99], [121, 41, 118, 105, 54, 50, 45, 37, 106, 48, 49, 101, 114, 109, 107, 104, 113, 43, 93, 87, 23, 42, 40, 89, 30, 60, 77, 26, 18, 85, 82, 21, 117, 25, 55, 13, 34, 111, 112, 14, 103, 83, 78, 10, 84, 94, 16, 63, 47, 74, 15, 19, 20, 79, 124, 76, 81, 8, 80, 100, 39, 29, 17, 12, 119, 102, 115, 11, 33, 57, 56, 86, 64, 53, 0, 126, 110, 75, 32, 65, 22, 127, 90, 97, 122, 68, 69, 72, 62, 27, 58, 61, 44, 52, 5, 98, 4, 125, 7, 73, 3, 6, 9, 51, 67, 38, 91, 71, 1, 116, 59, 66, 24, 70, 123, 108, 35, 2, 36, 88, 120, 96, 46, 92, 28, 99, 31, 95], [121, 41, 118, 54, 105, 50, 45, 37, 106, 101, 48, 49, 114, 107, 109, 104, 43, 113, 93, 87, 23, 60, 42, 77, 85, 82, 89, 40, 30, 25, 18, 21, 26, 14, 78, 13, 19, 112, 79, 83, 111, 16, 103, 84, 117, 74, 20, 15, 63, 34, 17, 10, 11, 8, 81, 80, 124, 12, 76, 100, 115, 94, 47, 55, 86, 126, 53, 39, 125, 29, 57, 127, 110, 72, 61, 52, 75, 0, 119, 33, 102, 58, 62, 27, 22, 44, 64, 122, 32, 56, 7, 9, 120, 97, 3, 90, 91, 123, 68, 6, 73, 5, 69, 51, 4, 71, 70, 65, 59, 1, 67, 98, 24, 35, 66, 116, 36, 38, 96, 108, 2, 92, 88, 28, 46, 31, 99, 95], [121, 41, 54, 105, 118, 50, 37, 45, 101, 106, 49, 48, 114, 109, 107, 23, 104, 113, 93, 87, 43, 30, 82, 42, 60, 77, 18, 21, 25, 14, 89, 85, 13, 40, 78, 26, 16, 19, 15, 84, 117, 111, 83, 79, 20, 103, 63, 17, 34, 74, 8, 10, 81, 112, 80, 12, 76, 47, 94, 124, 29, 33, 110, 27, 11, 115, 86, 39, 57, 55, 52, 100, 22, 125, 126, 75, 119, 122, 127, 56, 58, 44, 90, 61, 91, 32, 102, 72, 53, 0, 59, 62, 97, 73, 69, 123, 98, 7, 96, 5, 64, 9, 71, 51, 38, 68, 6, 3, 116, 92, 35, 36, 24, 1, 88, 67, 70, 4, 120, 65, 46, 2, 31, 28, 108, 66, 95, 99], [121, 41, 105, 118, 54, 50, 45, 106, 37, 49, 101, 48, 109, 114, 107, 104, 113, 93, 43, 23, 42, 87, 82, 40, 18, 21, 30, 13, 85, 19, 77, 60, 89, 26, 14, 78, 25, 79, 16, 84, 83, 15, 103, 80, 17, 20, 74, 12, 81, 8, 76, 34, 117, 10, 124, 94, 111, 29, 86, 11, 63, 112, 55, 39, 119, 27, 75, 100, 57, 127, 33, 115, 22, 61, 44, 102, 0, 47, 64, 90, 1, 125, 52, 5, 91, 110, 72, 69, 58, 32, 123, 122, 70, 56, 7, 24, 53, 97, 65, 62, 126, 59, 71, 98, 73, 4, 88, 67, 9, 96, 38, 51, 36, 116, 68, 92, 3, 2, 6, 46, 120, 31, 35, 95, 66, 108, 28, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 101, 49, 109, 107, 114, 104, 43, 93, 113, 23, 87, 40, 42, 30, 89, 21, 19, 18, 82, 25, 85, 60, 26, 84, 13, 14, 111, 77, 16, 83, 94, 63, 20, 79, 103, 117, 15, 17, 78, 112, 55, 34, 80, 12, 74, 124, 81, 10, 57, 8, 29, 53, 76, 33, 127, 47, 110, 52, 27, 39, 86, 119, 11, 126, 90, 125, 58, 100, 22, 102, 97, 44, 51, 32, 62, 91, 115, 75, 56, 0, 64, 98, 7, 61, 123, 70, 122, 4, 72, 5, 36, 65, 71, 88, 116, 59, 69, 73, 68, 67, 38, 9, 24, 1, 96, 120, 35, 92, 3, 6, 108, 28, 66, 31, 95, 99, 46, 2], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 49, 48, 109, 107, 114, 104, 113, 93, 43, 87, 23, 30, 60, 42, 25, 82, 21, 85, 40, 89, 13, 18, 111, 19, 16, 14, 77, 117, 83, 26, 84, 15, 20, 103, 79, 78, 34, 112, 17, 55, 80, 10, 81, 94, 74, 76, 8, 12, 115, 124, 102, 119, 11, 29, 126, 63, 27, 39, 33, 100, 86, 125, 47, 127, 57, 44, 56, 53, 110, 90, 122, 61, 51, 32, 64, 58, 91, 22, 0, 97, 75, 69, 72, 70, 62, 52, 71, 123, 73, 3, 98, 5, 65, 9, 116, 36, 67, 1, 7, 4, 92, 88, 68, 96, 59, 6, 120, 28, 24, 35, 66, 38, 46, 31, 99, 108, 95, 2], [121, 41, 118, 54, 105, 50, 37, 45, 106, 48, 49, 101, 114, 109, 107, 104, 113, 43, 93, 23, 42, 87, 30, 40, 13, 82, 25, 85, 89, 60, 77, 21, 18, 111, 55, 83, 117, 15, 16, 26, 79, 14, 63, 20, 78, 34, 119, 112, 19, 10, 103, 84, 74, 115, 47, 94, 80, 53, 127, 81, 17, 12, 52, 8, 39, 29, 110, 76, 86, 11, 33, 100, 72, 64, 102, 126, 122, 57, 124, 56, 44, 69, 62, 61, 51, 75, 125, 0, 97, 32, 1, 7, 58, 90, 5, 22, 123, 27, 70, 71, 9, 65, 73, 91, 67, 6, 98, 4, 68, 3, 38, 35, 36, 120, 96, 59, 116, 2, 66, 28, 92, 24, 31, 46, 108, 88, 95, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 49, 114, 101, 107, 109, 104, 113, 93, 87, 43, 23, 42, 30, 82, 85, 25, 40, 21, 18, 89, 13, 77, 60, 83, 26, 14, 15, 79, 111, 19, 84, 103, 16, 34, 20, 112, 78, 117, 81, 80, 10, 74, 94, 63, 124, 55, 76, 17, 86, 47, 39, 127, 12, 8, 53, 119, 29, 72, 33, 115, 100, 52, 27, 11, 110, 64, 56, 69, 90, 102, 22, 126, 75, 62, 44, 123, 91, 73, 32, 122, 67, 71, 57, 97, 125, 5, 61, 0, 4, 70, 68, 3, 9, 51, 7, 58, 1, 6, 98, 120, 65, 96, 38, 36, 66, 24, 116, 2, 59, 108, 88, 46, 35, 92, 28, 31, 99, 95], [121, 41, 118, 54, 105, 50, 45, 37, 101, 106, 48, 49, 109, 114, 107, 104, 113, 23, 93, 43, 87, 60, 30, 18, 42, 40, 25, 89, 21, 82, 13, 85, 19, 16, 26, 14, 83, 77, 78, 124, 84, 79, 103, 117, 111, 94, 15, 34, 81, 80, 20, 72, 74, 17, 112, 63, 10, 12, 115, 76, 29, 55, 119, 86, 100, 33, 27, 47, 90, 11, 126, 125, 39, 127, 110, 53, 75, 44, 102, 8, 56, 57, 52, 62, 97, 91, 22, 32, 61, 58, 122, 59, 64, 6, 4, 123, 73, 69, 0, 71, 68, 7, 65, 9, 3, 24, 36, 5, 51, 96, 98, 116, 70, 67, 38, 46, 92, 88, 120, 35, 31, 108, 1, 2, 66, 28, 95, 99], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 49, 48, 109, 114, 107, 104, 113, 93, 43, 23, 60, 13, 40, 42, 87, 30, 18, 85, 25, 82, 89, 103, 21, 83, 26, 72, 78, 16, 14, 17, 117, 79, 19, 20, 15, 34, 77, 63, 74, 84, 81, 111, 124, 55, 10, 112, 80, 76, 12, 119, 47, 126, 29, 11, 75, 86, 0, 64, 27, 94, 115, 100, 127, 8, 125, 44, 39, 90, 61, 102, 33, 110, 22, 53, 6, 57, 52, 56, 69, 5, 58, 7, 65, 51, 71, 1, 9, 32, 68, 62, 97, 73, 116, 122, 91, 4, 3, 59, 96, 67, 70, 98, 36, 123, 88, 35, 24, 38, 66, 92, 31, 120, 46, 2, 99, 28, 95, 108], [121, 41, 118, 105, 54, 50, 45, 37, 49, 106, 101, 48, 109, 114, 107, 104, 43, 113, 93, 23, 87, 30, 42, 60, 25, 40, 89, 18, 21, 82, 85, 26, 13, 83, 111, 15, 16, 78, 72, 79, 19, 20, 112, 117, 77, 84, 63, 14, 124, 74, 103, 17, 34, 10, 81, 80, 55, 29, 47, 86, 94, 115, 12, 119, 76, 39, 127, 27, 57, 90, 33, 11, 110, 100, 53, 75, 126, 52, 61, 32, 8, 102, 62, 22, 69, 58, 6, 56, 0, 125, 44, 91, 64, 71, 73, 5, 9, 7, 122, 123, 51, 65, 4, 3, 1, 67, 36, 88, 97, 70, 68, 96, 24, 38, 120, 46, 116, 59, 92, 28, 98, 35, 2, 31, 66, 108, 99, 95], [121, 41, 118, 54, 105, 50, 45, 37, 106, 49, 48, 101, 114, 107, 109, 104, 113, 93, 43, 23, 30, 60, 42, 25, 87, 85, 40, 89, 82, 18, 13, 26, 83, 21, 112, 77, 14, 111, 117, 16, 94, 78, 19, 15, 79, 34, 20, 84, 63, 74, 72, 29, 124, 55, 80, 115, 10, 17, 53, 57, 126, 47, 86, 76, 39, 103, 81, 33, 100, 119, 110, 62, 125, 127, 12, 56, 11, 90, 22, 27, 52, 6, 64, 75, 102, 61, 32, 8, 122, 44, 97, 91, 7, 51, 123, 58, 9, 73, 69, 0, 38, 68, 4, 71, 5, 3, 36, 59, 67, 46, 1, 120, 116, 98, 96, 88, 24, 70, 35, 65, 92, 108, 66, 28, 31, 99, 2, 95], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 114, 109, 48, 107, 104, 113, 23, 93, 87, 60, 43, 42, 30, 25, 89, 18, 82, 85, 13, 83, 40, 21, 26, 16, 117, 19, 78, 14, 77, 84, 81, 79, 34, 15, 74, 112, 72, 20, 124, 80, 10, 111, 94, 17, 100, 12, 115, 39, 55, 76, 119, 103, 63, 110, 33, 11, 29, 90, 57, 86, 47, 127, 123, 27, 53, 102, 58, 22, 52, 56, 8, 125, 44, 32, 75, 122, 0, 91, 126, 9, 7, 71, 97, 62, 61, 5, 69, 68, 51, 6, 98, 64, 120, 1, 70, 36, 59, 4, 108, 38, 116, 73, 35, 65, 88, 24, 46, 96, 92, 67, 2, 31, 3, 99, 95, 28, 66], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 109, 48, 114, 107, 104, 113, 23, 93, 43, 42, 87, 30, 60, 85, 83, 40, 25, 13, 89, 82, 18, 21, 26, 19, 111, 78, 77, 16, 117, 84, 14, 79, 15, 20, 103, 80, 72, 34, 124, 10, 112, 17, 81, 29, 74, 94, 55, 76, 119, 12, 33, 63, 57, 75, 90, 8, 127, 100, 47, 110, 126, 11, 39, 86, 22, 115, 61, 52, 32, 27, 53, 62, 102, 125, 56, 64, 58, 7, 51, 0, 91, 122, 44, 71, 5, 69, 70, 97, 9, 59, 1, 67, 116, 73, 98, 6, 123, 36, 38, 3, 4, 46, 65, 120, 96, 24, 68, 99, 28, 92, 31, 108, 35, 88, 95, 2, 66], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 101, 109, 48, 114, 107, 104, 43, 113, 23, 30, 87, 93, 89, 42, 60, 40, 25, 18, 117, 85, 13, 63, 82, 21, 83, 112, 26, 94, 16, 111, 77, 78, 119, 103, 20, 34, 55, 15, 84, 19, 80, 79, 124, 14, 33, 57, 74, 81, 29, 102, 10, 17, 52, 39, 72, 127, 90, 8, 12, 115, 53, 100, 97, 47, 76, 86, 110, 126, 32, 27, 44, 11, 64, 58, 51, 0, 62, 91, 56, 75, 22, 71, 69, 61, 7, 122, 70, 98, 5, 1, 67, 4, 3, 68, 35, 9, 36, 65, 125, 123, 59, 120, 73, 92, 24, 95, 38, 31, 6, 116, 99, 46, 28, 108, 88, 96, 2, 66], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 48, 109, 107, 114, 113, 104, 93, 23, 43, 30, 25, 87, 60, 42, 18, 77, 40, 26, 21, 89, 13, 85, 83, 19, 82, 117, 80, 111, 112, 15, 84, 14, 124, 10, 78, 34, 79, 16, 103, 74, 63, 76, 94, 55, 100, 72, 81, 17, 8, 33, 115, 39, 29, 12, 20, 126, 86, 27, 102, 11, 110, 62, 53, 75, 57, 56, 44, 97, 61, 127, 22, 70, 119, 47, 52, 71, 51, 58, 90, 69, 0, 122, 64, 5, 125, 123, 91, 32, 73, 7, 4, 9, 65, 1, 36, 98, 35, 120, 68, 59, 24, 46, 96, 38, 3, 116, 6, 67, 31, 88, 92, 108, 99, 2, 28, 95, 66], [121, 41, 118, 54, 105, 50, 45, 37, 106, 49, 101, 48, 109, 114, 107, 104, 113, 93, 43, 42, 23, 30, 60, 87, 89, 40, 25, 18, 13, 77, 85, 82, 21, 26, 103, 117, 14, 111, 74, 83, 19, 63, 79, 112, 119, 16, 34, 15, 81, 78, 84, 80, 94, 29, 33, 55, 124, 20, 10, 57, 17, 8, 76, 62, 110, 86, 127, 39, 53, 47, 12, 11, 72, 75, 126, 100, 22, 90, 102, 58, 115, 56, 9, 71, 123, 70, 27, 44, 52, 69, 5, 97, 125, 32, 51, 64, 120, 91, 7, 122, 3, 0, 61, 4, 59, 38, 73, 67, 68, 98, 46, 96, 36, 1, 6, 88, 65, 99, 35, 31, 116, 108, 28, 2, 92, 66, 24, 95], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 48, 101, 109, 114, 107, 104, 113, 87, 93, 23, 43, 42, 30, 60, 89, 40, 25, 82, 85, 26, 13, 18, 21, 77, 19, 117, 83, 14, 79, 103, 124, 34, 16, 112, 78, 111, 63, 15, 74, 84, 119, 29, 94, 33, 55, 20, 17, 110, 80, 81, 10, 76, 86, 8, 127, 12, 100, 39, 90, 115, 57, 75, 102, 47, 126, 52, 32, 53, 91, 62, 27, 56, 22, 44, 58, 11, 64, 97, 72, 61, 51, 123, 0, 71, 5, 59, 125, 69, 9, 70, 1, 98, 122, 116, 73, 38, 7, 88, 120, 46, 96, 67, 36, 4, 3, 35, 65, 68, 28, 92, 31, 99, 6, 24, 95, 2, 108, 66], [121, 41, 118, 54, 105, 50, 37, 45, 101, 49, 48, 106, 114, 109, 107, 113, 104, 43, 93, 23, 42, 30, 87, 60, 25, 40, 89, 18, 82, 13, 26, 85, 117, 77, 21, 78, 34, 94, 19, 124, 103, 83, 14, 55, 84, 15, 79, 16, 74, 111, 80, 20, 17, 8, 81, 29, 10, 112, 119, 76, 47, 115, 86, 90, 39, 33, 11, 12, 56, 63, 126, 100, 102, 75, 44, 127, 22, 110, 53, 57, 0, 32, 91, 61, 27, 58, 97, 52, 72, 64, 73, 4, 62, 51, 69, 5, 1, 122, 125, 9, 123, 68, 38, 71, 6, 59, 7, 35, 65, 24, 96, 98, 46, 70, 88, 36, 3, 67, 116, 31, 92, 66, 28, 120, 95, 99, 2, 108], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 48, 49, 109, 114, 107, 104, 113, 43, 93, 60, 23, 42, 30, 25, 87, 40, 85, 13, 117, 21, 89, 82, 18, 26, 19, 77, 78, 103, 111, 74, 124, 55, 94, 16, 79, 8, 14, 63, 84, 83, 112, 34, 20, 127, 17, 10, 15, 119, 80, 29, 33, 126, 47, 12, 56, 53, 81, 100, 86, 39, 75, 115, 110, 76, 62, 102, 11, 51, 90, 27, 97, 125, 61, 22, 44, 58, 32, 72, 7, 57, 64, 52, 73, 6, 122, 69, 9, 4, 123, 5, 91, 68, 71, 0, 96, 3, 98, 35, 116, 38, 120, 36, 67, 59, 65, 46, 108, 1, 88, 70, 28, 31, 66, 99, 24, 95, 92, 2], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 101, 48, 109, 114, 104, 107, 113, 43, 93, 23, 60, 30, 42, 87, 25, 13, 40, 82, 89, 21, 85, 18, 78, 19, 117, 26, 34, 77, 74, 15, 79, 124, 111, 83, 112, 14, 16, 20, 80, 84, 103, 10, 63, 17, 8, 39, 94, 115, 33, 100, 55, 29, 12, 76, 81, 110, 11, 86, 57, 56, 126, 22, 44, 90, 127, 53, 75, 119, 27, 52, 47, 58, 72, 102, 62, 123, 122, 32, 64, 97, 125, 73, 6, 71, 38, 5, 91, 69, 61, 0, 36, 9, 59, 120, 7, 51, 4, 67, 31, 1, 98, 68, 3, 88, 116, 70, 65, 46, 96, 35, 28, 108, 99, 92, 24, 2, 95, 66], [121, 41, 118, 105, 54, 50, 45, 37, 106, 101, 49, 48, 109, 114, 107, 104, 113, 43, 93, 23, 60, 42, 87, 40, 25, 82, 13, 85, 30, 89, 18, 21, 26, 77, 117, 14, 19, 74, 83, 34, 15, 78, 79, 63, 112, 80, 84, 17, 103, 8, 111, 94, 10, 12, 20, 16, 124, 76, 29, 110, 55, 39, 100, 81, 33, 86, 119, 127, 115, 52, 75, 47, 53, 56, 27, 57, 11, 126, 102, 22, 72, 90, 97, 125, 58, 0, 122, 6, 44, 64, 7, 123, 32, 71, 51, 62, 69, 91, 59, 73, 9, 5, 61, 98, 1, 120, 65, 96, 116, 88, 68, 4, 3, 38, 24, 92, 36, 67, 46, 35, 108, 28, 31, 70, 2, 99, 95, 66], [121, 41, 118, 105, 54, 50, 45, 37, 106, 101, 48, 49, 114, 109, 107, 104, 93, 113, 43, 23, 42, 30, 87, 60, 25, 40, 85, 82, 26, 89, 21, 77, 13, 83, 18, 103, 14, 19, 111, 20, 34, 17, 117, 55, 84, 124, 16, 78, 79, 94, 112, 29, 15, 63, 127, 86, 80, 76, 126, 119, 10, 74, 110, 33, 39, 53, 58, 123, 56, 27, 90, 8, 122, 12, 115, 81, 47, 57, 11, 52, 22, 102, 44, 72, 97, 0, 100, 32, 125, 62, 38, 61, 75, 9, 51, 64, 59, 7, 4, 116, 98, 5, 69, 6, 36, 68, 67, 3, 96, 65, 91, 24, 71, 1, 120, 73, 70, 92, 35, 88, 31, 28, 46, 95, 66, 99, 2, 108], [121, 41, 118, 105, 54, 50, 45, 37, 106, 48, 49, 101, 109, 107, 114, 104, 113, 43, 93, 42, 87, 23, 60, 30, 40, 13, 82, 77, 85, 21, 25, 18, 19, 89, 78, 14, 34, 15, 117, 83, 74, 26, 103, 111, 112, 10, 20, 72, 124, 16, 79, 80, 17, 8, 12, 81, 84, 76, 47, 110, 64, 75, 56, 127, 115, 100, 55, 86, 29, 63, 94, 119, 11, 126, 39, 0, 53, 22, 122, 52, 102, 7, 61, 5, 57, 33, 58, 90, 68, 69, 9, 1, 62, 27, 97, 44, 65, 32, 73, 123, 59, 125, 98, 3, 71, 91, 4, 6, 67, 70, 51, 116, 66, 38, 36, 120, 88, 96, 35, 2, 24, 46, 108, 31, 92, 28, 99, 95], [41, 121, 118, 105, 54, 50, 45, 37, 106, 49, 48, 101, 109, 107, 114, 104, 113, 43, 93, 42, 23, 30, 87, 60, 40, 89, 25, 85, 55, 19, 77, 13, 112, 18, 21, 34, 117, 26, 103, 82, 78, 83, 111, 124, 94, 63, 15, 20, 10, 84, 16, 14, 74, 115, 79, 80, 29, 56, 100, 72, 39, 76, 33, 17, 81, 127, 47, 110, 53, 8, 126, 12, 102, 52, 86, 75, 119, 0, 11, 90, 123, 22, 57, 7, 27, 32, 44, 58, 97, 5, 125, 122, 62, 51, 70, 64, 61, 9, 91, 71, 69, 68, 38, 120, 65, 1, 67, 4, 96, 73, 92, 6, 59, 88, 98, 36, 116, 35, 3, 28, 108, 24, 66, 2, 46, 31, 95, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 49, 101, 114, 107, 109, 104, 113, 43, 93, 23, 42, 60, 87, 40, 25, 30, 89, 18, 77, 85, 21, 13, 82, 14, 78, 19, 83, 63, 26, 111, 117, 15, 79, 34, 10, 112, 115, 74, 103, 20, 119, 80, 72, 124, 16, 84, 17, 81, 11, 39, 94, 55, 12, 86, 76, 56, 100, 47, 126, 33, 110, 29, 102, 127, 44, 8, 0, 57, 75, 27, 122, 52, 7, 62, 64, 22, 90, 123, 53, 61, 73, 125, 70, 3, 68, 5, 67, 97, 69, 65, 51, 71, 9, 58, 32, 1, 91, 120, 4, 59, 98, 96, 2, 6, 36, 108, 35, 38, 88, 46, 92, 28, 99, 24, 116, 66, 95, 31], [41, 121, 118, 105, 54, 50, 45, 106, 37, 101, 48, 49, 114, 109, 107, 104, 113, 43, 60, 23, 87, 93, 42, 40, 30, 89, 25, 18, 13, 85, 21, 77, 82, 78, 26, 34, 19, 83, 14, 10, 80, 111, 119, 63, 79, 72, 15, 112, 103, 117, 55, 74, 20, 124, 16, 100, 115, 17, 84, 81, 29, 94, 12, 47, 76, 11, 52, 39, 102, 110, 57, 53, 75, 33, 22, 27, 62, 44, 86, 126, 0, 123, 5, 127, 73, 90, 122, 8, 7, 125, 59, 97, 64, 70, 4, 32, 61, 38, 56, 116, 58, 51, 69, 68, 65, 3, 91, 9, 71, 1, 88, 67, 98, 96, 120, 35, 6, 36, 2, 92, 99, 46, 28, 31, 108, 95, 66, 24], [121, 41, 118, 105, 54, 50, 45, 106, 37, 49, 48, 101, 109, 107, 104, 114, 113, 43, 93, 23, 60, 42, 87, 40, 13, 30, 25, 89, 85, 18, 19, 77, 82, 21, 14, 26, 111, 78, 72, 80, 10, 83, 34, 79, 103, 15, 63, 16, 117, 20, 74, 119, 124, 112, 84, 81, 12, 47, 11, 17, 76, 55, 127, 94, 39, 27, 75, 100, 126, 29, 52, 115, 110, 33, 86, 44, 53, 56, 22, 102, 58, 61, 125, 90, 70, 57, 51, 8, 62, 123, 32, 7, 122, 0, 69, 9, 73, 64, 59, 5, 71, 91, 4, 1, 97, 67, 3, 65, 68, 120, 98, 24, 6, 96, 116, 38, 35, 92, 36, 88, 46, 2, 31, 99, 66, 28, 95, 108]], "model.layers.8.self_attn.q_proj": [[60, 102, 116, 51, 123, 117, 91, 34, 52, 33, 118, 124, 115, 62, 119, 18, 126, 63, 48, 97, 58, 57, 50, 79, 54, 55, 53, 56, 61, 85, 49, 121, 59, 125, 94, 111, 122, 35, 101, 127, 45, 112, 120, 77, 44, 105, 43, 113, 89, 114, 73, 47, 26, 110, 46, 42, 13, 90, 21, 19, 109, 24, 28, 108, 107, 22, 12, 82, 14, 83, 96, 70, 37, 20, 87, 106, 39, 16, 40, 81, 72, 74, 84, 41, 103, 80, 7, 99, 30, 27, 32, 88, 31, 11, 15, 25, 36, 17, 95, 29, 104, 93, 23, 86, 92, 10, 100, 98, 76, 1, 4, 5, 3, 9, 75, 78, 38, 69, 64, 71, 67, 68, 66, 8, 6, 2, 65, 0], [117, 60, 51, 123, 116, 52, 122, 63, 126, 53, 55, 49, 48, 125, 57, 121, 120, 56, 110, 113, 111, 61, 47, 112, 114, 127, 54, 118, 119, 45, 46, 124, 62, 115, 109, 50, 58, 59, 44, 108, 43, 105, 107, 36, 42, 106, 41, 104, 40, 101, 39, 103, 102, 37, 35, 100, 24, 86, 98, 38, 95, 34, 99, 33, 87, 22, 96, 90, 66, 32, 0, 94, 31, 81, 5, 30, 97, 8, 84, 65, 19, 27, 93, 23, 85, 69, 29, 82, 75, 72, 91, 28, 21, 17, 15, 20, 89, 67, 88, 78, 92, 83, 2, 76, 13, 14, 64, 18, 25, 79, 80, 6, 70, 9, 26, 73, 68, 12, 77, 11, 1, 4, 74, 3, 16, 7, 71, 10], [102, 116, 60, 51, 33, 123, 117, 91, 52, 115, 85, 84, 15, 124, 70, 118, 79, 18, 7, 62, 34, 73, 12, 38, 119, 27, 26, 74, 89, 88, 11, 87, 59, 24, 94, 90, 58, 50, 20, 21, 93, 69, 30, 22, 3, 57, 54, 72, 126, 77, 4, 14, 82, 121, 35, 9, 98, 81, 61, 42, 37, 1, 13, 63, 56, 48, 92, 53, 10, 32, 23, 76, 25, 2, 28, 16, 64, 68, 55, 80, 31, 41, 105, 5, 17, 44, 95, 36, 101, 75, 49, 39, 67, 71, 103, 108, 66, 43, 29, 99, 120, 106, 96, 19, 127, 107, 46, 45, 125, 83, 78, 114, 97, 113, 40, 111, 100, 47, 104, 109, 6, 110, 8, 112, 86, 122, 65, 0], [116, 102, 60, 51, 123, 117, 115, 52, 62, 124, 119, 118, 34, 38, 39, 19, 58, 79, 57, 126, 85, 54, 50, 37, 43, 61, 48, 89, 53, 56, 121, 55, 94, 44, 104, 46, 42, 31, 107, 73, 108, 110, 125, 41, 40, 70, 72, 33, 111, 95, 36, 106, 109, 87, 63, 45, 49, 101, 59, 127, 103, 23, 105, 113, 120, 47, 32, 35, 28, 90, 114, 91, 13, 122, 96, 21, 112, 88, 100, 25, 11, 99, 74, 30, 27, 5, 93, 98, 92, 77, 29, 26, 1, 12, 17, 7, 24, 20, 14, 66, 80, 64, 10, 4, 3, 15, 76, 82, 16, 67, 78, 97, 18, 68, 22, 75, 71, 83, 69, 81, 84, 65, 2, 9, 86, 8, 0, 6], [38, 34, 119, 60, 18, 92, 78, 77, 9, 24, 88, 72, 85, 8, 75, 124, 68, 3, 21, 55, 59, 70, 80, 123, 69, 115, 28, 48, 71, 19, 22, 29, 37, 44, 47, 26, 61, 58, 117, 76, 4, 35, 73, 10, 95, 83, 125, 64, 0, 118, 14, 16, 79, 66, 12, 2, 62, 13, 31, 87, 93, 82, 46, 65, 57, 104, 49, 89, 42, 98, 103, 27, 15, 81, 84, 56, 63, 107, 67, 111, 30, 94, 110, 36, 6, 100, 7, 50, 25, 116, 17, 106, 105, 32, 101, 39, 53, 23, 120, 74, 86, 97, 113, 112, 43, 40, 20, 99, 41, 96, 126, 90, 1, 52, 11, 33, 121, 54, 108, 127, 114, 91, 5, 122, 45, 109, 51, 102], [38, 34, 59, 60, 119, 29, 18, 92, 88, 24, 115, 47, 77, 79, 9, 78, 68, 85, 1, 80, 55, 71, 124, 75, 22, 83, 70, 61, 65, 110, 44, 72, 64, 117, 35, 37, 69, 19, 123, 76, 125, 98, 3, 12, 58, 5, 16, 8, 73, 52, 21, 28, 95, 13, 7, 10, 100, 103, 27, 67, 0, 6, 48, 66, 93, 86, 106, 118, 26, 17, 4, 14, 112, 30, 20, 90, 46, 81, 63, 91, 32, 82, 107, 15, 2, 104, 102, 23, 94, 42, 74, 36, 11, 25, 99, 89, 39, 45, 54, 62, 49, 116, 120, 97, 50, 41, 96, 126, 31, 87, 113, 33, 43, 56, 127, 114, 122, 53, 105, 40, 84, 121, 101, 111, 51, 109, 108, 57], [38, 34, 60, 119, 29, 18, 85, 77, 88, 59, 78, 24, 37, 80, 9, 92, 124, 75, 71, 55, 69, 44, 21, 47, 8, 125, 3, 61, 93, 72, 16, 68, 19, 117, 35, 118, 79, 123, 22, 81, 95, 106, 11, 76, 115, 13, 32, 48, 58, 31, 10, 1, 83, 120, 65, 27, 63, 98, 122, 7, 116, 52, 84, 97, 89, 14, 26, 82, 99, 112, 107, 36, 94, 66, 64, 50, 42, 25, 30, 53, 28, 33, 100, 70, 40, 103, 87, 126, 110, 62, 51, 96, 104, 12, 41, 101, 39, 20, 54, 91, 86, 43, 49, 109, 45, 127, 23, 46, 114, 111, 57, 56, 108, 74, 113, 6, 17, 90, 15, 67, 121, 4, 0, 105, 73, 5, 2, 102], [38, 60, 119, 34, 29, 28, 88, 115, 85, 102, 93, 59, 19, 125, 80, 95, 27, 78, 92, 47, 87, 52, 98, 55, 18, 33, 44, 37, 35, 123, 118, 117, 24, 26, 83, 110, 22, 62, 58, 54, 46, 12, 89, 25, 61, 42, 48, 104, 103, 84, 30, 113, 40, 45, 97, 100, 101, 106, 91, 109, 31, 112, 116, 10, 81, 94, 114, 63, 21, 99, 107, 51, 49, 50, 105, 126, 96, 16, 111, 108, 127, 121, 124, 32, 57, 122, 43, 36, 56, 17, 120, 53, 41, 20, 86, 79, 39, 76, 23, 90, 8, 14, 74, 77, 71, 15, 75, 72, 11, 13, 82, 9, 7, 66, 2, 67, 70, 5, 69, 3, 4, 6, 73, 0, 68, 1, 64, 65], [59, 60, 116, 80, 55, 50, 73, 72, 117, 12, 62, 77, 10, 84, 120, 26, 54, 125, 103, 69, 19, 13, 24, 127, 68, 78, 15, 75, 93, 7, 21, 82, 17, 51, 90, 124, 6, 57, 49, 113, 3, 98, 56, 122, 121, 1, 66, 114, 16, 18, 112, 46, 58, 76, 70, 119, 36, 126, 123, 52, 28, 34, 118, 53, 88, 110, 61, 63, 0, 47, 29, 71, 109, 43, 27, 115, 96, 32, 104, 44, 107, 45, 108, 100, 92, 48, 102, 38, 42, 97, 106, 94, 105, 79, 111, 41, 101, 87, 89, 40, 37, 31, 33, 99, 95, 11, 91, 85, 35, 30, 86, 74, 9, 20, 81, 65, 83, 39, 23, 14, 4, 25, 8, 2, 22, 64, 67, 5], [59, 116, 60, 103, 55, 73, 117, 15, 87, 62, 80, 120, 125, 82, 54, 84, 10, 90, 127, 72, 13, 19, 69, 68, 76, 7, 6, 3, 78, 77, 51, 50, 124, 57, 66, 0, 75, 17, 113, 1, 49, 12, 21, 37, 56, 70, 121, 122, 11, 114, 119, 46, 123, 52, 36, 94, 74, 93, 32, 38, 88, 110, 58, 61, 118, 39, 43, 31, 112, 126, 28, 96, 47, 26, 104, 99, 53, 71, 42, 89, 65, 106, 101, 107, 115, 40, 100, 109, 111, 86, 83, 97, 102, 63, 108, 44, 105, 41, 81, 24, 33, 35, 30, 45, 95, 48, 8, 22, 92, 4, 98, 27, 20, 5, 16, 18, 79, 25, 91, 14, 34, 2, 29, 64, 23, 9, 85, 67], [103, 59, 116, 60, 20, 26, 16, 96, 84, 98, 24, 29, 89, 114, 93, 14, 10, 55, 125, 94, 91, 99, 88, 62, 19, 117, 70, 1, 87, 33, 18, 50, 31, 28, 80, 90, 68, 17, 57, 92, 52, 127, 69, 12, 95, 3, 25, 54, 27, 49, 23, 124, 72, 97, 120, 123, 66, 30, 74, 110, 71, 35, 22, 100, 36, 38, 32, 34, 113, 21, 0, 67, 64, 104, 102, 53, 82, 39, 78, 37, 76, 101, 40, 42, 51, 6, 119, 7, 108, 4, 122, 105, 83, 43, 81, 58, 75, 41, 118, 106, 44, 109, 48, 107, 45, 121, 47, 46, 9, 112, 56, 63, 73, 61, 111, 126, 115, 15, 5, 79, 85, 11, 8, 13, 65, 86, 77, 2], [59, 50, 60, 116, 19, 21, 15, 82, 80, 89, 76, 84, 13, 73, 88, 78, 87, 120, 62, 117, 17, 125, 54, 55, 7, 51, 90, 72, 127, 77, 6, 10, 3, 124, 61, 56, 121, 122, 58, 0, 63, 123, 112, 126, 118, 119, 11, 69, 57, 113, 53, 115, 47, 68, 75, 49, 66, 48, 111, 74, 46, 92, 86, 52, 110, 45, 109, 114, 108, 44, 1, 107, 93, 43, 106, 42, 105, 8, 41, 23, 104, 40, 27, 29, 65, 12, 100, 81, 103, 4, 28, 30, 38, 37, 102, 36, 39, 5, 70, 99, 20, 101, 14, 2, 83, 32, 35, 98, 97, 96, 22, 33, 95, 31, 67, 64, 26, 94, 71, 34, 24, 25, 85, 91, 16, 79, 9, 18], [104, 118, 34, 95, 50, 25, 70, 73, 51, 56, 45, 23, 55, 114, 49, 62, 5, 84, 3, 7, 40, 109, 117, 18, 121, 52, 74, 63, 54, 110, 124, 82, 58, 68, 14, 86, 126, 76, 15, 60, 8, 119, 112, 65, 17, 31, 123, 53, 80, 108, 97, 59, 127, 16, 27, 71, 125, 20, 35, 115, 46, 107, 89, 47, 32, 93, 120, 77, 44, 39, 111, 42, 29, 101, 61, 48, 90, 38, 41, 2, 113, 122, 100, 88, 30, 106, 92, 12, 102, 22, 37, 33, 103, 36, 105, 19, 13, 57, 94, 43, 116, 96, 28, 9, 91, 78, 85, 87, 81, 75, 99, 79, 21, 24, 83, 98, 10, 11, 26, 67, 0, 72, 6, 69, 4, 1, 66, 64], [104, 118, 34, 95, 84, 25, 73, 12, 40, 23, 18, 51, 16, 9, 49, 5, 65, 68, 70, 56, 1, 69, 66, 54, 8, 89, 80, 67, 63, 6, 13, 117, 15, 64, 62, 87, 60, 78, 45, 11, 50, 55, 3, 27, 58, 74, 14, 7, 20, 124, 72, 79, 2, 110, 24, 10, 71, 109, 76, 77, 82, 81, 127, 0, 28, 52, 4, 123, 22, 114, 31, 93, 83, 125, 97, 42, 91, 75, 126, 85, 112, 94, 86, 90, 21, 35, 46, 88, 33, 47, 19, 59, 121, 100, 38, 48, 39, 61, 119, 122, 17, 41, 26, 92, 29, 44, 101, 96, 108, 107, 111, 53, 113, 106, 36, 103, 32, 98, 120, 37, 99, 30, 102, 105, 43, 116, 115, 57], [118, 104, 34, 95, 50, 25, 23, 45, 52, 62, 84, 56, 117, 58, 49, 51, 18, 54, 16, 63, 114, 109, 123, 60, 55, 124, 106, 112, 12, 59, 27, 125, 57, 91, 121, 15, 126, 61, 110, 42, 32, 41, 31, 127, 119, 48, 43, 53, 33, 22, 39, 47, 115, 108, 111, 37, 113, 122, 120, 35, 28, 46, 116, 88, 107, 44, 99, 105, 93, 73, 103, 36, 97, 102, 38, 101, 92, 30, 82, 100, 86, 5, 87, 79, 96, 90, 13, 94, 85, 24, 89, 17, 78, 29, 9, 8, 70, 83, 26, 14, 74, 98, 21, 76, 20, 81, 80, 19, 7, 75, 3, 10, 77, 40, 72, 11, 69, 68, 65, 2, 71, 67, 6, 64, 4, 66, 1, 0], [104, 118, 34, 95, 25, 16, 12, 56, 23, 84, 18, 45, 15, 55, 69, 51, 78, 50, 117, 72, 58, 9, 40, 63, 8, 5, 49, 109, 66, 114, 67, 124, 54, 79, 71, 6, 60, 76, 4, 13, 110, 10, 80, 87, 86, 73, 31, 81, 119, 27, 62, 91, 108, 89, 75, 14, 1, 20, 70, 85, 52, 53, 112, 64, 29, 123, 11, 7, 19, 82, 42, 44, 59, 93, 121, 83, 61, 127, 39, 126, 90, 96, 94, 97, 32, 125, 74, 77, 103, 2, 21, 98, 107, 24, 41, 88, 111, 17, 22, 35, 106, 26, 57, 47, 115, 101, 92, 48, 43, 33, 30, 37, 68, 100, 122, 46, 28, 3, 36, 116, 105, 38, 102, 113, 120, 0, 99, 65], [41, 54, 59, 99, 79, 88, 101, 33, 13, 22, 90, 91, 31, 27, 73, 82, 39, 11, 123, 21, 84, 26, 71, 0, 66, 16, 40, 109, 94, 6, 74, 108, 2, 83, 28, 12, 38, 107, 68, 29, 126, 105, 51, 62, 7, 18, 95, 60, 76, 87, 34, 52, 20, 1, 98, 23, 9, 15, 32, 80, 78, 122, 67, 96, 125, 8, 110, 42, 55, 65, 4, 75, 57, 70, 36, 30, 106, 85, 49, 89, 45, 50, 46, 3, 124, 103, 56, 64, 77, 19, 127, 43, 17, 44, 58, 92, 100, 47, 104, 121, 97, 113, 118, 86, 112, 14, 115, 93, 114, 119, 10, 25, 120, 63, 102, 61, 5, 24, 81, 53, 72, 48, 69, 116, 111, 117, 37, 35], [41, 54, 99, 52, 91, 33, 126, 123, 27, 105, 113, 88, 108, 59, 61, 43, 36, 40, 101, 13, 125, 109, 22, 84, 107, 49, 16, 21, 57, 44, 118, 56, 45, 34, 115, 51, 63, 62, 17, 106, 50, 98, 31, 112, 127, 42, 25, 47, 120, 90, 39, 55, 100, 46, 124, 38, 121, 110, 114, 116, 58, 103, 29, 111, 96, 122, 94, 60, 95, 48, 53, 77, 71, 119, 117, 23, 104, 32, 102, 28, 93, 37, 82, 19, 4, 92, 85, 30, 81, 75, 20, 87, 89, 35, 26, 83, 97, 76, 68, 74, 10, 80, 73, 70, 18, 11, 24, 14, 72, 69, 12, 79, 78, 67, 6, 0, 2, 7, 8, 86, 15, 64, 65, 9, 5, 66, 1, 3], [41, 54, 99, 126, 33, 52, 91, 105, 27, 40, 101, 88, 21, 61, 89, 56, 125, 13, 45, 110, 123, 44, 34, 113, 82, 115, 36, 109, 85, 38, 59, 43, 107, 84, 22, 31, 42, 112, 81, 124, 94, 108, 114, 111, 120, 39, 116, 127, 57, 103, 16, 96, 58, 50, 63, 18, 118, 62, 60, 74, 90, 53, 100, 122, 17, 55, 37, 106, 23, 98, 104, 48, 87, 32, 49, 46, 121, 93, 28, 77, 51, 117, 19, 47, 119, 102, 25, 26, 97, 80, 29, 95, 68, 30, 83, 92, 35, 71, 24, 79, 75, 20, 11, 76, 4, 10, 78, 14, 12, 72, 86, 15, 70, 73, 7, 6, 1, 8, 9, 65, 64, 69, 2, 67, 5, 66, 0, 3], [41, 99, 59, 54, 88, 82, 31, 123, 27, 22, 91, 105, 79, 84, 13, 52, 21, 108, 87, 101, 12, 46, 33, 8, 18, 74, 83, 26, 39, 19, 81, 9, 94, 95, 118, 126, 10, 89, 77, 38, 17, 6, 92, 127, 20, 25, 55, 28, 15, 56, 40, 14, 69, 90, 70, 115, 44, 85, 68, 24, 16, 30, 97, 5, 86, 100, 23, 61, 98, 43, 32, 109, 110, 96, 104, 120, 113, 124, 76, 111, 29, 34, 122, 42, 75, 125, 80, 93, 107, 119, 57, 102, 58, 106, 36, 103, 47, 1, 63, 112, 114, 45, 72, 121, 4, 78, 50, 60, 117, 11, 3, 116, 53, 65, 49, 48, 71, 73, 37, 51, 62, 35, 67, 66, 0, 7, 64, 2], [103, 33, 59, 31, 82, 20, 23, 120, 81, 90, 114, 11, 76, 13, 14, 87, 22, 78, 7, 18, 71, 10, 21, 106, 83, 112, 16, 12, 15, 73, 124, 19, 119, 52, 77, 67, 96, 123, 38, 91, 25, 125, 41, 48, 40, 43, 107, 37, 94, 104, 98, 97, 88, 118, 27, 75, 86, 42, 51, 121, 84, 53, 117, 54, 55, 35, 58, 32, 3, 24, 30, 72, 80, 69, 111, 28, 34, 109, 126, 101, 47, 56, 100, 46, 8, 44, 85, 29, 93, 105, 115, 26, 79, 92, 108, 9, 1, 99, 62, 60, 36, 122, 127, 45, 61, 68, 113, 89, 4, 17, 57, 2, 39, 74, 63, 49, 5, 65, 102, 66, 110, 6, 95, 116, 50, 70, 64, 0], [103, 33, 59, 20, 31, 23, 120, 90, 83, 114, 82, 29, 16, 78, 19, 13, 84, 27, 88, 37, 94, 24, 32, 118, 81, 11, 72, 30, 17, 97, 7, 126, 48, 38, 92, 127, 96, 117, 21, 8, 77, 124, 56, 36, 87, 106, 25, 76, 22, 40, 9, 51, 12, 91, 119, 62, 105, 57, 41, 115, 10, 43, 45, 113, 52, 26, 110, 123, 14, 102, 79, 107, 15, 98, 54, 28, 93, 55, 34, 89, 61, 109, 18, 66, 44, 74, 80, 100, 95, 42, 99, 125, 85, 101, 47, 121, 63, 49, 35, 60, 73, 104, 68, 111, 58, 39, 108, 71, 4, 122, 53, 86, 75, 46, 112, 65, 116, 70, 50, 67, 2, 3, 6, 69, 5, 64, 0, 1], [103, 33, 59, 120, 31, 25, 13, 20, 11, 82, 78, 52, 16, 87, 10, 73, 23, 81, 7, 3, 65, 2, 66, 70, 90, 0, 5, 68, 114, 76, 69, 6, 19, 83, 18, 80, 106, 107, 67, 71, 27, 8, 75, 119, 9, 72, 96, 51, 77, 117, 48, 39, 28, 74, 105, 1, 43, 126, 4, 118, 22, 24, 37, 41, 123, 100, 46, 17, 53, 36, 127, 94, 56, 109, 115, 101, 44, 85, 40, 26, 92, 30, 54, 32, 104, 21, 84, 113, 62, 49, 47, 15, 12, 34, 14, 45, 121, 60, 93, 98, 91, 42, 88, 79, 50, 124, 29, 57, 111, 58, 110, 122, 125, 61, 116, 63, 99, 108, 35, 86, 102, 38, 89, 55, 112, 95, 64, 97], [103, 33, 114, 59, 31, 81, 120, 25, 76, 20, 78, 64, 15, 23, 82, 7, 21, 90, 16, 9, 3, 1, 83, 11, 5, 48, 106, 13, 65, 24, 66, 67, 27, 75, 115, 37, 47, 126, 77, 19, 74, 52, 44, 84, 55, 4, 10, 28, 53, 107, 93, 87, 22, 97, 71, 92, 29, 98, 39, 12, 43, 0, 109, 40, 96, 124, 46, 127, 51, 119, 17, 121, 88, 32, 105, 38, 2, 26, 100, 118, 112, 99, 111, 69, 8, 79, 104, 41, 86, 116, 58, 113, 73, 62, 45, 42, 110, 91, 18, 56, 94, 70, 14, 125, 6, 89, 54, 35, 57, 80, 108, 72, 63, 101, 123, 68, 122, 30, 50, 60, 61, 49, 34, 117, 36, 95, 102, 85], [44, 102, 50, 63, 98, 93, 108, 40, 123, 88, 80, 107, 43, 13, 31, 120, 119, 90, 7, 53, 115, 125, 57, 22, 29, 10, 114, 126, 19, 51, 84, 26, 20, 116, 92, 112, 109, 39, 55, 28, 12, 124, 127, 49, 121, 21, 45, 48, 36, 17, 18, 8, 101, 61, 52, 14, 46, 56, 85, 54, 47, 78, 118, 122, 99, 16, 113, 94, 59, 110, 74, 83, 60, 6, 58, 117, 70, 3, 97, 111, 68, 105, 81, 62, 25, 23, 4, 104, 42, 87, 32, 41, 91, 103, 30, 76, 79, 35, 106, 33, 15, 37, 82, 11, 27, 9, 96, 24, 89, 75, 66, 0, 95, 71, 73, 100, 86, 72, 67, 77, 65, 1, 2, 64, 69, 5, 38, 34], [44, 63, 102, 50, 98, 93, 40, 123, 107, 26, 108, 80, 31, 57, 115, 120, 43, 84, 90, 10, 127, 125, 53, 13, 22, 88, 116, 21, 126, 45, 19, 109, 121, 23, 51, 119, 112, 61, 124, 114, 104, 29, 18, 55, 47, 68, 56, 101, 54, 49, 36, 14, 62, 46, 52, 48, 122, 92, 17, 99, 58, 7, 60, 72, 105, 118, 12, 28, 8, 39, 97, 30, 78, 91, 59, 87, 110, 111, 117, 20, 42, 113, 77, 15, 103, 27, 81, 41, 106, 32, 95, 25, 85, 89, 94, 79, 83, 16, 24, 9, 66, 96, 2, 35, 37, 4, 76, 100, 75, 82, 11, 33, 5, 3, 73, 70, 1, 0, 74, 86, 6, 69, 65, 38, 34, 71, 67, 64], [44, 63, 102, 50, 108, 98, 120, 107, 40, 57, 53, 119, 51, 123, 88, 93, 26, 109, 125, 112, 58, 31, 127, 114, 43, 126, 61, 121, 116, 101, 118, 45, 48, 49, 54, 115, 36, 85, 47, 59, 56, 124, 21, 111, 122, 113, 55, 23, 95, 117, 80, 60, 39, 84, 52, 25, 62, 78, 104, 8, 46, 87, 41, 42, 100, 14, 19, 13, 18, 105, 90, 29, 89, 106, 110, 92, 35, 28, 32, 94, 99, 22, 103, 97, 37, 12, 91, 30, 17, 96, 33, 34, 81, 38, 66, 72, 82, 68, 24, 71, 83, 27, 20, 77, 7, 10, 74, 79, 2, 65, 16, 75, 11, 70, 15, 3, 73, 76, 9, 86, 6, 67, 69, 0, 4, 64, 5, 1], [44, 63, 102, 50, 108, 98, 40, 43, 115, 123, 120, 88, 125, 116, 31, 57, 26, 107, 112, 93, 114, 58, 53, 28, 127, 121, 84, 51, 19, 119, 101, 95, 52, 126, 56, 55, 49, 54, 36, 47, 109, 61, 39, 45, 59, 21, 104, 124, 46, 97, 62, 111, 8, 118, 48, 29, 90, 122, 23, 78, 14, 60, 113, 30, 117, 42, 110, 80, 22, 103, 12, 87, 92, 105, 99, 20, 37, 94, 33, 100, 106, 18, 32, 13, 17, 41, 27, 35, 38, 91, 81, 89, 72, 96, 24, 83, 15, 2, 77, 85, 66, 79, 25, 76, 82, 34, 7, 71, 73, 10, 68, 75, 9, 65, 16, 11, 6, 4, 70, 67, 0, 74, 69, 86, 3, 5, 1, 64], [54, 102, 122, 127, 120, 33, 28, 20, 125, 87, 92, 25, 116, 15, 12, 30, 97, 114, 47, 89, 63, 17, 96, 21, 22, 38, 32, 82, 19, 49, 94, 99, 7, 52, 48, 93, 123, 50, 16, 80, 13, 62, 115, 27, 91, 95, 121, 83, 74, 75, 72, 14, 88, 84, 31, 60, 85, 81, 55, 26, 51, 29, 10, 11, 59, 108, 35, 34, 113, 100, 111, 77, 90, 36, 98, 78, 23, 45, 118, 79, 37, 103, 24, 126, 53, 101, 57, 109, 58, 110, 112, 39, 117, 18, 105, 124, 56, 107, 41, 104, 43, 69, 42, 119, 40, 44, 46, 106, 76, 9, 2, 73, 61, 8, 71, 4, 86, 67, 6, 0, 70, 5, 68, 3, 1, 65, 66, 64], [54, 102, 122, 125, 127, 116, 25, 20, 114, 38, 33, 63, 30, 79, 47, 87, 120, 62, 123, 50, 52, 74, 48, 49, 53, 60, 55, 118, 126, 115, 28, 22, 58, 15, 121, 59, 7, 31, 57, 46, 51, 17, 94, 117, 92, 124, 113, 84, 76, 111, 70, 41, 44, 103, 89, 110, 35, 104, 32, 43, 12, 36, 107, 112, 39, 93, 34, 61, 119, 98, 96, 109, 99, 67, 40, 37, 56, 88, 105, 108, 95, 45, 2, 91, 106, 29, 77, 101, 83, 42, 97, 13, 69, 27, 100, 81, 19, 80, 72, 11, 90, 86, 21, 85, 82, 26, 73, 78, 23, 14, 75, 24, 71, 68, 18, 4, 0, 8, 16, 65, 10, 1, 9, 6, 5, 66, 64, 3], [122, 102, 54, 127, 125, 116, 120, 114, 87, 25, 33, 38, 92, 30, 12, 63, 47, 28, 20, 48, 22, 123, 62, 53, 17, 50, 59, 55, 118, 52, 60, 80, 49, 72, 82, 126, 57, 89, 24, 124, 51, 115, 76, 15, 58, 121, 91, 29, 81, 67, 94, 113, 97, 70, 109, 7, 0, 111, 44, 46, 21, 112, 2, 77, 103, 61, 56, 74, 119, 117, 69, 88, 110, 93, 108, 43, 79, 39, 65, 84, 107, 8, 36, 14, 1, 45, 73, 78, 83, 68, 32, 23, 90, 31, 27, 105, 40, 95, 96, 104, 6, 9, 37, 106, 16, 42, 3, 86, 85, 18, 100, 26, 10, 11, 41, 35, 34, 98, 101, 71, 19, 75, 13, 64, 4, 66, 5, 99], [122, 54, 102, 127, 116, 125, 114, 120, 38, 70, 25, 55, 63, 74, 53, 47, 59, 48, 62, 123, 118, 49, 103, 50, 60, 20, 67, 44, 52, 51, 124, 115, 126, 39, 57, 56, 58, 121, 113, 2, 45, 43, 33, 117, 76, 111, 7, 112, 110, 13, 46, 87, 119, 107, 109, 61, 93, 42, 108, 106, 36, 0, 105, 40, 101, 28, 104, 79, 92, 22, 41, 89, 37, 30, 35, 100, 19, 94, 26, 96, 90, 81, 88, 32, 34, 31, 29, 97, 71, 78, 23, 77, 99, 80, 69, 98, 27, 95, 84, 72, 10, 65, 4, 24, 1, 8, 12, 91, 68, 15, 73, 86, 17, 83, 11, 85, 9, 6, 14, 18, 82, 64, 21, 16, 5, 66, 75, 3]], "model.layers.8.self_attn.k_proj": [[116, 60, 38, 22, 51, 97, 94, 123, 117, 28, 89, 90, 84, 124, 18, 62, 16, 119, 81, 91, 36, 77, 52, 118, 87, 58, 15, 74, 35, 7, 54, 86, 19, 59, 50, 121, 61, 57, 21, 80, 42, 56, 99, 78, 126, 53, 100, 10, 73, 13, 48, 70, 41, 46, 75, 108, 125, 44, 127, 76, 40, 14, 114, 107, 103, 115, 104, 47, 98, 37, 63, 55, 106, 109, 105, 110, 24, 111, 26, 120, 43, 101, 45, 112, 49, 113, 39, 31, 122, 9, 95, 68, 67, 32, 85, 96, 82, 29, 1, 88, 30, 34, 64, 17, 102, 4, 66, 5, 23, 6, 92, 27, 20, 93, 71, 12, 83, 72, 3, 33, 65, 25, 11, 0, 2, 79, 8, 69], [60, 119, 98, 102, 88, 18, 85, 9, 80, 28, 78, 77, 64, 75, 71, 65, 59, 99, 3, 19, 93, 108, 68, 12, 69, 23, 115, 125, 2, 70, 61, 95, 111, 43, 51, 104, 29, 11, 38, 91, 5, 124, 123, 22, 117, 47, 37, 52, 20, 58, 46, 42, 24, 30, 10, 31, 49, 33, 66, 103, 36, 118, 53, 67, 40, 90, 113, 4, 15, 17, 122, 86, 81, 7, 114, 109, 6, 107, 83, 41, 48, 8, 92, 25, 63, 62, 54, 127, 32, 96, 13, 110, 94, 34, 0, 89, 84, 44, 106, 105, 100, 112, 120, 126, 26, 79, 97, 76, 121, 57, 55, 39, 82, 35, 116, 87, 16, 50, 45, 73, 101, 72, 56, 27, 74, 21, 1, 14], [59, 22, 60, 116, 39, 34, 114, 120, 54, 86, 124, 127, 51, 50, 35, 62, 55, 117, 113, 94, 57, 121, 56, 125, 122, 123, 49, 119, 52, 46, 29, 99, 61, 118, 43, 126, 58, 53, 110, 115, 112, 63, 103, 111, 47, 106, 108, 109, 76, 37, 18, 48, 28, 45, 44, 107, 42, 40, 81, 104, 96, 41, 14, 102, 38, 101, 105, 11, 85, 91, 95, 98, 15, 97, 100, 13, 74, 27, 26, 36, 73, 20, 25, 16, 82, 7, 32, 24, 30, 23, 78, 33, 92, 72, 6, 19, 8, 79, 80, 31, 89, 69, 88, 93, 90, 83, 66, 3, 68, 87, 17, 75, 0, 4, 1, 9, 5, 10, 67, 21, 12, 65, 84, 77, 2, 70, 71, 64], [40, 118, 98, 31, 23, 18, 25, 84, 16, 12, 56, 51, 78, 72, 10, 15, 9, 117, 55, 69, 71, 49, 63, 0, 54, 67, 62, 6, 1, 109, 4, 110, 60, 58, 45, 50, 127, 27, 124, 13, 2, 64, 66, 113, 74, 46, 8, 17, 47, 106, 42, 83, 123, 112, 44, 34, 19, 53, 91, 75, 126, 26, 24, 125, 52, 114, 61, 122, 29, 97, 68, 77, 30, 48, 11, 94, 108, 111, 59, 38, 119, 93, 107, 99, 120, 86, 103, 88, 39, 28, 121, 7, 21, 101, 33, 90, 100, 43, 105, 115, 41, 32, 96, 35, 92, 36, 102, 14, 37, 116, 57, 65, 89, 82, 85, 20, 70, 87, 22, 76, 79, 3, 80, 95, 5, 73, 81, 104], [105, 54, 59, 35, 95, 22, 91, 88, 21, 64, 110, 123, 68, 18, 97, 13, 16, 92, 74, 1, 6, 79, 37, 45, 71, 62, 52, 98, 44, 82, 41, 55, 107, 66, 83, 108, 126, 77, 57, 11, 50, 99, 48, 120, 127, 104, 58, 112, 53, 8, 90, 117, 56, 94, 122, 5, 40, 125, 60, 63, 51, 81, 121, 67, 103, 61, 106, 124, 114, 34, 12, 111, 115, 65, 3, 26, 113, 102, 116, 46, 42, 119, 30, 96, 47, 93, 73, 100, 15, 25, 109, 118, 36, 78, 17, 87, 43, 20, 14, 39, 2, 75, 38, 32, 0, 101, 76, 89, 84, 49, 29, 23, 19, 69, 72, 9, 33, 80, 31, 28, 24, 70, 7, 27, 10, 86, 4, 85], [39, 59, 120, 97, 23, 95, 83, 16, 11, 114, 20, 0, 78, 13, 82, 7, 76, 8, 73, 2, 42, 65, 48, 15, 50, 126, 69, 68, 3, 104, 107, 52, 25, 32, 27, 43, 117, 105, 119, 45, 62, 51, 4, 115, 53, 112, 118, 40, 9, 89, 81, 90, 88, 86, 58, 44, 98, 55, 1, 85, 109, 21, 10, 113, 79, 127, 66, 5, 54, 28, 30, 37, 35, 63, 122, 75, 47, 124, 46, 24, 102, 29, 61, 34, 106, 38, 94, 108, 121, 67, 91, 70, 57, 125, 49, 26, 110, 101, 22, 36, 96, 123, 72, 56, 60, 14, 18, 93, 99, 111, 100, 41, 116, 6, 84, 92, 19, 64, 17, 74, 12, 87, 77, 31, 80, 71, 103, 33], [108, 38, 22, 63, 34, 95, 50, 29, 23, 26, 18, 88, 44, 92, 104, 17, 19, 120, 20, 66, 115, 12, 114, 57, 125, 80, 13, 49, 107, 37, 123, 69, 53, 98, 112, 111, 8, 127, 45, 47, 121, 126, 51, 78, 113, 10, 55, 61, 35, 56, 52, 14, 54, 62, 91, 59, 119, 117, 65, 79, 85, 116, 109, 122, 7, 60, 64, 103, 118, 83, 124, 89, 46, 110, 105, 27, 70, 93, 58, 15, 9, 39, 100, 106, 48, 11, 40, 5, 97, 72, 42, 3, 28, 33, 94, 25, 41, 99, 96, 36, 77, 32, 30, 87, 84, 67, 43, 82, 24, 75, 76, 0, 21, 81, 31, 73, 101, 6, 4, 86, 1, 16, 90, 71, 2, 74, 68, 102], [122, 54, 38, 22, 97, 127, 63, 30, 114, 120, 125, 53, 47, 92, 50, 62, 126, 116, 82, 123, 51, 118, 59, 52, 124, 25, 57, 115, 46, 87, 55, 48, 58, 113, 44, 49, 111, 61, 56, 60, 121, 119, 117, 109, 12, 108, 98, 64, 43, 110, 112, 71, 15, 100, 17, 45, 40, 107, 20, 103, 106, 80, 32, 104, 105, 101, 41, 75, 39, 78, 42, 66, 36, 19, 67, 10, 35, 37, 102, 34, 94, 68, 5, 91, 99, 31, 77, 14, 3, 83, 8, 96, 18, 81, 73, 93, 24, 21, 26, 65, 95, 9, 11, 79, 88, 69, 29, 90, 33, 85, 70, 4, 16, 23, 27, 28, 89, 13, 6, 86, 1, 74, 72, 0, 2, 7, 84, 76]], "model.layers.8.self_attn.qk_proj": [[59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 114, 108, 50, 38, 44, 125, 105, 117, 123, 40, 41, 102, 82, 86, 127, 47, 95, 34, 53, 87, 98, 18, 55, 62, 58, 39, 57, 52, 49, 124, 24, 103, 126, 88, 23, 115, 80, 97, 16, 78, 61, 84, 27, 20, 22, 89, 29, 13, 14, 104, 110, 83, 77, 19, 107, 113, 12, 92, 85, 31, 99, 76, 93, 46, 21, 11, 43, 56, 48, 15, 75, 111, 112, 109, 25, 33, 42, 9, 90, 121, 73, 79, 28, 71, 7, 106, 8, 26, 35, 45, 30, 81, 94, 10, 74, 64, 17, 36, 91, 5, 37, 3, 4, 0, 70, 72, 32, 67, 68, 69, 101, 100, 66, 2, 6, 65, 96, 1], [59, 60, 118, 54, 116, 122, 119, 120, 51, 63, 114, 108, 50, 38, 117, 44, 40, 105, 125, 123, 53, 102, 86, 41, 18, 103, 98, 127, 62, 82, 115, 87, 52, 126, 39, 88, 95, 24, 124, 34, 57, 84, 58, 97, 80, 23, 104, 55, 47, 20, 16, 22, 107, 78, 27, 77, 89, 49, 56, 14, 92, 19, 25, 85, 31, 13, 61, 83, 93, 12, 76, 21, 79, 42, 29, 99, 110, 48, 113, 11, 43, 75, 28, 9, 106, 26, 109, 8, 46, 73, 71, 45, 112, 15, 35, 37, 90, 121, 33, 111, 7, 94, 30, 36, 91, 17, 64, 10, 5, 81, 74, 0, 65, 4, 32, 67, 2, 1, 3, 70, 66, 101, 68, 69, 6, 100, 96, 72], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 108, 38, 50, 114, 123, 44, 40, 117, 105, 53, 125, 102, 86, 34, 98, 39, 41, 95, 18, 24, 97, 87, 62, 82, 57, 127, 88, 23, 52, 115, 55, 84, 124, 22, 103, 20, 126, 49, 27, 104, 58, 89, 31, 47, 113, 16, 80, 93, 78, 29, 92, 77, 25, 56, 61, 48, 99, 33, 109, 85, 13, 75, 26, 45, 19, 21, 112, 76, 107, 43, 110, 14, 90, 35, 11, 94, 79, 12, 37, 83, 28, 46, 91, 111, 30, 42, 106, 15, 9, 7, 121, 8, 71, 81, 17, 36, 73, 32, 74, 10, 69, 100, 101, 64, 65, 0, 3, 72, 4, 66, 2, 68, 5, 6, 1, 67, 70, 96], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 114, 108, 50, 38, 123, 44, 40, 117, 125, 105, 127, 86, 95, 41, 98, 53, 87, 102, 24, 115, 18, 62, 39, 34, 126, 82, 49, 103, 124, 88, 97, 23, 58, 47, 57, 84, 113, 52, 20, 22, 93, 55, 29, 80, 31, 16, 27, 14, 89, 25, 104, 83, 99, 77, 92, 110, 19, 48, 13, 78, 43, 109, 107, 61, 45, 11, 56, 85, 33, 21, 28, 46, 42, 15, 76, 37, 90, 26, 12, 75, 35, 73, 30, 94, 7, 36, 106, 79, 121, 112, 8, 111, 64, 9, 32, 71, 17, 91, 101, 81, 74, 10, 67, 3, 69, 4, 1, 2, 66, 0, 5, 65, 72, 96, 68, 70, 100, 6], [59, 60, 118, 54, 119, 122, 116, 120, 51, 63, 114, 108, 38, 50, 44, 117, 123, 40, 105, 125, 102, 98, 127, 57, 53, 95, 39, 24, 18, 82, 62, 41, 124, 87, 103, 34, 86, 23, 49, 113, 126, 115, 55, 80, 104, 88, 27, 97, 78, 84, 20, 58, 47, 52, 22, 77, 29, 16, 89, 93, 19, 46, 31, 110, 13, 14, 112, 45, 11, 12, 76, 92, 56, 107, 85, 28, 99, 83, 61, 48, 79, 109, 43, 21, 25, 73, 90, 35, 42, 37, 71, 0, 33, 9, 75, 111, 15, 26, 121, 7, 64, 36, 30, 106, 94, 17, 74, 72, 1, 3, 5, 32, 8, 4, 68, 66, 67, 10, 2, 81, 69, 91, 65, 101, 6, 96, 70, 100], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 114, 108, 50, 38, 44, 123, 125, 117, 40, 105, 86, 102, 53, 127, 41, 95, 98, 82, 18, 24, 62, 124, 39, 87, 88, 80, 103, 16, 115, 34, 84, 57, 55, 52, 20, 78, 22, 47, 58, 23, 14, 126, 77, 49, 31, 83, 97, 104, 29, 27, 13, 107, 113, 76, 19, 89, 85, 28, 11, 12, 110, 92, 93, 99, 48, 42, 56, 109, 79, 75, 15, 25, 43, 61, 45, 46, 21, 30, 73, 112, 33, 9, 71, 90, 35, 37, 7, 72, 111, 121, 26, 106, 17, 81, 5, 94, 8, 36, 74, 10, 32, 64, 2, 0, 68, 91, 3, 69, 6, 67, 4, 65, 66, 101, 100, 1, 96, 70], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 108, 114, 50, 38, 123, 117, 44, 40, 125, 105, 127, 86, 53, 95, 18, 87, 98, 82, 41, 102, 115, 24, 52, 88, 62, 84, 34, 126, 124, 22, 47, 16, 58, 20, 23, 39, 80, 97, 55, 57, 103, 104, 78, 29, 83, 13, 14, 113, 89, 27, 112, 49, 92, 107, 110, 19, 77, 76, 99, 48, 56, 31, 93, 11, 111, 21, 85, 12, 75, 61, 15, 28, 109, 9, 73, 79, 121, 43, 90, 33, 42, 94, 25, 81, 30, 46, 17, 7, 35, 26, 72, 71, 45, 106, 37, 74, 10, 91, 101, 32, 5, 8, 36, 68, 6, 2, 64, 100, 3, 69, 67, 4, 70, 65, 66, 96, 0, 1], [59, 60, 118, 54, 116, 119, 122, 63, 120, 108, 51, 114, 38, 50, 44, 117, 40, 105, 123, 86, 125, 87, 82, 18, 102, 127, 115, 95, 98, 88, 124, 41, 24, 34, 39, 22, 52, 58, 16, 20, 53, 80, 84, 107, 57, 23, 14, 47, 103, 104, 13, 27, 55, 56, 97, 76, 78, 62, 12, 126, 89, 83, 48, 77, 92, 85, 121, 29, 49, 93, 21, 31, 19, 75, 110, 113, 79, 99, 28, 11, 106, 15, 112, 73, 46, 9, 111, 42, 25, 30, 94, 90, 61, 109, 37, 43, 91, 71, 35, 45, 72, 33, 81, 26, 7, 10, 74, 17, 0, 64, 66, 32, 69, 36, 101, 5, 4, 65, 68, 6, 2, 67, 100, 8, 3, 1, 70, 96], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 108, 50, 38, 114, 117, 123, 44, 105, 40, 125, 102, 127, 18, 98, 95, 62, 82, 24, 87, 115, 34, 39, 41, 86, 84, 88, 52, 16, 20, 103, 57, 22, 107, 126, 58, 89, 124, 80, 55, 97, 104, 27, 47, 23, 48, 13, 29, 78, 14, 92, 53, 46, 49, 77, 28, 21, 93, 19, 31, 110, 99, 25, 76, 83, 43, 56, 11, 113, 37, 85, 111, 94, 9, 109, 75, 12, 42, 15, 79, 112, 45, 33, 91, 61, 106, 35, 72, 30, 71, 73, 26, 90, 121, 7, 17, 101, 74, 81, 32, 64, 67, 36, 10, 1, 0, 5, 4, 68, 3, 69, 2, 66, 70, 65, 6, 8, 96, 100], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 108, 114, 50, 117, 38, 123, 125, 44, 105, 40, 18, 102, 53, 41, 95, 82, 98, 62, 86, 87, 115, 103, 24, 88, 127, 39, 16, 49, 124, 52, 34, 80, 23, 126, 22, 84, 20, 47, 78, 58, 55, 13, 31, 97, 92, 19, 27, 89, 29, 77, 113, 107, 104, 14, 21, 75, 76, 42, 83, 57, 93, 61, 25, 12, 85, 110, 43, 99, 46, 33, 109, 26, 28, 79, 56, 112, 48, 111, 9, 45, 11, 7, 71, 72, 73, 121, 94, 15, 90, 37, 30, 35, 106, 91, 17, 36, 74, 81, 32, 10, 69, 3, 101, 0, 1, 64, 67, 68, 66, 100, 70, 5, 4, 6, 2, 65, 96, 8], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 44, 123, 125, 40, 117, 105, 102, 95, 127, 58, 55, 41, 98, 18, 47, 87, 39, 86, 82, 24, 57, 53, 52, 126, 124, 16, 103, 88, 115, 34, 49, 80, 23, 78, 22, 13, 20, 62, 84, 27, 14, 104, 107, 97, 113, 19, 61, 29, 77, 83, 31, 89, 93, 109, 28, 110, 33, 121, 75, 46, 42, 43, 11, 12, 25, 85, 99, 76, 56, 21, 92, 71, 48, 7, 15, 79, 111, 45, 73, 26, 9, 112, 35, 72, 30, 106, 37, 94, 90, 10, 0, 64, 5, 81, 101, 67, 74, 36, 17, 91, 4, 66, 32, 68, 65, 2, 69, 1, 70, 3, 96, 8, 100, 6], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 117, 125, 44, 40, 123, 105, 127, 18, 55, 102, 86, 98, 87, 82, 95, 24, 53, 41, 39, 22, 16, 88, 78, 47, 62, 126, 124, 57, 80, 34, 84, 23, 52, 20, 29, 49, 103, 27, 58, 113, 13, 97, 115, 104, 46, 31, 83, 14, 121, 89, 110, 77, 28, 93, 43, 76, 107, 11, 75, 19, 12, 112, 92, 61, 21, 73, 99, 48, 15, 25, 90, 79, 85, 9, 109, 45, 42, 106, 33, 72, 7, 71, 56, 81, 111, 35, 30, 0, 26, 37, 36, 64, 69, 91, 10, 17, 67, 74, 3, 4, 94, 65, 8, 68, 5, 70, 66, 2, 1, 6, 101, 32, 100, 96], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 114, 50, 108, 38, 117, 44, 125, 123, 105, 40, 18, 102, 41, 127, 98, 52, 87, 34, 82, 86, 95, 88, 115, 22, 57, 23, 55, 124, 62, 24, 126, 39, 80, 84, 20, 16, 97, 58, 53, 78, 103, 110, 104, 47, 27, 89, 107, 61, 92, 49, 113, 29, 43, 13, 31, 14, 19, 28, 93, 85, 83, 77, 99, 21, 76, 25, 75, 11, 12, 56, 15, 73, 112, 109, 42, 45, 121, 46, 33, 111, 90, 9, 48, 79, 35, 26, 37, 106, 91, 7, 94, 30, 71, 17, 36, 8, 72, 81, 74, 10, 32, 69, 101, 100, 66, 68, 3, 70, 5, 4, 0, 67, 65, 64, 1, 6, 2, 96], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 114, 50, 108, 125, 38, 44, 117, 123, 105, 40, 53, 102, 41, 55, 18, 86, 95, 82, 34, 62, 127, 87, 98, 52, 124, 16, 88, 80, 103, 22, 115, 23, 24, 47, 84, 20, 58, 57, 39, 13, 104, 14, 78, 61, 49, 85, 107, 43, 110, 12, 27, 92, 31, 89, 97, 113, 19, 93, 126, 83, 21, 45, 76, 11, 77, 28, 75, 79, 29, 99, 121, 15, 112, 48, 56, 25, 42, 9, 73, 46, 7, 71, 90, 33, 8, 106, 30, 109, 26, 94, 0, 64, 111, 35, 10, 72, 91, 17, 81, 37, 74, 69, 65, 66, 4, 67, 32, 5, 68, 70, 2, 36, 101, 1, 6, 3, 100, 96], [59, 60, 54, 118, 116, 122, 119, 120, 63, 51, 108, 50, 114, 38, 40, 44, 117, 123, 105, 102, 125, 86, 95, 98, 18, 82, 53, 124, 41, 127, 87, 55, 84, 34, 88, 39, 62, 24, 52, 104, 57, 23, 58, 47, 49, 103, 97, 20, 16, 80, 115, 126, 14, 22, 110, 31, 78, 43, 56, 46, 113, 27, 61, 13, 29, 99, 89, 12, 107, 45, 106, 85, 77, 93, 19, 48, 83, 28, 92, 109, 75, 15, 112, 76, 25, 37, 79, 33, 21, 7, 94, 11, 73, 90, 35, 42, 71, 111, 8, 121, 91, 9, 30, 36, 26, 17, 81, 10, 0, 101, 72, 32, 67, 3, 69, 64, 68, 74, 5, 65, 100, 2, 4, 6, 66, 1, 70, 96], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 125, 44, 40, 123, 105, 102, 117, 86, 52, 124, 18, 127, 98, 115, 82, 57, 41, 88, 95, 103, 39, 87, 78, 62, 24, 34, 61, 107, 16, 20, 126, 23, 47, 53, 110, 97, 104, 22, 55, 80, 84, 49, 29, 31, 27, 89, 109, 19, 83, 113, 45, 85, 99, 58, 46, 93, 48, 14, 13, 21, 25, 43, 77, 28, 76, 92, 121, 11, 56, 37, 12, 42, 112, 75, 8, 106, 9, 79, 33, 30, 15, 7, 35, 73, 36, 90, 26, 71, 111, 17, 91, 94, 32, 67, 10, 81, 74, 68, 5, 64, 0, 65, 4, 2, 3, 101, 69, 1, 70, 96, 72, 66, 100, 6], [59, 60, 54, 118, 116, 122, 119, 63, 120, 51, 108, 114, 50, 38, 117, 44, 40, 123, 105, 125, 86, 18, 102, 82, 41, 98, 52, 87, 95, 22, 34, 124, 88, 24, 115, 39, 127, 16, 23, 62, 20, 80, 55, 104, 49, 103, 84, 126, 110, 58, 48, 56, 97, 78, 57, 27, 53, 93, 89, 83, 92, 29, 14, 47, 19, 45, 31, 107, 13, 77, 61, 85, 28, 113, 21, 76, 11, 12, 33, 42, 9, 99, 112, 75, 25, 46, 79, 43, 121, 8, 15, 90, 37, 73, 35, 71, 7, 30, 109, 106, 94, 26, 17, 10, 111, 91, 5, 81, 74, 36, 101, 32, 4, 64, 6, 0, 72, 68, 100, 66, 69, 70, 65, 2, 96, 1, 67, 3], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 108, 50, 38, 114, 44, 40, 105, 117, 86, 125, 123, 102, 127, 41, 82, 98, 87, 18, 124, 34, 22, 24, 95, 39, 88, 62, 97, 115, 23, 52, 20, 16, 55, 103, 80, 84, 126, 110, 27, 56, 49, 57, 93, 58, 104, 19, 29, 31, 78, 48, 89, 83, 92, 47, 13, 113, 85, 14, 99, 25, 28, 12, 107, 76, 53, 61, 21, 77, 112, 26, 75, 42, 45, 46, 15, 111, 7, 90, 11, 33, 109, 79, 43, 30, 9, 35, 94, 8, 73, 37, 106, 71, 121, 17, 91, 32, 36, 5, 81, 101, 6, 74, 10, 100, 0, 64, 67, 72, 4, 68, 96, 66, 3, 70, 2, 69, 65, 1], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 117, 40, 44, 123, 105, 125, 98, 86, 102, 87, 62, 34, 95, 82, 97, 39, 127, 124, 57, 41, 18, 52, 115, 24, 49, 88, 22, 55, 103, 23, 58, 53, 80, 126, 104, 29, 84, 16, 110, 56, 45, 20, 89, 61, 99, 25, 27, 31, 14, 26, 48, 46, 47, 83, 113, 93, 109, 92, 19, 33, 107, 28, 43, 42, 13, 78, 112, 12, 85, 77, 37, 35, 75, 15, 21, 73, 121, 76, 8, 9, 106, 36, 30, 94, 90, 11, 7, 79, 32, 91, 17, 71, 111, 101, 69, 3, 81, 74, 64, 10, 67, 0, 65, 4, 96, 5, 72, 6, 68, 66, 2, 1, 100, 70], [59, 60, 54, 118, 116, 122, 119, 63, 120, 51, 50, 108, 114, 38, 117, 44, 125, 40, 105, 102, 123, 86, 127, 98, 95, 41, 52, 82, 34, 18, 103, 39, 87, 115, 24, 57, 88, 55, 124, 62, 22, 23, 104, 80, 20, 16, 107, 126, 97, 27, 89, 110, 84, 56, 61, 58, 14, 83, 47, 53, 49, 12, 77, 31, 113, 25, 78, 93, 19, 99, 13, 28, 109, 48, 92, 29, 21, 46, 45, 30, 75, 76, 85, 112, 33, 43, 106, 42, 11, 73, 37, 26, 71, 35, 79, 121, 15, 9, 90, 7, 36, 17, 8, 91, 94, 111, 101, 74, 32, 10, 72, 69, 64, 81, 67, 65, 0, 2, 68, 96, 4, 5, 6, 100, 70, 66, 3, 1], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 50, 114, 38, 105, 44, 40, 86, 117, 125, 123, 102, 127, 18, 41, 124, 87, 126, 39, 52, 82, 88, 98, 55, 95, 22, 103, 49, 24, 115, 57, 34, 16, 23, 80, 20, 78, 62, 84, 53, 47, 56, 83, 104, 27, 97, 93, 13, 77, 29, 89, 14, 19, 110, 111, 61, 107, 113, 85, 75, 58, 31, 12, 48, 99, 76, 28, 21, 25, 109, 9, 71, 45, 92, 106, 79, 43, 11, 90, 37, 33, 26, 46, 42, 30, 121, 112, 73, 35, 7, 15, 36, 81, 94, 8, 32, 72, 67, 74, 17, 91, 64, 10, 68, 69, 3, 2, 0, 5, 101, 6, 1, 70, 65, 4, 66, 96, 100], [59, 60, 118, 54, 119, 122, 116, 120, 63, 51, 108, 50, 38, 114, 117, 44, 123, 105, 40, 86, 125, 87, 18, 102, 41, 98, 82, 55, 62, 124, 115, 39, 52, 34, 24, 95, 22, 57, 88, 103, 80, 127, 126, 23, 97, 47, 16, 84, 29, 48, 20, 78, 53, 89, 110, 27, 49, 61, 93, 56, 43, 92, 19, 58, 31, 42, 25, 83, 107, 104, 77, 113, 14, 75, 13, 99, 76, 85, 28, 112, 121, 12, 21, 46, 90, 106, 109, 79, 26, 11, 33, 30, 9, 15, 35, 73, 94, 111, 37, 45, 36, 71, 72, 8, 17, 7, 91, 81, 74, 5, 0, 101, 32, 64, 10, 3, 100, 68, 2, 67, 65, 69, 70, 96, 4, 1, 66, 6], [59, 60, 118, 54, 119, 116, 122, 63, 120, 51, 108, 114, 38, 50, 117, 44, 123, 105, 125, 40, 124, 41, 86, 98, 34, 102, 18, 39, 87, 127, 95, 62, 115, 52, 82, 24, 22, 55, 23, 57, 103, 88, 47, 16, 84, 53, 49, 97, 126, 80, 61, 20, 48, 58, 27, 104, 78, 56, 89, 13, 45, 19, 93, 113, 14, 92, 77, 31, 76, 110, 29, 83, 99, 107, 11, 85, 42, 12, 112, 28, 25, 75, 21, 26, 37, 121, 73, 43, 9, 46, 106, 111, 109, 33, 90, 79, 30, 15, 94, 7, 35, 72, 71, 36, 91, 17, 101, 32, 74, 0, 81, 64, 69, 100, 10, 65, 3, 8, 5, 68, 2, 96, 66, 67, 6, 4, 70, 1], [59, 60, 118, 54, 119, 116, 122, 63, 120, 51, 108, 114, 50, 38, 117, 125, 44, 40, 105, 123, 86, 102, 98, 127, 62, 95, 124, 87, 52, 41, 88, 82, 39, 18, 103, 115, 23, 34, 24, 57, 53, 16, 22, 126, 49, 84, 92, 110, 55, 58, 80, 97, 27, 47, 25, 20, 107, 56, 89, 31, 78, 83, 99, 42, 14, 104, 19, 121, 93, 77, 29, 48, 21, 13, 113, 12, 43, 45, 76, 28, 75, 85, 46, 73, 11, 72, 61, 15, 33, 109, 106, 30, 35, 112, 90, 37, 79, 26, 7, 111, 36, 91, 71, 9, 32, 94, 68, 17, 81, 0, 74, 69, 65, 70, 64, 100, 5, 67, 101, 3, 2, 10, 1, 66, 4, 6, 8, 96], [59, 60, 54, 118, 116, 119, 122, 63, 120, 51, 108, 50, 114, 38, 44, 117, 40, 125, 123, 105, 86, 102, 127, 98, 52, 124, 18, 41, 82, 39, 95, 87, 55, 57, 24, 34, 62, 88, 126, 22, 16, 97, 115, 53, 84, 49, 110, 47, 103, 58, 23, 80, 20, 83, 14, 29, 99, 104, 46, 48, 43, 56, 77, 89, 13, 61, 113, 78, 27, 93, 31, 25, 28, 73, 45, 76, 92, 85, 21, 107, 111, 19, 106, 121, 12, 75, 11, 112, 109, 72, 33, 15, 9, 42, 90, 35, 37, 79, 26, 71, 36, 94, 7, 81, 91, 74, 32, 5, 17, 30, 101, 10, 0, 64, 100, 68, 4, 70, 67, 2, 3, 69, 1, 96, 6, 8, 65, 66], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 38, 50, 114, 44, 117, 40, 125, 105, 123, 102, 86, 52, 98, 18, 127, 124, 82, 87, 95, 22, 39, 41, 88, 126, 115, 34, 55, 62, 24, 103, 97, 16, 23, 57, 84, 80, 20, 110, 47, 83, 58, 89, 27, 14, 53, 45, 56, 78, 49, 12, 19, 92, 29, 93, 61, 13, 107, 77, 104, 46, 21, 42, 43, 121, 99, 85, 28, 31, 76, 75, 9, 48, 15, 11, 79, 26, 112, 35, 73, 25, 106, 71, 90, 33, 72, 109, 113, 30, 37, 17, 36, 74, 94, 7, 81, 91, 32, 111, 64, 0, 101, 10, 100, 69, 4, 5, 66, 65, 8, 70, 3, 96, 6, 67, 2, 1, 68], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 50, 108, 114, 38, 40, 123, 105, 44, 117, 125, 62, 102, 124, 98, 87, 41, 86, 55, 57, 95, 82, 126, 34, 22, 52, 47, 127, 24, 39, 18, 88, 103, 97, 115, 84, 27, 89, 104, 23, 20, 53, 16, 49, 29, 93, 80, 78, 99, 48, 31, 61, 58, 83, 92, 42, 121, 21, 45, 110, 13, 107, 25, 77, 14, 85, 46, 19, 111, 12, 43, 90, 33, 11, 75, 26, 112, 28, 37, 9, 56, 15, 94, 30, 109, 76, 113, 35, 91, 7, 73, 106, 72, 79, 71, 17, 32, 101, 100, 67, 74, 69, 36, 0, 10, 4, 2, 68, 1, 66, 64, 65, 5, 81, 3, 70, 96, 6, 8], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 114, 50, 38, 44, 40, 105, 123, 117, 125, 86, 82, 87, 41, 127, 124, 102, 95, 98, 52, 39, 126, 18, 24, 34, 62, 88, 57, 22, 55, 23, 47, 16, 80, 115, 58, 20, 97, 84, 78, 83, 103, 104, 27, 56, 14, 93, 89, 29, 12, 92, 110, 19, 13, 107, 77, 76, 75, 49, 53, 21, 121, 45, 48, 61, 85, 11, 99, 46, 31, 43, 42, 109, 79, 15, 28, 9, 7, 73, 71, 111, 112, 113, 106, 25, 30, 0, 26, 94, 64, 72, 37, 33, 90, 35, 69, 17, 65, 74, 3, 8, 6, 4, 68, 66, 91, 1, 67, 10, 101, 2, 81, 36, 32, 5, 70, 96, 100], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 114, 108, 50, 38, 40, 117, 105, 44, 123, 102, 98, 127, 87, 125, 57, 41, 124, 18, 82, 95, 39, 52, 126, 86, 34, 62, 88, 24, 22, 103, 55, 97, 115, 58, 23, 84, 49, 20, 27, 16, 80, 104, 53, 31, 56, 89, 99, 47, 93, 78, 113, 29, 107, 83, 13, 121, 14, 110, 12, 77, 85, 25, 48, 109, 61, 43, 92, 45, 19, 28, 26, 106, 46, 21, 33, 90, 11, 94, 37, 75, 30, 76, 9, 111, 42, 35, 71, 15, 73, 112, 79, 36, 72, 91, 7, 8, 32, 64, 69, 74, 10, 17, 0, 66, 67, 1, 68, 6, 65, 101, 81, 3, 4, 5, 2, 96, 70, 100], [59, 60, 118, 54, 122, 116, 119, 120, 51, 63, 114, 108, 50, 38, 44, 125, 40, 105, 123, 117, 102, 86, 98, 52, 87, 41, 18, 127, 34, 82, 95, 124, 126, 88, 53, 57, 39, 115, 58, 55, 24, 80, 20, 47, 16, 103, 78, 97, 84, 22, 14, 23, 19, 31, 56, 62, 93, 13, 113, 49, 77, 27, 76, 42, 48, 83, 75, 43, 121, 89, 110, 85, 21, 12, 107, 29, 99, 92, 61, 104, 45, 25, 11, 46, 73, 109, 35, 15, 79, 9, 28, 106, 26, 30, 8, 90, 71, 7, 33, 112, 37, 94, 111, 17, 64, 72, 0, 10, 74, 66, 69, 65, 4, 81, 100, 3, 91, 36, 5, 1, 6, 68, 32, 67, 70, 101, 2, 96], [59, 60, 118, 54, 116, 122, 119, 120, 51, 63, 114, 108, 50, 38, 44, 117, 40, 123, 125, 105, 55, 41, 102, 98, 52, 86, 95, 127, 126, 34, 124, 18, 53, 82, 115, 49, 87, 58, 39, 57, 84, 24, 23, 88, 80, 103, 62, 97, 47, 22, 20, 16, 113, 14, 31, 104, 89, 27, 110, 48, 99, 45, 19, 29, 78, 61, 107, 77, 85, 92, 12, 56, 121, 93, 21, 13, 76, 83, 11, 43, 109, 112, 42, 106, 73, 28, 75, 25, 33, 15, 8, 46, 9, 111, 26, 71, 79, 90, 35, 94, 37, 30, 7, 64, 69, 17, 74, 10, 81, 68, 32, 5, 1, 0, 2, 72, 4, 36, 91, 67, 101, 6, 70, 66, 3, 65, 100, 96], [59, 60, 54, 118, 116, 122, 119, 120, 63, 51, 108, 114, 50, 38, 123, 44, 125, 117, 40, 105, 86, 41, 52, 127, 98, 102, 55, 87, 82, 34, 95, 58, 115, 18, 126, 62, 22, 24, 53, 124, 23, 88, 80, 47, 39, 103, 16, 20, 57, 84, 97, 49, 27, 14, 12, 77, 104, 19, 92, 93, 89, 21, 13, 15, 48, 83, 110, 76, 31, 78, 61, 45, 85, 11, 107, 113, 43, 42, 29, 73, 33, 9, 99, 121, 28, 75, 109, 25, 56, 46, 26, 8, 79, 71, 90, 35, 7, 37, 106, 30, 94, 112, 111, 81, 74, 64, 17, 91, 5, 10, 0, 32, 68, 36, 1, 65, 67, 69, 72, 4, 6, 101, 70, 2, 100, 66, 3, 96]], "model.layers.9.self_attn.q_proj": [[110, 101, 46, 124, 63, 59, 28, 32, 113, 25, 89, 61, 19, 115, 27, 87, 16, 60, 57, 49, 122, 83, 85, 26, 78, 22, 47, 114, 55, 58, 56, 53, 96, 99, 37, 92, 108, 41, 23, 54, 48, 125, 30, 79, 51, 11, 119, 67, 84, 1, 24, 118, 69, 35, 62, 91, 120, 123, 116, 70, 43, 88, 97, 117, 121, 68, 106, 112, 90, 126, 109, 29, 40, 2, 77, 127, 45, 111, 8, 17, 86, 0, 74, 50, 82, 107, 15, 5, 42, 9, 66, 21, 18, 10, 52, 14, 6, 104, 105, 94, 36, 93, 100, 81, 65, 102, 44, 34, 33, 76, 20, 80, 31, 103, 3, 64, 7, 38, 4, 71, 95, 75, 73, 12, 13, 39, 98, 72], [110, 124, 101, 63, 46, 113, 25, 59, 80, 28, 24, 61, 89, 115, 57, 122, 60, 56, 83, 114, 32, 53, 58, 47, 125, 27, 22, 51, 41, 54, 119, 49, 78, 55, 40, 11, 112, 118, 62, 48, 123, 43, 96, 120, 106, 69, 23, 121, 37, 108, 52, 117, 127, 50, 42, 109, 19, 104, 45, 100, 111, 116, 35, 126, 105, 16, 107, 85, 91, 87, 102, 30, 38, 39, 31, 44, 33, 92, 34, 103, 84, 36, 86, 20, 17, 94, 88, 26, 97, 73, 98, 29, 99, 93, 8, 90, 95, 77, 72, 14, 18, 1, 21, 12, 67, 81, 5, 79, 76, 15, 82, 70, 4, 75, 2, 0, 10, 9, 7, 13, 6, 74, 66, 3, 68, 71, 64, 65], [110, 124, 46, 101, 63, 59, 89, 61, 11, 37, 113, 57, 25, 122, 60, 115, 69, 56, 53, 19, 54, 58, 47, 55, 32, 114, 120, 48, 16, 51, 49, 41, 117, 118, 116, 28, 112, 50, 62, 27, 109, 24, 119, 22, 121, 111, 125, 92, 127, 99, 126, 123, 43, 87, 45, 52, 82, 30, 107, 108, 39, 100, 72, 35, 103, 42, 105, 106, 1, 96, 104, 83, 40, 0, 4, 44, 23, 78, 2, 86, 77, 80, 26, 102, 15, 38, 29, 34, 85, 97, 90, 84, 67, 73, 36, 31, 33, 14, 88, 94, 5, 6, 98, 95, 18, 17, 9, 93, 91, 7, 21, 8, 20, 68, 79, 12, 81, 75, 66, 10, 13, 76, 71, 70, 74, 64, 3, 65], [124, 110, 101, 59, 46, 89, 63, 61, 19, 28, 115, 25, 83, 60, 113, 56, 27, 57, 23, 114, 122, 24, 118, 54, 78, 53, 55, 22, 125, 119, 47, 49, 58, 120, 48, 51, 37, 26, 107, 93, 32, 50, 81, 96, 117, 121, 99, 112, 11, 106, 43, 62, 123, 92, 127, 41, 35, 109, 108, 17, 116, 111, 103, 45, 42, 36, 104, 52, 44, 82, 126, 40, 100, 105, 30, 39, 38, 85, 88, 95, 14, 12, 34, 86, 87, 8, 98, 97, 31, 33, 102, 29, 16, 91, 94, 77, 69, 84, 18, 21, 73, 20, 15, 90, 80, 75, 76, 67, 13, 5, 79, 72, 9, 6, 74, 7, 1, 10, 70, 4, 3, 71, 2, 0, 65, 68, 66, 64], [117, 41, 59, 61, 97, 126, 101, 39, 31, 87, 55, 43, 42, 44, 28, 100, 124, 112, 110, 116, 35, 92, 102, 113, 96, 122, 57, 86, 36, 121, 127, 89, 54, 58, 114, 63, 38, 49, 52, 115, 93, 125, 47, 94, 51, 60, 53, 105, 99, 120, 16, 80, 48, 111, 32, 45, 50, 84, 46, 98, 29, 104, 19, 56, 118, 109, 123, 108, 40, 62, 27, 78, 33, 24, 107, 23, 74, 18, 26, 17, 25, 83, 37, 85, 75, 30, 91, 22, 20, 119, 21, 34, 14, 106, 12, 88, 9, 68, 81, 13, 103, 15, 77, 10, 3, 7, 76, 73, 95, 11, 71, 79, 4, 82, 66, 8, 90, 2, 70, 5, 6, 0, 72, 69, 64, 67, 65, 1], [41, 64, 107, 101, 117, 31, 67, 63, 1, 56, 69, 55, 26, 97, 42, 126, 112, 87, 105, 2, 72, 10, 39, 18, 84, 79, 76, 98, 71, 124, 65, 43, 106, 28, 78, 4, 35, 9, 111, 6, 44, 100, 53, 70, 29, 93, 23, 114, 80, 12, 121, 3, 74, 58, 86, 66, 20, 92, 94, 57, 5, 81, 11, 89, 59, 127, 33, 110, 102, 99, 83, 38, 75, 7, 48, 125, 109, 8, 61, 19, 15, 17, 77, 27, 21, 0, 46, 113, 88, 62, 118, 68, 50, 85, 45, 116, 32, 54, 104, 47, 24, 25, 120, 52, 122, 96, 40, 13, 51, 115, 103, 60, 90, 30, 34, 14, 73, 119, 123, 49, 82, 36, 16, 108, 91, 22, 95, 37], [41, 117, 26, 87, 31, 42, 101, 97, 56, 98, 18, 124, 35, 84, 105, 79, 15, 23, 12, 76, 55, 39, 57, 78, 43, 20, 80, 7, 59, 112, 93, 106, 118, 44, 74, 90, 61, 102, 10, 89, 100, 63, 28, 83, 114, 126, 107, 82, 58, 9, 53, 25, 104, 92, 69, 81, 14, 48, 8, 96, 36, 21, 127, 95, 86, 70, 24, 49, 121, 110, 99, 11, 29, 60, 111, 22, 46, 27, 17, 62, 85, 51, 122, 109, 16, 4, 52, 120, 125, 32, 5, 19, 77, 33, 13, 123, 40, 54, 45, 91, 2, 30, 67, 94, 71, 47, 3, 73, 115, 38, 113, 116, 72, 88, 64, 108, 50, 37, 103, 66, 65, 75, 68, 34, 0, 119, 6, 1], [117, 56, 41, 61, 107, 43, 97, 39, 31, 126, 35, 44, 118, 100, 42, 57, 110, 89, 121, 102, 36, 105, 46, 87, 63, 48, 101, 94, 52, 124, 92, 28, 59, 53, 113, 50, 86, 55, 38, 98, 54, 122, 49, 51, 29, 112, 111, 25, 45, 115, 108, 83, 62, 123, 125, 127, 80, 40, 47, 96, 104, 19, 99, 114, 27, 58, 116, 120, 60, 33, 109, 34, 88, 26, 18, 32, 93, 85, 9, 23, 106, 22, 64, 21, 24, 16, 119, 84, 30, 17, 103, 13, 91, 37, 77, 14, 90, 1, 2, 81, 69, 78, 67, 76, 95, 11, 79, 4, 82, 66, 73, 71, 68, 7, 70, 10, 75, 72, 20, 5, 12, 74, 6, 8, 65, 15, 3, 0], [105, 98, 86, 18, 84, 53, 79, 81, 88, 13, 111, 41, 76, 119, 74, 51, 72, 55, 29, 10, 117, 52, 7, 61, 2, 63, 123, 114, 68, 110, 107, 71, 6, 126, 31, 59, 70, 26, 125, 116, 5, 15, 4, 17, 48, 28, 120, 50, 118, 8, 127, 108, 77, 75, 90, 12, 27, 1, 3, 82, 24, 85, 92, 11, 109, 21, 122, 42, 124, 20, 38, 106, 9, 62, 40, 57, 93, 78, 25, 113, 33, 49, 22, 66, 80, 56, 16, 67, 58, 39, 30, 32, 60, 19, 23, 102, 91, 37, 54, 46, 87, 89, 64, 100, 96, 43, 115, 36, 83, 101, 14, 112, 47, 94, 95, 121, 103, 44, 35, 97, 99, 104, 0, 45, 34, 69, 73, 65], [105, 98, 84, 79, 88, 53, 18, 72, 41, 86, 111, 5, 81, 51, 2, 13, 63, 15, 76, 10, 29, 55, 52, 6, 68, 114, 119, 126, 74, 116, 110, 12, 123, 48, 92, 7, 117, 61, 28, 26, 107, 71, 9, 59, 27, 11, 118, 4, 87, 90, 31, 3, 93, 8, 85, 127, 125, 24, 122, 70, 109, 78, 1, 58, 20, 108, 50, 120, 124, 77, 91, 38, 66, 42, 49, 17, 22, 40, 39, 60, 30, 102, 69, 95, 67, 57, 54, 34, 106, 83, 23, 37, 62, 89, 32, 101, 16, 82, 21, 75, 96, 36, 19, 56, 33, 35, 121, 43, 112, 103, 113, 80, 46, 73, 94, 25, 44, 99, 14, 115, 104, 100, 0, 97, 45, 64, 47, 65], [105, 98, 84, 41, 72, 86, 53, 6, 18, 2, 79, 12, 4, 1, 76, 68, 71, 111, 74, 51, 13, 0, 3, 67, 81, 88, 66, 5, 114, 126, 63, 61, 123, 119, 8, 117, 26, 52, 116, 55, 48, 118, 125, 107, 127, 29, 65, 110, 59, 27, 10, 92, 93, 7, 85, 90, 20, 17, 109, 34, 120, 24, 9, 87, 50, 28, 122, 124, 22, 58, 62, 78, 70, 80, 49, 42, 32, 57, 108, 31, 30, 106, 83, 101, 35, 25, 56, 54, 21, 102, 77, 91, 14, 82, 113, 95, 64, 89, 15, 112, 38, 75, 97, 39, 36, 43, 115, 60, 40, 33, 121, 23, 19, 103, 37, 94, 11, 100, 46, 104, 45, 99, 69, 16, 96, 47, 44, 73], [105, 98, 74, 3, 70, 0, 86, 53, 111, 4, 1, 81, 18, 79, 76, 61, 5, 41, 67, 66, 119, 13, 117, 71, 8, 84, 2, 51, 110, 123, 68, 114, 55, 12, 52, 72, 126, 50, 125, 65, 120, 116, 107, 63, 17, 6, 9, 20, 59, 57, 64, 118, 88, 29, 109, 48, 10, 127, 49, 56, 38, 62, 73, 124, 34, 26, 7, 22, 40, 106, 78, 87, 23, 30, 15, 16, 28, 11, 24, 69, 93, 92, 19, 77, 89, 80, 83, 14, 60, 31, 94, 113, 75, 42, 39, 32, 82, 85, 108, 122, 54, 25, 43, 58, 121, 27, 90, 21, 96, 102, 115, 44, 101, 99, 112, 91, 36, 46, 95, 35, 33, 103, 104, 45, 37, 100, 47, 97], [125, 62, 104, 48, 63, 119, 122, 120, 59, 108, 55, 57, 118, 54, 121, 116, 58, 60, 27, 124, 113, 114, 115, 117, 84, 123, 111, 61, 97, 56, 53, 50, 43, 51, 47, 30, 44, 87, 52, 49, 126, 127, 107, 45, 109, 25, 110, 89, 12, 46, 21, 105, 28, 22, 72, 99, 112, 103, 41, 36, 38, 94, 106, 34, 5, 98, 90, 39, 102, 101, 20, 100, 1, 37, 92, 14, 9, 66, 96, 68, 42, 19, 16, 83, 35, 32, 79, 17, 29, 33, 95, 2, 85, 69, 31, 81, 0, 23, 40, 3, 91, 77, 86, 88, 18, 4, 93, 6, 15, 64, 82, 71, 76, 13, 11, 26, 24, 10, 73, 80, 78, 75, 74, 7, 8, 70, 65, 67], [62, 125, 48, 104, 63, 119, 122, 120, 59, 108, 97, 115, 55, 58, 54, 57, 60, 117, 114, 116, 121, 52, 124, 113, 50, 118, 47, 123, 61, 56, 111, 84, 36, 53, 127, 51, 89, 49, 109, 126, 44, 87, 5, 43, 72, 45, 14, 96, 69, 103, 46, 112, 107, 12, 30, 41, 110, 1, 106, 102, 105, 68, 2, 27, 32, 101, 66, 37, 28, 39, 38, 42, 24, 88, 98, 21, 25, 20, 22, 19, 35, 95, 99, 92, 100, 31, 64, 34, 23, 40, 83, 3, 93, 4, 15, 90, 91, 71, 76, 29, 85, 75, 79, 0, 6, 18, 94, 78, 9, 8, 17, 73, 77, 81, 13, 16, 7, 26, 10, 80, 33, 86, 11, 70, 74, 67, 82, 65], [104, 125, 62, 48, 63, 97, 119, 30, 21, 57, 122, 60, 59, 28, 127, 115, 118, 120, 116, 108, 55, 85, 84, 27, 54, 121, 113, 126, 123, 53, 114, 49, 56, 25, 111, 58, 117, 61, 22, 89, 124, 87, 52, 47, 9, 112, 109, 90, 44, 107, 51, 50, 45, 43, 40, 35, 5, 79, 46, 36, 105, 103, 94, 66, 1, 18, 68, 24, 41, 78, 92, 16, 12, 34, 15, 2, 98, 99, 102, 38, 95, 110, 72, 39, 19, 32, 3, 14, 96, 69, 11, 106, 101, 64, 31, 42, 37, 0, 6, 93, 91, 73, 100, 88, 83, 17, 23, 75, 20, 8, 29, 81, 4, 77, 26, 33, 13, 70, 71, 74, 7, 10, 80, 86, 76, 67, 65, 82], [104, 62, 125, 90, 30, 97, 63, 122, 84, 18, 19, 115, 23, 60, 80, 22, 16, 119, 27, 74, 79, 38, 57, 49, 26, 36, 78, 55, 20, 54, 59, 39, 82, 13, 124, 99, 120, 108, 94, 116, 123, 112, 45, 87, 21, 121, 111, 118, 114, 28, 83, 24, 113, 61, 126, 33, 88, 50, 58, 44, 127, 56, 17, 75, 48, 107, 10, 53, 102, 92, 34, 81, 100, 91, 109, 117, 96, 43, 51, 101, 47, 77, 103, 52, 14, 46, 25, 93, 15, 105, 37, 41, 31, 106, 6, 42, 110, 73, 98, 85, 70, 40, 5, 89, 12, 35, 32, 76, 95, 86, 68, 8, 29, 4, 7, 11, 71, 3, 9, 64, 1, 66, 65, 72, 67, 69, 0, 2], [103, 124, 55, 56, 21, 33, 15, 57, 29, 113, 81, 83, 91, 88, 27, 122, 123, 125, 93, 41, 10, 25, 35, 49, 98, 114, 24, 12, 78, 61, 105, 60, 48, 30, 109, 86, 115, 22, 52, 120, 112, 85, 53, 51, 110, 58, 31, 116, 43, 62, 45, 17, 94, 108, 44, 84, 100, 42, 107, 102, 70, 119, 111, 90, 126, 23, 18, 76, 118, 121, 117, 47, 36, 46, 4, 106, 127, 26, 89, 54, 63, 32, 50, 59, 20, 19, 11, 99, 8, 28, 40, 79, 101, 96, 13, 104, 37, 80, 38, 82, 95, 92, 16, 77, 34, 87, 72, 6, 65, 14, 2, 75, 97, 9, 5, 68, 39, 73, 74, 71, 7, 69, 0, 66, 67, 1, 64, 3], [103, 124, 55, 57, 88, 33, 56, 21, 105, 81, 113, 93, 123, 29, 15, 83, 23, 41, 122, 91, 125, 19, 49, 70, 78, 27, 114, 10, 12, 44, 86, 24, 126, 117, 45, 90, 98, 109, 2, 22, 58, 76, 52, 74, 4, 108, 120, 30, 60, 118, 107, 115, 48, 111, 106, 84, 110, 43, 65, 5, 85, 51, 35, 53, 119, 32, 18, 38, 39, 102, 17, 112, 99, 95, 34, 100, 104, 79, 47, 68, 54, 116, 92, 101, 42, 121, 20, 97, 61, 62, 31, 26, 46, 28, 96, 127, 87, 36, 11, 6, 50, 82, 94, 73, 37, 14, 89, 75, 59, 77, 8, 25, 16, 67, 40, 7, 63, 80, 0, 13, 9, 69, 72, 71, 1, 66, 64, 3], [103, 56, 124, 55, 57, 33, 88, 29, 113, 21, 15, 93, 105, 23, 86, 10, 77, 81, 114, 123, 91, 122, 64, 67, 125, 120, 78, 109, 0, 45, 85, 83, 60, 49, 61, 51, 65, 41, 52, 115, 53, 58, 119, 13, 110, 116, 112, 118, 27, 34, 2, 108, 48, 73, 24, 84, 71, 62, 42, 63, 89, 30, 50, 90, 14, 8, 6, 46, 126, 117, 107, 43, 70, 35, 26, 11, 17, 44, 121, 47, 111, 80, 3, 127, 20, 54, 59, 102, 7, 104, 100, 87, 37, 38, 82, 79, 106, 40, 22, 31, 9, 16, 97, 99, 36, 1, 18, 28, 101, 98, 66, 76, 12, 32, 25, 94, 68, 96, 4, 75, 92, 95, 5, 74, 19, 72, 69, 39], [124, 103, 55, 57, 41, 123, 88, 56, 125, 70, 114, 74, 122, 33, 19, 113, 21, 93, 110, 115, 23, 45, 49, 109, 120, 118, 52, 58, 116, 61, 15, 53, 108, 2, 112, 48, 83, 51, 60, 43, 35, 117, 81, 62, 29, 119, 126, 105, 73, 44, 50, 91, 121, 84, 127, 65, 111, 47, 46, 12, 107, 86, 42, 54, 63, 98, 30, 68, 106, 39, 14, 22, 59, 78, 100, 79, 27, 40, 102, 101, 10, 90, 5, 4, 38, 104, 67, 96, 99, 92, 24, 13, 36, 37, 34, 75, 26, 69, 20, 32, 8, 97, 85, 87, 17, 7, 28, 80, 25, 82, 18, 31, 76, 11, 94, 0, 71, 95, 16, 6, 77, 89, 9, 72, 3, 64, 1, 66], [57, 126, 39, 34, 61, 45, 95, 111, 120, 125, 60, 21, 51, 63, 58, 87, 52, 90, 54, 113, 114, 119, 117, 123, 109, 55, 127, 48, 50, 62, 118, 112, 122, 19, 106, 53, 56, 49, 46, 72, 59, 43, 115, 124, 110, 5, 82, 116, 108, 100, 41, 22, 36, 121, 47, 105, 17, 104, 28, 44, 26, 92, 107, 102, 88, 35, 73, 9, 42, 79, 14, 24, 16, 40, 38, 37, 77, 76, 78, 89, 99, 101, 27, 18, 30, 97, 75, 10, 32, 71, 93, 83, 98, 1, 33, 94, 13, 81, 66, 96, 29, 86, 25, 91, 103, 85, 8, 67, 11, 4, 2, 0, 20, 12, 74, 31, 70, 23, 15, 84, 69, 65, 3, 6, 7, 80, 68, 64], [39, 126, 57, 34, 95, 90, 19, 87, 60, 120, 21, 26, 61, 17, 92, 11, 111, 48, 79, 28, 31, 5, 71, 72, 22, 10, 45, 63, 117, 84, 27, 9, 58, 70, 102, 42, 4, 67, 100, 125, 127, 105, 40, 36, 108, 51, 83, 113, 99, 16, 101, 13, 38, 73, 41, 54, 96, 75, 46, 35, 53, 77, 124, 122, 52, 106, 66, 50, 44, 37, 49, 81, 59, 47, 112, 118, 62, 23, 104, 109, 119, 33, 43, 116, 85, 115, 114, 121, 110, 30, 32, 55, 123, 29, 97, 107, 25, 56, 0, 24, 20, 89, 65, 93, 15, 7, 80, 94, 18, 98, 91, 74, 1, 88, 76, 78, 6, 68, 12, 14, 82, 103, 69, 86, 8, 2, 3, 64], [39, 57, 126, 34, 95, 87, 60, 21, 19, 90, 61, 16, 120, 73, 28, 26, 22, 79, 75, 17, 77, 48, 111, 125, 5, 58, 51, 83, 13, 63, 9, 92, 45, 104, 74, 33, 82, 113, 118, 27, 112, 122, 109, 127, 18, 123, 7, 71, 78, 20, 10, 89, 117, 62, 23, 119, 31, 30, 49, 50, 36, 115, 53, 56, 81, 94, 102, 54, 24, 84, 76, 52, 70, 25, 67, 47, 96, 68, 55, 110, 114, 91, 44, 88, 85, 116, 108, 41, 40, 8, 15, 6, 2, 46, 38, 121, 99, 80, 1, 105, 100, 37, 106, 14, 29, 97, 124, 93, 42, 98, 35, 12, 32, 72, 101, 107, 59, 0, 3, 43, 86, 11, 69, 103, 64, 4, 65, 66], [39, 57, 126, 34, 120, 87, 95, 90, 19, 60, 26, 92, 11, 61, 21, 111, 36, 17, 72, 22, 5, 48, 117, 63, 125, 71, 9, 4, 127, 67, 58, 83, 16, 79, 75, 70, 66, 73, 52, 51, 27, 113, 13, 122, 91, 119, 53, 50, 115, 45, 46, 31, 10, 28, 124, 65, 123, 23, 55, 118, 109, 54, 56, 114, 40, 62, 20, 121, 15, 18, 32, 49, 0, 77, 7, 102, 81, 44, 47, 82, 37, 85, 112, 101, 38, 106, 43, 110, 100, 35, 24, 116, 80, 107, 105, 78, 99, 33, 76, 41, 74, 94, 59, 1, 14, 84, 108, 97, 30, 104, 86, 42, 88, 96, 68, 93, 89, 6, 25, 98, 8, 12, 3, 2, 69, 29, 64, 103], [59, 61, 53, 101, 27, 88, 76, 78, 18, 16, 25, 20, 22, 85, 29, 69, 7, 96, 28, 75, 9, 30, 119, 62, 100, 56, 87, 66, 39, 14, 126, 83, 82, 117, 90, 37, 2, 71, 43, 67, 3, 12, 91, 72, 21, 79, 80, 74, 77, 73, 84, 0, 33, 24, 26, 93, 81, 65, 15, 23, 19, 6, 17, 1, 31, 54, 94, 8, 92, 36, 50, 10, 89, 5, 97, 105, 111, 41, 95, 13, 35, 106, 108, 116, 107, 42, 70, 103, 68, 113, 86, 32, 124, 48, 49, 4, 11, 99, 44, 60, 98, 64, 34, 55, 102, 63, 115, 120, 46, 52, 125, 104, 121, 123, 122, 110, 45, 57, 58, 127, 40, 47, 109, 118, 38, 114, 112, 51], [53, 59, 61, 100, 119, 56, 107, 50, 124, 52, 126, 113, 123, 116, 60, 54, 62, 115, 117, 120, 122, 58, 23, 125, 63, 110, 121, 57, 55, 111, 77, 45, 108, 39, 49, 114, 37, 43, 127, 48, 109, 112, 47, 118, 90, 51, 42, 46, 103, 104, 44, 38, 92, 41, 40, 101, 106, 32, 98, 105, 36, 84, 33, 99, 28, 31, 102, 35, 34, 17, 73, 26, 97, 95, 3, 94, 15, 64, 2, 11, 30, 12, 5, 74, 14, 96, 82, 70, 13, 29, 93, 83, 24, 85, 27, 7, 1, 25, 69, 19, 68, 87, 72, 66, 21, 71, 91, 79, 80, 76, 9, 8, 4, 20, 75, 16, 81, 67, 89, 18, 78, 6, 88, 22, 86, 65, 0, 10], [59, 53, 61, 101, 88, 71, 75, 18, 78, 27, 16, 76, 2, 22, 69, 25, 20, 9, 68, 117, 56, 0, 119, 79, 5, 74, 81, 66, 85, 91, 93, 14, 7, 96, 72, 11, 83, 10, 87, 24, 82, 90, 28, 30, 15, 6, 13, 80, 39, 3, 4, 89, 43, 29, 21, 32, 84, 12, 116, 23, 111, 33, 26, 70, 19, 35, 64, 41, 92, 94, 1, 8, 31, 100, 77, 49, 73, 67, 62, 17, 54, 37, 65, 99, 95, 103, 55, 126, 86, 124, 108, 38, 98, 50, 113, 34, 36, 97, 115, 60, 122, 105, 42, 44, 104, 123, 110, 47, 107, 102, 48, 121, 63, 52, 120, 106, 46, 58, 40, 45, 57, 125, 109, 127, 51, 112, 118, 114], [61, 53, 119, 124, 113, 63, 54, 47, 52, 57, 115, 117, 114, 120, 56, 50, 58, 111, 55, 118, 127, 125, 59, 62, 123, 121, 51, 110, 45, 49, 122, 116, 48, 112, 109, 126, 44, 43, 60, 108, 46, 42, 106, 107, 105, 32, 103, 36, 40, 101, 41, 100, 37, 33, 39, 104, 99, 86, 22, 102, 35, 23, 38, 34, 29, 95, 92, 98, 28, 97, 31, 96, 90, 83, 17, 94, 26, 19, 81, 20, 93, 30, 27, 25, 82, 16, 88, 87, 21, 18, 24, 80, 89, 14, 84, 85, 15, 91, 13, 78, 77, 79, 76, 67, 72, 74, 75, 70, 12, 10, 9, 71, 11, 1, 68, 0, 73, 69, 8, 6, 3, 2, 4, 5, 66, 64, 7, 65], [127, 124, 102, 33, 62, 51, 94, 56, 22, 18, 24, 121, 60, 116, 59, 37, 119, 39, 113, 122, 125, 114, 27, 91, 53, 93, 99, 30, 110, 38, 55, 126, 32, 25, 61, 58, 57, 49, 104, 118, 95, 112, 54, 28, 123, 48, 117, 47, 63, 50, 97, 46, 2, 111, 115, 108, 7, 120, 84, 29, 52, 45, 109, 31, 44, 21, 101, 71, 75, 107, 34, 106, 89, 3, 0, 90, 82, 85, 70, 35, 42, 98, 92, 103, 40, 78, 6, 105, 36, 41, 67, 65, 73, 83, 15, 43, 23, 20, 26, 14, 16, 87, 69, 88, 80, 68, 72, 96, 19, 100, 1, 76, 4, 64, 12, 77, 8, 66, 17, 79, 11, 10, 81, 74, 13, 5, 86, 9], [124, 102, 127, 47, 24, 33, 62, 94, 18, 51, 91, 84, 22, 75, 25, 36, 28, 121, 114, 56, 15, 37, 26, 60, 40, 110, 112, 116, 32, 53, 59, 122, 126, 119, 38, 27, 58, 79, 89, 20, 109, 107, 13, 125, 113, 11, 57, 87, 30, 123, 111, 34, 63, 55, 103, 35, 117, 48, 100, 49, 93, 46, 29, 54, 41, 45, 108, 9, 105, 61, 7, 42, 118, 31, 39, 85, 101, 88, 115, 106, 50, 80, 98, 96, 92, 99, 52, 120, 82, 90, 23, 104, 19, 3, 67, 97, 95, 44, 83, 17, 69, 21, 0, 16, 10, 76, 5, 71, 12, 74, 81, 2, 43, 14, 78, 86, 72, 68, 77, 73, 8, 4, 65, 66, 64, 6, 1, 70], [102, 127, 124, 94, 24, 33, 84, 13, 30, 93, 19, 22, 9, 25, 62, 18, 28, 17, 15, 92, 82, 88, 97, 16, 83, 73, 91, 38, 11, 23, 56, 51, 20, 79, 42, 85, 47, 112, 36, 89, 77, 69, 27, 46, 81, 31, 87, 95, 90, 106, 29, 67, 110, 119, 34, 2, 32, 80, 35, 86, 96, 8, 5, 0, 76, 26, 14, 12, 39, 122, 7, 74, 21, 78, 37, 10, 72, 108, 99, 71, 120, 3, 60, 59, 100, 114, 70, 123, 65, 107, 75, 63, 98, 116, 40, 68, 6, 103, 104, 105, 126, 101, 111, 4, 52, 55, 1, 57, 58, 54, 41, 66, 109, 45, 118, 50, 125, 64, 48, 117, 121, 53, 113, 115, 44, 43, 49, 61], [124, 127, 102, 47, 51, 62, 121, 56, 85, 59, 116, 119, 114, 60, 110, 113, 125, 58, 126, 53, 91, 122, 55, 123, 61, 112, 54, 48, 49, 63, 57, 50, 117, 118, 18, 109, 115, 108, 26, 41, 120, 111, 107, 101, 46, 105, 40, 45, 52, 21, 33, 38, 44, 25, 104, 42, 43, 36, 106, 39, 86, 22, 73, 99, 35, 37, 82, 76, 24, 34, 100, 30, 103, 79, 97, 94, 92, 3, 15, 98, 32, 29, 95, 96, 31, 27, 77, 90, 28, 89, 23, 75, 93, 65, 69, 7, 68, 84, 88, 17, 83, 12, 20, 81, 87, 0, 1, 72, 10, 66, 67, 19, 16, 80, 11, 64, 70, 6, 9, 5, 78, 71, 13, 14, 4, 74, 2, 8]], "model.layers.9.self_attn.k_proj": [[46, 37, 110, 96, 22, 63, 124, 99, 59, 113, 28, 49, 56, 122, 55, 114, 115, 54, 58, 57, 51, 61, 60, 119, 123, 53, 120, 48, 62, 25, 47, 85, 50, 121, 125, 112, 118, 127, 19, 117, 126, 116, 78, 43, 52, 8, 42, 108, 111, 109, 105, 107, 27, 40, 45, 65, 44, 30, 41, 106, 77, 26, 104, 17, 16, 103, 23, 39, 94, 102, 36, 7, 38, 4, 5, 3, 97, 95, 31, 29, 90, 81, 33, 35, 32, 82, 67, 74, 100, 93, 2, 21, 64, 18, 98, 101, 34, 76, 84, 24, 20, 14, 1, 87, 9, 73, 11, 10, 92, 91, 66, 13, 88, 12, 79, 15, 0, 80, 69, 75, 72, 6, 68, 70, 86, 71, 83, 89], [105, 117, 95, 56, 37, 58, 43, 106, 112, 34, 92, 23, 33, 108, 63, 114, 55, 29, 110, 103, 38, 111, 99, 22, 89, 26, 9, 78, 124, 46, 107, 119, 84, 50, 122, 87, 18, 82, 127, 2, 126, 0, 60, 51, 83, 4, 20, 14, 121, 98, 54, 65, 88, 12, 57, 120, 59, 69, 67, 16, 90, 115, 52, 40, 44, 109, 118, 62, 31, 36, 10, 42, 48, 70, 79, 24, 47, 76, 80, 41, 8, 30, 25, 100, 113, 96, 32, 27, 53, 101, 6, 125, 45, 17, 123, 64, 11, 91, 102, 15, 21, 86, 49, 116, 85, 13, 7, 104, 94, 61, 3, 97, 81, 72, 71, 1, 75, 5, 74, 66, 93, 35, 77, 73, 68, 19, 28, 39], [41, 0, 34, 47, 18, 74, 13, 81, 86, 117, 76, 8, 70, 53, 65, 55, 66, 79, 105, 71, 116, 84, 3, 4, 50, 63, 51, 112, 126, 59, 119, 111, 123, 43, 61, 127, 69, 44, 88, 106, 114, 46, 125, 93, 118, 67, 49, 87, 48, 98, 92, 85, 45, 115, 26, 90, 68, 122, 104, 52, 2, 27, 54, 42, 6, 124, 7, 73, 121, 120, 60, 30, 58, 29, 1, 31, 38, 83, 57, 110, 11, 95, 24, 37, 62, 10, 25, 77, 64, 75, 32, 12, 103, 5, 14, 19, 107, 9, 36, 72, 35, 80, 102, 39, 15, 91, 16, 28, 33, 109, 40, 17, 101, 97, 23, 89, 100, 56, 78, 20, 94, 113, 108, 96, 99, 82, 21, 22], [40, 125, 62, 22, 33, 94, 48, 63, 100, 122, 90, 18, 119, 117, 92, 57, 127, 16, 114, 60, 29, 52, 49, 47, 86, 19, 54, 78, 59, 87, 45, 58, 115, 109, 110, 120, 10, 84, 56, 123, 46, 106, 113, 108, 112, 89, 70, 61, 43, 99, 24, 116, 80, 126, 53, 124, 50, 51, 96, 107, 111, 118, 28, 77, 4, 121, 39, 2, 21, 38, 79, 55, 8, 44, 105, 42, 35, 41, 0, 74, 37, 23, 101, 88, 17, 102, 32, 91, 103, 31, 98, 36, 97, 81, 82, 34, 27, 75, 93, 95, 1, 30, 7, 11, 26, 3, 15, 73, 20, 13, 83, 76, 69, 25, 12, 65, 14, 72, 71, 67, 6, 104, 85, 64, 9, 5, 68, 66], [124, 39, 55, 56, 97, 93, 113, 88, 21, 83, 57, 91, 15, 122, 81, 114, 123, 125, 78, 105, 44, 117, 6, 11, 52, 23, 18, 10, 66, 4, 45, 64, 9, 43, 109, 69, 126, 76, 106, 22, 49, 12, 110, 47, 116, 53, 112, 120, 51, 80, 107, 61, 46, 58, 118, 62, 42, 32, 54, 1, 14, 41, 82, 119, 3, 50, 121, 127, 115, 27, 60, 111, 34, 89, 108, 33, 48, 101, 100, 38, 20, 63, 72, 102, 36, 84, 99, 59, 90, 25, 98, 35, 13, 37, 26, 96, 92, 19, 104, 94, 30, 95, 86, 40, 16, 73, 28, 87, 77, 7, 29, 85, 31, 5, 8, 71, 67, 75, 24, 103, 65, 68, 79, 0, 17, 2, 70, 74], [126, 57, 103, 31, 98, 61, 87, 90, 22, 60, 16, 92, 21, 120, 19, 17, 111, 48, 77, 79, 75, 125, 51, 7, 63, 58, 113, 74, 118, 84, 122, 53, 3, 127, 73, 115, 117, 6, 45, 55, 123, 65, 64, 100, 56, 119, 82, 62, 52, 116, 66, 49, 112, 50, 114, 99, 110, 46, 69, 44, 121, 88, 54, 38, 47, 76, 105, 124, 40, 109, 107, 68, 4, 28, 59, 15, 13, 36, 32, 91, 104, 41, 97, 43, 106, 101, 35, 20, 29, 30, 1, 102, 18, 42, 86, 10, 27, 108, 33, 5, 93, 80, 14, 25, 24, 37, 96, 94, 85, 23, 70, 34, 67, 89, 12, 81, 78, 83, 8, 2, 71, 11, 72, 9, 26, 39, 95, 0], [59, 61, 53, 37, 22, 16, 27, 18, 78, 32, 20, 119, 75, 88, 29, 25, 56, 76, 71, 85, 69, 79, 60, 116, 10, 8, 34, 2, 9, 81, 126, 98, 50, 105, 19, 113, 117, 15, 1, 108, 36, 96, 123, 122, 107, 103, 124, 87, 26, 115, 74, 39, 62, 120, 3, 0, 55, 52, 43, 13, 90, 111, 63, 70, 28, 54, 35, 6, 21, 38, 125, 109, 46, 106, 112, 30, 121, 49, 4, 47, 57, 65, 41, 17, 94, 58, 23, 104, 33, 110, 127, 77, 45, 68, 118, 48, 51, 114, 83, 97, 44, 99, 42, 93, 102, 5, 86, 72, 92, 95, 40, 89, 31, 80, 100, 73, 24, 67, 12, 64, 14, 82, 84, 7, 66, 91, 11, 101], [127, 124, 38, 22, 97, 51, 62, 30, 121, 24, 15, 114, 59, 111, 110, 18, 56, 60, 116, 112, 9, 119, 63, 91, 13, 122, 125, 93, 64, 48, 113, 17, 84, 11, 53, 5, 57, 126, 47, 106, 55, 54, 58, 102, 118, 94, 49, 117, 61, 123, 50, 66, 44, 16, 100, 19, 45, 115, 43, 101, 109, 120, 76, 25, 99, 88, 104, 108, 46, 28, 29, 10, 52, 23, 42, 103, 40, 105, 98, 41, 107, 37, 68, 26, 83, 1, 35, 14, 89, 95, 6, 87, 8, 79, 36, 71, 21, 81, 7, 96, 65, 34, 78, 74, 31, 33, 12, 39, 80, 85, 90, 86, 72, 32, 20, 3, 67, 75, 77, 92, 4, 27, 82, 69, 73, 2, 70, 0]], "model.layers.9.self_attn.qk_proj": [[124, 61, 127, 53, 125, 126, 62, 41, 57, 59, 110, 105, 117, 46, 56, 55, 51, 119, 114, 86, 63, 22, 113, 48, 98, 102, 82, 111, 112, 122, 116, 60, 52, 18, 123, 49, 58, 118, 26, 84, 50, 43, 39, 121, 28, 20, 17, 88, 47, 42, 81, 24, 101, 15, 38, 23, 29, 31, 79, 76, 103, 37, 13, 87, 97, 77, 34, 93, 33, 115, 45, 120, 74, 21, 10, 40, 0, 94, 64, 12, 54, 27, 66, 108, 107, 109, 89, 106, 16, 2, 7, 71, 44, 67, 91, 30, 19, 70, 90, 83, 78, 3, 95, 80, 92, 25, 32, 104, 85, 96, 72, 35, 8, 68, 6, 14, 99, 69, 75, 11, 9, 1, 36, 4, 100, 73, 5, 65], [124, 61, 127, 53, 126, 62, 125, 41, 59, 57, 110, 105, 117, 46, 56, 55, 63, 51, 86, 119, 114, 22, 48, 113, 111, 60, 112, 122, 98, 116, 49, 43, 82, 88, 118, 28, 24, 18, 102, 84, 123, 50, 37, 103, 26, 47, 52, 58, 39, 42, 15, 101, 87, 23, 77, 17, 20, 76, 115, 34, 45, 81, 31, 38, 79, 10, 0, 107, 13, 121, 33, 40, 97, 29, 120, 27, 54, 21, 64, 74, 94, 93, 30, 44, 12, 109, 83, 70, 108, 7, 89, 16, 90, 91, 25, 68, 106, 92, 96, 104, 100, 71, 80, 78, 85, 35, 95, 36, 66, 19, 8, 32, 4, 14, 11, 65, 99, 2, 5, 9, 75, 72, 69, 1, 3, 6, 73, 67], [124, 61, 127, 53, 126, 62, 41, 125, 59, 57, 105, 117, 110, 46, 56, 55, 63, 51, 119, 114, 86, 22, 113, 111, 48, 102, 98, 116, 60, 103, 112, 37, 122, 123, 47, 88, 39, 43, 115, 82, 18, 84, 28, 121, 118, 52, 38, 26, 34, 24, 49, 50, 20, 45, 120, 101, 87, 27, 23, 40, 17, 97, 31, 76, 42, 93, 109, 15, 81, 58, 94, 108, 54, 33, 79, 29, 106, 64, 70, 91, 77, 32, 74, 12, 44, 95, 0, 89, 10, 90, 30, 66, 107, 21, 100, 13, 83, 16, 4, 96, 7, 78, 99, 71, 8, 80, 85, 25, 92, 36, 104, 2, 35, 19, 11, 3, 68, 6, 9, 72, 67, 1, 14, 5, 75, 69, 65, 73], [124, 127, 53, 61, 125, 126, 62, 41, 57, 59, 110, 105, 46, 117, 56, 55, 63, 51, 114, 48, 86, 119, 22, 122, 111, 112, 118, 98, 60, 113, 116, 50, 49, 121, 102, 45, 18, 82, 47, 52, 43, 28, 88, 123, 38, 37, 58, 84, 115, 39, 103, 26, 17, 34, 87, 101, 42, 24, 31, 20, 54, 29, 120, 81, 93, 107, 33, 97, 23, 40, 27, 15, 94, 79, 108, 76, 0, 70, 44, 77, 92, 10, 106, 89, 13, 109, 96, 64, 12, 71, 74, 25, 66, 30, 80, 91, 35, 32, 95, 85, 21, 7, 8, 90, 100, 14, 104, 2, 83, 16, 19, 67, 69, 99, 36, 3, 78, 68, 11, 65, 72, 73, 6, 1, 4, 75, 9, 5], [124, 127, 53, 61, 125, 126, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 51, 63, 119, 114, 116, 86, 22, 48, 98, 113, 82, 60, 18, 112, 39, 45, 102, 111, 123, 50, 122, 49, 84, 42, 24, 47, 118, 26, 38, 28, 121, 101, 52, 37, 88, 120, 34, 20, 103, 17, 115, 43, 58, 33, 87, 81, 64, 31, 40, 77, 29, 76, 23, 15, 97, 54, 93, 107, 2, 79, 44, 10, 108, 0, 12, 89, 7, 66, 13, 74, 27, 92, 109, 21, 104, 91, 96, 90, 80, 70, 94, 71, 95, 30, 106, 83, 35, 8, 78, 4, 68, 16, 85, 19, 6, 25, 65, 14, 9, 5, 75, 99, 72, 32, 11, 67, 1, 69, 100, 3, 73, 36], [124, 127, 61, 53, 126, 125, 62, 41, 57, 59, 110, 105, 117, 56, 46, 55, 51, 63, 119, 86, 114, 48, 22, 111, 113, 82, 98, 116, 58, 112, 49, 18, 122, 60, 26, 52, 102, 123, 88, 118, 28, 84, 103, 50, 47, 24, 39, 77, 45, 101, 13, 42, 43, 120, 15, 17, 20, 38, 87, 81, 79, 33, 31, 121, 34, 23, 76, 37, 54, 10, 29, 40, 97, 74, 12, 44, 7, 115, 107, 0, 83, 104, 21, 27, 2, 94, 89, 80, 106, 108, 96, 8, 71, 109, 30, 92, 64, 85, 19, 14, 16, 91, 93, 25, 95, 6, 4, 11, 78, 90, 66, 3, 75, 35, 70, 67, 9, 68, 32, 72, 69, 99, 100, 5, 65, 36, 73, 1], [124, 127, 61, 53, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 51, 86, 22, 63, 119, 114, 48, 112, 116, 82, 111, 18, 52, 26, 49, 122, 118, 60, 84, 98, 20, 102, 120, 123, 28, 58, 88, 50, 39, 24, 101, 79, 81, 29, 15, 76, 17, 103, 121, 43, 113, 38, 13, 47, 31, 42, 37, 12, 34, 45, 106, 77, 10, 87, 21, 33, 27, 115, 8, 107, 23, 44, 40, 54, 94, 19, 97, 80, 93, 83, 74, 89, 85, 91, 25, 14, 109, 108, 92, 96, 7, 3, 30, 90, 0, 67, 71, 95, 6, 72, 16, 99, 78, 64, 11, 66, 36, 104, 32, 35, 2, 69, 9, 100, 5, 68, 75, 70, 1, 4, 65, 73], [124, 127, 61, 53, 126, 125, 41, 62, 59, 57, 110, 105, 117, 56, 46, 55, 51, 63, 22, 86, 119, 114, 48, 98, 112, 18, 82, 88, 122, 116, 84, 28, 26, 24, 111, 20, 60, 113, 118, 79, 15, 103, 102, 58, 50, 123, 38, 52, 43, 49, 34, 101, 17, 37, 87, 31, 42, 76, 29, 45, 13, 83, 77, 81, 39, 10, 120, 12, 21, 121, 47, 23, 97, 40, 94, 27, 74, 33, 0, 104, 64, 19, 54, 107, 115, 108, 7, 30, 80, 93, 25, 106, 6, 96, 89, 85, 16, 14, 66, 90, 92, 44, 8, 91, 109, 78, 100, 11, 2, 99, 35, 95, 68, 75, 71, 65, 72, 32, 36, 70, 4, 1, 69, 9, 5, 3, 73, 67], [124, 127, 53, 61, 126, 125, 41, 62, 59, 57, 110, 117, 105, 56, 46, 51, 55, 63, 114, 119, 22, 86, 122, 48, 98, 111, 88, 18, 60, 112, 49, 26, 84, 118, 82, 113, 116, 50, 20, 24, 39, 28, 47, 37, 38, 58, 103, 43, 102, 123, 52, 29, 94, 15, 121, 17, 79, 87, 101, 34, 27, 77, 31, 33, 120, 81, 13, 107, 97, 45, 12, 76, 23, 42, 0, 115, 40, 85, 30, 21, 74, 54, 10, 6, 19, 96, 83, 93, 25, 89, 78, 64, 92, 108, 44, 90, 16, 104, 80, 91, 68, 7, 106, 67, 109, 2, 32, 66, 95, 3, 100, 71, 4, 36, 1, 72, 70, 99, 65, 14, 35, 11, 69, 75, 8, 5, 9, 73], [124, 127, 53, 61, 126, 41, 125, 59, 62, 57, 105, 110, 117, 56, 46, 55, 63, 51, 114, 22, 86, 119, 48, 112, 116, 122, 111, 18, 98, 118, 113, 88, 60, 102, 50, 58, 82, 84, 28, 52, 79, 24, 26, 49, 38, 103, 81, 47, 15, 87, 20, 39, 34, 17, 123, 43, 29, 37, 23, 120, 107, 121, 97, 74, 76, 101, 45, 42, 31, 33, 13, 77, 27, 12, 54, 115, 21, 93, 44, 94, 0, 10, 40, 78, 64, 6, 90, 92, 83, 104, 66, 96, 7, 19, 80, 85, 25, 109, 108, 30, 91, 106, 16, 68, 71, 89, 70, 2, 72, 4, 95, 67, 32, 14, 99, 100, 5, 35, 69, 8, 3, 65, 36, 11, 73, 75, 1, 9], [124, 53, 127, 61, 125, 126, 41, 57, 62, 59, 105, 110, 117, 46, 56, 51, 55, 63, 119, 114, 22, 86, 48, 113, 111, 116, 122, 112, 98, 102, 18, 60, 82, 118, 49, 52, 50, 39, 47, 58, 20, 123, 88, 42, 31, 26, 84, 87, 15, 24, 37, 28, 103, 79, 121, 64, 81, 17, 107, 43, 34, 120, 101, 54, 77, 38, 93, 29, 12, 76, 97, 13, 66, 33, 45, 23, 27, 7, 115, 10, 40, 0, 74, 71, 2, 78, 16, 44, 21, 94, 90, 25, 6, 92, 70, 30, 72, 95, 104, 109, 85, 108, 19, 106, 35, 96, 83, 5, 68, 89, 91, 32, 75, 69, 80, 65, 4, 9, 14, 11, 3, 67, 8, 100, 99, 73, 1, 36], [124, 127, 53, 61, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 51, 55, 63, 119, 114, 86, 22, 48, 122, 111, 82, 60, 18, 116, 52, 113, 98, 39, 112, 102, 49, 118, 26, 58, 50, 88, 47, 123, 20, 15, 84, 28, 24, 120, 101, 17, 13, 77, 81, 42, 121, 29, 37, 33, 12, 43, 79, 40, 87, 31, 103, 107, 54, 38, 16, 27, 76, 0, 64, 21, 44, 34, 74, 23, 19, 10, 45, 97, 78, 93, 115, 71, 94, 66, 30, 72, 89, 108, 7, 104, 92, 2, 5, 70, 96, 90, 3, 83, 25, 106, 91, 67, 85, 109, 68, 1, 95, 35, 32, 80, 69, 14, 8, 11, 4, 100, 6, 75, 9, 73, 99, 65, 36], [124, 127, 61, 53, 126, 125, 62, 41, 57, 59, 110, 105, 117, 56, 46, 55, 51, 63, 119, 22, 86, 114, 60, 112, 116, 18, 48, 113, 122, 88, 111, 49, 118, 98, 82, 24, 102, 50, 28, 52, 123, 20, 43, 39, 26, 84, 101, 58, 103, 15, 47, 38, 37, 29, 81, 13, 87, 120, 17, 33, 121, 97, 45, 79, 34, 107, 27, 31, 12, 42, 115, 21, 77, 74, 104, 93, 23, 40, 16, 76, 94, 10, 54, 91, 19, 44, 83, 89, 30, 25, 96, 90, 85, 108, 64, 70, 78, 72, 92, 100, 106, 32, 95, 71, 80, 109, 0, 75, 14, 36, 35, 7, 68, 66, 2, 99, 69, 4, 3, 67, 11, 5, 8, 6, 65, 9, 1, 73], [124, 127, 53, 61, 126, 125, 62, 41, 57, 59, 110, 117, 105, 56, 46, 55, 63, 51, 86, 119, 22, 114, 48, 112, 111, 98, 113, 60, 18, 58, 116, 122, 82, 88, 49, 28, 15, 118, 26, 47, 123, 13, 120, 24, 43, 102, 50, 84, 52, 103, 79, 20, 34, 64, 101, 87, 81, 39, 21, 42, 45, 29, 23, 94, 17, 38, 40, 121, 74, 77, 10, 83, 104, 70, 12, 0, 33, 31, 37, 93, 107, 54, 27, 97, 89, 19, 7, 71, 76, 16, 25, 14, 90, 108, 44, 2, 92, 80, 91, 72, 115, 78, 85, 95, 30, 106, 66, 75, 4, 73, 11, 32, 96, 5, 68, 35, 6, 69, 9, 8, 65, 36, 109, 1, 100, 99, 3, 67], [124, 127, 53, 61, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 63, 51, 119, 86, 114, 48, 22, 60, 111, 112, 118, 113, 52, 123, 49, 82, 98, 47, 18, 101, 116, 102, 120, 122, 88, 39, 58, 26, 28, 24, 50, 103, 13, 20, 38, 84, 37, 34, 45, 43, 33, 79, 15, 87, 23, 42, 17, 121, 97, 31, 81, 40, 107, 27, 77, 29, 108, 7, 12, 21, 0, 54, 16, 19, 44, 74, 10, 64, 94, 70, 93, 104, 92, 76, 89, 106, 25, 71, 91, 72, 83, 90, 66, 115, 2, 80, 95, 14, 85, 78, 96, 32, 30, 3, 67, 5, 4, 9, 100, 99, 35, 109, 1, 68, 6, 11, 36, 75, 69, 73, 65, 8], [124, 127, 53, 61, 126, 125, 41, 57, 62, 59, 110, 105, 117, 56, 46, 55, 63, 51, 86, 119, 22, 114, 113, 48, 112, 60, 111, 18, 102, 98, 52, 122, 118, 49, 39, 116, 43, 82, 123, 84, 101, 28, 50, 47, 58, 120, 88, 103, 26, 20, 42, 24, 29, 33, 13, 17, 15, 81, 97, 38, 45, 34, 27, 37, 79, 12, 121, 31, 40, 23, 107, 87, 77, 94, 16, 108, 104, 10, 54, 64, 30, 21, 0, 91, 25, 7, 76, 115, 74, 90, 83, 89, 44, 35, 93, 92, 71, 70, 69, 96, 85, 19, 32, 66, 106, 67, 2, 78, 72, 95, 80, 100, 109, 99, 6, 68, 3, 75, 14, 4, 8, 5, 1, 36, 9, 11, 65, 73], [124, 127, 126, 53, 61, 125, 41, 57, 62, 59, 110, 105, 56, 46, 117, 55, 51, 63, 22, 86, 114, 48, 119, 112, 52, 18, 111, 116, 102, 123, 60, 113, 98, 28, 84, 49, 122, 82, 88, 118, 50, 101, 103, 20, 26, 39, 79, 24, 15, 47, 58, 17, 120, 37, 13, 38, 81, 107, 104, 31, 45, 33, 27, 121, 29, 43, 34, 10, 40, 21, 12, 74, 23, 94, 42, 19, 91, 87, 77, 16, 97, 32, 90, 115, 54, 25, 108, 76, 106, 93, 78, 35, 71, 83, 44, 64, 89, 85, 80, 96, 99, 0, 30, 92, 14, 6, 95, 8, 7, 68, 72, 109, 69, 4, 66, 70, 100, 75, 9, 5, 2, 65, 36, 1, 73, 67, 11, 3], [124, 127, 53, 126, 61, 125, 41, 59, 62, 57, 110, 105, 117, 56, 46, 55, 51, 22, 86, 63, 119, 114, 111, 48, 112, 122, 123, 82, 98, 52, 28, 18, 116, 49, 101, 102, 24, 84, 88, 50, 47, 26, 58, 38, 60, 20, 113, 118, 17, 103, 37, 81, 42, 15, 39, 79, 34, 31, 29, 43, 33, 21, 23, 13, 87, 97, 120, 40, 107, 45, 12, 121, 27, 74, 10, 89, 93, 77, 19, 90, 94, 83, 16, 104, 0, 25, 7, 91, 115, 54, 6, 78, 80, 92, 108, 76, 30, 44, 96, 106, 64, 85, 99, 2, 95, 71, 66, 14, 35, 75, 32, 8, 100, 67, 9, 3, 109, 36, 68, 70, 4, 73, 1, 72, 11, 69, 5, 65], [124, 127, 53, 61, 126, 41, 125, 59, 62, 57, 105, 110, 117, 46, 56, 55, 51, 86, 114, 63, 119, 22, 48, 118, 52, 111, 98, 122, 116, 60, 39, 112, 37, 123, 18, 47, 102, 49, 113, 103, 101, 82, 43, 58, 88, 84, 38, 115, 24, 34, 26, 81, 28, 31, 42, 50, 120, 107, 17, 121, 33, 20, 79, 13, 87, 40, 91, 15, 45, 54, 23, 106, 29, 108, 27, 97, 94, 0, 99, 64, 12, 93, 32, 89, 44, 10, 6, 21, 76, 19, 83, 95, 92, 85, 90, 74, 16, 100, 2, 35, 109, 96, 77, 30, 80, 25, 14, 66, 7, 104, 68, 36, 8, 4, 71, 67, 70, 1, 3, 75, 78, 9, 69, 5, 65, 73, 72, 11], [124, 127, 53, 61, 126, 125, 57, 41, 62, 59, 110, 105, 117, 46, 56, 55, 51, 86, 63, 119, 22, 114, 48, 112, 111, 113, 116, 60, 123, 49, 118, 98, 102, 103, 18, 39, 122, 88, 52, 47, 101, 58, 28, 26, 82, 50, 33, 84, 43, 24, 37, 121, 31, 38, 79, 15, 34, 87, 20, 97, 13, 42, 29, 17, 81, 21, 120, 107, 27, 45, 23, 115, 54, 74, 12, 94, 25, 10, 6, 93, 104, 40, 89, 19, 90, 92, 30, 77, 32, 91, 64, 76, 0, 108, 35, 83, 80, 7, 44, 95, 85, 96, 78, 106, 16, 8, 4, 109, 71, 68, 14, 2, 75, 66, 9, 70, 99, 100, 36, 5, 11, 72, 69, 1, 3, 65, 67, 73], [124, 61, 53, 127, 126, 125, 41, 57, 62, 59, 110, 105, 117, 56, 46, 55, 51, 63, 86, 119, 22, 114, 48, 112, 111, 116, 98, 18, 60, 102, 84, 52, 123, 122, 58, 118, 50, 113, 88, 82, 49, 101, 17, 42, 28, 26, 43, 13, 47, 121, 39, 20, 15, 33, 27, 81, 38, 79, 24, 29, 103, 120, 10, 87, 107, 31, 45, 77, 40, 23, 37, 115, 12, 54, 94, 34, 74, 95, 6, 30, 90, 25, 7, 93, 16, 64, 21, 104, 71, 66, 76, 78, 0, 2, 83, 89, 97, 91, 44, 5, 80, 85, 19, 92, 108, 106, 8, 3, 96, 75, 4, 9, 14, 69, 67, 70, 100, 109, 35, 32, 72, 36, 99, 68, 11, 73, 65, 1], [124, 127, 53, 61, 125, 126, 59, 41, 62, 57, 110, 117, 105, 56, 46, 55, 51, 63, 119, 86, 22, 114, 48, 112, 98, 111, 118, 58, 52, 18, 116, 102, 49, 50, 37, 82, 88, 101, 42, 122, 20, 60, 113, 115, 123, 38, 39, 84, 24, 28, 81, 47, 103, 29, 26, 43, 15, 17, 34, 54, 33, 120, 45, 87, 12, 121, 13, 31, 79, 64, 23, 40, 27, 93, 107, 106, 94, 97, 21, 44, 92, 10, 108, 77, 89, 16, 74, 30, 90, 76, 91, 83, 95, 25, 71, 109, 7, 19, 0, 96, 35, 78, 85, 8, 100, 66, 104, 80, 99, 70, 4, 2, 32, 14, 75, 6, 5, 36, 3, 11, 67, 69, 1, 72, 68, 9, 73, 65], [124, 127, 61, 53, 126, 125, 41, 62, 57, 59, 110, 105, 117, 56, 46, 63, 55, 51, 119, 86, 22, 114, 48, 98, 112, 111, 18, 37, 118, 38, 60, 24, 116, 84, 102, 82, 52, 58, 113, 20, 39, 123, 101, 88, 50, 115, 120, 28, 103, 45, 15, 81, 49, 26, 43, 42, 23, 31, 47, 122, 87, 29, 27, 17, 34, 33, 94, 40, 97, 12, 13, 10, 0, 54, 91, 107, 79, 77, 121, 44, 109, 93, 21, 104, 108, 74, 64, 83, 90, 76, 106, 95, 70, 19, 80, 71, 25, 96, 66, 30, 7, 92, 16, 78, 2, 85, 89, 32, 14, 8, 100, 68, 35, 99, 4, 75, 72, 73, 5, 69, 36, 3, 1, 11, 65, 9, 6, 67], [124, 127, 53, 61, 126, 125, 62, 41, 57, 59, 110, 105, 56, 117, 46, 63, 55, 51, 86, 119, 114, 22, 48, 58, 111, 98, 18, 116, 112, 60, 113, 118, 52, 82, 49, 88, 45, 50, 103, 123, 39, 84, 38, 102, 122, 37, 43, 42, 47, 28, 26, 20, 101, 24, 81, 27, 54, 79, 15, 87, 13, 120, 23, 31, 10, 40, 77, 33, 17, 107, 94, 29, 121, 34, 115, 97, 83, 85, 30, 76, 64, 12, 70, 71, 104, 91, 21, 44, 93, 108, 95, 90, 25, 74, 89, 7, 19, 92, 16, 80, 3, 66, 0, 106, 14, 35, 78, 2, 96, 68, 69, 8, 4, 32, 109, 9, 75, 100, 11, 67, 5, 99, 73, 36, 72, 1, 6, 65], [124, 127, 61, 53, 126, 125, 62, 57, 41, 59, 110, 117, 105, 46, 56, 55, 51, 63, 86, 119, 22, 114, 48, 60, 111, 113, 118, 98, 52, 82, 122, 47, 123, 49, 116, 18, 102, 112, 39, 58, 88, 101, 84, 26, 37, 50, 20, 28, 43, 17, 24, 45, 38, 121, 81, 31, 103, 27, 42, 23, 34, 79, 29, 97, 15, 33, 54, 87, 76, 115, 120, 12, 108, 13, 21, 40, 94, 77, 70, 91, 10, 107, 106, 74, 93, 0, 44, 30, 83, 25, 89, 2, 7, 96, 16, 19, 32, 85, 71, 92, 64, 90, 95, 80, 14, 109, 100, 73, 78, 8, 35, 99, 66, 72, 104, 4, 3, 75, 69, 36, 5, 67, 6, 11, 65, 1, 68, 9], [124, 61, 127, 53, 126, 125, 59, 57, 62, 41, 110, 105, 117, 56, 46, 55, 63, 86, 51, 119, 22, 114, 48, 60, 113, 112, 58, 49, 18, 111, 98, 122, 88, 116, 118, 52, 82, 50, 102, 101, 84, 26, 123, 39, 24, 121, 81, 38, 15, 37, 20, 34, 45, 103, 28, 17, 79, 115, 76, 42, 120, 29, 33, 47, 13, 31, 43, 107, 27, 87, 40, 77, 74, 21, 23, 10, 25, 12, 97, 93, 94, 54, 90, 30, 83, 89, 64, 91, 80, 44, 7, 70, 85, 96, 92, 19, 78, 104, 108, 100, 0, 99, 71, 106, 35, 109, 16, 95, 36, 32, 72, 8, 73, 14, 75, 6, 68, 66, 69, 11, 2, 4, 5, 9, 1, 67, 3, 65], [124, 127, 61, 53, 126, 125, 41, 59, 62, 57, 110, 105, 117, 56, 46, 55, 63, 51, 119, 86, 22, 114, 122, 48, 58, 60, 112, 98, 88, 82, 116, 52, 111, 49, 120, 26, 43, 34, 18, 50, 102, 113, 121, 24, 101, 45, 28, 27, 123, 37, 40, 84, 42, 118, 38, 103, 31, 20, 29, 15, 47, 115, 39, 17, 94, 81, 23, 87, 21, 33, 79, 97, 91, 77, 107, 12, 109, 13, 93, 96, 76, 74, 0, 54, 104, 95, 108, 44, 80, 89, 25, 10, 90, 106, 64, 85, 30, 19, 3, 99, 78, 100, 70, 83, 7, 92, 68, 71, 2, 36, 72, 66, 67, 16, 35, 5, 6, 32, 4, 8, 73, 14, 65, 69, 1, 11, 9, 75], [124, 127, 53, 61, 126, 125, 59, 57, 41, 62, 110, 117, 105, 56, 46, 55, 63, 51, 119, 86, 22, 114, 111, 48, 98, 58, 82, 112, 113, 116, 49, 60, 18, 122, 102, 26, 84, 52, 118, 28, 79, 88, 120, 50, 121, 103, 123, 47, 20, 43, 101, 34, 45, 33, 29, 24, 39, 76, 81, 0, 42, 15, 17, 38, 37, 77, 87, 31, 94, 64, 54, 27, 10, 40, 23, 74, 12, 2, 97, 93, 7, 13, 107, 83, 21, 71, 115, 66, 6, 19, 80, 85, 44, 4, 109, 72, 106, 95, 89, 104, 96, 91, 30, 1, 92, 78, 16, 99, 25, 5, 108, 90, 14, 35, 32, 67, 9, 75, 68, 11, 100, 36, 73, 65, 70, 69, 3, 8], [124, 127, 61, 53, 125, 126, 41, 57, 62, 59, 105, 110, 117, 46, 56, 63, 55, 51, 86, 119, 22, 114, 113, 48, 111, 98, 112, 52, 122, 102, 116, 18, 47, 37, 49, 82, 123, 60, 118, 58, 50, 20, 26, 39, 101, 103, 24, 88, 34, 81, 42, 87, 38, 45, 33, 31, 120, 29, 28, 43, 15, 0, 76, 40, 121, 84, 97, 23, 17, 27, 21, 77, 79, 93, 107, 94, 7, 44, 74, 64, 2, 10, 92, 13, 106, 6, 115, 19, 89, 91, 12, 96, 90, 54, 95, 109, 80, 30, 108, 100, 16, 71, 104, 72, 66, 25, 32, 5, 78, 14, 69, 35, 85, 36, 83, 11, 9, 99, 4, 67, 75, 1, 3, 65, 68, 8, 73, 70], [124, 127, 61, 53, 126, 125, 41, 62, 57, 59, 110, 105, 56, 117, 46, 55, 51, 63, 86, 119, 114, 113, 22, 111, 48, 52, 58, 112, 60, 116, 98, 18, 102, 47, 49, 82, 50, 103, 26, 84, 88, 123, 37, 24, 28, 15, 122, 118, 38, 20, 39, 87, 101, 31, 81, 43, 42, 34, 121, 77, 97, 33, 17, 45, 40, 13, 29, 76, 10, 27, 79, 0, 23, 54, 74, 115, 108, 94, 12, 107, 6, 21, 89, 64, 99, 83, 2, 104, 72, 80, 90, 30, 120, 93, 44, 25, 71, 16, 109, 67, 96, 7, 91, 66, 19, 3, 106, 14, 92, 85, 95, 4, 78, 35, 75, 32, 36, 11, 5, 65, 100, 1, 73, 69, 8, 68, 70, 9], [124, 127, 61, 53, 126, 125, 41, 59, 62, 57, 110, 105, 117, 46, 56, 55, 63, 51, 119, 114, 22, 86, 60, 113, 48, 98, 111, 112, 52, 116, 18, 58, 122, 102, 82, 84, 118, 24, 47, 50, 88, 28, 103, 123, 43, 15, 26, 121, 49, 42, 39, 101, 20, 38, 45, 31, 17, 87, 97, 120, 81, 76, 23, 34, 29, 77, 37, 13, 40, 54, 33, 0, 27, 21, 79, 7, 44, 115, 12, 94, 74, 10, 89, 107, 106, 96, 93, 6, 83, 108, 72, 25, 2, 30, 64, 95, 71, 80, 68, 92, 104, 16, 35, 91, 4, 14, 85, 90, 19, 66, 75, 36, 100, 11, 78, 99, 32, 109, 9, 73, 70, 5, 69, 65, 1, 8, 67, 3], [124, 127, 61, 126, 53, 125, 62, 57, 59, 41, 110, 105, 56, 117, 46, 55, 51, 63, 119, 114, 86, 22, 48, 113, 60, 98, 18, 111, 116, 112, 24, 49, 82, 103, 123, 84, 88, 118, 122, 52, 58, 28, 26, 102, 50, 15, 20, 17, 76, 81, 121, 79, 13, 45, 39, 38, 43, 34, 87, 101, 23, 77, 31, 94, 21, 115, 47, 97, 29, 74, 37, 12, 10, 33, 42, 120, 40, 27, 54, 107, 44, 7, 83, 30, 0, 93, 89, 64, 71, 25, 91, 78, 11, 92, 6, 16, 90, 104, 96, 108, 72, 80, 66, 19, 85, 14, 5, 95, 70, 2, 75, 67, 4, 32, 3, 69, 36, 8, 106, 109, 99, 35, 100, 65, 68, 73, 9, 1]], "model.layers.10.self_attn.q_proj": [[121, 122, 26, 65, 36, 83, 50, 79, 21, 24, 81, 63, 89, 29, 94, 9, 0, 64, 68, 16, 6, 19, 99, 1, 91, 78, 8, 17, 23, 2, 58, 77, 75, 111, 69, 124, 86, 12, 114, 71, 95, 82, 85, 44, 52, 14, 120, 76, 30, 37, 67, 51, 96, 5, 118, 57, 31, 54, 70, 11, 123, 74, 66, 93, 20, 84, 47, 27, 60, 127, 61, 40, 4, 13, 10, 3, 18, 125, 73, 80, 102, 87, 53, 49, 117, 115, 119, 62, 113, 116, 100, 72, 112, 55, 45, 7, 15, 126, 59, 90, 48, 105, 104, 110, 92, 41, 88, 33, 56, 46, 109, 35, 42, 25, 103, 22, 39, 107, 97, 38, 106, 108, 43, 34, 32, 28, 101, 98], [121, 122, 63, 87, 50, 36, 75, 52, 118, 28, 12, 124, 17, 120, 94, 123, 60, 61, 54, 125, 51, 33, 73, 29, 57, 62, 127, 119, 40, 117, 34, 44, 115, 53, 113, 102, 15, 49, 47, 58, 111, 19, 112, 55, 23, 70, 126, 110, 114, 59, 14, 109, 46, 105, 24, 56, 97, 116, 45, 104, 99, 38, 31, 32, 21, 71, 16, 86, 13, 39, 107, 48, 76, 106, 80, 37, 35, 85, 103, 25, 101, 20, 88, 91, 100, 92, 96, 98, 108, 93, 41, 5, 18, 84, 66, 95, 67, 26, 22, 89, 90, 42, 64, 27, 83, 30, 43, 79, 11, 3, 74, 82, 4, 1, 8, 77, 72, 81, 7, 65, 10, 9, 0, 6, 2, 69, 78, 68], [50, 121, 122, 114, 63, 56, 62, 115, 120, 60, 58, 51, 49, 116, 84, 87, 59, 119, 53, 11, 113, 52, 15, 127, 44, 17, 117, 109, 45, 55, 124, 61, 126, 54, 47, 57, 123, 48, 107, 14, 111, 46, 72, 118, 108, 125, 110, 43, 112, 7, 24, 2, 27, 78, 106, 25, 99, 101, 75, 41, 4, 85, 42, 16, 77, 80, 19, 82, 10, 105, 1, 81, 3, 74, 104, 18, 13, 12, 9, 28, 83, 94, 40, 37, 97, 6, 71, 90, 73, 102, 103, 21, 39, 22, 29, 38, 23, 76, 30, 89, 93, 33, 5, 36, 0, 35, 32, 96, 34, 98, 70, 91, 95, 69, 31, 26, 92, 100, 86, 8, 65, 88, 79, 20, 66, 68, 67, 64], [121, 122, 50, 63, 14, 118, 52, 84, 124, 11, 123, 87, 120, 61, 97, 125, 15, 114, 54, 57, 62, 101, 60, 82, 47, 16, 17, 53, 119, 85, 28, 111, 115, 51, 112, 127, 117, 102, 58, 110, 40, 104, 55, 44, 109, 45, 75, 32, 59, 105, 46, 77, 113, 107, 49, 27, 106, 39, 37, 103, 126, 25, 33, 38, 81, 48, 116, 12, 34, 19, 41, 74, 80, 90, 108, 71, 72, 83, 24, 9, 13, 99, 91, 4, 2, 7, 56, 89, 21, 42, 78, 35, 6, 94, 36, 10, 96, 3, 76, 0, 73, 43, 92, 79, 93, 5, 30, 100, 23, 31, 98, 1, 22, 95, 29, 86, 26, 70, 88, 20, 66, 69, 18, 8, 67, 68, 65, 64], [103, 124, 113, 24, 82, 49, 21, 91, 31, 15, 32, 11, 62, 13, 83, 19, 16, 28, 50, 80, 72, 99, 27, 81, 63, 73, 93, 4, 76, 17, 88, 7, 75, 9, 14, 18, 74, 8, 85, 79, 12, 78, 26, 98, 77, 87, 86, 84, 64, 89, 100, 68, 1, 20, 25, 29, 45, 23, 35, 53, 92, 46, 70, 66, 90, 22, 71, 94, 10, 69, 33, 30, 34, 104, 95, 40, 6, 123, 101, 5, 67, 0, 60, 61, 122, 3, 110, 96, 59, 39, 109, 51, 111, 126, 2, 65, 117, 97, 112, 127, 116, 43, 55, 58, 54, 120, 125, 115, 105, 102, 44, 36, 37, 57, 48, 38, 56, 108, 41, 114, 121, 106, 107, 118, 52, 42, 47, 119], [62, 103, 124, 113, 49, 122, 61, 115, 57, 45, 40, 87, 26, 50, 119, 112, 53, 32, 48, 58, 104, 109, 29, 54, 121, 120, 126, 63, 59, 117, 118, 60, 90, 95, 108, 110, 93, 55, 127, 46, 116, 52, 125, 51, 94, 123, 111, 56, 114, 99, 47, 96, 44, 105, 42, 107, 31, 106, 41, 27, 43, 14, 33, 19, 17, 91, 38, 97, 28, 23, 102, 98, 21, 30, 20, 101, 74, 100, 81, 35, 36, 37, 84, 34, 24, 39, 92, 86, 78, 25, 22, 89, 70, 6, 13, 66, 88, 68, 15, 69, 83, 7, 18, 75, 85, 67, 9, 16, 79, 64, 77, 10, 5, 8, 72, 3, 12, 4, 82, 11, 71, 80, 65, 2, 0, 1, 73, 76], [113, 124, 103, 49, 62, 50, 53, 46, 26, 122, 45, 61, 32, 40, 120, 51, 59, 123, 63, 87, 126, 112, 109, 5, 58, 117, 57, 30, 47, 100, 35, 127, 115, 111, 90, 8, 108, 17, 54, 104, 121, 105, 56, 114, 68, 125, 33, 95, 48, 60, 1, 99, 52, 116, 83, 118, 97, 106, 7, 94, 31, 107, 101, 41, 25, 64, 110, 119, 43, 34, 80, 91, 75, 79, 6, 29, 93, 38, 44, 3, 36, 9, 42, 77, 92, 55, 102, 12, 98, 28, 37, 27, 0, 66, 23, 14, 96, 76, 78, 84, 20, 21, 2, 82, 86, 88, 24, 15, 19, 39, 74, 73, 89, 10, 85, 81, 67, 11, 22, 70, 71, 16, 65, 69, 4, 72, 13, 18], [113, 124, 62, 40, 103, 50, 49, 46, 122, 83, 59, 112, 126, 87, 5, 53, 8, 127, 109, 45, 77, 120, 61, 117, 1, 78, 26, 123, 6, 68, 80, 55, 63, 54, 75, 35, 58, 125, 42, 51, 57, 7, 114, 9, 115, 3, 60, 17, 111, 118, 76, 52, 56, 47, 15, 116, 119, 108, 48, 110, 121, 44, 41, 82, 30, 43, 107, 104, 32, 106, 0, 73, 105, 12, 101, 64, 33, 100, 38, 66, 102, 36, 37, 79, 11, 89, 85, 34, 2, 23, 71, 90, 84, 31, 99, 94, 24, 92, 95, 67, 29, 81, 98, 96, 97, 20, 21, 28, 25, 93, 14, 10, 19, 39, 74, 86, 65, 88, 91, 27, 22, 70, 69, 13, 4, 16, 18, 72], [102, 56, 49, 113, 23, 89, 55, 91, 19, 58, 85, 47, 37, 31, 20, 52, 18, 33, 14, 127, 95, 92, 122, 80, 124, 115, 117, 29, 125, 59, 120, 42, 112, 107, 123, 75, 61, 57, 46, 108, 28, 116, 114, 82, 50, 45, 121, 73, 48, 84, 109, 44, 77, 88, 41, 63, 51, 99, 39, 40, 53, 38, 103, 126, 34, 43, 97, 118, 32, 16, 13, 106, 11, 87, 100, 54, 104, 98, 94, 15, 86, 27, 105, 83, 60, 119, 111, 26, 35, 110, 78, 79, 62, 8, 93, 21, 30, 101, 5, 22, 25, 36, 90, 96, 74, 72, 6, 81, 10, 24, 71, 76, 17, 4, 7, 9, 12, 67, 66, 69, 65, 70, 0, 3, 1, 68, 2, 64], [102, 56, 113, 49, 85, 23, 91, 19, 80, 77, 9, 31, 11, 89, 58, 4, 13, 55, 6, 70, 68, 72, 47, 2, 73, 95, 75, 16, 21, 83, 52, 65, 8, 82, 64, 87, 39, 48, 27, 42, 7, 53, 15, 78, 37, 25, 109, 110, 124, 22, 26, 69, 125, 90, 112, 46, 5, 33, 81, 84, 79, 71, 50, 99, 88, 32, 120, 63, 12, 106, 17, 119, 35, 44, 127, 10, 61, 92, 30, 96, 107, 123, 29, 14, 0, 101, 76, 24, 28, 66, 118, 54, 40, 116, 57, 94, 115, 59, 114, 98, 45, 126, 60, 122, 41, 86, 67, 121, 18, 100, 20, 105, 36, 104, 43, 117, 111, 34, 93, 74, 97, 51, 62, 103, 108, 1, 3, 38], [102, 47, 113, 56, 49, 89, 91, 23, 85, 19, 31, 55, 58, 9, 80, 82, 77, 11, 14, 0, 120, 46, 68, 121, 72, 59, 51, 33, 66, 95, 83, 127, 37, 124, 114, 70, 52, 53, 88, 110, 78, 87, 84, 3, 48, 99, 10, 15, 112, 81, 57, 126, 60, 73, 62, 16, 35, 42, 1, 21, 109, 34, 20, 125, 32, 116, 108, 27, 79, 118, 5, 61, 12, 100, 96, 29, 7, 97, 30, 2, 93, 25, 44, 106, 40, 92, 71, 36, 104, 17, 43, 24, 39, 122, 123, 63, 22, 119, 67, 103, 69, 18, 26, 38, 13, 115, 4, 65, 117, 90, 50, 8, 54, 111, 98, 45, 105, 94, 64, 86, 101, 41, 74, 76, 28, 107, 75, 6], [102, 113, 56, 49, 23, 80, 19, 89, 9, 85, 77, 11, 70, 68, 31, 2, 55, 64, 83, 14, 1, 16, 72, 21, 91, 37, 87, 6, 58, 75, 25, 67, 82, 3, 66, 125, 13, 120, 47, 33, 27, 48, 52, 127, 7, 73, 42, 112, 78, 12, 110, 65, 8, 29, 53, 59, 5, 17, 57, 46, 24, 32, 44, 88, 22, 10, 81, 76, 116, 51, 50, 92, 124, 30, 114, 121, 40, 122, 61, 95, 109, 28, 39, 4, 84, 74, 93, 123, 90, 119, 99, 26, 43, 35, 86, 115, 18, 63, 79, 15, 62, 45, 106, 100, 118, 69, 126, 105, 103, 20, 107, 41, 104, 60, 71, 117, 94, 98, 101, 97, 54, 36, 108, 34, 96, 111, 0, 38], [52, 62, 122, 58, 102, 125, 61, 53, 60, 127, 123, 112, 121, 119, 63, 115, 114, 50, 57, 48, 117, 126, 124, 118, 56, 49, 46, 54, 55, 113, 120, 47, 116, 44, 111, 36, 110, 51, 107, 45, 59, 109, 108, 35, 42, 30, 105, 43, 106, 104, 89, 25, 40, 41, 101, 103, 100, 23, 39, 33, 38, 37, 96, 32, 92, 99, 98, 34, 95, 97, 94, 31, 87, 28, 27, 91, 75, 29, 26, 15, 93, 24, 90, 17, 84, 21, 9, 20, 82, 85, 88, 70, 2, 6, 11, 19, 81, 80, 77, 12, 0, 79, 16, 14, 74, 86, 7, 65, 10, 67, 73, 69, 3, 66, 13, 5, 4, 64, 83, 8, 18, 76, 71, 1, 72, 68, 22, 78], [58, 62, 52, 122, 60, 121, 63, 118, 125, 55, 112, 127, 115, 51, 114, 117, 126, 123, 50, 110, 46, 59, 61, 113, 119, 111, 43, 120, 56, 42, 48, 124, 47, 116, 49, 57, 45, 109, 104, 96, 53, 40, 41, 105, 54, 108, 106, 107, 37, 101, 39, 102, 32, 103, 44, 38, 100, 29, 25, 23, 35, 98, 36, 34, 99, 84, 33, 27, 95, 30, 19, 97, 86, 24, 94, 20, 90, 22, 31, 92, 28, 89, 93, 81, 91, 80, 17, 16, 88, 83, 13, 79, 77, 26, 87, 85, 15, 21, 73, 78, 76, 11, 18, 82, 14, 70, 75, 9, 72, 12, 6, 10, 0, 74, 66, 68, 7, 2, 71, 8, 3, 67, 69, 1, 65, 5, 4, 64], [62, 58, 52, 101, 121, 122, 60, 127, 125, 112, 55, 115, 118, 117, 123, 63, 51, 53, 57, 119, 114, 50, 113, 61, 126, 102, 111, 120, 124, 48, 46, 59, 110, 36, 116, 56, 42, 54, 109, 47, 45, 44, 106, 49, 108, 93, 105, 43, 107, 40, 41, 100, 96, 104, 90, 30, 39, 103, 38, 99, 94, 35, 34, 23, 29, 25, 28, 27, 97, 80, 95, 86, 98, 33, 19, 16, 85, 92, 89, 31, 91, 21, 32, 73, 75, 24, 87, 37, 83, 26, 9, 88, 81, 11, 3, 77, 13, 22, 20, 82, 68, 70, 84, 15, 71, 5, 69, 10, 74, 1, 78, 14, 17, 7, 18, 76, 8, 79, 6, 67, 64, 66, 4, 12, 2, 65, 72, 0], [58, 52, 62, 101, 24, 90, 34, 82, 93, 14, 12, 85, 19, 80, 121, 32, 60, 21, 86, 30, 37, 15, 20, 76, 78, 96, 9, 122, 18, 79, 69, 83, 16, 125, 67, 74, 10, 88, 94, 72, 55, 5, 8, 127, 100, 84, 29, 77, 87, 7, 17, 71, 1, 81, 112, 102, 26, 13, 25, 23, 118, 75, 89, 91, 113, 27, 68, 117, 0, 2, 114, 64, 98, 70, 120, 124, 11, 4, 61, 66, 115, 126, 73, 104, 119, 110, 92, 51, 63, 42, 28, 107, 123, 111, 31, 3, 116, 46, 50, 59, 45, 65, 57, 56, 53, 48, 95, 6, 47, 106, 97, 41, 33, 43, 99, 103, 109, 38, 40, 35, 105, 22, 108, 39, 36, 54, 49, 44], [118, 127, 26, 101, 17, 10, 86, 15, 77, 70, 98, 88, 82, 20, 74, 93, 6, 29, 68, 89, 13, 104, 30, 22, 72, 87, 2, 76, 85, 79, 81, 27, 92, 84, 90, 54, 24, 67, 37, 28, 34, 71, 18, 11, 7, 52, 19, 78, 3, 9, 80, 1, 12, 40, 91, 109, 59, 65, 126, 75, 95, 73, 97, 0, 66, 16, 42, 63, 64, 23, 5, 21, 83, 4, 94, 105, 62, 31, 14, 47, 113, 69, 121, 32, 33, 96, 25, 39, 43, 35, 36, 8, 38, 107, 50, 110, 99, 44, 41, 53, 103, 108, 100, 48, 56, 115, 122, 112, 46, 116, 124, 58, 102, 117, 45, 111, 55, 114, 49, 120, 61, 119, 106, 123, 125, 60, 51, 57], [118, 127, 101, 98, 37, 126, 88, 86, 62, 39, 30, 29, 50, 20, 63, 91, 104, 52, 26, 113, 72, 48, 121, 54, 112, 56, 119, 93, 82, 38, 97, 116, 43, 125, 59, 122, 44, 110, 60, 92, 66, 83, 117, 0, 35, 51, 25, 40, 109, 58, 77, 123, 53, 46, 102, 45, 55, 15, 69, 115, 41, 120, 108, 28, 31, 57, 61, 49, 1, 47, 90, 114, 99, 24, 11, 84, 111, 89, 106, 75, 36, 124, 94, 33, 103, 8, 6, 4, 79, 23, 68, 76, 80, 96, 18, 107, 73, 105, 42, 74, 3, 64, 13, 17, 100, 22, 32, 81, 78, 2, 95, 85, 16, 87, 19, 9, 27, 12, 14, 34, 21, 5, 71, 7, 65, 70, 10, 67], [127, 118, 62, 101, 126, 56, 104, 113, 63, 54, 119, 48, 112, 52, 82, 40, 60, 121, 44, 50, 122, 116, 98, 43, 51, 123, 59, 117, 88, 58, 97, 125, 120, 91, 37, 106, 110, 55, 57, 47, 109, 46, 53, 45, 114, 115, 61, 39, 86, 42, 111, 29, 49, 107, 23, 41, 8, 124, 108, 105, 100, 35, 102, 93, 36, 38, 103, 83, 96, 72, 31, 92, 26, 99, 33, 66, 30, 87, 28, 4, 95, 89, 32, 34, 80, 90, 25, 94, 22, 27, 16, 18, 68, 24, 20, 15, 19, 0, 77, 85, 12, 84, 2, 69, 78, 13, 14, 73, 21, 75, 11, 74, 71, 5, 76, 79, 81, 7, 17, 1, 64, 6, 9, 65, 3, 70, 10, 67], [127, 118, 101, 126, 56, 97, 88, 82, 62, 29, 63, 54, 52, 40, 113, 48, 44, 28, 23, 119, 109, 60, 43, 98, 112, 36, 122, 117, 50, 41, 121, 106, 45, 116, 59, 8, 86, 51, 37, 123, 115, 104, 47, 125, 91, 32, 26, 114, 58, 110, 120, 105, 89, 46, 39, 61, 102, 111, 55, 107, 93, 30, 57, 53, 108, 72, 42, 100, 38, 16, 20, 4, 92, 49, 12, 99, 124, 103, 35, 18, 34, 31, 33, 94, 77, 83, 13, 27, 15, 71, 79, 96, 95, 87, 81, 22, 14, 25, 80, 24, 75, 84, 21, 11, 85, 76, 69, 74, 68, 0, 6, 66, 90, 73, 1, 17, 19, 7, 3, 78, 9, 5, 2, 70, 10, 64, 65, 67], [42, 96, 106, 120, 45, 125, 26, 23, 20, 118, 62, 109, 78, 52, 98, 80, 67, 63, 124, 90, 122, 32, 82, 71, 126, 7, 34, 101, 94, 58, 27, 56, 87, 14, 73, 55, 77, 116, 12, 86, 97, 123, 13, 81, 48, 85, 91, 115, 35, 57, 103, 104, 114, 43, 50, 16, 49, 18, 99, 110, 111, 39, 76, 33, 22, 64, 65, 117, 24, 119, 37, 10, 1, 53, 92, 59, 31, 47, 60, 28, 112, 84, 17, 19, 46, 121, 83, 44, 127, 21, 29, 38, 89, 113, 54, 30, 74, 5, 107, 70, 61, 93, 41, 51, 95, 105, 8, 4, 102, 88, 15, 108, 100, 40, 25, 36, 3, 6, 68, 79, 9, 75, 66, 69, 72, 11, 2, 0], [42, 96, 106, 125, 45, 23, 26, 62, 20, 109, 52, 98, 80, 124, 118, 126, 120, 34, 63, 78, 90, 82, 55, 27, 94, 122, 101, 77, 56, 43, 116, 46, 111, 58, 103, 81, 44, 35, 119, 114, 97, 104, 53, 7, 123, 87, 48, 127, 115, 50, 99, 49, 107, 60, 39, 38, 41, 91, 113, 117, 110, 59, 47, 30, 100, 54, 121, 102, 40, 85, 37, 57, 67, 61, 108, 105, 36, 112, 31, 95, 18, 93, 16, 51, 28, 32, 33, 29, 73, 13, 92, 21, 14, 86, 84, 71, 24, 25, 17, 19, 12, 5, 22, 89, 10, 88, 83, 74, 9, 11, 1, 65, 69, 79, 75, 15, 76, 64, 3, 8, 72, 4, 6, 68, 70, 66, 0, 2], [42, 96, 125, 106, 52, 45, 23, 20, 26, 82, 80, 78, 120, 98, 118, 109, 5, 62, 73, 7, 77, 12, 124, 90, 101, 86, 126, 103, 67, 63, 122, 32, 94, 35, 27, 87, 9, 56, 16, 1, 55, 58, 21, 69, 85, 91, 83, 64, 84, 76, 114, 123, 75, 3, 50, 34, 13, 18, 116, 115, 22, 11, 29, 70, 17, 40, 19, 43, 15, 48, 10, 39, 97, 81, 14, 74, 30, 110, 4, 79, 37, 31, 89, 44, 108, 104, 24, 49, 25, 71, 47, 66, 59, 92, 99, 60, 113, 28, 95, 112, 107, 88, 51, 8, 111, 100, 117, 72, 121, 57, 93, 6, 119, 105, 38, 41, 33, 36, 65, 54, 127, 68, 61, 0, 2, 102, 46, 53], [42, 96, 45, 106, 125, 23, 124, 26, 20, 52, 109, 98, 120, 62, 118, 126, 80, 122, 82, 115, 94, 56, 103, 116, 55, 78, 34, 90, 63, 58, 43, 48, 87, 114, 97, 101, 50, 60, 77, 110, 39, 41, 35, 104, 13, 123, 31, 47, 108, 53, 107, 99, 119, 113, 30, 19, 121, 27, 117, 33, 57, 49, 51, 40, 7, 61, 38, 102, 54, 92, 44, 85, 100, 67, 46, 32, 111, 93, 95, 36, 59, 37, 105, 81, 91, 112, 18, 28, 16, 29, 84, 127, 71, 86, 24, 14, 25, 73, 21, 89, 88, 79, 76, 12, 22, 74, 72, 69, 83, 17, 11, 15, 10, 9, 5, 68, 75, 65, 1, 8, 4, 66, 6, 64, 70, 0, 3, 2], [102, 62, 126, 98, 87, 26, 110, 85, 51, 41, 30, 19, 38, 45, 50, 22, 15, 79, 103, 94, 90, 84, 12, 93, 31, 17, 16, 113, 21, 109, 69, 105, 112, 20, 125, 127, 23, 8, 53, 65, 83, 25, 32, 117, 119, 14, 88, 63, 6, 55, 99, 58, 1, 116, 122, 75, 46, 57, 60, 96, 100, 108, 34, 121, 124, 52, 80, 120, 115, 74, 59, 106, 24, 89, 114, 49, 97, 111, 10, 76, 104, 118, 0, 2, 95, 64, 27, 86, 47, 40, 73, 123, 48, 81, 36, 92, 70, 54, 67, 82, 28, 42, 66, 5, 35, 29, 91, 78, 18, 77, 9, 13, 56, 44, 72, 39, 11, 101, 71, 68, 107, 61, 33, 37, 4, 43, 7, 3], [102, 126, 62, 98, 87, 17, 85, 110, 90, 26, 22, 19, 50, 8, 14, 75, 45, 103, 41, 15, 94, 21, 93, 89, 6, 51, 24, 113, 109, 2, 30, 84, 88, 83, 68, 92, 12, 38, 27, 58, 20, 31, 72, 79, 78, 112, 122, 23, 25, 74, 80, 32, 4, 71, 16, 117, 119, 53, 49, 127, 77, 29, 47, 7, 35, 81, 76, 60, 105, 10, 111, 28, 55, 120, 116, 52, 82, 0, 124, 46, 48, 5, 91, 13, 99, 125, 34, 67, 39, 36, 33, 121, 96, 97, 57, 95, 86, 63, 37, 118, 65, 115, 18, 44, 100, 66, 104, 11, 73, 108, 59, 42, 114, 43, 101, 9, 61, 69, 123, 54, 56, 3, 106, 64, 107, 70, 1, 40], [102, 126, 98, 62, 26, 41, 45, 87, 85, 19, 17, 93, 38, 103, 22, 15, 94, 51, 21, 50, 12, 113, 92, 110, 32, 75, 14, 30, 8, 100, 90, 31, 79, 83, 84, 88, 89, 20, 86, 35, 36, 54, 55, 25, 117, 127, 125, 119, 76, 13, 77, 81, 123, 46, 82, 23, 6, 2, 16, 53, 109, 39, 18, 27, 121, 59, 116, 70, 74, 97, 69, 80, 114, 47, 111, 52, 57, 105, 29, 118, 78, 58, 24, 49, 11, 63, 95, 99, 108, 122, 96, 115, 112, 10, 48, 91, 104, 124, 42, 66, 33, 72, 120, 40, 60, 101, 34, 37, 7, 43, 73, 65, 28, 1, 4, 107, 67, 5, 71, 44, 106, 61, 68, 9, 56, 3, 64, 0], [62, 102, 126, 98, 87, 41, 110, 26, 85, 38, 50, 45, 22, 105, 103, 109, 30, 113, 94, 63, 112, 93, 58, 125, 84, 121, 53, 127, 119, 15, 122, 31, 51, 116, 59, 57, 19, 52, 114, 55, 47, 97, 117, 118, 90, 44, 46, 43, 60, 49, 111, 108, 56, 42, 106, 54, 120, 124, 48, 115, 101, 36, 107, 32, 40, 123, 104, 100, 99, 61, 35, 37, 25, 39, 20, 96, 12, 17, 83, 95, 79, 92, 88, 27, 28, 33, 91, 29, 16, 8, 89, 23, 74, 21, 73, 34, 6, 75, 24, 10, 18, 82, 14, 69, 78, 86, 80, 71, 2, 77, 76, 5, 13, 81, 9, 7, 1, 65, 68, 11, 72, 70, 67, 0, 64, 66, 3, 4], [120, 122, 54, 37, 53, 127, 19, 101, 87, 17, 113, 35, 91, 58, 93, 60, 123, 124, 13, 115, 50, 117, 55, 56, 89, 63, 51, 118, 116, 32, 75, 45, 74, 114, 15, 119, 57, 14, 125, 112, 80, 49, 33, 36, 62, 52, 48, 100, 98, 12, 61, 97, 103, 47, 34, 110, 96, 90, 44, 73, 106, 59, 43, 126, 111, 41, 7, 26, 38, 18, 39, 121, 102, 85, 46, 108, 99, 104, 109, 107, 105, 29, 95, 42, 31, 27, 21, 20, 83, 81, 40, 72, 23, 94, 28, 92, 25, 30, 88, 24, 86, 5, 70, 68, 82, 67, 65, 2, 22, 0, 78, 77, 79, 84, 11, 9, 76, 16, 71, 8, 3, 10, 4, 69, 6, 1, 64, 66], [120, 122, 37, 13, 19, 17, 75, 14, 87, 15, 80, 74, 32, 85, 20, 53, 12, 18, 54, 73, 113, 127, 93, 58, 123, 26, 88, 115, 7, 30, 60, 50, 35, 124, 51, 55, 91, 45, 117, 72, 101, 116, 56, 63, 118, 94, 119, 57, 114, 90, 98, 62, 125, 112, 52, 25, 92, 49, 48, 86, 24, 61, 23, 36, 126, 33, 31, 96, 59, 47, 46, 34, 44, 82, 111, 110, 43, 70, 79, 41, 121, 21, 109, 99, 84, 29, 81, 108, 106, 100, 5, 42, 103, 27, 105, 104, 107, 16, 39, 97, 40, 95, 38, 102, 78, 77, 28, 22, 89, 68, 11, 83, 76, 67, 9, 65, 10, 2, 8, 71, 3, 0, 4, 6, 69, 64, 1, 66], [122, 120, 37, 84, 7, 16, 67, 74, 66, 69, 19, 75, 1, 68, 72, 24, 5, 70, 73, 12, 94, 78, 71, 11, 8, 53, 6, 64, 23, 13, 4, 3, 82, 88, 15, 54, 14, 80, 20, 90, 32, 30, 58, 92, 87, 93, 18, 113, 26, 115, 79, 77, 10, 123, 127, 50, 51, 60, 76, 124, 17, 117, 55, 56, 85, 21, 116, 96, 83, 63, 45, 118, 35, 9, 25, 65, 98, 0, 114, 119, 57, 28, 112, 101, 62, 49, 52, 29, 99, 48, 86, 125, 31, 61, 126, 59, 36, 81, 46, 91, 41, 44, 111, 47, 43, 22, 121, 110, 27, 95, 100, 109, 108, 2, 97, 33, 34, 106, 89, 105, 42, 102, 104, 39, 107, 103, 40, 38], [120, 122, 54, 87, 19, 53, 127, 123, 113, 60, 58, 124, 125, 115, 117, 55, 51, 116, 50, 17, 62, 86, 56, 27, 57, 119, 118, 45, 94, 49, 25, 52, 85, 114, 61, 96, 48, 28, 63, 23, 126, 112, 59, 44, 47, 111, 121, 24, 46, 109, 81, 43, 110, 90, 41, 106, 108, 84, 42, 107, 39, 83, 105, 82, 103, 35, 104, 40, 21, 37, 98, 38, 88, 36, 102, 89, 91, 79, 20, 65, 100, 101, 77, 16, 80, 97, 34, 99, 22, 78, 0, 33, 13, 10, 30, 29, 26, 76, 95, 31, 64, 93, 11, 18, 2, 9, 14, 15, 32, 3, 92, 71, 68, 4, 8, 7, 12, 75, 5, 6, 67, 69, 74, 1, 73, 70, 72, 66]], "model.layers.10.self_attn.k_proj": [[122, 121, 100, 118, 86, 63, 114, 123, 54, 61, 124, 119, 62, 52, 125, 115, 57, 108, 111, 117, 120, 60, 112, 31, 53, 126, 51, 127, 116, 59, 47, 58, 110, 55, 102, 46, 48, 83, 0, 45, 109, 49, 113, 56, 98, 29, 105, 104, 77, 9, 40, 44, 76, 103, 39, 38, 91, 106, 80, 81, 24, 6, 107, 35, 101, 34, 41, 37, 43, 21, 42, 79, 97, 26, 23, 1, 30, 22, 28, 36, 99, 32, 89, 88, 84, 3, 69, 50, 10, 95, 96, 92, 78, 19, 93, 33, 4, 20, 90, 82, 11, 85, 87, 7, 27, 25, 94, 14, 2, 18, 72, 8, 74, 5, 16, 71, 13, 66, 15, 67, 73, 65, 68, 70, 12, 75, 64, 17], [113, 124, 39, 50, 53, 96, 93, 21, 123, 120, 49, 15, 62, 126, 125, 63, 51, 112, 11, 87, 24, 82, 122, 58, 55, 114, 48, 117, 13, 91, 104, 116, 46, 61, 73, 30, 54, 115, 57, 71, 52, 95, 84, 26, 81, 118, 76, 121, 59, 127, 47, 80, 60, 111, 108, 19, 110, 86, 8, 99, 109, 56, 40, 45, 119, 100, 92, 89, 3, 10, 44, 35, 14, 1, 2, 98, 43, 68, 107, 79, 6, 106, 41, 27, 88, 0, 105, 42, 18, 38, 5, 102, 101, 36, 37, 33, 28, 23, 97, 77, 16, 17, 32, 94, 69, 34, 78, 66, 70, 7, 25, 31, 22, 72, 64, 4, 74, 29, 9, 83, 65, 90, 85, 20, 67, 12, 75, 103], [113, 56, 38, 80, 19, 77, 85, 23, 89, 11, 91, 9, 95, 58, 70, 72, 68, 55, 64, 14, 76, 1, 66, 20, 46, 123, 78, 111, 127, 44, 59, 106, 116, 112, 47, 17, 57, 53, 121, 69, 114, 117, 115, 71, 125, 51, 101, 109, 86, 118, 126, 42, 79, 124, 97, 52, 74, 48, 108, 45, 18, 96, 120, 24, 28, 82, 15, 37, 50, 33, 98, 39, 60, 122, 29, 119, 31, 93, 92, 32, 43, 3, 62, 103, 90, 61, 99, 34, 67, 5, 63, 12, 40, 49, 26, 88, 110, 35, 7, 30, 94, 100, 54, 104, 36, 81, 41, 105, 22, 8, 107, 10, 75, 25, 27, 87, 6, 84, 21, 0, 13, 83, 2, 73, 102, 16, 65, 4], [58, 62, 86, 37, 52, 96, 121, 29, 90, 14, 82, 17, 19, 12, 80, 60, 127, 77, 10, 72, 118, 125, 117, 24, 119, 55, 51, 20, 115, 63, 114, 126, 57, 120, 123, 79, 7, 100, 124, 50, 56, 44, 53, 48, 116, 98, 34, 59, 113, 46, 111, 110, 43, 45, 61, 25, 54, 93, 76, 112, 99, 109, 106, 70, 47, 9, 40, 49, 36, 15, 41, 122, 108, 105, 107, 11, 69, 104, 38, 42, 39, 102, 2, 23, 8, 97, 3, 85, 91, 21, 27, 35, 4, 95, 103, 31, 84, 83, 101, 6, 28, 33, 18, 74, 94, 30, 81, 65, 0, 92, 13, 22, 1, 16, 71, 87, 88, 68, 64, 32, 78, 5, 26, 89, 75, 73, 67, 66], [118, 127, 37, 126, 86, 63, 62, 10, 52, 34, 17, 48, 113, 30, 26, 119, 56, 121, 122, 15, 54, 70, 77, 112, 60, 43, 29, 50, 104, 59, 88, 51, 20, 120, 117, 116, 41, 110, 96, 115, 61, 123, 0, 3, 109, 57, 58, 55, 125, 47, 45, 53, 114, 108, 111, 49, 124, 106, 76, 103, 107, 46, 44, 82, 42, 71, 100, 2, 83, 38, 9, 65, 40, 68, 1, 72, 99, 32, 105, 92, 85, 39, 75, 35, 33, 95, 97, 36, 11, 91, 102, 14, 23, 31, 4, 80, 94, 67, 7, 64, 5, 73, 78, 69, 19, 25, 21, 27, 101, 93, 98, 81, 24, 90, 28, 89, 16, 12, 66, 79, 84, 87, 22, 8, 18, 74, 13, 6], [106, 32, 52, 26, 23, 20, 82, 125, 120, 42, 118, 45, 80, 78, 109, 126, 73, 58, 124, 77, 62, 112, 34, 12, 123, 114, 115, 50, 56, 55, 59, 54, 7, 5, 63, 85, 116, 47, 81, 122, 3, 51, 37, 119, 61, 39, 43, 103, 30, 104, 6, 1, 110, 127, 117, 101, 0, 107, 108, 11, 27, 48, 86, 65, 91, 46, 66, 13, 44, 57, 75, 74, 49, 35, 70, 111, 40, 60, 2, 22, 76, 64, 105, 98, 53, 79, 121, 102, 113, 33, 94, 29, 72, 71, 92, 36, 41, 83, 68, 8, 19, 25, 88, 14, 24, 100, 21, 4, 69, 38, 31, 97, 89, 10, 99, 95, 15, 28, 93, 96, 84, 90, 9, 17, 18, 87, 16, 67], [126, 62, 38, 34, 26, 94, 45, 85, 19, 41, 87, 22, 17, 110, 51, 84, 15, 50, 113, 12, 103, 75, 8, 14, 1, 127, 53, 93, 57, 112, 119, 117, 55, 116, 120, 39, 124, 125, 111, 122, 109, 59, 114, 118, 69, 63, 74, 52, 121, 102, 6, 29, 60, 115, 66, 28, 106, 5, 95, 73, 24, 108, 49, 32, 48, 58, 77, 123, 97, 46, 43, 33, 68, 105, 104, 54, 56, 16, 36, 47, 27, 35, 31, 40, 86, 61, 100, 44, 91, 64, 107, 4, 71, 67, 25, 78, 18, 42, 80, 9, 101, 99, 72, 23, 7, 37, 90, 3, 98, 82, 89, 92, 96, 70, 88, 0, 30, 21, 76, 13, 81, 20, 10, 83, 79, 11, 2, 65], [122, 120, 101, 24, 30, 82, 54, 127, 32, 79, 28, 84, 117, 26, 21, 113, 115, 124, 123, 55, 16, 27, 53, 56, 78, 60, 116, 50, 11, 77, 62, 114, 118, 9, 76, 51, 89, 63, 119, 57, 112, 52, 49, 125, 48, 93, 45, 72, 58, 126, 47, 59, 46, 44, 2, 70, 23, 61, 108, 81, 109, 10, 111, 110, 99, 121, 12, 65, 43, 5, 41, 106, 107, 29, 22, 35, 42, 39, 71, 104, 8, 105, 3, 103, 40, 14, 96, 97, 100, 4, 102, 38, 95, 98, 75, 83, 36, 80, 18, 69, 34, 68, 73, 33, 86, 6, 37, 31, 20, 92, 15, 7, 64, 87, 0, 91, 13, 90, 94, 25, 67, 88, 74, 17, 19, 85, 1, 66]], "model.layers.10.self_attn.qk_proj": [[122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 124, 106, 42, 54, 50, 90, 45, 110, 102, 55, 63, 117, 37, 125, 51, 116, 123, 53, 49, 60, 114, 87, 115, 85, 61, 38, 93, 57, 46, 23, 101, 59, 80, 83, 47, 32, 21, 22, 41, 112, 96, 34, 119, 103, 19, 16, 26, 25, 44, 13, 109, 20, 48, 104, 14, 77, 82, 86, 39, 43, 88, 27, 18, 84, 24, 100, 12, 11, 94, 98, 75, 81, 78, 30, 89, 108, 15, 29, 17, 111, 9, 91, 79, 73, 76, 31, 40, 107, 36, 35, 95, 97, 72, 6, 74, 105, 92, 10, 70, 99, 28, 71, 8, 68, 33, 64, 69, 65, 2, 1, 7, 3, 67, 5, 4, 0, 66], [122, 113, 62, 120, 118, 56, 121, 127, 126, 58, 52, 124, 106, 42, 54, 50, 117, 90, 116, 45, 37, 55, 110, 102, 60, 87, 114, 63, 123, 125, 85, 49, 51, 115, 23, 61, 38, 53, 101, 41, 93, 46, 103, 21, 80, 83, 96, 22, 34, 112, 104, 32, 19, 109, 59, 20, 26, 111, 27, 48, 43, 13, 119, 57, 25, 47, 98, 24, 84, 16, 86, 77, 88, 18, 14, 89, 39, 100, 30, 81, 108, 79, 12, 82, 31, 94, 78, 11, 75, 17, 29, 44, 9, 40, 91, 73, 95, 76, 15, 70, 92, 72, 97, 33, 74, 36, 105, 10, 35, 99, 6, 107, 69, 4, 8, 0, 68, 28, 65, 71, 64, 7, 5, 3, 67, 2, 66, 1], [122, 113, 62, 120, 118, 127, 121, 56, 126, 58, 52, 106, 124, 42, 54, 50, 116, 102, 55, 110, 37, 125, 51, 45, 90, 38, 60, 53, 114, 49, 63, 123, 87, 101, 117, 23, 93, 57, 115, 34, 96, 32, 119, 26, 85, 27, 43, 41, 103, 83, 22, 104, 19, 109, 21, 48, 112, 24, 25, 46, 61, 111, 16, 80, 59, 30, 94, 20, 84, 47, 86, 100, 88, 13, 77, 95, 18, 89, 39, 31, 98, 91, 29, 81, 14, 82, 35, 108, 78, 75, 12, 79, 92, 44, 11, 99, 15, 17, 40, 9, 70, 36, 73, 33, 76, 72, 107, 105, 28, 74, 97, 71, 4, 10, 1, 6, 7, 68, 64, 69, 8, 5, 65, 0, 66, 67, 3, 2], [122, 113, 62, 120, 118, 121, 127, 126, 56, 58, 52, 106, 124, 42, 54, 55, 117, 50, 90, 102, 45, 125, 60, 51, 114, 123, 37, 110, 116, 87, 63, 49, 53, 112, 23, 93, 115, 46, 119, 101, 38, 85, 96, 19, 57, 27, 47, 41, 61, 43, 109, 34, 22, 16, 104, 25, 83, 32, 59, 21, 80, 24, 44, 100, 26, 77, 13, 48, 111, 39, 88, 103, 30, 81, 18, 86, 94, 82, 84, 89, 98, 78, 20, 108, 17, 91, 12, 14, 40, 75, 95, 92, 11, 31, 35, 15, 105, 79, 9, 29, 107, 72, 70, 36, 73, 33, 74, 76, 28, 99, 97, 10, 71, 4, 6, 8, 0, 69, 7, 65, 68, 1, 5, 64, 67, 66, 3, 2], [122, 113, 62, 118, 120, 121, 126, 56, 127, 58, 52, 124, 106, 42, 55, 110, 54, 50, 90, 123, 45, 125, 102, 51, 37, 60, 116, 53, 49, 114, 117, 63, 87, 38, 115, 47, 61, 93, 46, 57, 119, 109, 96, 112, 23, 85, 59, 26, 101, 34, 103, 83, 21, 111, 80, 19, 16, 48, 22, 41, 77, 32, 13, 43, 100, 104, 18, 25, 39, 82, 14, 89, 81, 86, 24, 30, 27, 44, 94, 108, 20, 84, 88, 78, 11, 98, 36, 91, 17, 29, 9, 79, 75, 15, 40, 35, 73, 31, 12, 76, 95, 72, 70, 74, 105, 107, 97, 33, 4, 92, 10, 68, 8, 7, 28, 6, 71, 99, 65, 64, 0, 1, 67, 69, 2, 5, 66, 3], [122, 113, 62, 118, 120, 127, 121, 56, 126, 58, 52, 124, 106, 42, 54, 55, 90, 45, 110, 125, 116, 114, 60, 50, 117, 87, 63, 102, 123, 37, 53, 51, 46, 85, 93, 61, 49, 38, 96, 119, 23, 19, 115, 80, 109, 22, 103, 21, 41, 57, 34, 83, 77, 112, 101, 47, 16, 13, 111, 48, 20, 26, 32, 59, 25, 18, 104, 14, 100, 82, 98, 27, 88, 12, 15, 86, 11, 78, 39, 108, 24, 17, 81, 30, 75, 79, 89, 84, 43, 9, 31, 76, 105, 94, 73, 35, 29, 44, 91, 40, 107, 36, 10, 72, 92, 74, 70, 95, 33, 6, 97, 8, 28, 99, 68, 4, 69, 5, 71, 67, 64, 0, 1, 66, 7, 2, 65, 3], [122, 113, 62, 120, 118, 56, 127, 126, 121, 58, 52, 106, 124, 42, 90, 45, 125, 102, 55, 116, 37, 110, 54, 114, 87, 60, 50, 53, 85, 101, 51, 63, 23, 115, 61, 38, 46, 123, 49, 119, 93, 117, 22, 112, 80, 59, 19, 83, 16, 103, 21, 20, 48, 32, 96, 77, 34, 109, 41, 47, 57, 26, 111, 13, 100, 18, 25, 86, 82, 24, 27, 81, 78, 104, 84, 44, 88, 11, 75, 15, 14, 39, 12, 30, 98, 89, 94, 43, 9, 105, 108, 17, 31, 91, 76, 79, 29, 73, 40, 35, 36, 95, 107, 92, 74, 72, 99, 10, 6, 33, 70, 97, 8, 71, 28, 68, 7, 69, 66, 4, 5, 1, 65, 0, 3, 64, 2, 67], [113, 122, 62, 120, 118, 126, 56, 127, 121, 58, 106, 52, 124, 42, 90, 54, 45, 50, 37, 125, 117, 87, 102, 55, 23, 49, 46, 110, 60, 85, 123, 116, 83, 53, 61, 38, 119, 93, 114, 112, 21, 101, 22, 51, 115, 19, 80, 16, 77, 47, 63, 109, 111, 96, 86, 26, 32, 20, 25, 57, 82, 103, 41, 59, 13, 18, 84, 24, 78, 34, 27, 81, 98, 48, 39, 30, 100, 44, 17, 108, 15, 14, 79, 88, 75, 11, 104, 12, 9, 76, 89, 91, 105, 73, 107, 31, 43, 29, 40, 94, 92, 95, 36, 35, 6, 97, 72, 74, 99, 33, 10, 71, 70, 8, 68, 7, 28, 69, 65, 5, 0, 4, 64, 3, 1, 2, 66, 67], [122, 113, 62, 118, 120, 127, 56, 121, 126, 58, 52, 106, 124, 42, 90, 54, 45, 50, 37, 110, 123, 102, 55, 125, 87, 117, 116, 23, 60, 49, 38, 101, 53, 21, 93, 85, 83, 115, 114, 46, 47, 112, 63, 109, 57, 51, 59, 32, 16, 26, 22, 61, 19, 34, 96, 41, 111, 20, 25, 98, 119, 80, 77, 39, 27, 104, 24, 84, 86, 13, 82, 100, 78, 30, 108, 88, 43, 44, 18, 81, 48, 94, 91, 40, 17, 15, 29, 103, 11, 31, 75, 79, 105, 92, 89, 6, 14, 76, 95, 36, 9, 12, 73, 107, 97, 33, 35, 74, 99, 10, 28, 8, 72, 71, 68, 70, 1, 64, 4, 65, 0, 7, 69, 66, 2, 5, 67, 3], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 52, 124, 106, 42, 117, 90, 116, 125, 123, 50, 55, 102, 45, 37, 87, 60, 23, 51, 63, 54, 110, 101, 115, 114, 38, 83, 53, 112, 85, 49, 109, 21, 93, 59, 47, 61, 80, 46, 111, 96, 16, 26, 77, 22, 82, 57, 119, 41, 19, 32, 103, 20, 13, 24, 34, 86, 27, 100, 108, 30, 25, 98, 104, 84, 14, 48, 18, 78, 81, 91, 39, 43, 88, 17, 79, 94, 12, 75, 44, 89, 73, 29, 11, 76, 15, 92, 107, 9, 40, 6, 36, 31, 97, 105, 95, 8, 33, 10, 99, 74, 35, 72, 68, 70, 28, 71, 69, 4, 7, 67, 0, 5, 65, 3, 1, 2, 64, 66], [122, 113, 62, 118, 126, 120, 121, 56, 127, 58, 52, 124, 106, 42, 123, 54, 125, 55, 110, 45, 102, 51, 90, 60, 50, 116, 117, 37, 63, 49, 87, 114, 53, 112, 119, 23, 115, 101, 85, 96, 93, 109, 57, 59, 38, 80, 111, 21, 83, 61, 19, 46, 47, 22, 77, 34, 16, 25, 41, 26, 20, 86, 13, 44, 18, 32, 82, 98, 48, 103, 100, 39, 30, 14, 81, 108, 43, 91, 24, 27, 15, 29, 78, 17, 11, 84, 12, 94, 89, 104, 73, 9, 8, 36, 79, 75, 76, 40, 10, 88, 6, 31, 74, 92, 107, 70, 105, 97, 95, 35, 68, 33, 28, 71, 99, 4, 65, 72, 1, 69, 0, 7, 64, 3, 66, 5, 2, 67], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 124, 106, 42, 123, 90, 51, 54, 55, 102, 110, 49, 45, 116, 125, 117, 50, 60, 87, 37, 114, 63, 112, 23, 53, 93, 38, 85, 101, 80, 109, 115, 83, 111, 59, 57, 96, 19, 46, 22, 47, 34, 20, 61, 21, 48, 25, 16, 119, 103, 77, 13, 41, 26, 32, 18, 39, 86, 14, 43, 98, 81, 82, 100, 11, 108, 75, 94, 24, 78, 27, 84, 17, 79, 15, 88, 76, 44, 9, 104, 30, 40, 12, 29, 36, 89, 73, 91, 10, 8, 107, 70, 74, 31, 6, 95, 35, 72, 97, 4, 92, 68, 7, 33, 99, 0, 69, 71, 1, 28, 65, 66, 105, 64, 5, 2, 3, 67], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 106, 52, 124, 42, 54, 50, 90, 45, 102, 37, 110, 117, 55, 51, 116, 87, 23, 125, 85, 123, 38, 60, 57, 53, 101, 59, 63, 19, 46, 21, 119, 115, 93, 80, 26, 83, 32, 114, 22, 96, 47, 49, 109, 77, 86, 16, 103, 41, 20, 111, 112, 39, 98, 48, 61, 34, 25, 27, 24, 18, 13, 81, 82, 100, 91, 78, 17, 84, 15, 14, 89, 30, 88, 12, 43, 94, 76, 79, 11, 75, 104, 44, 29, 95, 9, 40, 108, 31, 36, 107, 8, 73, 92, 35, 33, 70, 97, 105, 74, 99, 72, 10, 71, 4, 28, 6, 7, 69, 68, 0, 65, 5, 3, 1, 66, 67, 64, 2], [122, 113, 62, 118, 126, 120, 127, 56, 121, 58, 52, 106, 124, 42, 54, 117, 125, 55, 90, 45, 50, 123, 87, 102, 110, 60, 116, 37, 63, 46, 114, 49, 51, 23, 85, 119, 57, 93, 38, 115, 83, 77, 19, 101, 112, 47, 53, 61, 21, 59, 109, 16, 96, 80, 22, 111, 86, 13, 20, 25, 34, 81, 26, 18, 41, 82, 27, 48, 103, 78, 14, 98, 39, 32, 76, 24, 100, 30, 17, 43, 75, 12, 15, 84, 11, 9, 73, 44, 79, 8, 104, 70, 91, 108, 29, 94, 40, 88, 31, 89, 72, 74, 92, 97, 35, 107, 36, 6, 95, 10, 7, 64, 71, 69, 33, 105, 4, 0, 65, 68, 99, 5, 66, 1, 2, 28, 67, 3], [122, 113, 62, 120, 118, 127, 126, 56, 58, 121, 52, 124, 106, 42, 54, 110, 55, 102, 125, 123, 50, 117, 45, 37, 90, 60, 51, 116, 87, 49, 46, 114, 61, 38, 23, 57, 63, 101, 112, 96, 109, 85, 115, 93, 59, 19, 41, 32, 83, 21, 26, 111, 22, 119, 47, 80, 53, 77, 34, 103, 20, 48, 25, 27, 98, 16, 86, 24, 100, 43, 39, 82, 13, 18, 44, 78, 81, 17, 94, 84, 15, 79, 89, 30, 107, 14, 40, 108, 29, 70, 75, 91, 104, 73, 9, 76, 11, 12, 88, 36, 31, 8, 72, 95, 35, 74, 105, 92, 97, 10, 33, 6, 28, 71, 99, 64, 1, 4, 69, 68, 0, 7, 65, 5, 67, 2, 66, 3], [122, 113, 62, 118, 120, 127, 126, 56, 121, 58, 52, 106, 124, 42, 55, 54, 45, 110, 90, 102, 125, 123, 117, 60, 37, 114, 51, 50, 87, 63, 46, 109, 61, 116, 85, 93, 101, 115, 49, 103, 53, 23, 38, 112, 47, 96, 57, 22, 83, 80, 59, 19, 111, 21, 26, 20, 32, 86, 77, 16, 41, 82, 34, 98, 13, 25, 100, 119, 39, 27, 24, 44, 78, 81, 18, 43, 48, 14, 30, 84, 89, 104, 75, 17, 15, 11, 76, 79, 29, 73, 12, 40, 108, 91, 94, 70, 9, 31, 88, 105, 74, 33, 10, 72, 36, 92, 107, 35, 95, 97, 99, 8, 71, 6, 68, 28, 4, 7, 0, 1, 66, 69, 64, 67, 5, 3, 2, 65], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 52, 106, 124, 42, 117, 37, 102, 54, 90, 125, 87, 45, 116, 110, 50, 55, 123, 60, 23, 85, 46, 63, 112, 51, 114, 101, 93, 80, 38, 115, 49, 96, 59, 19, 47, 83, 21, 53, 103, 32, 111, 109, 27, 57, 119, 22, 26, 86, 16, 20, 77, 61, 24, 34, 25, 98, 81, 82, 48, 100, 18, 39, 41, 78, 44, 14, 13, 11, 79, 30, 17, 91, 84, 15, 88, 43, 9, 94, 40, 89, 12, 75, 104, 108, 29, 76, 31, 95, 73, 72, 36, 10, 35, 74, 92, 70, 28, 33, 107, 8, 6, 99, 7, 105, 97, 71, 68, 4, 69, 66, 5, 0, 64, 67, 1, 3, 65, 2], [122, 113, 62, 118, 120, 126, 56, 127, 121, 58, 106, 52, 124, 42, 117, 102, 37, 110, 90, 45, 87, 54, 125, 50, 60, 55, 112, 116, 85, 51, 23, 38, 46, 49, 123, 114, 101, 63, 53, 93, 115, 21, 47, 109, 26, 19, 83, 96, 80, 103, 59, 32, 61, 111, 57, 119, 16, 27, 22, 18, 41, 44, 24, 39, 34, 25, 86, 98, 77, 20, 84, 82, 100, 108, 17, 15, 43, 81, 88, 78, 30, 13, 91, 11, 79, 14, 104, 75, 40, 76, 89, 48, 31, 94, 9, 95, 72, 29, 73, 36, 107, 92, 12, 10, 35, 33, 6, 99, 74, 28, 97, 8, 70, 105, 7, 71, 5, 4, 68, 64, 0, 1, 65, 69, 67, 2, 3, 66], [122, 113, 62, 118, 120, 126, 127, 121, 56, 58, 52, 124, 106, 42, 117, 55, 102, 37, 125, 45, 60, 116, 110, 54, 90, 51, 123, 50, 87, 46, 38, 101, 114, 49, 115, 23, 53, 63, 32, 112, 93, 47, 96, 26, 59, 119, 85, 21, 22, 109, 44, 57, 61, 34, 19, 83, 103, 43, 41, 16, 100, 111, 88, 24, 27, 80, 20, 91, 39, 98, 86, 25, 94, 18, 108, 82, 77, 89, 78, 107, 84, 48, 104, 81, 30, 40, 14, 17, 13, 95, 29, 79, 31, 15, 11, 75, 6, 9, 36, 35, 99, 72, 28, 33, 76, 92, 12, 73, 105, 74, 97, 10, 70, 7, 69, 68, 71, 8, 0, 1, 3, 4, 5, 64, 65, 67, 66, 2], [122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 106, 124, 42, 117, 54, 60, 37, 55, 102, 45, 90, 51, 125, 87, 110, 50, 116, 123, 114, 101, 46, 61, 85, 63, 112, 23, 96, 93, 59, 38, 49, 22, 103, 47, 109, 111, 119, 53, 32, 34, 115, 41, 19, 21, 26, 27, 16, 83, 57, 20, 24, 77, 18, 25, 44, 86, 80, 39, 48, 98, 100, 30, 88, 43, 13, 89, 82, 14, 108, 78, 75, 81, 84, 31, 79, 94, 104, 29, 11, 6, 17, 73, 15, 95, 9, 35, 72, 105, 92, 76, 33, 91, 40, 12, 36, 107, 74, 99, 28, 8, 97, 10, 68, 7, 5, 71, 70, 4, 64, 66, 1, 65, 69, 0, 67, 3, 2], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 106, 52, 124, 42, 117, 55, 60, 125, 110, 54, 90, 51, 102, 114, 50, 45, 123, 112, 87, 37, 115, 53, 85, 116, 93, 23, 119, 61, 109, 46, 63, 38, 22, 47, 49, 103, 101, 59, 83, 111, 19, 21, 96, 80, 57, 77, 26, 34, 86, 44, 27, 98, 16, 39, 32, 13, 82, 24, 14, 25, 20, 48, 100, 81, 18, 108, 17, 41, 84, 30, 88, 11, 9, 89, 75, 72, 79, 91, 43, 6, 78, 104, 29, 94, 15, 76, 73, 36, 40, 31, 107, 12, 92, 74, 10, 35, 8, 33, 99, 95, 70, 68, 28, 97, 105, 4, 5, 7, 71, 66, 0, 1, 64, 67, 69, 65, 3, 2], [122, 113, 62, 118, 120, 126, 127, 121, 56, 58, 52, 106, 124, 42, 117, 55, 110, 45, 102, 54, 90, 51, 125, 37, 46, 87, 50, 114, 115, 112, 60, 63, 61, 123, 38, 116, 53, 101, 49, 85, 23, 119, 47, 109, 22, 93, 83, 57, 26, 21, 59, 19, 96, 41, 111, 80, 44, 103, 34, 18, 32, 16, 24, 27, 77, 39, 43, 48, 100, 13, 25, 88, 20, 84, 17, 94, 98, 82, 30, 91, 86, 78, 89, 11, 14, 76, 104, 75, 29, 79, 81, 36, 107, 31, 15, 9, 95, 73, 99, 35, 40, 92, 108, 12, 6, 33, 72, 74, 10, 97, 28, 105, 68, 70, 8, 7, 64, 71, 66, 0, 67, 65, 4, 5, 69, 2, 1, 3], [122, 113, 62, 118, 126, 120, 56, 127, 121, 58, 52, 106, 124, 42, 54, 125, 117, 110, 37, 55, 50, 90, 102, 53, 60, 45, 123, 114, 51, 115, 87, 63, 38, 61, 112, 101, 49, 23, 109, 116, 57, 85, 59, 119, 111, 46, 26, 21, 96, 19, 93, 22, 34, 47, 83, 41, 48, 32, 103, 80, 16, 13, 98, 44, 27, 25, 77, 91, 24, 84, 86, 82, 30, 94, 29, 18, 20, 40, 39, 100, 107, 17, 81, 108, 78, 43, 31, 88, 11, 76, 14, 89, 75, 9, 79, 36, 35, 104, 95, 15, 73, 99, 12, 92, 10, 70, 8, 28, 97, 72, 74, 68, 33, 105, 7, 6, 64, 0, 71, 1, 65, 4, 5, 67, 69, 2, 66, 3], [122, 113, 62, 118, 120, 126, 56, 127, 121, 58, 106, 52, 124, 42, 117, 60, 54, 125, 90, 45, 87, 50, 37, 55, 110, 102, 51, 123, 114, 49, 61, 46, 63, 116, 38, 119, 93, 23, 115, 85, 22, 101, 96, 21, 59, 83, 109, 53, 112, 103, 26, 111, 80, 34, 57, 16, 25, 47, 19, 27, 32, 41, 13, 98, 77, 24, 18, 30, 20, 48, 108, 43, 44, 39, 104, 82, 17, 86, 88, 75, 100, 81, 14, 84, 29, 78, 31, 91, 11, 94, 12, 73, 15, 9, 107, 40, 70, 92, 79, 76, 89, 36, 35, 33, 95, 99, 74, 8, 72, 10, 105, 97, 68, 6, 28, 4, 71, 69, 7, 0, 5, 64, 1, 67, 65, 66, 2, 3], [122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 106, 124, 42, 55, 125, 117, 123, 102, 50, 116, 37, 60, 90, 45, 110, 87, 54, 51, 61, 63, 46, 53, 38, 115, 101, 23, 114, 112, 93, 57, 85, 119, 22, 96, 83, 49, 47, 26, 109, 103, 48, 21, 80, 59, 111, 32, 27, 19, 16, 25, 20, 44, 24, 18, 34, 88, 41, 86, 43, 13, 98, 82, 77, 91, 104, 100, 94, 17, 84, 39, 30, 11, 14, 81, 40, 9, 89, 75, 79, 15, 78, 70, 29, 12, 107, 76, 73, 35, 33, 31, 36, 95, 99, 74, 92, 97, 108, 10, 8, 72, 71, 28, 7, 4, 6, 5, 69, 68, 0, 105, 65, 64, 66, 3, 67, 1, 2], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 106, 52, 124, 42, 117, 90, 50, 125, 102, 45, 60, 54, 55, 123, 87, 37, 23, 49, 63, 110, 46, 112, 116, 51, 47, 114, 38, 85, 109, 93, 101, 83, 119, 96, 115, 103, 59, 86, 22, 57, 16, 41, 32, 26, 111, 53, 21, 27, 20, 19, 61, 80, 34, 98, 13, 48, 82, 77, 100, 24, 25, 44, 14, 30, 75, 17, 18, 84, 11, 79, 9, 81, 43, 88, 12, 78, 39, 91, 94, 40, 89, 73, 31, 108, 15, 104, 70, 92, 8, 76, 36, 29, 10, 33, 107, 95, 35, 97, 74, 72, 99, 105, 6, 71, 7, 28, 68, 4, 1, 64, 5, 69, 2, 0, 65, 67, 66, 3], [122, 113, 62, 120, 118, 126, 127, 121, 56, 58, 52, 106, 124, 42, 125, 50, 54, 123, 90, 55, 60, 45, 110, 102, 37, 117, 114, 112, 87, 116, 53, 115, 38, 49, 23, 51, 101, 109, 57, 63, 93, 119, 19, 96, 85, 32, 47, 21, 59, 46, 41, 61, 22, 34, 83, 80, 26, 27, 94, 86, 111, 44, 103, 25, 24, 16, 43, 20, 91, 77, 39, 84, 13, 108, 18, 88, 40, 100, 30, 98, 31, 17, 82, 48, 104, 14, 29, 89, 81, 11, 95, 107, 15, 12, 36, 79, 73, 75, 78, 35, 9, 8, 76, 92, 99, 70, 10, 33, 28, 105, 97, 74, 7, 4, 6, 72, 71, 68, 65, 1, 0, 69, 5, 64, 67, 2, 66, 3], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 106, 124, 42, 54, 90, 60, 123, 117, 45, 37, 50, 125, 102, 55, 49, 114, 87, 110, 116, 112, 115, 93, 57, 23, 38, 53, 51, 109, 85, 83, 63, 119, 101, 46, 61, 21, 96, 19, 59, 80, 41, 16, 22, 20, 47, 98, 27, 34, 77, 82, 13, 32, 25, 26, 100, 103, 111, 44, 30, 24, 86, 81, 39, 18, 14, 48, 108, 43, 78, 73, 11, 94, 84, 75, 9, 17, 79, 91, 104, 12, 29, 15, 89, 88, 76, 8, 31, 36, 40, 92, 10, 74, 107, 70, 6, 71, 33, 35, 7, 0, 4, 97, 72, 68, 65, 64, 28, 105, 1, 99, 95, 2, 5, 69, 67, 3, 66], [122, 113, 62, 118, 120, 127, 126, 121, 56, 58, 52, 106, 124, 42, 55, 54, 102, 51, 123, 90, 37, 50, 125, 110, 45, 53, 87, 114, 117, 116, 63, 60, 38, 49, 61, 57, 115, 101, 85, 23, 112, 93, 103, 83, 96, 32, 47, 46, 109, 22, 111, 41, 34, 59, 26, 19, 24, 27, 77, 21, 80, 16, 20, 82, 98, 17, 94, 86, 39, 43, 88, 30, 48, 13, 44, 100, 18, 14, 119, 29, 25, 84, 91, 78, 89, 31, 75, 79, 11, 108, 9, 73, 104, 35, 81, 6, 12, 36, 40, 107, 95, 15, 92, 8, 76, 33, 97, 10, 99, 74, 70, 105, 71, 4, 68, 72, 28, 7, 1, 64, 2, 5, 69, 65, 0, 66, 67, 3], [122, 113, 62, 120, 118, 127, 56, 126, 121, 58, 52, 124, 106, 42, 54, 90, 102, 50, 37, 125, 55, 60, 117, 45, 110, 87, 51, 116, 123, 46, 114, 53, 49, 101, 59, 57, 38, 61, 85, 93, 112, 63, 23, 109, 83, 22, 96, 115, 47, 32, 103, 21, 80, 119, 26, 19, 77, 34, 13, 41, 44, 16, 25, 20, 14, 18, 98, 81, 27, 100, 24, 11, 30, 78, 48, 75, 86, 82, 111, 17, 108, 39, 43, 9, 79, 12, 91, 94, 84, 76, 6, 73, 8, 89, 15, 40, 104, 31, 29, 88, 36, 107, 10, 72, 33, 92, 99, 74, 35, 97, 95, 4, 71, 70, 28, 105, 68, 64, 7, 2, 65, 5, 1, 0, 3, 69, 66, 67], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 106, 124, 42, 54, 55, 90, 102, 45, 110, 37, 125, 117, 123, 87, 50, 51, 116, 57, 60, 63, 46, 53, 101, 49, 114, 23, 85, 38, 61, 93, 115, 83, 59, 112, 103, 109, 47, 32, 21, 22, 16, 96, 19, 20, 13, 41, 80, 24, 77, 34, 48, 25, 100, 82, 27, 111, 26, 86, 104, 88, 14, 17, 44, 43, 94, 119, 78, 18, 39, 11, 12, 30, 81, 98, 73, 84, 31, 108, 15, 89, 76, 75, 29, 79, 35, 6, 91, 40, 9, 92, 107, 36, 95, 8, 10, 99, 97, 68, 72, 74, 33, 71, 105, 7, 4, 5, 70, 0, 1, 64, 28, 65, 67, 66, 69, 3, 2], [122, 113, 62, 120, 118, 126, 56, 121, 127, 58, 52, 106, 124, 42, 54, 117, 45, 90, 50, 55, 125, 102, 63, 87, 60, 37, 116, 123, 51, 110, 114, 115, 85, 38, 53, 101, 93, 49, 23, 57, 61, 46, 83, 21, 59, 47, 109, 80, 119, 20, 13, 96, 22, 34, 77, 103, 111, 48, 19, 112, 25, 41, 82, 32, 26, 86, 27, 24, 16, 44, 43, 14, 30, 75, 100, 78, 89, 18, 11, 104, 98, 84, 88, 81, 17, 12, 9, 76, 73, 15, 39, 79, 94, 29, 6, 31, 40, 8, 91, 108, 105, 92, 107, 10, 72, 35, 71, 99, 95, 36, 74, 97, 70, 4, 33, 68, 69, 28, 7, 5, 64, 0, 1, 65, 66, 3, 2, 67]], "model.layers.11.self_attn.q_proj": [[45, 36, 96, 109, 92, 23, 86, 62, 48, 28, 14, 20, 10, 16, 61, 81, 71, 54, 56, 25, 89, 73, 18, 111, 22, 32, 70, 66, 4, 58, 47, 67, 125, 76, 17, 83, 88, 94, 42, 37, 38, 11, 12, 84, 29, 107, 30, 82, 78, 27, 80, 87, 95, 74, 21, 24, 19, 75, 13, 97, 64, 6, 1, 50, 9, 5, 102, 91, 15, 46, 77, 55, 85, 40, 116, 53, 41, 79, 43, 90, 49, 113, 115, 39, 122, 60, 51, 100, 3, 34, 98, 101, 69, 126, 112, 26, 123, 59, 93, 33, 114, 7, 108, 120, 105, 72, 119, 31, 63, 118, 103, 106, 99, 124, 35, 110, 44, 52, 8, 57, 127, 121, 2, 104, 117, 65, 0, 68], [45, 36, 109, 96, 62, 92, 28, 25, 54, 86, 48, 4, 89, 81, 7, 3, 47, 102, 23, 100, 40, 20, 32, 43, 14, 22, 1, 111, 8, 38, 58, 10, 101, 66, 46, 12, 116, 41, 50, 125, 107, 55, 64, 18, 114, 56, 126, 122, 51, 49, 74, 120, 42, 5, 57, 97, 112, 59, 118, 127, 123, 19, 39, 108, 113, 37, 115, 105, 99, 53, 63, 31, 104, 0, 52, 121, 35, 29, 61, 119, 124, 34, 106, 44, 110, 60, 33, 117, 30, 11, 98, 103, 95, 17, 9, 68, 91, 93, 90, 70, 84, 65, 75, 27, 24, 80, 94, 26, 15, 85, 72, 88, 78, 21, 82, 2, 6, 73, 16, 69, 79, 76, 87, 83, 71, 77, 67, 13], [45, 36, 109, 48, 100, 25, 86, 96, 92, 125, 111, 58, 54, 56, 22, 50, 107, 46, 43, 113, 55, 47, 41, 122, 23, 116, 115, 114, 120, 101, 32, 119, 112, 53, 118, 57, 28, 62, 60, 117, 38, 127, 121, 40, 59, 97, 49, 51, 20, 63, 42, 39, 110, 126, 102, 52, 124, 44, 123, 108, 10, 81, 103, 105, 14, 106, 4, 7, 61, 88, 104, 89, 15, 37, 24, 83, 31, 99, 98, 33, 35, 34, 91, 78, 30, 12, 19, 93, 29, 3, 95, 74, 84, 85, 21, 94, 90, 66, 17, 80, 18, 16, 87, 1, 8, 9, 27, 75, 26, 82, 11, 76, 70, 79, 65, 64, 0, 73, 68, 72, 5, 13, 77, 71, 6, 69, 2, 67], [45, 36, 96, 109, 25, 28, 62, 92, 86, 48, 23, 20, 54, 89, 14, 81, 111, 61, 56, 58, 10, 18, 97, 107, 47, 32, 41, 22, 46, 125, 12, 102, 80, 100, 43, 30, 16, 4, 50, 42, 113, 53, 40, 38, 39, 9, 115, 101, 37, 66, 1, 55, 64, 71, 31, 122, 3, 17, 44, 114, 126, 105, 91, 116, 21, 108, 5, 87, 70, 51, 93, 112, 94, 8, 34, 29, 60, 49, 57, 110, 90, 83, 95, 7, 85, 121, 19, 118, 98, 120, 84, 78, 127, 24, 52, 123, 99, 88, 106, 35, 15, 76, 124, 26, 27, 104, 119, 33, 59, 79, 103, 11, 117, 63, 82, 75, 72, 74, 73, 0, 77, 68, 65, 69, 6, 13, 2, 67], [58, 123, 63, 37, 117, 26, 87, 33, 88, 95, 86, 76, 17, 20, 78, 80, 83, 122, 81, 91, 126, 118, 30, 92, 0, 85, 12, 124, 67, 5, 9, 97, 82, 69, 79, 24, 90, 93, 1, 60, 70, 22, 18, 29, 14, 2, 23, 3, 25, 96, 27, 21, 61, 71, 16, 45, 120, 84, 50, 73, 75, 125, 77, 94, 72, 15, 32, 28, 109, 31, 19, 115, 89, 62, 11, 52, 56, 13, 54, 8, 113, 55, 127, 10, 59, 49, 112, 7, 57, 74, 47, 66, 100, 46, 68, 116, 111, 121, 6, 48, 103, 53, 64, 34, 106, 114, 38, 110, 40, 35, 98, 101, 65, 36, 119, 107, 105, 42, 102, 39, 51, 43, 104, 108, 41, 44, 99, 4], [123, 63, 58, 37, 117, 122, 126, 118, 60, 62, 33, 112, 115, 124, 61, 50, 52, 125, 54, 59, 30, 56, 45, 49, 101, 48, 47, 114, 119, 116, 57, 113, 55, 127, 39, 111, 99, 53, 110, 109, 40, 105, 121, 100, 106, 108, 43, 51, 46, 44, 104, 42, 120, 41, 10, 103, 102, 16, 88, 107, 34, 38, 86, 35, 21, 96, 36, 92, 32, 98, 97, 84, 95, 26, 18, 77, 93, 19, 28, 8, 29, 91, 94, 83, 14, 24, 22, 31, 65, 13, 68, 89, 15, 7, 27, 12, 17, 25, 23, 87, 82, 64, 70, 67, 90, 85, 75, 80, 79, 5, 4, 73, 78, 66, 71, 20, 72, 9, 69, 11, 81, 1, 76, 74, 3, 2, 0, 6], [63, 123, 58, 117, 60, 126, 118, 124, 122, 10, 37, 50, 62, 115, 54, 61, 68, 125, 52, 59, 65, 45, 112, 116, 56, 113, 8, 49, 127, 13, 16, 121, 48, 114, 55, 47, 7, 64, 57, 111, 53, 46, 40, 67, 21, 15, 14, 119, 17, 51, 120, 84, 12, 109, 75, 78, 110, 43, 99, 77, 100, 70, 73, 108, 5, 104, 44, 18, 105, 42, 106, 33, 39, 107, 102, 4, 41, 92, 103, 19, 101, 38, 1, 30, 66, 96, 24, 86, 9, 93, 95, 36, 88, 0, 28, 97, 34, 35, 87, 22, 80, 98, 83, 29, 79, 32, 31, 69, 91, 26, 25, 71, 76, 81, 27, 20, 23, 89, 72, 94, 74, 85, 90, 11, 82, 3, 2, 6], [58, 123, 63, 37, 122, 117, 126, 118, 60, 88, 33, 10, 112, 115, 61, 54, 62, 124, 50, 125, 52, 101, 30, 49, 59, 45, 16, 56, 55, 116, 48, 39, 111, 40, 113, 127, 114, 57, 93, 47, 8, 106, 53, 110, 100, 119, 120, 28, 92, 18, 51, 77, 19, 84, 86, 109, 108, 121, 46, 65, 21, 96, 26, 14, 68, 43, 104, 12, 44, 102, 105, 99, 42, 13, 91, 15, 41, 103, 24, 38, 95, 35, 32, 107, 97, 34, 83, 29, 36, 7, 5, 17, 98, 70, 31, 64, 75, 67, 25, 73, 22, 94, 89, 78, 66, 27, 82, 87, 90, 20, 80, 79, 71, 9, 4, 76, 23, 85, 1, 81, 72, 2, 11, 74, 0, 69, 3, 6], [102, 46, 110, 19, 91, 88, 125, 86, 114, 113, 95, 43, 111, 122, 27, 12, 10, 50, 63, 16, 18, 60, 52, 31, 107, 56, 17, 42, 54, 45, 123, 93, 71, 126, 119, 55, 69, 120, 118, 76, 83, 99, 44, 116, 41, 48, 121, 59, 51, 108, 58, 112, 47, 124, 105, 61, 57, 109, 36, 70, 53, 100, 39, 24, 104, 49, 34, 117, 4, 103, 127, 115, 67, 30, 106, 40, 62, 97, 23, 94, 101, 33, 22, 35, 9, 32, 80, 72, 82, 96, 37, 29, 89, 79, 98, 28, 14, 81, 20, 25, 87, 26, 90, 21, 11, 92, 74, 85, 65, 84, 78, 15, 38, 66, 13, 75, 77, 2, 73, 8, 0, 6, 5, 68, 7, 3, 1, 64], [102, 46, 110, 91, 86, 88, 125, 19, 16, 95, 113, 27, 69, 76, 122, 111, 10, 12, 71, 70, 72, 67, 43, 4, 85, 18, 93, 82, 114, 66, 9, 31, 24, 58, 90, 13, 17, 14, 117, 83, 23, 100, 22, 49, 121, 79, 65, 116, 74, 99, 63, 28, 124, 45, 29, 120, 80, 30, 41, 42, 54, 44, 81, 123, 56, 39, 78, 57, 60, 35, 126, 8, 52, 92, 2, 11, 112, 25, 53, 94, 103, 75, 47, 40, 59, 119, 61, 38, 36, 104, 107, 51, 127, 21, 55, 84, 115, 118, 32, 98, 0, 96, 101, 62, 108, 97, 87, 89, 33, 15, 26, 48, 50, 105, 37, 34, 106, 20, 109, 5, 6, 73, 77, 7, 68, 3, 64, 1], [102, 46, 110, 19, 27, 86, 88, 122, 95, 1, 78, 3, 7, 74, 76, 67, 91, 71, 24, 64, 65, 16, 5, 93, 4, 73, 125, 22, 18, 114, 9, 14, 68, 69, 111, 0, 82, 12, 11, 8, 31, 72, 113, 2, 70, 79, 99, 10, 66, 83, 43, 6, 75, 77, 45, 80, 17, 124, 121, 112, 40, 21, 20, 58, 116, 120, 42, 56, 63, 119, 50, 13, 123, 25, 57, 23, 47, 89, 15, 33, 81, 44, 104, 54, 90, 32, 85, 107, 103, 87, 34, 29, 118, 28, 105, 51, 36, 39, 97, 96, 38, 100, 101, 126, 94, 49, 84, 26, 92, 52, 117, 106, 62, 59, 108, 41, 30, 37, 35, 61, 115, 109, 60, 48, 98, 53, 127, 55], [102, 46, 110, 91, 88, 86, 19, 16, 95, 27, 18, 125, 122, 76, 10, 93, 31, 113, 114, 24, 111, 74, 71, 12, 69, 78, 70, 22, 72, 14, 83, 43, 4, 17, 82, 30, 67, 79, 80, 94, 60, 99, 29, 15, 58, 9, 75, 120, 23, 100, 39, 21, 66, 6, 13, 89, 90, 49, 38, 11, 116, 33, 42, 52, 126, 32, 81, 123, 36, 85, 107, 97, 65, 28, 59, 61, 41, 103, 92, 45, 73, 112, 54, 84, 50, 63, 87, 25, 53, 51, 56, 35, 44, 26, 101, 119, 48, 77, 37, 0, 121, 55, 2, 34, 106, 124, 127, 117, 109, 98, 57, 104, 115, 20, 40, 105, 62, 96, 47, 108, 118, 5, 8, 7, 68, 3, 64, 1], [63, 59, 106, 36, 14, 42, 114, 121, 60, 15, 126, 117, 47, 12, 97, 48, 52, 124, 123, 100, 120, 11, 82, 125, 122, 109, 113, 84, 115, 49, 56, 116, 58, 16, 53, 119, 62, 110, 33, 54, 44, 111, 51, 85, 61, 24, 57, 45, 55, 46, 118, 83, 127, 108, 112, 107, 43, 104, 50, 105, 72, 0, 41, 92, 17, 103, 40, 39, 65, 102, 37, 69, 38, 99, 81, 10, 89, 101, 67, 91, 34, 98, 6, 9, 7, 21, 35, 18, 73, 88, 93, 22, 68, 31, 86, 13, 29, 2, 96, 32, 94, 64, 95, 19, 27, 66, 25, 30, 28, 23, 1, 26, 4, 90, 8, 20, 75, 87, 76, 70, 77, 80, 78, 79, 5, 3, 71, 74], [59, 63, 100, 97, 42, 36, 60, 121, 93, 87, 79, 88, 73, 27, 76, 20, 117, 33, 47, 52, 82, 15, 106, 114, 92, 78, 75, 12, 124, 68, 86, 6, 116, 125, 14, 25, 85, 8, 67, 105, 126, 81, 18, 83, 104, 80, 11, 28, 48, 95, 29, 69, 56, 26, 31, 110, 7, 19, 65, 2, 90, 72, 91, 22, 10, 84, 24, 17, 123, 45, 120, 34, 115, 38, 98, 96, 89, 113, 37, 99, 16, 109, 111, 23, 53, 107, 103, 49, 102, 35, 46, 74, 21, 127, 30, 94, 62, 122, 108, 77, 39, 13, 32, 50, 101, 112, 119, 41, 58, 54, 44, 70, 40, 43, 64, 71, 51, 57, 9, 55, 5, 61, 118, 0, 3, 66, 4, 1], [59, 63, 100, 97, 14, 68, 36, 6, 76, 42, 60, 73, 7, 12, 10, 67, 69, 72, 8, 11, 87, 88, 121, 2, 93, 82, 27, 65, 25, 52, 33, 20, 13, 17, 117, 81, 16, 15, 47, 78, 106, 79, 70, 74, 29, 83, 21, 80, 86, 22, 95, 77, 116, 126, 85, 48, 71, 105, 114, 124, 123, 91, 89, 26, 9, 64, 5, 84, 125, 23, 115, 110, 19, 90, 56, 75, 104, 31, 92, 24, 49, 3, 113, 122, 120, 109, 18, 127, 111, 119, 45, 46, 107, 98, 53, 28, 58, 38, 35, 54, 37, 44, 51, 55, 62, 94, 103, 41, 50, 96, 34, 61, 39, 40, 57, 43, 108, 0, 112, 118, 102, 4, 32, 99, 30, 101, 66, 1], [59, 63, 100, 97, 36, 42, 121, 106, 60, 114, 93, 27, 73, 20, 117, 33, 78, 76, 52, 12, 125, 123, 88, 126, 86, 48, 124, 116, 120, 87, 6, 47, 113, 110, 122, 99, 67, 82, 58, 68, 111, 51, 115, 79, 25, 8, 109, 49, 53, 54, 62, 37, 119, 118, 55, 104, 56, 112, 46, 45, 69, 81, 98, 14, 92, 18, 57, 105, 44, 29, 61, 7, 127, 108, 10, 102, 2, 50, 15, 31, 107, 80, 22, 65, 43, 84, 35, 85, 39, 90, 41, 34, 83, 91, 75, 17, 103, 38, 95, 96, 94, 32, 28, 101, 24, 26, 40, 30, 89, 23, 11, 19, 5, 72, 74, 13, 9, 16, 64, 21, 0, 71, 77, 70, 66, 3, 4, 1], [101, 121, 116, 94, 57, 17, 71, 19, 2, 52, 78, 21, 3, 5, 75, 24, 64, 88, 9, 66, 76, 69, 124, 30, 1, 7, 80, 6, 118, 18, 115, 127, 117, 0, 67, 53, 11, 63, 70, 37, 120, 50, 125, 14, 33, 87, 72, 73, 29, 60, 83, 105, 97, 12, 82, 104, 54, 61, 79, 91, 59, 81, 16, 74, 122, 20, 4, 96, 43, 93, 46, 51, 113, 22, 106, 112, 77, 110, 65, 27, 40, 86, 15, 85, 111, 45, 13, 58, 126, 25, 26, 23, 55, 41, 84, 39, 48, 42, 89, 107, 102, 28, 31, 68, 10, 90, 62, 8, 92, 103, 123, 114, 49, 108, 99, 44, 56, 35, 100, 109, 34, 47, 38, 95, 32, 98, 119, 36], [101, 121, 116, 94, 57, 24, 21, 52, 19, 17, 78, 76, 80, 30, 88, 9, 75, 5, 37, 7, 15, 71, 11, 18, 69, 97, 29, 67, 33, 124, 63, 118, 66, 82, 117, 53, 127, 3, 73, 16, 115, 105, 22, 0, 120, 50, 60, 46, 86, 6, 81, 14, 2, 72, 1, 28, 26, 54, 83, 106, 20, 45, 51, 59, 43, 79, 122, 12, 85, 89, 91, 74, 77, 90, 39, 58, 104, 87, 125, 42, 13, 114, 41, 70, 110, 113, 112, 23, 61, 25, 96, 10, 109, 92, 93, 56, 95, 108, 31, 84, 49, 123, 32, 40, 102, 107, 62, 48, 126, 98, 4, 8, 99, 27, 103, 65, 111, 100, 35, 47, 38, 34, 55, 64, 119, 44, 36, 68], [101, 116, 121, 94, 52, 37, 30, 21, 57, 24, 19, 85, 80, 33, 117, 115, 105, 16, 50, 127, 124, 114, 62, 113, 26, 104, 59, 53, 125, 58, 118, 110, 60, 43, 51, 39, 49, 95, 63, 97, 107, 123, 54, 120, 55, 41, 45, 61, 91, 76, 17, 102, 46, 48, 42, 112, 56, 111, 28, 126, 122, 109, 38, 108, 103, 89, 119, 78, 36, 32, 73, 40, 22, 93, 99, 47, 44, 100, 106, 20, 35, 23, 27, 96, 15, 12, 18, 98, 87, 34, 92, 31, 86, 29, 88, 83, 79, 90, 25, 81, 82, 7, 9, 84, 14, 74, 5, 11, 66, 70, 72, 8, 75, 10, 13, 67, 77, 4, 6, 69, 2, 71, 3, 64, 68, 65, 0, 1], [101, 116, 121, 94, 57, 21, 24, 19, 76, 52, 17, 80, 78, 30, 6, 75, 9, 3, 33, 88, 71, 37, 124, 2, 127, 67, 29, 117, 97, 60, 115, 69, 83, 118, 46, 7, 18, 53, 64, 63, 50, 1, 23, 0, 16, 54, 90, 22, 106, 73, 70, 82, 14, 68, 12, 77, 26, 81, 122, 72, 112, 85, 61, 43, 13, 93, 58, 59, 104, 86, 98, 27, 79, 120, 42, 105, 20, 11, 74, 96, 84, 28, 39, 35, 125, 40, 65, 91, 25, 110, 123, 15, 107, 95, 41, 10, 92, 111, 32, 51, 99, 89, 87, 109, 103, 31, 48, 45, 62, 36, 126, 34, 113, 38, 44, 4, 55, 114, 49, 47, 56, 8, 100, 119, 102, 108, 5, 66], [40, 126, 97, 25, 89, 95, 87, 86, 28, 92, 121, 84, 15, 83, 115, 63, 82, 122, 55, 17, 49, 44, 53, 37, 119, 62, 29, 50, 38, 104, 45, 111, 91, 107, 61, 101, 46, 43, 41, 35, 9, 114, 123, 120, 102, 118, 23, 52, 117, 30, 54, 58, 109, 60, 12, 124, 48, 75, 57, 13, 42, 47, 116, 36, 103, 32, 98, 20, 34, 56, 105, 18, 85, 39, 51, 127, 110, 125, 90, 106, 16, 67, 112, 21, 59, 99, 1, 31, 96, 108, 113, 5, 80, 27, 74, 93, 79, 22, 100, 94, 71, 88, 11, 24, 0, 19, 26, 68, 77, 33, 73, 81, 6, 72, 70, 64, 14, 4, 66, 78, 7, 2, 69, 76, 65, 10, 8, 3], [40, 126, 97, 95, 89, 92, 86, 25, 120, 61, 15, 122, 28, 87, 82, 52, 83, 62, 17, 53, 84, 9, 60, 29, 121, 45, 63, 59, 12, 58, 118, 16, 107, 54, 21, 123, 114, 116, 85, 49, 47, 13, 77, 23, 117, 30, 8, 55, 33, 20, 115, 46, 127, 125, 50, 18, 79, 119, 36, 27, 56, 22, 57, 111, 80, 112, 90, 124, 44, 102, 110, 113, 106, 43, 93, 101, 91, 31, 41, 108, 109, 48, 38, 72, 24, 51, 35, 81, 68, 104, 4, 70, 19, 103, 71, 75, 100, 88, 96, 105, 37, 76, 6, 42, 2, 98, 39, 11, 78, 34, 26, 73, 14, 32, 99, 69, 66, 10, 94, 74, 1, 5, 0, 67, 64, 7, 3, 65], [126, 40, 97, 95, 86, 121, 101, 28, 82, 122, 62, 58, 120, 25, 53, 115, 45, 61, 102, 111, 84, 52, 54, 44, 114, 127, 60, 49, 92, 59, 118, 56, 63, 125, 50, 83, 46, 37, 112, 47, 113, 87, 89, 15, 43, 123, 57, 117, 51, 119, 55, 100, 29, 18, 12, 9, 116, 33, 108, 68, 30, 21, 109, 35, 48, 107, 110, 27, 73, 71, 91, 66, 38, 69, 80, 36, 93, 78, 124, 106, 19, 72, 10, 103, 99, 41, 42, 76, 14, 79, 65, 24, 90, 77, 23, 85, 16, 17, 31, 2, 13, 20, 105, 104, 5, 7, 34, 26, 3, 67, 64, 39, 98, 11, 1, 32, 94, 22, 75, 96, 88, 70, 74, 0, 8, 4, 81, 6], [126, 40, 97, 121, 61, 122, 71, 67, 49, 62, 89, 1, 54, 101, 60, 25, 53, 52, 59, 5, 45, 43, 95, 127, 63, 58, 114, 125, 120, 92, 28, 47, 56, 68, 50, 12, 46, 0, 86, 118, 84, 102, 111, 55, 57, 112, 82, 115, 117, 116, 123, 113, 66, 88, 108, 64, 81, 15, 73, 44, 51, 11, 109, 107, 77, 105, 119, 110, 106, 100, 30, 87, 24, 48, 42, 38, 19, 9, 41, 76, 14, 124, 17, 69, 2, 34, 36, 18, 27, 6, 4, 20, 78, 94, 35, 13, 72, 32, 39, 23, 103, 37, 99, 98, 16, 91, 85, 80, 3, 7, 74, 29, 83, 70, 104, 75, 21, 22, 93, 65, 10, 96, 31, 90, 26, 33, 79, 8], [104, 111, 30, 119, 26, 19, 127, 24, 94, 21, 79, 62, 9, 113, 61, 123, 12, 40, 81, 32, 86, 122, 47, 87, 118, 63, 78, 114, 54, 83, 68, 36, 49, 110, 89, 88, 126, 107, 90, 85, 100, 105, 115, 74, 56, 96, 71, 43, 4, 18, 70, 15, 57, 46, 17, 76, 125, 55, 121, 51, 120, 106, 42, 23, 37, 80, 117, 73, 3, 14, 97, 20, 112, 1, 103, 53, 92, 69, 45, 50, 5, 95, 101, 59, 66, 48, 38, 98, 39, 41, 22, 124, 102, 93, 28, 91, 108, 116, 10, 75, 52, 82, 58, 33, 99, 67, 77, 64, 35, 84, 8, 34, 31, 16, 60, 29, 44, 13, 2, 109, 27, 72, 11, 7, 25, 0, 65, 6], [104, 111, 119, 30, 62, 127, 24, 19, 21, 26, 94, 79, 12, 113, 47, 9, 123, 81, 122, 40, 32, 54, 70, 1, 61, 96, 86, 87, 49, 90, 118, 4, 91, 107, 67, 110, 15, 63, 20, 57, 120, 82, 105, 23, 64, 68, 43, 126, 85, 124, 103, 22, 56, 114, 45, 121, 36, 53, 46, 42, 55, 78, 58, 52, 112, 100, 109, 117, 18, 88, 89, 108, 71, 59, 98, 101, 2, 17, 115, 28, 83, 106, 74, 13, 41, 37, 34, 80, 27, 48, 66, 5, 50, 116, 125, 51, 75, 95, 60, 38, 84, 76, 72, 93, 8, 39, 99, 44, 73, 35, 102, 33, 3, 29, 16, 6, 10, 7, 0, 14, 31, 25, 97, 92, 11, 77, 69, 65], [104, 111, 119, 30, 127, 24, 62, 26, 94, 19, 21, 79, 113, 9, 61, 32, 122, 107, 12, 81, 86, 87, 54, 123, 40, 63, 47, 36, 105, 49, 70, 103, 117, 43, 114, 110, 56, 118, 100, 57, 126, 41, 83, 108, 85, 89, 96, 120, 90, 46, 37, 53, 28, 45, 95, 68, 88, 48, 42, 74, 115, 51, 112, 121, 39, 101, 38, 55, 1, 18, 125, 15, 4, 20, 78, 58, 76, 23, 17, 35, 91, 99, 59, 13, 98, 116, 52, 106, 44, 124, 93, 109, 50, 34, 102, 29, 97, 71, 67, 73, 64, 60, 82, 31, 33, 92, 72, 22, 14, 77, 80, 27, 66, 25, 3, 75, 84, 10, 5, 16, 2, 11, 8, 69, 0, 6, 7, 65], [104, 111, 30, 119, 24, 26, 19, 94, 79, 127, 81, 12, 61, 9, 21, 62, 32, 113, 78, 40, 118, 87, 90, 47, 96, 91, 54, 22, 123, 88, 100, 36, 4, 23, 85, 86, 18, 124, 15, 89, 107, 74, 110, 80, 63, 95, 72, 17, 53, 43, 122, 57, 83, 126, 98, 70, 120, 114, 45, 50, 49, 56, 20, 42, 37, 58, 125, 101, 34, 102, 105, 46, 112, 55, 14, 103, 35, 48, 16, 84, 7, 29, 76, 115, 97, 93, 25, 31, 99, 5, 82, 27, 51, 75, 41, 117, 109, 39, 59, 121, 38, 28, 44, 33, 92, 71, 2, 106, 52, 116, 108, 11, 60, 10, 6, 13, 68, 67, 1, 8, 77, 73, 64, 66, 69, 3, 0, 65], [59, 119, 46, 104, 51, 113, 61, 48, 56, 115, 98, 120, 116, 109, 57, 52, 49, 55, 53, 124, 60, 126, 63, 42, 112, 117, 122, 110, 50, 54, 118, 58, 111, 45, 47, 107, 121, 62, 106, 114, 123, 37, 125, 105, 41, 43, 127, 44, 39, 108, 36, 101, 38, 30, 24, 103, 33, 93, 86, 96, 89, 40, 35, 92, 102, 26, 88, 32, 95, 90, 97, 99, 100, 29, 27, 34, 80, 28, 91, 25, 20, 94, 31, 22, 87, 84, 16, 17, 18, 82, 85, 75, 13, 77, 23, 19, 21, 11, 68, 5, 83, 8, 76, 3, 14, 78, 9, 7, 81, 6, 79, 73, 2, 74, 0, 4, 1, 10, 66, 64, 15, 72, 12, 69, 65, 67, 70, 71], [46, 59, 104, 51, 53, 98, 117, 93, 91, 113, 48, 119, 49, 36, 37, 26, 114, 121, 55, 89, 61, 127, 110, 41, 30, 33, 120, 23, 57, 112, 109, 97, 54, 90, 115, 38, 107, 50, 63, 44, 83, 122, 56, 111, 94, 78, 84, 95, 106, 82, 52, 125, 118, 108, 45, 60, 99, 123, 116, 39, 102, 62, 43, 58, 27, 101, 100, 103, 105, 47, 32, 34, 35, 124, 25, 126, 96, 92, 21, 28, 18, 87, 42, 88, 80, 24, 75, 85, 22, 31, 40, 86, 81, 29, 15, 19, 13, 17, 20, 16, 77, 11, 14, 72, 79, 66, 9, 76, 74, 68, 8, 4, 73, 2, 64, 0, 70, 6, 3, 12, 5, 65, 1, 69, 71, 10, 67, 7], [46, 59, 48, 104, 51, 119, 61, 117, 113, 123, 98, 53, 55, 57, 56, 109, 126, 50, 114, 120, 37, 116, 63, 52, 93, 115, 60, 112, 49, 124, 122, 127, 42, 54, 110, 121, 118, 106, 44, 58, 45, 47, 41, 105, 111, 125, 43, 38, 62, 32, 29, 85, 89, 108, 36, 103, 107, 35, 31, 92, 39, 90, 95, 33, 84, 102, 91, 86, 97, 101, 96, 80, 26, 100, 99, 21, 28, 82, 30, 40, 25, 88, 34, 23, 20, 27, 24, 78, 68, 18, 22, 94, 75, 87, 16, 17, 11, 3, 77, 2, 83, 8, 14, 13, 0, 72, 5, 4, 65, 6, 9, 19, 73, 81, 74, 1, 79, 66, 70, 15, 10, 7, 76, 69, 67, 64, 71, 12], [104, 59, 98, 46, 83, 79, 23, 21, 81, 37, 76, 74, 48, 91, 8, 5, 69, 12, 7, 72, 78, 119, 71, 2, 3, 64, 66, 110, 67, 15, 84, 40, 19, 26, 25, 62, 10, 85, 14, 16, 27, 17, 18, 89, 87, 93, 13, 75, 31, 90, 20, 1, 51, 30, 94, 0, 73, 88, 77, 22, 86, 11, 29, 24, 70, 65, 92, 82, 9, 80, 4, 53, 6, 28, 32, 68, 96, 33, 95, 52, 61, 100, 108, 34, 99, 116, 115, 112, 39, 35, 102, 106, 56, 120, 54, 97, 36, 55, 101, 103, 113, 45, 111, 117, 50, 44, 38, 118, 124, 127, 122, 123, 109, 126, 58, 49, 121, 105, 60, 41, 42, 43, 47, 114, 57, 63, 107, 125]], "model.layers.11.self_attn.k_proj": [[109, 45, 32, 100, 54, 48, 92, 25, 56, 20, 23, 81, 47, 14, 62, 58, 107, 22, 16, 61, 112, 76, 85, 18, 69, 0, 68, 65, 120, 125, 86, 30, 55, 10, 73, 31, 122, 42, 59, 12, 7, 50, 71, 123, 72, 111, 118, 51, 2, 57, 43, 46, 126, 116, 38, 19, 3, 53, 113, 63, 40, 37, 13, 114, 102, 41, 124, 60, 108, 95, 119, 115, 93, 104, 127, 117, 44, 110, 39, 121, 49, 80, 52, 94, 11, 33, 101, 106, 97, 75, 99, 90, 91, 87, 96, 98, 17, 35, 105, 15, 28, 74, 77, 6, 103, 34, 29, 67, 27, 26, 24, 84, 5, 66, 21, 83, 88, 79, 36, 9, 82, 8, 64, 89, 78, 1, 70, 4], [58, 123, 63, 101, 126, 118, 117, 31, 61, 60, 50, 124, 86, 125, 52, 91, 62, 115, 20, 17, 121, 97, 55, 47, 56, 113, 49, 120, 111, 127, 112, 122, 59, 57, 48, 54, 53, 116, 45, 26, 119, 114, 78, 46, 76, 109, 51, 110, 108, 88, 73, 43, 75, 107, 44, 106, 32, 40, 103, 66, 42, 69, 37, 104, 41, 80, 105, 82, 39, 102, 99, 79, 36, 38, 89, 100, 70, 98, 93, 87, 96, 30, 34, 35, 33, 92, 19, 72, 27, 94, 25, 21, 74, 83, 29, 67, 13, 28, 64, 65, 5, 81, 95, 6, 90, 23, 71, 24, 11, 7, 16, 4, 9, 18, 85, 12, 8, 15, 10, 22, 84, 77, 1, 68, 0, 3, 14, 2], [110, 38, 46, 86, 91, 78, 88, 125, 18, 122, 19, 16, 31, 76, 111, 7, 114, 73, 3, 74, 1, 43, 5, 113, 64, 45, 121, 58, 49, 47, 120, 93, 56, 124, 6, 9, 42, 68, 17, 40, 15, 116, 119, 57, 63, 72, 62, 50, 13, 44, 123, 112, 99, 54, 103, 67, 35, 27, 118, 61, 59, 75, 11, 105, 41, 36, 2, 126, 108, 101, 107, 96, 90, 30, 70, 51, 55, 20, 106, 115, 82, 104, 8, 127, 21, 48, 60, 109, 100, 52, 117, 81, 87, 29, 34, 26, 94, 97, 37, 89, 79, 98, 53, 32, 66, 33, 24, 85, 39, 92, 95, 10, 28, 22, 80, 65, 25, 23, 84, 69, 14, 12, 71, 83, 0, 77, 4, 102], [59, 63, 36, 33, 121, 87, 117, 124, 93, 19, 106, 25, 60, 126, 95, 114, 81, 27, 26, 56, 125, 20, 116, 113, 47, 48, 42, 21, 53, 119, 52, 110, 86, 49, 80, 54, 123, 46, 16, 79, 58, 77, 111, 10, 98, 104, 100, 115, 122, 18, 45, 120, 55, 88, 78, 39, 51, 50, 7, 101, 40, 13, 118, 76, 41, 112, 108, 127, 105, 61, 35, 62, 109, 22, 102, 44, 34, 9, 43, 57, 38, 4, 107, 103, 69, 72, 28, 99, 37, 70, 14, 32, 74, 85, 92, 75, 3, 11, 0, 94, 96, 91, 30, 31, 73, 15, 97, 66, 17, 67, 89, 82, 71, 90, 24, 83, 6, 29, 65, 1, 23, 68, 12, 84, 2, 8, 64, 5], [121, 116, 37, 21, 24, 80, 78, 9, 75, 19, 17, 30, 57, 76, 64, 71, 69, 6, 2, 3, 97, 46, 53, 94, 124, 66, 127, 118, 65, 60, 74, 1, 59, 120, 115, 5, 63, 70, 43, 4, 122, 125, 79, 41, 93, 82, 73, 40, 50, 13, 91, 117, 42, 8, 104, 113, 88, 61, 7, 101, 0, 90, 67, 89, 33, 112, 15, 58, 92, 12, 28, 110, 107, 45, 87, 114, 34, 27, 86, 32, 108, 111, 22, 96, 99, 26, 109, 126, 55, 54, 23, 39, 31, 68, 51, 11, 10, 48, 105, 49, 84, 47, 123, 18, 72, 62, 106, 35, 29, 38, 25, 100, 20, 95, 102, 119, 44, 56, 98, 103, 14, 77, 36, 52, 16, 83, 85, 81], [126, 104, 33, 86, 28, 122, 121, 31, 62, 54, 118, 25, 60, 58, 17, 61, 53, 115, 87, 52, 127, 120, 50, 15, 63, 56, 114, 47, 49, 125, 108, 84, 113, 112, 3, 59, 100, 57, 64, 45, 46, 65, 37, 9, 82, 111, 42, 12, 109, 110, 117, 43, 75, 92, 123, 55, 116, 7, 70, 69, 107, 48, 51, 99, 106, 13, 29, 41, 83, 16, 124, 85, 105, 44, 0, 38, 102, 74, 119, 1, 39, 91, 94, 36, 8, 103, 66, 68, 19, 78, 89, 2, 4, 26, 34, 98, 101, 88, 32, 27, 96, 93, 11, 35, 81, 71, 95, 21, 90, 10, 24, 30, 6, 80, 14, 18, 22, 97, 72, 77, 23, 67, 79, 76, 20, 73, 5, 40], [40, 111, 47, 119, 94, 24, 127, 113, 26, 21, 62, 19, 110, 81, 122, 78, 114, 49, 123, 79, 118, 56, 63, 32, 12, 55, 120, 121, 105, 57, 61, 58, 126, 115, 43, 108, 54, 59, 42, 107, 87, 106, 9, 46, 18, 45, 125, 124, 104, 112, 48, 69, 116, 117, 51, 53, 37, 7, 39, 11, 41, 100, 60, 52, 109, 50, 38, 16, 36, 80, 44, 86, 73, 96, 28, 103, 99, 95, 102, 3, 101, 33, 91, 10, 30, 93, 89, 0, 97, 65, 71, 4, 35, 90, 13, 75, 76, 98, 23, 20, 27, 34, 82, 92, 31, 29, 15, 68, 22, 88, 25, 8, 84, 17, 74, 2, 14, 77, 66, 83, 85, 6, 72, 64, 70, 5, 1, 67], [40, 59, 46, 34, 110, 74, 76, 79, 7, 83, 81, 6, 21, 8, 5, 78, 2, 23, 93, 3, 0, 48, 1, 119, 9, 62, 27, 91, 117, 52, 115, 116, 77, 30, 45, 4, 89, 57, 49, 90, 64, 84, 56, 113, 126, 58, 122, 55, 120, 51, 47, 123, 118, 65, 108, 54, 109, 60, 42, 121, 124, 127, 112, 53, 61, 92, 63, 43, 125, 50, 44, 68, 37, 101, 38, 80, 111, 107, 15, 97, 72, 70, 88, 75, 114, 82, 87, 73, 105, 31, 41, 36, 102, 22, 18, 33, 95, 66, 86, 28, 103, 99, 106, 12, 39, 96, 104, 85, 11, 13, 94, 35, 67, 98, 32, 25, 29, 100, 24, 71, 20, 17, 19, 16, 26, 14, 10, 69]], "model.layers.11.self_attn.qk_proj": [[59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 104, 60, 57, 40, 94, 56, 37, 125, 122, 62, 61, 101, 113, 114, 36, 88, 48, 24, 118, 127, 119, 27, 47, 22, 124, 83, 21, 49, 30, 19, 78, 53, 115, 76, 17, 14, 55, 120, 100, 51, 89, 81, 12, 85, 33, 52, 102, 95, 80, 50, 86, 32, 43, 16, 28, 97, 108, 87, 91, 9, 90, 23, 112, 42, 7, 92, 71, 38, 29, 31, 44, 93, 25, 41, 73, 107, 15, 10, 82, 96, 75, 69, 74, 34, 20, 79, 3, 11, 106, 5, 18, 67, 2, 84, 105, 0, 26, 6, 66, 98, 64, 99, 39, 70, 68, 72, 103, 13, 77, 4, 1, 65, 35, 8], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 54, 117, 60, 104, 57, 94, 40, 56, 113, 122, 101, 127, 37, 36, 119, 114, 88, 118, 125, 24, 61, 62, 22, 49, 27, 53, 30, 48, 76, 52, 21, 83, 115, 124, 47, 55, 78, 120, 100, 19, 81, 51, 95, 43, 14, 12, 42, 86, 108, 17, 85, 50, 33, 97, 80, 91, 90, 16, 89, 28, 9, 23, 112, 102, 92, 44, 38, 25, 107, 7, 31, 20, 79, 32, 87, 93, 71, 34, 73, 29, 105, 15, 96, 10, 106, 18, 69, 41, 74, 6, 82, 26, 5, 75, 84, 11, 0, 98, 2, 64, 66, 99, 67, 72, 39, 103, 3, 1, 70, 65, 68, 77, 4, 35, 8, 13], [59, 126, 121, 46, 63, 111, 110, 116, 58, 45, 123, 109, 54, 117, 104, 40, 94, 122, 61, 37, 101, 36, 56, 125, 57, 60, 113, 114, 24, 88, 62, 119, 118, 48, 127, 124, 27, 53, 30, 47, 21, 100, 52, 51, 22, 83, 19, 55, 115, 50, 49, 33, 102, 95, 43, 42, 76, 28, 78, 120, 108, 17, 86, 91, 89, 85, 38, 112, 106, 81, 32, 12, 23, 97, 14, 92, 93, 31, 90, 80, 87, 44, 107, 16, 29, 96, 25, 9, 73, 105, 34, 41, 71, 20, 7, 79, 84, 98, 10, 99, 26, 69, 11, 82, 18, 74, 15, 75, 6, 5, 66, 0, 103, 64, 2, 3, 67, 39, 70, 1, 65, 68, 35, 72, 4, 8, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 60, 104, 40, 57, 94, 36, 37, 122, 61, 56, 62, 114, 127, 101, 113, 24, 49, 119, 88, 118, 124, 48, 30, 27, 52, 47, 115, 125, 120, 95, 19, 22, 55, 21, 53, 50, 100, 83, 42, 28, 89, 14, 76, 108, 78, 102, 43, 81, 90, 17, 51, 12, 97, 92, 33, 85, 91, 32, 16, 31, 86, 80, 44, 38, 93, 87, 105, 107, 23, 64, 25, 73, 71, 96, 75, 29, 106, 112, 26, 9, 41, 7, 69, 2, 67, 74, 20, 79, 34, 0, 66, 11, 18, 98, 99, 82, 10, 3, 39, 5, 84, 6, 15, 103, 70, 68, 72, 35, 65, 4, 1, 8, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 54, 104, 40, 122, 61, 57, 60, 56, 94, 127, 36, 37, 118, 62, 113, 101, 119, 24, 125, 114, 88, 48, 49, 52, 124, 83, 21, 30, 120, 19, 76, 22, 115, 47, 55, 27, 78, 53, 12, 95, 85, 80, 28, 100, 42, 43, 50, 17, 51, 16, 102, 81, 89, 108, 33, 14, 112, 23, 97, 91, 92, 90, 32, 107, 25, 86, 7, 31, 71, 105, 93, 29, 44, 73, 9, 87, 41, 96, 0, 38, 15, 75, 106, 64, 82, 79, 20, 74, 66, 2, 10, 98, 67, 18, 84, 69, 26, 99, 34, 70, 11, 65, 3, 5, 103, 6, 39, 8, 1, 72, 68, 13, 77, 4, 35], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 104, 60, 40, 57, 127, 122, 56, 113, 94, 114, 61, 36, 118, 37, 48, 101, 62, 88, 24, 119, 49, 22, 125, 83, 30, 52, 21, 27, 78, 19, 124, 12, 120, 76, 53, 81, 85, 95, 47, 14, 97, 17, 80, 115, 55, 43, 42, 16, 89, 28, 50, 51, 108, 100, 92, 102, 33, 86, 91, 25, 90, 107, 32, 73, 23, 9, 31, 96, 87, 79, 71, 7, 105, 44, 29, 20, 15, 93, 74, 82, 10, 112, 75, 69, 38, 99, 41, 26, 67, 18, 34, 11, 106, 70, 103, 84, 66, 5, 98, 3, 0, 64, 68, 6, 8, 2, 65, 39, 4, 1, 35, 77, 13, 72], [59, 126, 46, 63, 121, 116, 111, 110, 58, 123, 45, 109, 117, 54, 60, 104, 94, 40, 57, 36, 37, 113, 101, 24, 114, 127, 122, 88, 48, 21, 61, 118, 119, 56, 83, 19, 125, 27, 22, 62, 78, 30, 53, 49, 14, 17, 52, 95, 85, 76, 81, 12, 120, 124, 89, 86, 16, 100, 80, 55, 50, 28, 32, 115, 42, 43, 47, 112, 33, 92, 91, 87, 9, 102, 23, 97, 107, 51, 108, 38, 90, 71, 20, 96, 93, 31, 7, 11, 82, 75, 73, 25, 44, 29, 5, 41, 74, 18, 15, 34, 99, 10, 79, 105, 26, 67, 106, 98, 84, 69, 3, 39, 70, 64, 103, 2, 66, 8, 1, 6, 13, 0, 68, 4, 35, 72, 65, 77], [59, 126, 46, 121, 63, 116, 111, 110, 58, 45, 123, 109, 104, 40, 54, 94, 117, 60, 113, 24, 101, 37, 122, 36, 27, 88, 83, 57, 114, 127, 61, 22, 21, 47, 30, 52, 19, 118, 76, 56, 119, 125, 12, 62, 53, 48, 124, 78, 86, 81, 85, 49, 95, 17, 100, 14, 89, 115, 80, 91, 33, 55, 16, 28, 120, 50, 102, 92, 23, 42, 87, 43, 51, 7, 90, 32, 44, 9, 112, 108, 20, 38, 25, 82, 11, 97, 93, 75, 31, 29, 71, 15, 79, 73, 107, 106, 18, 10, 26, 5, 84, 64, 34, 41, 69, 74, 2, 70, 96, 0, 105, 3, 98, 8, 66, 99, 103, 67, 39, 1, 65, 4, 72, 6, 13, 68, 77, 35], [59, 126, 121, 46, 63, 111, 116, 110, 58, 45, 123, 109, 117, 104, 40, 94, 54, 60, 56, 36, 57, 122, 88, 101, 62, 37, 24, 114, 61, 119, 27, 49, 125, 118, 83, 113, 22, 19, 30, 21, 52, 124, 48, 100, 47, 127, 78, 55, 51, 89, 120, 85, 115, 17, 14, 76, 81, 12, 43, 86, 53, 80, 33, 95, 102, 38, 91, 28, 50, 42, 90, 108, 16, 23, 32, 107, 112, 92, 106, 93, 97, 44, 87, 31, 73, 9, 34, 7, 71, 15, 25, 18, 29, 41, 20, 10, 96, 11, 79, 105, 5, 82, 69, 98, 75, 26, 103, 84, 2, 99, 64, 0, 74, 66, 70, 67, 65, 3, 8, 6, 1, 4, 39, 72, 68, 77, 13, 35], [59, 126, 46, 121, 63, 111, 116, 110, 58, 123, 45, 109, 104, 54, 56, 60, 117, 40, 122, 57, 113, 94, 101, 37, 61, 62, 88, 36, 24, 114, 125, 127, 27, 119, 48, 83, 118, 19, 49, 47, 115, 21, 52, 22, 30, 120, 124, 76, 78, 85, 100, 51, 43, 86, 33, 12, 17, 14, 89, 80, 53, 55, 42, 81, 95, 108, 91, 92, 32, 16, 28, 97, 23, 50, 73, 31, 87, 38, 9, 90, 7, 102, 93, 15, 25, 105, 71, 112, 96, 79, 44, 107, 41, 69, 10, 18, 29, 67, 75, 84, 74, 11, 34, 3, 82, 20, 6, 106, 103, 26, 66, 64, 98, 0, 2, 5, 70, 8, 99, 1, 13, 68, 4, 39, 77, 72, 65, 35], [59, 126, 63, 121, 46, 111, 110, 116, 58, 123, 45, 109, 104, 117, 122, 54, 40, 113, 56, 61, 60, 125, 94, 62, 57, 37, 48, 114, 101, 24, 127, 88, 118, 36, 124, 119, 49, 21, 115, 47, 19, 52, 78, 83, 120, 30, 33, 22, 27, 76, 95, 12, 100, 50, 80, 28, 17, 85, 53, 14, 102, 43, 42, 16, 51, 81, 97, 86, 89, 73, 55, 91, 23, 25, 92, 108, 41, 32, 90, 7, 106, 38, 74, 9, 71, 29, 107, 66, 96, 79, 15, 10, 18, 6, 31, 87, 11, 34, 69, 93, 82, 20, 2, 0, 5, 75, 105, 64, 98, 26, 112, 84, 70, 67, 8, 3, 44, 13, 99, 4, 68, 1, 65, 72, 103, 77, 39, 35], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 109, 45, 117, 104, 54, 40, 122, 60, 118, 56, 61, 113, 88, 57, 62, 114, 94, 48, 36, 101, 127, 37, 124, 24, 49, 19, 125, 21, 78, 83, 119, 30, 12, 22, 52, 115, 27, 76, 80, 17, 14, 47, 81, 16, 95, 120, 85, 86, 28, 100, 91, 33, 89, 50, 43, 102, 55, 108, 97, 51, 71, 53, 73, 32, 23, 90, 42, 7, 92, 10, 9, 11, 15, 38, 25, 41, 107, 82, 112, 105, 20, 87, 31, 74, 29, 18, 79, 96, 106, 69, 75, 5, 6, 0, 93, 84, 34, 3, 99, 67, 64, 1, 44, 98, 2, 26, 66, 65, 8, 72, 4, 70, 103, 13, 77, 39, 68, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 54, 117, 104, 40, 60, 122, 94, 24, 57, 88, 113, 48, 56, 49, 36, 101, 37, 114, 62, 61, 27, 124, 119, 125, 127, 19, 118, 83, 22, 78, 21, 12, 30, 55, 17, 115, 14, 80, 47, 50, 81, 76, 85, 52, 86, 120, 100, 108, 89, 95, 28, 53, 43, 91, 16, 33, 51, 42, 92, 97, 23, 25, 31, 38, 79, 32, 90, 44, 105, 15, 73, 112, 82, 9, 102, 29, 71, 87, 93, 74, 41, 7, 10, 96, 84, 69, 18, 34, 11, 107, 20, 75, 106, 26, 103, 3, 6, 99, 98, 67, 39, 5, 72, 0, 8, 2, 66, 65, 1, 70, 64, 13, 77, 35, 68, 4], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 54, 60, 40, 117, 114, 94, 122, 57, 48, 127, 37, 36, 61, 113, 49, 47, 56, 62, 101, 50, 21, 24, 88, 12, 27, 119, 125, 124, 22, 118, 78, 19, 14, 76, 83, 30, 52, 80, 97, 53, 95, 17, 28, 81, 55, 16, 120, 86, 115, 89, 100, 51, 73, 85, 9, 108, 112, 92, 43, 7, 25, 90, 71, 33, 91, 102, 10, 15, 87, 44, 79, 23, 107, 32, 31, 69, 29, 82, 42, 38, 18, 20, 64, 96, 6, 74, 11, 0, 106, 34, 93, 84, 75, 26, 41, 5, 66, 99, 67, 105, 2, 70, 1, 3, 98, 72, 39, 103, 65, 68, 4, 35, 13, 8, 77], [59, 126, 46, 121, 63, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 114, 113, 94, 122, 61, 56, 57, 36, 49, 37, 88, 24, 101, 48, 125, 127, 118, 62, 119, 27, 19, 21, 124, 30, 78, 22, 12, 83, 17, 47, 55, 50, 52, 14, 16, 115, 76, 89, 80, 100, 51, 95, 85, 86, 53, 81, 97, 91, 28, 120, 108, 102, 33, 92, 43, 23, 107, 31, 9, 38, 42, 25, 32, 7, 90, 44, 41, 71, 112, 73, 93, 29, 15, 11, 87, 105, 74, 82, 10, 106, 18, 20, 79, 84, 34, 96, 69, 6, 98, 5, 99, 75, 0, 3, 26, 72, 70, 66, 39, 67, 64, 103, 2, 35, 65, 68, 4, 8, 1, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 54, 60, 104, 40, 127, 56, 57, 61, 122, 94, 48, 37, 36, 114, 113, 118, 88, 124, 101, 49, 24, 62, 83, 30, 27, 19, 78, 52, 115, 119, 21, 12, 47, 125, 22, 81, 55, 100, 120, 17, 16, 95, 97, 50, 85, 80, 89, 14, 86, 43, 53, 76, 51, 102, 108, 91, 33, 107, 112, 90, 42, 28, 41, 32, 73, 23, 7, 9, 71, 31, 25, 10, 74, 29, 92, 96, 105, 34, 15, 44, 11, 106, 87, 18, 75, 20, 82, 98, 79, 69, 99, 93, 38, 67, 0, 84, 64, 3, 26, 5, 6, 70, 103, 66, 72, 2, 68, 35, 4, 65, 1, 8, 13, 39, 77], [59, 126, 46, 63, 121, 116, 111, 110, 58, 45, 123, 109, 104, 117, 60, 40, 54, 122, 49, 56, 24, 101, 57, 114, 94, 37, 88, 61, 27, 125, 48, 36, 127, 83, 30, 19, 113, 22, 62, 47, 85, 78, 21, 119, 118, 120, 12, 55, 52, 81, 16, 100, 14, 124, 86, 43, 17, 95, 76, 80, 89, 115, 97, 50, 33, 42, 9, 108, 51, 91, 28, 53, 112, 90, 31, 32, 23, 92, 73, 25, 41, 71, 44, 15, 107, 87, 74, 102, 106, 82, 20, 79, 38, 75, 7, 29, 93, 105, 96, 10, 34, 98, 18, 11, 84, 26, 72, 69, 99, 70, 5, 0, 3, 103, 64, 35, 67, 2, 4, 68, 39, 65, 66, 1, 77, 13, 6, 8], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 104, 117, 60, 54, 40, 94, 56, 57, 127, 114, 101, 24, 122, 113, 36, 37, 118, 88, 119, 49, 27, 19, 48, 62, 125, 83, 21, 22, 61, 78, 12, 30, 55, 124, 47, 52, 86, 14, 95, 50, 120, 100, 115, 17, 81, 80, 85, 16, 97, 42, 76, 89, 33, 28, 91, 43, 44, 23, 53, 112, 31, 102, 51, 108, 9, 90, 32, 38, 25, 92, 20, 41, 107, 73, 82, 71, 106, 87, 18, 93, 105, 7, 79, 15, 84, 29, 74, 11, 96, 34, 26, 10, 98, 64, 69, 5, 67, 75, 103, 70, 2, 0, 66, 72, 99, 3, 35, 77, 1, 39, 6, 8, 65, 4, 68, 13], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 49, 104, 56, 36, 122, 57, 94, 118, 61, 54, 119, 114, 40, 60, 101, 127, 88, 125, 113, 37, 48, 24, 83, 27, 52, 62, 124, 30, 120, 19, 85, 115, 43, 55, 47, 50, 22, 17, 21, 14, 102, 51, 53, 81, 95, 108, 12, 100, 78, 33, 76, 38, 80, 112, 16, 86, 42, 97, 89, 32, 41, 91, 28, 107, 31, 105, 93, 25, 92, 106, 23, 90, 87, 82, 29, 73, 7, 96, 71, 99, 18, 44, 69, 20, 34, 0, 84, 9, 79, 11, 26, 10, 103, 70, 67, 74, 66, 98, 5, 64, 3, 35, 15, 75, 2, 39, 1, 68, 6, 8, 65, 77, 72, 4, 13], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 114, 56, 94, 57, 49, 127, 101, 37, 36, 122, 24, 118, 113, 61, 88, 27, 22, 48, 125, 119, 124, 115, 30, 19, 83, 62, 52, 21, 14, 95, 47, 120, 81, 100, 53, 89, 12, 51, 17, 97, 50, 78, 55, 108, 33, 76, 86, 28, 85, 102, 16, 43, 91, 112, 80, 90, 25, 42, 31, 73, 92, 23, 93, 107, 106, 105, 41, 38, 44, 32, 9, 87, 96, 29, 20, 7, 82, 18, 34, 79, 10, 11, 5, 71, 98, 69, 15, 74, 84, 26, 70, 99, 103, 75, 64, 2, 0, 66, 72, 67, 6, 35, 3, 39, 8, 77, 65, 68, 4, 1, 13], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 60, 54, 40, 56, 113, 114, 48, 94, 122, 118, 57, 49, 125, 36, 127, 124, 24, 88, 37, 61, 101, 62, 19, 52, 22, 27, 83, 119, 47, 21, 30, 78, 55, 12, 95, 81, 115, 76, 16, 14, 120, 100, 17, 89, 86, 108, 80, 97, 85, 50, 33, 51, 28, 42, 91, 53, 102, 9, 112, 90, 43, 23, 105, 41, 29, 32, 20, 7, 107, 25, 73, 92, 31, 106, 71, 38, 93, 74, 87, 18, 79, 10, 5, 44, 96, 82, 69, 75, 11, 98, 70, 64, 3, 34, 84, 66, 15, 26, 6, 67, 0, 2, 68, 99, 103, 39, 72, 77, 8, 4, 13, 65, 1, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 113, 94, 60, 40, 54, 114, 56, 118, 48, 125, 122, 57, 88, 49, 101, 37, 36, 24, 119, 61, 52, 124, 127, 62, 21, 27, 83, 30, 19, 22, 100, 33, 47, 43, 115, 55, 78, 50, 81, 17, 12, 95, 108, 85, 80, 120, 89, 53, 14, 102, 16, 97, 86, 91, 76, 28, 106, 90, 32, 51, 42, 112, 38, 31, 92, 107, 87, 23, 105, 9, 93, 7, 71, 96, 25, 20, 74, 11, 82, 29, 18, 73, 79, 34, 10, 41, 15, 99, 75, 26, 67, 6, 103, 0, 2, 84, 98, 69, 64, 44, 3, 5, 66, 35, 8, 72, 39, 70, 1, 77, 65, 68, 13, 4], [59, 126, 46, 63, 121, 116, 110, 111, 58, 45, 123, 109, 104, 40, 54, 117, 60, 56, 94, 113, 125, 122, 61, 37, 118, 57, 48, 24, 101, 119, 36, 88, 62, 49, 114, 22, 30, 127, 124, 83, 19, 100, 52, 27, 85, 78, 21, 33, 47, 115, 12, 43, 17, 91, 95, 120, 108, 50, 16, 97, 14, 76, 80, 28, 89, 53, 55, 81, 102, 86, 51, 32, 42, 106, 112, 38, 92, 23, 31, 105, 87, 107, 9, 96, 93, 90, 41, 73, 25, 71, 7, 79, 20, 34, 26, 74, 18, 29, 44, 10, 82, 99, 84, 15, 11, 75, 6, 5, 64, 0, 103, 69, 98, 67, 2, 1, 39, 66, 8, 72, 4, 3, 70, 13, 65, 77, 35, 68], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 104, 117, 56, 54, 60, 40, 113, 94, 36, 49, 57, 48, 122, 118, 125, 24, 114, 62, 37, 127, 101, 119, 88, 61, 22, 83, 124, 76, 47, 30, 27, 120, 52, 19, 97, 21, 95, 78, 100, 33, 85, 115, 80, 28, 14, 81, 55, 91, 16, 17, 50, 53, 86, 43, 12, 102, 108, 89, 73, 42, 23, 51, 107, 92, 31, 25, 90, 7, 32, 9, 112, 44, 93, 105, 41, 71, 79, 87, 38, 10, 29, 96, 82, 20, 74, 18, 26, 5, 6, 34, 84, 99, 15, 69, 67, 75, 103, 106, 11, 3, 66, 0, 98, 8, 2, 64, 70, 39, 68, 4, 65, 72, 1, 13, 35, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 54, 117, 40, 94, 56, 62, 122, 57, 114, 60, 125, 101, 36, 48, 118, 24, 113, 37, 61, 88, 49, 127, 119, 21, 124, 83, 27, 19, 22, 86, 30, 52, 100, 120, 81, 115, 47, 78, 14, 17, 76, 85, 50, 91, 33, 55, 42, 32, 89, 102, 16, 53, 43, 95, 12, 51, 80, 97, 108, 41, 107, 92, 23, 28, 112, 90, 87, 25, 31, 73, 7, 38, 71, 29, 93, 20, 82, 74, 9, 106, 96, 105, 44, 75, 26, 18, 99, 34, 79, 10, 84, 15, 5, 69, 3, 98, 6, 11, 8, 64, 39, 70, 103, 66, 67, 0, 2, 35, 68, 1, 13, 65, 77, 4, 72], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 104, 117, 40, 54, 94, 56, 57, 125, 60, 122, 113, 36, 24, 101, 62, 37, 88, 114, 49, 48, 119, 19, 118, 127, 22, 27, 83, 21, 86, 47, 100, 30, 124, 61, 95, 52, 115, 33, 17, 43, 78, 76, 55, 120, 85, 14, 81, 80, 12, 91, 97, 16, 108, 89, 42, 53, 102, 28, 51, 92, 41, 50, 90, 23, 20, 31, 9, 25, 32, 38, 87, 107, 7, 29, 105, 112, 82, 71, 11, 106, 79, 44, 73, 93, 96, 74, 18, 10, 69, 15, 84, 98, 5, 34, 75, 26, 2, 99, 0, 70, 64, 103, 6, 39, 67, 8, 66, 13, 3, 35, 72, 77, 68, 1, 65, 4], [59, 126, 46, 121, 63, 111, 116, 110, 58, 45, 123, 109, 104, 54, 40, 125, 117, 113, 122, 94, 56, 57, 62, 101, 37, 119, 60, 36, 88, 24, 61, 114, 118, 49, 48, 22, 127, 19, 21, 47, 30, 27, 124, 83, 52, 55, 115, 43, 17, 33, 50, 100, 86, 78, 12, 108, 95, 28, 76, 14, 80, 51, 85, 97, 89, 53, 91, 81, 120, 42, 16, 112, 102, 90, 23, 38, 31, 106, 92, 87, 32, 71, 93, 9, 41, 29, 7, 73, 79, 105, 25, 44, 74, 84, 18, 75, 96, 34, 20, 82, 5, 26, 15, 67, 10, 64, 2, 0, 66, 69, 107, 11, 103, 98, 3, 70, 99, 39, 6, 8, 35, 68, 4, 1, 65, 72, 13, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 117, 54, 40, 57, 94, 122, 36, 118, 37, 113, 56, 60, 114, 61, 125, 62, 48, 101, 88, 24, 49, 47, 119, 124, 83, 30, 19, 27, 22, 50, 52, 21, 76, 17, 127, 78, 95, 120, 85, 12, 28, 55, 100, 81, 115, 89, 14, 43, 33, 86, 80, 53, 97, 16, 108, 9, 91, 51, 102, 73, 32, 23, 7, 71, 92, 90, 25, 87, 42, 75, 107, 41, 106, 93, 29, 64, 38, 5, 18, 74, 112, 96, 10, 15, 31, 70, 2, 105, 82, 66, 44, 11, 3, 79, 20, 0, 84, 69, 34, 67, 26, 99, 65, 6, 1, 39, 8, 98, 103, 72, 68, 77, 4, 13, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 54, 104, 40, 61, 60, 57, 56, 113, 118, 122, 101, 37, 48, 36, 94, 62, 125, 114, 127, 88, 24, 47, 119, 21, 49, 83, 124, 22, 19, 30, 120, 27, 52, 76, 89, 53, 95, 78, 100, 115, 51, 33, 43, 55, 12, 50, 17, 102, 28, 81, 80, 108, 42, 86, 85, 31, 90, 97, 14, 32, 16, 23, 91, 87, 38, 93, 73, 112, 7, 92, 96, 41, 9, 71, 106, 25, 44, 20, 15, 107, 105, 79, 5, 26, 34, 18, 70, 99, 10, 29, 75, 84, 82, 0, 69, 3, 74, 64, 11, 66, 67, 2, 98, 1, 65, 103, 8, 39, 68, 6, 35, 72, 4, 77, 13], [59, 126, 46, 63, 121, 116, 111, 110, 58, 123, 45, 109, 117, 104, 40, 54, 60, 36, 56, 57, 113, 118, 61, 88, 94, 37, 101, 24, 122, 47, 62, 83, 125, 49, 114, 127, 27, 22, 48, 119, 21, 124, 30, 76, 19, 78, 95, 17, 52, 80, 43, 12, 85, 14, 115, 53, 108, 50, 55, 100, 16, 81, 33, 28, 120, 42, 89, 73, 51, 23, 32, 31, 9, 102, 86, 87, 90, 91, 97, 44, 71, 38, 92, 112, 25, 7, 106, 74, 93, 107, 34, 79, 29, 96, 41, 10, 67, 3, 64, 20, 5, 70, 18, 15, 11, 69, 75, 82, 84, 26, 103, 105, 0, 66, 65, 6, 99, 98, 72, 2, 4, 1, 39, 8, 13, 68, 77, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 122, 57, 118, 113, 127, 94, 61, 56, 37, 101, 48, 36, 88, 125, 114, 24, 27, 119, 62, 124, 83, 21, 47, 49, 19, 22, 50, 76, 89, 30, 55, 14, 100, 78, 86, 120, 12, 81, 85, 42, 52, 115, 95, 53, 80, 33, 17, 16, 108, 51, 87, 112, 32, 102, 28, 97, 90, 43, 31, 23, 9, 107, 91, 92, 38, 71, 73, 41, 7, 20, 79, 18, 105, 15, 82, 74, 44, 25, 96, 11, 75, 10, 26, 34, 5, 29, 93, 99, 64, 106, 3, 98, 70, 69, 103, 84, 67, 2, 0, 66, 6, 8, 1, 72, 65, 39, 4, 13, 68, 35, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 104, 54, 60, 40, 94, 122, 113, 127, 57, 88, 56, 36, 61, 37, 125, 48, 27, 24, 118, 101, 76, 62, 114, 22, 120, 83, 124, 119, 19, 49, 80, 30, 47, 85, 115, 14, 12, 50, 86, 55, 21, 52, 81, 17, 100, 78, 89, 95, 16, 28, 33, 108, 97, 91, 51, 42, 9, 73, 53, 90, 32, 43, 102, 87, 23, 112, 7, 71, 31, 92, 79, 107, 82, 74, 38, 44, 15, 41, 29, 10, 25, 20, 93, 69, 75, 18, 84, 3, 26, 11, 96, 105, 106, 5, 99, 34, 6, 70, 98, 0, 39, 67, 72, 64, 2, 66, 65, 1, 4, 103, 13, 8, 35, 68, 77]], "model.layers.12.self_attn.q_proj": [[102, 127, 49, 95, 21, 24, 16, 83, 80, 82, 88, 77, 28, 6, 51, 99, 44, 37, 38, 72, 2, 31, 74, 27, 106, 22, 75, 86, 13, 53, 42, 108, 26, 98, 112, 10, 93, 96, 18, 43, 62, 20, 94, 103, 91, 79, 4, 117, 90, 61, 85, 78, 29, 101, 104, 14, 40, 19, 25, 81, 11, 32, 92, 64, 89, 60, 17, 30, 36, 56, 12, 109, 125, 9, 113, 48, 111, 52, 116, 70, 87, 41, 115, 105, 84, 71, 76, 33, 15, 55, 114, 65, 8, 73, 118, 100, 45, 97, 54, 107, 34, 68, 39, 69, 0, 123, 35, 5, 66, 7, 110, 67, 58, 122, 126, 1, 47, 121, 59, 23, 120, 50, 57, 46, 119, 63, 124, 3], [127, 102, 49, 95, 51, 108, 24, 31, 26, 38, 98, 19, 83, 21, 96, 61, 106, 44, 42, 116, 97, 111, 46, 43, 109, 28, 125, 105, 29, 55, 115, 114, 93, 62, 112, 53, 40, 37, 52, 41, 126, 101, 22, 27, 54, 60, 121, 58, 45, 103, 107, 110, 117, 82, 123, 120, 104, 56, 113, 36, 124, 48, 119, 57, 50, 59, 32, 118, 94, 122, 20, 100, 88, 47, 92, 91, 13, 33, 99, 39, 84, 34, 86, 90, 85, 63, 15, 25, 77, 17, 30, 35, 79, 18, 80, 87, 89, 23, 81, 16, 14, 6, 78, 72, 12, 70, 76, 74, 64, 8, 4, 75, 11, 10, 73, 71, 9, 65, 2, 68, 66, 67, 1, 7, 0, 69, 3, 5], [102, 49, 127, 95, 21, 108, 82, 88, 77, 83, 24, 51, 96, 44, 86, 98, 99, 80, 113, 37, 28, 27, 6, 16, 31, 74, 117, 26, 42, 90, 72, 2, 62, 19, 46, 41, 53, 105, 116, 79, 8, 94, 22, 119, 25, 18, 121, 112, 85, 20, 61, 14, 38, 101, 109, 65, 29, 93, 68, 92, 12, 104, 36, 76, 43, 81, 115, 64, 10, 55, 125, 111, 56, 23, 87, 45, 91, 40, 126, 58, 107, 120, 13, 100, 59, 50, 52, 60, 97, 122, 39, 30, 4, 32, 110, 54, 71, 103, 78, 123, 34, 118, 17, 57, 33, 114, 106, 124, 11, 47, 35, 84, 63, 69, 89, 3, 48, 70, 15, 75, 7, 5, 73, 9, 0, 67, 1, 66], [102, 49, 127, 95, 21, 24, 83, 28, 82, 80, 77, 11, 99, 38, 88, 31, 96, 20, 74, 90, 42, 53, 51, 26, 113, 35, 86, 60, 105, 14, 70, 6, 68, 108, 44, 15, 22, 18, 16, 75, 98, 101, 97, 56, 79, 58, 27, 37, 62, 2, 114, 29, 85, 25, 103, 81, 19, 8, 43, 106, 110, 112, 87, 92, 9, 116, 61, 117, 5, 12, 67, 89, 10, 17, 34, 109, 30, 120, 111, 55, 13, 78, 73, 76, 36, 3, 94, 48, 46, 0, 121, 124, 119, 23, 115, 93, 39, 91, 107, 50, 122, 71, 100, 41, 123, 72, 33, 126, 65, 59, 1, 57, 125, 47, 52, 69, 104, 45, 118, 54, 32, 40, 7, 84, 64, 4, 63, 66], [45, 101, 109, 26, 28, 21, 24, 83, 88, 62, 31, 78, 95, 97, 72, 82, 124, 75, 87, 113, 4, 122, 30, 49, 44, 121, 53, 93, 94, 27, 98, 51, 16, 116, 85, 80, 110, 77, 66, 57, 111, 69, 102, 0, 18, 25, 86, 105, 106, 14, 63, 92, 68, 36, 115, 119, 100, 107, 118, 19, 114, 23, 76, 35, 34, 79, 61, 81, 15, 59, 41, 33, 47, 103, 9, 52, 91, 60, 112, 46, 120, 1, 8, 117, 39, 50, 13, 67, 104, 125, 48, 127, 10, 89, 40, 58, 32, 70, 71, 22, 56, 90, 20, 38, 11, 123, 54, 55, 42, 17, 29, 43, 108, 12, 37, 126, 99, 84, 96, 74, 6, 2, 73, 5, 7, 65, 3, 64], [45, 101, 109, 28, 62, 21, 26, 24, 31, 88, 83, 87, 97, 78, 122, 124, 75, 95, 16, 107, 53, 72, 113, 82, 98, 49, 116, 94, 17, 25, 91, 85, 47, 93, 121, 76, 102, 30, 77, 86, 34, 57, 106, 51, 59, 6, 103, 112, 22, 108, 44, 117, 36, 84, 29, 33, 10, 61, 52, 74, 63, 39, 118, 60, 27, 43, 12, 13, 18, 71, 123, 96, 48, 114, 23, 110, 105, 79, 41, 67, 92, 115, 42, 81, 111, 56, 66, 80, 69, 8, 58, 55, 50, 127, 14, 125, 0, 126, 120, 100, 35, 20, 119, 40, 11, 104, 54, 89, 68, 4, 46, 19, 32, 99, 38, 70, 37, 5, 7, 90, 73, 9, 1, 15, 2, 65, 3, 64], [45, 101, 109, 28, 26, 122, 21, 24, 83, 87, 88, 31, 97, 95, 78, 121, 62, 124, 113, 30, 75, 51, 53, 82, 116, 93, 86, 16, 110, 102, 98, 25, 36, 63, 106, 72, 59, 34, 79, 49, 112, 57, 35, 107, 47, 111, 38, 44, 52, 58, 105, 29, 39, 100, 27, 118, 61, 125, 123, 114, 115, 55, 41, 40, 48, 37, 119, 15, 92, 94, 50, 85, 6, 13, 46, 60, 103, 108, 127, 33, 99, 126, 74, 42, 12, 32, 104, 54, 20, 0, 84, 22, 14, 117, 56, 43, 120, 80, 18, 19, 90, 91, 23, 96, 8, 11, 68, 17, 89, 81, 69, 77, 76, 10, 67, 66, 9, 5, 70, 71, 73, 3, 1, 7, 4, 65, 2, 64], [45, 101, 109, 26, 28, 24, 88, 83, 21, 82, 93, 87, 121, 62, 95, 75, 78, 31, 124, 122, 16, 97, 98, 116, 30, 113, 34, 51, 53, 57, 77, 110, 49, 63, 107, 61, 125, 115, 18, 112, 41, 105, 22, 106, 94, 103, 71, 36, 111, 92, 86, 11, 118, 29, 37, 85, 40, 59, 9, 27, 67, 47, 50, 4, 127, 33, 100, 108, 44, 19, 117, 123, 32, 35, 126, 1, 60, 0, 48, 80, 69, 46, 23, 42, 84, 120, 55, 6, 79, 52, 91, 20, 81, 14, 72, 58, 68, 54, 5, 56, 8, 114, 13, 43, 66, 74, 102, 39, 90, 99, 15, 12, 119, 104, 76, 96, 17, 65, 25, 38, 89, 2, 73, 10, 70, 7, 3, 64], [43, 97, 107, 24, 91, 26, 31, 22, 83, 81, 100, 85, 103, 75, 33, 87, 30, 20, 50, 15, 88, 110, 13, 7, 114, 90, 49, 127, 95, 66, 93, 86, 84, 12, 94, 19, 102, 92, 98, 61, 113, 48, 70, 18, 35, 0, 115, 40, 118, 25, 38, 54, 1, 63, 36, 16, 80, 28, 117, 56, 58, 27, 101, 77, 42, 51, 99, 82, 32, 34, 120, 29, 55, 125, 71, 96, 4, 89, 21, 79, 17, 6, 39, 47, 112, 41, 111, 116, 8, 119, 121, 123, 23, 106, 45, 62, 52, 3, 122, 53, 46, 11, 57, 37, 105, 73, 67, 10, 9, 109, 74, 126, 108, 78, 44, 104, 14, 76, 60, 59, 124, 72, 69, 68, 5, 2, 64, 65], [43, 31, 97, 107, 22, 13, 24, 26, 15, 91, 83, 20, 85, 33, 9, 115, 100, 110, 75, 73, 18, 81, 86, 84, 114, 50, 90, 70, 68, 54, 36, 95, 72, 25, 17, 79, 27, 77, 88, 63, 121, 19, 71, 42, 111, 61, 12, 127, 76, 23, 82, 93, 74, 87, 16, 66, 49, 65, 8, 38, 122, 46, 48, 10, 7, 103, 112, 14, 101, 47, 78, 117, 69, 34, 113, 67, 55, 118, 80, 29, 21, 116, 126, 123, 92, 124, 56, 89, 28, 96, 106, 6, 1, 108, 120, 40, 102, 94, 30, 98, 11, 105, 45, 35, 125, 51, 39, 32, 5, 62, 58, 41, 4, 37, 57, 60, 64, 59, 52, 99, 44, 109, 119, 104, 0, 53, 3, 2], [43, 107, 31, 97, 26, 106, 42, 20, 113, 61, 115, 125, 50, 33, 24, 48, 127, 117, 11, 38, 114, 121, 36, 100, 49, 126, 95, 54, 63, 116, 110, 111, 56, 57, 118, 55, 123, 46, 47, 60, 58, 91, 103, 77, 108, 39, 119, 109, 101, 6, 51, 124, 45, 44, 59, 122, 7, 120, 62, 35, 85, 112, 40, 53, 41, 52, 30, 73, 90, 92, 105, 22, 37, 17, 81, 104, 16, 10, 99, 96, 3, 5, 84, 75, 1, 32, 21, 98, 79, 34, 19, 102, 93, 29, 87, 94, 4, 72, 28, 23, 25, 88, 89, 27, 0, 83, 82, 86, 18, 78, 80, 2, 71, 15, 9, 14, 12, 76, 13, 66, 8, 64, 70, 68, 65, 67, 69, 74], [43, 97, 31, 107, 24, 26, 91, 22, 54, 81, 42, 20, 115, 83, 75, 33, 13, 50, 100, 15, 18, 85, 48, 61, 84, 120, 66, 114, 110, 93, 90, 6, 57, 70, 127, 30, 113, 12, 49, 63, 103, 118, 95, 80, 19, 125, 121, 88, 86, 56, 9, 116, 77, 47, 0, 46, 1, 117, 71, 60, 36, 28, 41, 111, 45, 82, 112, 55, 123, 108, 106, 124, 89, 126, 92, 58, 14, 8, 38, 51, 29, 40, 76, 11, 109, 87, 23, 119, 122, 94, 34, 17, 73, 27, 79, 64, 25, 101, 99, 67, 44, 3, 39, 35, 68, 21, 32, 98, 52, 37, 96, 105, 53, 65, 7, 16, 69, 62, 78, 104, 74, 102, 59, 10, 72, 5, 4, 2], [38, 64, 113, 97, 93, 49, 57, 1, 6, 78, 67, 16, 70, 3, 82, 73, 65, 11, 4, 77, 23, 12, 90, 2, 31, 14, 72, 19, 8, 94, 121, 126, 66, 75, 89, 62, 71, 59, 20, 13, 84, 9, 122, 86, 18, 103, 22, 87, 37, 83, 125, 53, 69, 7, 21, 80, 118, 27, 42, 39, 54, 10, 101, 44, 76, 33, 5, 91, 58, 88, 74, 26, 17, 29, 40, 0, 30, 68, 32, 119, 79, 25, 50, 63, 96, 85, 45, 35, 15, 60, 99, 36, 95, 108, 123, 92, 81, 124, 48, 109, 100, 116, 120, 61, 111, 114, 51, 24, 115, 106, 46, 56, 117, 127, 52, 47, 102, 34, 104, 98, 28, 43, 55, 41, 105, 110, 112, 107], [38, 57, 113, 97, 49, 93, 23, 82, 78, 16, 73, 88, 11, 21, 12, 20, 25, 102, 6, 29, 5, 39, 94, 77, 72, 9, 26, 83, 37, 31, 32, 68, 27, 13, 125, 28, 90, 30, 92, 63, 71, 121, 106, 85, 19, 34, 101, 44, 22, 91, 62, 8, 65, 15, 80, 4, 74, 75, 81, 84, 59, 79, 87, 14, 17, 76, 18, 24, 122, 66, 7, 89, 33, 105, 10, 2, 95, 109, 119, 3, 108, 60, 36, 96, 42, 86, 45, 35, 98, 41, 70, 54, 100, 126, 52, 40, 99, 55, 50, 47, 51, 58, 56, 69, 53, 103, 118, 120, 61, 107, 115, 123, 112, 43, 46, 116, 104, 114, 48, 110, 127, 64, 111, 124, 67, 1, 117, 0], [113, 101, 57, 38, 62, 49, 97, 123, 126, 121, 119, 53, 52, 56, 102, 59, 60, 114, 127, 118, 125, 26, 116, 117, 51, 106, 58, 46, 100, 103, 124, 63, 44, 61, 31, 48, 111, 45, 50, 112, 120, 47, 90, 115, 109, 54, 122, 55, 110, 107, 108, 105, 40, 22, 41, 39, 32, 21, 43, 84, 104, 42, 28, 93, 94, 37, 99, 25, 36, 81, 95, 34, 30, 89, 86, 35, 85, 17, 98, 23, 19, 27, 88, 83, 96, 91, 29, 20, 76, 24, 33, 92, 79, 87, 4, 74, 75, 8, 14, 82, 9, 5, 72, 80, 13, 70, 15, 12, 18, 10, 71, 2, 73, 77, 66, 7, 16, 6, 67, 69, 65, 64, 68, 1, 11, 78, 3, 0], [38, 57, 97, 23, 93, 49, 113, 82, 29, 78, 16, 25, 11, 102, 59, 31, 62, 90, 85, 33, 87, 26, 21, 39, 83, 126, 77, 19, 27, 71, 88, 80, 18, 94, 13, 125, 101, 95, 89, 28, 15, 20, 22, 121, 79, 96, 30, 100, 75, 32, 92, 91, 119, 61, 37, 84, 68, 14, 86, 53, 98, 127, 81, 24, 35, 58, 47, 17, 44, 36, 8, 108, 34, 76, 63, 10, 99, 50, 43, 115, 40, 118, 48, 110, 12, 103, 74, 55, 106, 7, 72, 111, 114, 60, 109, 122, 123, 51, 46, 54, 42, 4, 41, 116, 56, 104, 66, 112, 117, 52, 105, 5, 2, 120, 124, 70, 69, 6, 45, 107, 67, 73, 9, 1, 0, 3, 65, 64], [39, 48, 62, 112, 29, 21, 14, 80, 61, 81, 76, 11, 67, 87, 73, 7, 56, 84, 5, 93, 24, 27, 69, 71, 75, 1, 109, 12, 0, 25, 41, 2, 96, 118, 6, 89, 116, 17, 9, 79, 117, 85, 66, 52, 4, 13, 16, 86, 54, 94, 32, 82, 101, 18, 78, 50, 95, 113, 119, 15, 19, 120, 34, 126, 83, 57, 23, 77, 26, 49, 106, 31, 45, 127, 55, 42, 107, 122, 36, 111, 28, 88, 44, 3, 22, 104, 40, 63, 97, 115, 121, 102, 35, 59, 33, 72, 108, 30, 114, 125, 60, 92, 124, 105, 8, 123, 58, 99, 110, 74, 53, 43, 51, 37, 100, 90, 38, 46, 98, 20, 47, 65, 91, 70, 10, 68, 103, 64], [39, 62, 48, 112, 87, 61, 29, 21, 81, 14, 73, 80, 11, 76, 69, 56, 24, 93, 41, 71, 77, 79, 118, 3, 86, 27, 111, 18, 9, 96, 32, 94, 109, 127, 125, 50, 101, 126, 2, 92, 37, 70, 95, 33, 6, 54, 117, 82, 85, 121, 5, 36, 28, 1, 66, 113, 22, 78, 0, 123, 74, 20, 17, 97, 91, 49, 116, 59, 120, 31, 122, 89, 99, 107, 72, 23, 119, 52, 16, 45, 84, 115, 108, 51, 43, 55, 38, 53, 40, 106, 60, 124, 12, 57, 13, 104, 19, 114, 83, 34, 102, 30, 110, 75, 35, 15, 88, 42, 58, 44, 67, 10, 100, 105, 46, 63, 26, 98, 25, 47, 90, 8, 65, 7, 68, 103, 64, 4], [39, 62, 48, 112, 87, 29, 61, 11, 25, 80, 81, 21, 14, 73, 69, 76, 5, 93, 66, 2, 71, 0, 77, 18, 67, 118, 27, 3, 1, 24, 94, 33, 117, 72, 109, 75, 125, 111, 86, 56, 92, 32, 96, 31, 41, 82, 74, 106, 50, 7, 116, 95, 37, 79, 113, 85, 28, 30, 19, 97, 70, 89, 83, 36, 122, 63, 78, 17, 123, 121, 126, 34, 120, 43, 88, 16, 38, 35, 54, 6, 12, 40, 59, 65, 26, 45, 91, 10, 9, 90, 23, 99, 127, 84, 51, 104, 52, 46, 114, 58, 98, 119, 49, 102, 20, 103, 100, 44, 124, 110, 108, 57, 101, 13, 115, 8, 4, 68, 55, 53, 15, 42, 105, 107, 60, 47, 22, 64], [39, 62, 48, 112, 71, 29, 14, 80, 73, 81, 21, 76, 11, 3, 69, 2, 61, 65, 1, 67, 56, 109, 118, 0, 72, 64, 7, 4, 13, 24, 12, 85, 8, 37, 79, 117, 17, 96, 111, 121, 87, 77, 66, 78, 93, 95, 15, 84, 5, 16, 54, 126, 75, 68, 50, 19, 83, 99, 33, 120, 53, 123, 94, 10, 116, 124, 88, 9, 103, 18, 101, 127, 106, 41, 92, 86, 28, 89, 82, 74, 38, 35, 40, 31, 57, 27, 30, 51, 20, 63, 26, 52, 70, 45, 115, 43, 36, 22, 97, 91, 107, 119, 32, 125, 34, 102, 6, 104, 105, 23, 25, 60, 110, 113, 46, 122, 100, 90, 44, 59, 98, 49, 42, 55, 108, 114, 47, 58], [105, 58, 34, 109, 26, 41, 86, 104, 88, 18, 84, 126, 24, 30, 55, 78, 92, 16, 73, 77, 36, 69, 19, 32, 49, 87, 71, 95, 91, 75, 119, 67, 121, 79, 111, 28, 45, 48, 1, 44, 2, 120, 51, 5, 94, 15, 29, 98, 114, 31, 12, 25, 40, 50, 57, 82, 112, 60, 33, 113, 64, 124, 107, 14, 125, 74, 10, 8, 123, 63, 108, 13, 37, 127, 101, 66, 62, 0, 22, 115, 81, 6, 39, 38, 118, 122, 65, 61, 72, 100, 3, 59, 47, 93, 7, 52, 90, 4, 23, 102, 46, 11, 56, 97, 42, 106, 21, 43, 76, 103, 70, 17, 99, 83, 68, 116, 9, 110, 20, 80, 54, 117, 27, 35, 53, 85, 89, 96], [105, 34, 26, 41, 58, 86, 36, 121, 84, 32, 104, 16, 126, 92, 62, 18, 51, 88, 118, 24, 78, 47, 57, 25, 52, 50, 49, 119, 116, 48, 109, 28, 102, 30, 90, 100, 114, 107, 95, 120, 23, 39, 46, 122, 55, 21, 61, 75, 113, 31, 38, 103, 127, 42, 37, 43, 83, 63, 11, 60, 80, 20, 76, 97, 112, 117, 14, 77, 108, 87, 125, 56, 45, 111, 7, 101, 22, 93, 96, 94, 19, 82, 123, 106, 115, 9, 59, 40, 33, 29, 91, 8, 124, 110, 98, 27, 44, 74, 72, 35, 85, 99, 79, 53, 81, 6, 89, 12, 54, 73, 71, 17, 15, 68, 13, 5, 10, 66, 4, 3, 70, 65, 67, 1, 64, 69, 2, 0], [105, 34, 58, 84, 26, 41, 86, 126, 77, 109, 16, 104, 73, 121, 32, 88, 18, 78, 75, 71, 30, 36, 49, 70, 108, 98, 48, 3, 29, 4, 51, 81, 24, 55, 120, 118, 11, 114, 50, 62, 111, 122, 100, 39, 2, 60, 94, 45, 102, 82, 80, 85, 52, 69, 115, 101, 37, 79, 116, 57, 65, 119, 44, 28, 125, 90, 0, 23, 19, 113, 92, 43, 38, 20, 112, 61, 83, 63, 13, 21, 124, 14, 22, 12, 95, 17, 68, 127, 117, 6, 123, 87, 46, 31, 103, 9, 99, 59, 47, 96, 76, 15, 27, 40, 67, 74, 106, 97, 25, 72, 42, 1, 35, 56, 93, 110, 53, 33, 54, 10, 89, 107, 91, 5, 8, 7, 64, 66], [105, 34, 92, 26, 41, 84, 18, 86, 109, 104, 16, 24, 126, 121, 78, 75, 77, 57, 71, 101, 58, 60, 73, 51, 36, 52, 120, 90, 118, 122, 107, 55, 62, 48, 49, 66, 45, 32, 95, 44, 30, 119, 108, 116, 14, 111, 102, 28, 29, 37, 63, 88, 50, 125, 46, 47, 7, 124, 114, 113, 106, 20, 127, 112, 40, 94, 5, 100, 87, 25, 123, 11, 31, 4, 42, 98, 82, 56, 38, 43, 17, 39, 19, 115, 103, 54, 21, 59, 91, 33, 8, 110, 35, 6, 97, 85, 83, 61, 80, 81, 117, 76, 27, 22, 12, 93, 53, 23, 96, 15, 79, 89, 99, 72, 2, 65, 64, 0, 68, 13, 69, 9, 3, 74, 1, 10, 70, 67], [104, 34, 20, 22, 124, 89, 92, 79, 95, 49, 18, 75, 119, 25, 77, 62, 48, 17, 114, 40, 120, 107, 6, 118, 46, 121, 117, 58, 54, 111, 70, 96, 93, 50, 45, 61, 53, 47, 9, 100, 84, 59, 43, 109, 68, 94, 15, 41, 110, 12, 82, 86, 19, 11, 90, 72, 55, 81, 66, 122, 98, 52, 1, 60, 123, 33, 99, 23, 126, 8, 76, 4, 56, 102, 64, 103, 3, 2, 112, 88, 37, 16, 113, 97, 73, 27, 80, 38, 78, 21, 32, 13, 24, 30, 28, 87, 91, 108, 29, 106, 105, 101, 44, 26, 35, 74, 0, 83, 85, 31, 51, 67, 10, 63, 57, 127, 42, 36, 39, 125, 115, 116, 7, 14, 5, 71, 69, 65], [104, 34, 89, 22, 49, 20, 124, 95, 18, 79, 48, 62, 119, 77, 25, 40, 118, 92, 75, 107, 31, 121, 46, 120, 114, 54, 9, 41, 6, 17, 111, 16, 82, 117, 58, 12, 15, 97, 84, 60, 38, 56, 93, 45, 115, 81, 53, 112, 101, 28, 76, 2, 86, 94, 68, 103, 110, 63, 98, 36, 55, 47, 70, 4, 96, 72, 61, 90, 85, 52, 88, 100, 127, 102, 73, 21, 113, 1, 123, 78, 50, 109, 59, 126, 24, 10, 0, 64, 91, 26, 23, 13, 87, 11, 19, 32, 99, 105, 5, 125, 108, 3, 66, 80, 30, 106, 57, 37, 67, 35, 33, 39, 43, 29, 69, 51, 83, 122, 8, 7, 27, 116, 74, 42, 14, 44, 71, 65], [104, 34, 22, 124, 89, 18, 92, 20, 48, 79, 62, 49, 119, 77, 95, 118, 25, 75, 40, 114, 17, 96, 46, 12, 9, 82, 121, 50, 111, 54, 58, 61, 117, 47, 84, 41, 120, 100, 4, 56, 107, 6, 110, 13, 59, 45, 98, 38, 15, 86, 81, 94, 52, 37, 55, 53, 108, 43, 35, 68, 63, 1, 122, 78, 112, 90, 101, 97, 126, 109, 33, 93, 87, 71, 70, 76, 19, 28, 60, 7, 72, 88, 123, 125, 39, 51, 23, 73, 31, 115, 127, 74, 26, 113, 105, 103, 2, 8, 21, 80, 91, 99, 14, 11, 3, 30, 85, 42, 116, 106, 16, 24, 10, 29, 102, 36, 44, 27, 32, 66, 57, 65, 67, 83, 64, 0, 5, 69], [104, 34, 119, 89, 22, 92, 18, 49, 20, 124, 95, 62, 48, 79, 77, 46, 25, 40, 114, 75, 121, 118, 120, 109, 41, 17, 54, 97, 12, 50, 111, 93, 9, 58, 16, 98, 82, 31, 112, 84, 4, 59, 47, 100, 117, 107, 80, 45, 61, 33, 53, 103, 108, 56, 38, 94, 21, 60, 32, 43, 24, 86, 90, 123, 55, 52, 127, 81, 126, 122, 113, 76, 96, 51, 110, 19, 35, 15, 63, 99, 105, 115, 28, 102, 85, 13, 87, 39, 44, 88, 125, 91, 37, 83, 27, 6, 101, 26, 106, 29, 78, 42, 23, 30, 1, 0, 2, 73, 72, 116, 36, 68, 69, 74, 57, 10, 70, 5, 7, 14, 71, 11, 8, 67, 65, 3, 66, 64], [41, 34, 53, 88, 93, 105, 20, 17, 79, 22, 35, 77, 72, 29, 75, 48, 25, 84, 115, 6, 66, 91, 18, 27, 82, 125, 24, 51, 55, 58, 62, 127, 126, 50, 8, 89, 63, 61, 108, 68, 2, 81, 15, 124, 13, 38, 0, 4, 16, 117, 80, 43, 114, 70, 52, 11, 19, 57, 103, 44, 60, 78, 100, 47, 90, 59, 76, 73, 98, 42, 109, 46, 26, 49, 83, 23, 86, 85, 40, 31, 112, 92, 33, 96, 28, 12, 10, 113, 14, 116, 87, 45, 21, 107, 39, 118, 101, 37, 122, 121, 64, 102, 97, 9, 36, 94, 104, 106, 119, 30, 123, 110, 7, 32, 74, 95, 99, 56, 69, 120, 71, 54, 5, 1, 111, 3, 65, 67], [41, 34, 53, 22, 105, 20, 88, 93, 48, 17, 29, 79, 50, 77, 44, 126, 62, 127, 27, 108, 35, 115, 58, 84, 80, 60, 96, 117, 43, 38, 125, 72, 112, 114, 75, 31, 49, 57, 124, 86, 24, 101, 21, 15, 118, 55, 61, 54, 123, 52, 63, 99, 26, 121, 90, 91, 8, 51, 37, 82, 103, 30, 39, 56, 28, 106, 122, 87, 6, 92, 18, 25, 47, 119, 81, 113, 40, 36, 120, 111, 70, 32, 102, 83, 110, 46, 16, 95, 33, 107, 85, 45, 23, 100, 89, 94, 97, 66, 104, 42, 71, 116, 12, 19, 73, 59, 109, 13, 78, 0, 14, 98, 10, 11, 7, 76, 9, 74, 67, 5, 68, 4, 2, 3, 69, 64, 65, 1], [41, 34, 53, 105, 22, 93, 88, 20, 27, 66, 91, 0, 79, 77, 17, 68, 127, 75, 19, 82, 72, 58, 112, 2, 67, 43, 124, 115, 48, 126, 44, 64, 35, 62, 70, 18, 65, 61, 29, 1, 60, 59, 6, 49, 38, 71, 51, 57, 84, 80, 50, 125, 3, 63, 52, 9, 96, 16, 69, 108, 109, 45, 55, 118, 8, 7, 31, 47, 15, 114, 110, 23, 5, 119, 101, 120, 99, 113, 117, 76, 46, 24, 73, 40, 12, 11, 37, 103, 4, 122, 25, 123, 74, 107, 10, 100, 116, 39, 32, 106, 87, 83, 26, 78, 13, 90, 85, 42, 56, 33, 102, 86, 81, 121, 54, 14, 97, 36, 28, 98, 104, 21, 111, 92, 89, 95, 94, 30], [41, 34, 53, 22, 88, 105, 17, 20, 58, 91, 93, 27, 29, 127, 115, 77, 126, 43, 75, 79, 61, 51, 48, 63, 35, 124, 108, 62, 112, 114, 55, 38, 57, 31, 81, 84, 44, 50, 113, 70, 123, 49, 82, 15, 25, 122, 59, 66, 52, 45, 40, 86, 39, 60, 118, 32, 106, 100, 119, 107, 116, 72, 80, 109, 101, 24, 0, 110, 125, 42, 117, 47, 37, 18, 96, 103, 46, 120, 11, 56, 19, 26, 102, 16, 54, 104, 121, 13, 99, 111, 23, 33, 83, 90, 98, 36, 92, 6, 78, 28, 21, 67, 68, 73, 95, 30, 97, 94, 87, 10, 76, 89, 12, 14, 85, 9, 71, 74, 4, 5, 8, 1, 69, 3, 2, 7, 65, 64]], "model.layers.12.self_attn.k_proj": [[38, 49, 127, 31, 113, 24, 21, 83, 80, 28, 82, 77, 35, 72, 108, 74, 62, 106, 105, 66, 96, 64, 1, 90, 112, 30, 40, 25, 102, 16, 107, 6, 51, 37, 34, 57, 116, 75, 86, 60, 125, 48, 45, 109, 76, 58, 91, 55, 44, 27, 117, 126, 29, 120, 119, 5, 46, 124, 59, 42, 67, 4, 87, 111, 85, 122, 47, 94, 23, 114, 26, 73, 84, 56, 43, 17, 63, 118, 115, 14, 79, 0, 104, 71, 98, 54, 41, 61, 69, 50, 78, 39, 12, 52, 110, 36, 92, 100, 53, 32, 123, 70, 103, 11, 121, 20, 15, 93, 97, 68, 89, 18, 101, 88, 99, 22, 81, 33, 19, 65, 10, 9, 7, 3, 8, 13, 2, 95], [109, 37, 45, 28, 24, 21, 31, 26, 82, 83, 78, 16, 62, 115, 75, 116, 124, 117, 49, 113, 121, 53, 110, 57, 30, 63, 8, 41, 105, 23, 64, 61, 127, 51, 60, 55, 126, 2, 103, 120, 33, 40, 90, 15, 93, 112, 42, 100, 118, 125, 107, 44, 111, 77, 108, 54, 34, 59, 17, 50, 18, 46, 43, 39, 56, 47, 22, 58, 5, 114, 85, 73, 65, 86, 48, 87, 38, 123, 97, 99, 95, 52, 27, 19, 102, 81, 104, 9, 106, 94, 68, 92, 13, 119, 96, 29, 98, 3, 80, 4, 20, 76, 32, 36, 12, 25, 122, 71, 35, 79, 89, 91, 84, 72, 10, 70, 66, 11, 74, 1, 7, 69, 14, 6, 67, 88, 0, 101], [107, 33, 43, 95, 22, 91, 24, 15, 26, 50, 115, 83, 13, 85, 118, 81, 20, 48, 61, 18, 49, 75, 63, 100, 117, 47, 126, 110, 9, 36, 46, 70, 54, 42, 112, 127, 124, 64, 39, 93, 116, 58, 108, 2, 121, 51, 101, 113, 111, 114, 68, 72, 123, 71, 57, 119, 120, 60, 12, 102, 125, 55, 65, 56, 99, 38, 44, 59, 40, 109, 19, 122, 45, 104, 52, 62, 41, 98, 103, 96, 53, 106, 14, 37, 105, 87, 11, 82, 92, 94, 3, 32, 30, 34, 29, 88, 74, 27, 35, 23, 76, 28, 21, 5, 78, 80, 25, 89, 16, 77, 79, 90, 31, 10, 97, 8, 86, 4, 1, 84, 17, 6, 67, 73, 7, 66, 69, 0], [113, 57, 102, 33, 82, 29, 23, 16, 11, 0, 3, 78, 65, 6, 77, 22, 26, 38, 59, 2, 71, 125, 108, 126, 73, 95, 83, 64, 63, 53, 84, 62, 93, 58, 72, 85, 12, 118, 114, 28, 79, 49, 89, 40, 56, 5, 60, 45, 61, 36, 124, 39, 68, 88, 123, 15, 96, 30, 119, 127, 117, 112, 106, 74, 116, 115, 52, 81, 94, 122, 111, 120, 27, 86, 121, 47, 100, 44, 31, 46, 4, 32, 34, 55, 109, 43, 91, 50, 103, 24, 51, 13, 37, 54, 35, 99, 41, 92, 19, 76, 48, 110, 105, 97, 98, 21, 17, 80, 20, 9, 42, 104, 107, 87, 1, 8, 69, 10, 25, 67, 14, 101, 75, 70, 90, 18, 7, 66], [103, 62, 48, 21, 81, 93, 14, 76, 11, 80, 61, 73, 71, 69, 3, 87, 0, 2, 65, 56, 45, 24, 8, 41, 70, 27, 64, 7, 66, 112, 6, 18, 117, 125, 32, 116, 54, 95, 119, 10, 94, 121, 13, 37, 33, 25, 47, 63, 74, 118, 1, 35, 51, 5, 72, 83, 4, 108, 97, 111, 99, 55, 67, 122, 50, 9, 105, 19, 75, 115, 68, 59, 15, 86, 120, 114, 31, 12, 113, 44, 43, 100, 17, 42, 107, 57, 126, 30, 28, 78, 104, 101, 22, 88, 92, 96, 84, 38, 26, 109, 16, 49, 106, 102, 60, 77, 20, 52, 79, 98, 85, 29, 123, 58, 53, 110, 127, 36, 89, 124, 23, 46, 40, 90, 34, 91, 82, 39], [41, 98, 26, 88, 18, 45, 84, 58, 16, 86, 112, 44, 111, 119, 109, 78, 40, 60, 75, 105, 77, 63, 50, 55, 96, 71, 121, 92, 73, 0, 104, 30, 2, 124, 122, 43, 4, 57, 56, 36, 49, 65, 39, 115, 125, 123, 29, 107, 101, 53, 31, 126, 110, 51, 62, 120, 85, 34, 113, 127, 69, 17, 74, 76, 47, 48, 68, 52, 42, 114, 79, 102, 100, 70, 61, 87, 38, 118, 59, 19, 89, 91, 28, 106, 94, 37, 6, 15, 103, 27, 117, 54, 9, 93, 35, 83, 108, 97, 32, 46, 11, 72, 20, 95, 116, 67, 33, 22, 23, 99, 25, 8, 81, 13, 90, 80, 10, 21, 5, 24, 12, 14, 64, 82, 66, 1, 3, 7], [40, 98, 89, 18, 22, 43, 119, 20, 79, 124, 48, 92, 49, 77, 112, 50, 17, 75, 31, 62, 61, 110, 9, 12, 6, 109, 60, 72, 117, 53, 113, 118, 54, 45, 68, 111, 107, 59, 41, 46, 84, 34, 58, 121, 64, 120, 14, 114, 47, 63, 35, 115, 56, 127, 65, 74, 44, 96, 83, 1, 100, 97, 66, 13, 70, 123, 28, 94, 104, 93, 55, 85, 33, 16, 36, 26, 2, 38, 3, 23, 67, 42, 102, 8, 103, 105, 11, 126, 29, 88, 24, 90, 81, 52, 39, 91, 122, 125, 101, 30, 51, 32, 57, 76, 116, 106, 7, 108, 37, 10, 27, 86, 99, 80, 69, 78, 73, 71, 15, 21, 0, 5, 4, 25, 19, 95, 87, 82], [105, 98, 53, 20, 88, 29, 17, 79, 22, 77, 75, 124, 61, 41, 64, 51, 126, 44, 6, 72, 119, 2, 80, 82, 59, 91, 112, 60, 118, 38, 63, 4, 108, 117, 127, 55, 58, 43, 45, 50, 62, 57, 35, 115, 125, 73, 101, 122, 12, 111, 110, 49, 46, 28, 123, 107, 42, 48, 32, 104, 19, 26, 47, 10, 40, 109, 66, 113, 34, 52, 78, 103, 114, 120, 39, 87, 56, 96, 90, 21, 86, 9, 102, 95, 116, 5, 31, 37, 36, 65, 97, 25, 106, 54, 121, 71, 67, 81, 30, 100, 33, 74, 99, 89, 8, 27, 92, 76, 1, 94, 69, 70, 85, 23, 11, 93, 7, 68, 24, 16, 15, 18, 3, 83, 14, 13, 84, 0]], "model.layers.12.self_attn.qk_proj": [[62, 49, 105, 41, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 88, 40, 93, 53, 24, 18, 34, 82, 86, 90, 21, 104, 58, 124, 98, 75, 97, 22, 85, 80, 20, 16, 11, 84, 95, 92, 112, 29, 77, 31, 78, 61, 13, 14, 81, 17, 119, 103, 28, 87, 26, 25, 27, 19, 73, 15, 79, 126, 33, 83, 121, 37, 89, 12, 23, 9, 50, 60, 118, 76, 6, 116, 44, 55, 115, 96, 0, 51, 108, 56, 101, 111, 125, 7, 39, 110, 64, 91, 59, 63, 71, 35, 47, 122, 117, 72, 8, 3, 1, 2, 65, 69, 66, 52, 100, 114, 36, 70, 46, 120, 5, 123, 94, 30, 67, 32, 10, 54, 42, 99, 4, 68, 106, 74], [49, 62, 105, 41, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 34, 93, 88, 53, 24, 86, 124, 90, 21, 58, 104, 18, 82, 11, 75, 97, 112, 98, 29, 31, 92, 22, 20, 95, 80, 84, 77, 85, 16, 13, 78, 61, 119, 14, 28, 87, 26, 17, 25, 27, 103, 81, 73, 19, 15, 9, 23, 6, 60, 126, 111, 83, 89, 33, 79, 118, 50, 55, 108, 115, 125, 37, 121, 39, 12, 44, 0, 76, 101, 110, 72, 59, 51, 47, 63, 116, 7, 35, 117, 96, 64, 56, 71, 1, 123, 36, 122, 2, 91, 94, 65, 66, 32, 100, 3, 5, 30, 46, 4, 114, 8, 120, 106, 42, 10, 54, 69, 67, 70, 52, 99, 68, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 93, 40, 53, 88, 34, 24, 104, 58, 90, 86, 124, 112, 95, 21, 98, 20, 22, 18, 82, 31, 29, 97, 92, 61, 84, 85, 75, 80, 11, 119, 77, 16, 78, 103, 26, 28, 27, 13, 87, 126, 14, 33, 37, 17, 25, 23, 81, 115, 121, 44, 110, 118, 51, 50, 73, 59, 35, 60, 19, 125, 6, 116, 55, 15, 108, 117, 83, 89, 79, 9, 39, 101, 76, 12, 111, 30, 63, 0, 36, 47, 56, 96, 72, 7, 91, 65, 122, 114, 66, 100, 3, 46, 71, 70, 64, 120, 123, 52, 1, 94, 32, 5, 106, 4, 69, 2, 42, 54, 67, 10, 68, 99, 8, 74], [62, 49, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 24, 93, 88, 34, 90, 58, 86, 124, 112, 82, 104, 18, 21, 20, 98, 80, 31, 95, 29, 22, 77, 97, 92, 75, 16, 85, 11, 84, 119, 14, 61, 103, 78, 25, 13, 28, 17, 27, 121, 79, 23, 87, 126, 44, 26, 81, 33, 50, 60, 51, 111, 118, 9, 83, 108, 15, 64, 89, 73, 101, 12, 115, 19, 39, 59, 63, 116, 37, 35, 55, 47, 70, 72, 125, 56, 110, 6, 91, 117, 76, 114, 0, 65, 30, 100, 7, 71, 123, 96, 2, 42, 66, 94, 120, 36, 67, 32, 3, 1, 122, 4, 54, 46, 52, 68, 69, 106, 5, 99, 8, 74, 10], [62, 49, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 34, 88, 24, 58, 90, 93, 82, 124, 18, 104, 86, 80, 20, 75, 112, 98, 21, 31, 11, 16, 77, 84, 85, 61, 22, 13, 29, 92, 78, 97, 14, 95, 119, 103, 81, 26, 17, 25, 87, 28, 9, 23, 15, 79, 70, 83, 126, 121, 73, 27, 33, 50, 118, 111, 39, 19, 12, 0, 55, 89, 37, 7, 64, 72, 110, 76, 108, 59, 35, 2, 51, 44, 125, 116, 60, 47, 101, 91, 115, 6, 117, 1, 30, 56, 71, 120, 63, 66, 114, 123, 67, 122, 3, 42, 36, 65, 5, 96, 68, 69, 100, 52, 54, 46, 4, 94, 32, 106, 8, 99, 10, 74], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 40, 102, 53, 24, 34, 88, 90, 18, 86, 58, 75, 93, 21, 82, 20, 104, 124, 80, 77, 11, 98, 84, 31, 16, 85, 13, 78, 22, 14, 112, 92, 97, 61, 29, 119, 95, 17, 25, 81, 103, 28, 26, 87, 73, 15, 27, 9, 19, 121, 79, 83, 111, 12, 76, 70, 23, 89, 47, 50, 126, 37, 116, 55, 39, 72, 108, 118, 60, 56, 33, 115, 125, 117, 0, 44, 59, 35, 7, 64, 110, 51, 91, 65, 114, 94, 123, 63, 46, 66, 101, 2, 71, 96, 1, 6, 54, 52, 68, 120, 5, 122, 30, 99, 42, 36, 69, 67, 3, 100, 4, 106, 8, 74, 32, 10], [49, 62, 41, 105, 113, 57, 48, 127, 109, 45, 43, 107, 38, 102, 40, 88, 24, 53, 90, 18, 34, 86, 21, 82, 93, 20, 80, 104, 75, 58, 124, 84, 77, 22, 98, 16, 85, 92, 11, 13, 61, 29, 78, 31, 112, 14, 97, 17, 119, 81, 95, 23, 87, 26, 79, 28, 103, 25, 76, 15, 83, 27, 33, 121, 73, 9, 44, 50, 89, 12, 70, 19, 39, 126, 111, 37, 118, 55, 110, 116, 108, 60, 72, 51, 115, 47, 56, 117, 7, 59, 64, 122, 35, 101, 96, 63, 52, 94, 36, 91, 125, 67, 120, 0, 42, 1, 123, 71, 100, 66, 65, 2, 30, 46, 54, 3, 69, 114, 8, 5, 6, 4, 32, 99, 74, 106, 68, 10], [49, 62, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 88, 40, 102, 53, 34, 93, 24, 90, 18, 21, 86, 82, 104, 20, 85, 80, 22, 75, 84, 92, 77, 11, 16, 124, 98, 31, 58, 78, 97, 13, 29, 14, 61, 95, 119, 112, 17, 87, 81, 15, 28, 19, 23, 26, 79, 25, 33, 44, 70, 121, 103, 115, 83, 89, 12, 9, 27, 73, 118, 50, 111, 37, 126, 76, 116, 117, 110, 55, 7, 60, 0, 51, 108, 47, 59, 64, 63, 35, 8, 72, 125, 122, 39, 71, 91, 3, 120, 2, 65, 101, 36, 94, 100, 66, 114, 46, 54, 5, 96, 30, 56, 1, 123, 32, 6, 67, 52, 68, 42, 99, 69, 106, 10, 4, 74], [49, 62, 105, 41, 57, 113, 48, 127, 109, 45, 43, 107, 38, 40, 102, 93, 53, 88, 24, 34, 90, 18, 20, 86, 104, 85, 82, 21, 98, 22, 29, 92, 58, 80, 16, 31, 61, 124, 119, 112, 84, 75, 95, 78, 11, 77, 97, 14, 25, 23, 17, 26, 81, 28, 13, 87, 19, 121, 79, 44, 103, 15, 115, 111, 126, 37, 89, 33, 27, 9, 73, 110, 108, 50, 60, 39, 83, 47, 76, 96, 51, 55, 125, 70, 94, 12, 116, 63, 118, 64, 117, 59, 7, 35, 122, 30, 0, 91, 2, 36, 6, 56, 8, 42, 67, 66, 54, 1, 71, 114, 101, 65, 52, 46, 100, 120, 106, 3, 123, 72, 4, 32, 68, 99, 5, 69, 74, 10], [49, 62, 41, 57, 105, 113, 48, 127, 45, 109, 43, 107, 38, 53, 102, 40, 88, 24, 93, 34, 18, 90, 58, 104, 86, 85, 82, 22, 20, 75, 21, 16, 124, 92, 31, 77, 61, 80, 29, 78, 84, 112, 119, 11, 13, 95, 14, 97, 98, 81, 25, 23, 28, 26, 19, 15, 17, 87, 103, 27, 9, 121, 79, 126, 33, 83, 44, 89, 73, 50, 111, 60, 51, 47, 115, 108, 37, 8, 118, 6, 70, 76, 55, 12, 110, 125, 63, 7, 39, 71, 64, 0, 96, 67, 120, 36, 101, 30, 66, 59, 94, 4, 91, 114, 56, 117, 1, 116, 65, 123, 35, 32, 2, 3, 42, 54, 68, 5, 100, 122, 46, 106, 69, 74, 99, 72, 52, 10], [49, 62, 105, 41, 57, 113, 48, 45, 127, 43, 109, 107, 38, 102, 53, 40, 34, 24, 93, 88, 18, 82, 86, 58, 21, 90, 104, 75, 80, 16, 31, 77, 124, 61, 85, 20, 78, 22, 98, 112, 29, 92, 97, 11, 95, 84, 14, 119, 13, 17, 81, 25, 26, 6, 103, 9, 87, 28, 126, 23, 19, 27, 15, 79, 76, 83, 64, 8, 50, 73, 118, 89, 33, 115, 121, 111, 12, 37, 47, 7, 2, 116, 60, 71, 0, 51, 63, 120, 66, 91, 125, 1, 110, 44, 70, 114, 35, 67, 108, 55, 56, 5, 65, 101, 4, 30, 39, 59, 96, 123, 3, 122, 117, 36, 68, 69, 72, 42, 54, 94, 32, 52, 46, 74, 100, 99, 106, 10], [49, 62, 41, 105, 113, 57, 48, 109, 127, 45, 43, 107, 38, 40, 102, 53, 88, 34, 24, 93, 86, 18, 90, 58, 21, 82, 80, 75, 104, 77, 22, 16, 20, 98, 84, 92, 85, 78, 11, 31, 29, 112, 124, 61, 13, 14, 97, 95, 17, 81, 119, 26, 28, 103, 6, 15, 25, 87, 9, 73, 19, 76, 23, 12, 83, 89, 79, 126, 37, 121, 50, 111, 27, 8, 33, 64, 60, 44, 47, 0, 39, 71, 51, 115, 118, 2, 120, 55, 7, 66, 56, 108, 116, 125, 59, 35, 91, 3, 94, 65, 122, 63, 70, 123, 5, 117, 110, 72, 96, 1, 114, 36, 30, 101, 42, 69, 46, 100, 4, 67, 74, 52, 54, 68, 99, 32, 106, 10], [49, 62, 41, 105, 113, 57, 48, 127, 109, 45, 43, 107, 38, 102, 40, 53, 24, 93, 88, 104, 18, 34, 86, 21, 82, 90, 20, 22, 58, 85, 124, 80, 84, 92, 77, 31, 16, 98, 29, 11, 78, 75, 61, 95, 112, 97, 17, 119, 13, 14, 81, 26, 15, 25, 28, 121, 19, 89, 87, 23, 83, 37, 103, 79, 33, 126, 50, 27, 44, 9, 12, 6, 60, 111, 110, 108, 39, 73, 55, 115, 35, 118, 76, 51, 63, 8, 47, 91, 125, 101, 30, 116, 56, 123, 59, 117, 7, 122, 96, 94, 36, 114, 100, 64, 42, 2, 120, 0, 65, 54, 71, 52, 3, 1, 67, 66, 68, 70, 32, 99, 5, 4, 106, 72, 46, 69, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 40, 34, 102, 24, 88, 104, 18, 82, 93, 21, 90, 75, 20, 77, 11, 86, 80, 58, 124, 98, 78, 22, 13, 84, 85, 31, 61, 112, 14, 16, 97, 92, 29, 95, 17, 119, 81, 25, 6, 15, 9, 19, 126, 83, 79, 87, 27, 89, 121, 26, 28, 73, 23, 103, 8, 50, 60, 118, 111, 0, 108, 33, 44, 7, 115, 64, 76, 12, 110, 125, 66, 51, 55, 47, 71, 2, 116, 70, 37, 56, 39, 91, 1, 117, 101, 30, 123, 63, 36, 65, 68, 35, 122, 96, 5, 114, 120, 59, 67, 4, 54, 72, 52, 94, 69, 42, 100, 3, 46, 32, 74, 99, 10, 106], [49, 62, 41, 105, 113, 57, 48, 127, 45, 109, 43, 107, 38, 53, 102, 40, 34, 93, 88, 24, 104, 18, 82, 90, 58, 86, 21, 20, 75, 31, 98, 124, 80, 77, 16, 11, 78, 84, 29, 92, 95, 112, 85, 61, 22, 97, 13, 14, 81, 17, 119, 9, 103, 28, 25, 121, 126, 26, 23, 15, 83, 33, 73, 27, 79, 44, 87, 89, 19, 37, 50, 110, 108, 60, 6, 118, 12, 111, 116, 51, 8, 115, 47, 0, 55, 39, 56, 70, 76, 35, 71, 30, 63, 7, 101, 91, 125, 96, 123, 66, 120, 59, 64, 117, 65, 54, 94, 114, 100, 72, 36, 32, 46, 122, 5, 52, 69, 106, 2, 1, 74, 67, 4, 3, 42, 99, 68, 10], [49, 62, 105, 41, 113, 57, 48, 109, 43, 127, 45, 107, 38, 102, 40, 53, 34, 93, 58, 24, 18, 88, 82, 104, 21, 86, 90, 20, 75, 98, 124, 61, 16, 80, 22, 85, 78, 84, 11, 31, 77, 92, 17, 95, 29, 112, 13, 97, 14, 81, 121, 119, 103, 15, 26, 87, 9, 126, 73, 27, 23, 44, 25, 79, 111, 83, 19, 12, 60, 76, 70, 28, 50, 115, 89, 33, 37, 59, 108, 64, 118, 110, 7, 0, 39, 116, 51, 123, 47, 125, 66, 55, 67, 35, 8, 101, 6, 120, 71, 117, 63, 2, 91, 56, 1, 30, 54, 96, 72, 65, 52, 114, 3, 36, 4, 122, 94, 42, 69, 5, 100, 32, 46, 68, 74, 10, 99, 106], [49, 62, 41, 105, 57, 113, 45, 48, 43, 127, 109, 107, 38, 40, 53, 102, 34, 88, 93, 18, 24, 90, 82, 104, 86, 22, 21, 20, 29, 75, 61, 80, 98, 85, 58, 16, 84, 95, 92, 77, 31, 11, 97, 112, 124, 78, 17, 14, 81, 13, 28, 119, 79, 15, 87, 23, 25, 26, 73, 103, 126, 83, 19, 50, 121, 108, 9, 33, 70, 89, 27, 111, 37, 51, 60, 118, 39, 76, 44, 12, 115, 117, 116, 91, 101, 110, 55, 30, 125, 63, 120, 47, 72, 7, 0, 114, 123, 54, 71, 56, 122, 59, 2, 1, 35, 42, 36, 5, 100, 65, 96, 64, 4, 32, 46, 94, 66, 8, 3, 68, 6, 52, 67, 99, 10, 69, 74, 106], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 34, 93, 88, 24, 18, 82, 21, 104, 86, 90, 20, 98, 11, 58, 75, 22, 80, 16, 31, 95, 92, 97, 84, 124, 112, 29, 85, 77, 61, 13, 14, 81, 17, 119, 78, 26, 103, 87, 28, 79, 15, 121, 83, 126, 50, 25, 27, 70, 23, 33, 73, 44, 9, 89, 19, 108, 37, 60, 118, 39, 111, 110, 116, 76, 51, 47, 35, 30, 101, 71, 55, 12, 117, 64, 125, 72, 91, 115, 63, 0, 7, 59, 67, 100, 96, 114, 36, 65, 66, 56, 123, 6, 2, 122, 52, 46, 120, 32, 1, 54, 94, 106, 42, 8, 69, 68, 99, 5, 10, 4, 3, 74], [49, 41, 62, 105, 57, 113, 127, 48, 45, 109, 43, 107, 38, 40, 102, 93, 53, 88, 34, 104, 58, 24, 18, 82, 90, 20, 86, 112, 29, 98, 80, 85, 31, 97, 61, 21, 124, 84, 16, 119, 75, 95, 22, 92, 11, 77, 14, 103, 26, 81, 17, 126, 78, 87, 28, 121, 13, 25, 108, 39, 110, 118, 44, 9, 33, 27, 15, 50, 79, 89, 23, 37, 73, 70, 51, 111, 60, 63, 83, 19, 59, 125, 101, 116, 47, 55, 117, 56, 96, 94, 30, 36, 72, 35, 12, 115, 71, 91, 100, 64, 42, 76, 32, 0, 7, 123, 66, 120, 65, 122, 106, 46, 6, 67, 114, 54, 52, 1, 99, 2, 69, 3, 68, 4, 8, 10, 74, 5], [49, 62, 105, 41, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 93, 34, 88, 24, 86, 90, 18, 21, 58, 104, 82, 98, 20, 31, 80, 75, 124, 11, 29, 61, 95, 97, 22, 85, 92, 112, 84, 77, 16, 78, 13, 17, 119, 14, 28, 25, 81, 103, 121, 87, 15, 33, 27, 9, 26, 50, 19, 126, 83, 37, 111, 89, 79, 101, 73, 60, 116, 47, 118, 23, 63, 39, 72, 59, 12, 108, 76, 55, 51, 44, 70, 117, 96, 115, 110, 35, 30, 125, 0, 120, 64, 71, 7, 123, 56, 91, 114, 1, 36, 42, 6, 2, 54, 4, 94, 106, 100, 32, 52, 67, 122, 46, 65, 68, 99, 5, 69, 66, 3, 74, 8, 10], [49, 62, 41, 105, 57, 113, 48, 45, 127, 43, 109, 107, 38, 40, 53, 34, 102, 93, 24, 104, 88, 86, 18, 58, 82, 90, 21, 98, 80, 20, 11, 75, 124, 97, 22, 77, 31, 92, 16, 85, 13, 29, 84, 95, 112, 78, 61, 14, 81, 17, 119, 103, 121, 73, 15, 25, 9, 27, 126, 79, 83, 87, 26, 19, 23, 110, 28, 50, 89, 76, 116, 72, 12, 111, 47, 44, 33, 118, 115, 60, 51, 37, 108, 55, 6, 39, 101, 59, 70, 71, 64, 117, 0, 123, 125, 63, 7, 35, 120, 30, 52, 36, 2, 46, 1, 54, 66, 96, 69, 91, 65, 56, 114, 3, 122, 42, 32, 67, 68, 4, 99, 10, 100, 94, 74, 8, 106, 5], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 93, 34, 104, 88, 58, 24, 86, 90, 21, 18, 98, 82, 31, 97, 84, 20, 124, 95, 11, 85, 61, 22, 16, 80, 13, 29, 112, 92, 78, 75, 77, 14, 81, 28, 26, 17, 119, 121, 103, 15, 23, 126, 87, 27, 25, 110, 44, 19, 79, 9, 89, 73, 33, 111, 59, 115, 50, 83, 6, 37, 108, 12, 55, 76, 118, 47, 116, 91, 101, 35, 60, 96, 51, 63, 117, 39, 72, 64, 0, 71, 125, 123, 7, 52, 36, 42, 32, 114, 56, 122, 94, 120, 3, 30, 65, 46, 2, 70, 66, 1, 4, 100, 106, 68, 10, 8, 54, 99, 67, 69, 5, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 102, 40, 88, 93, 34, 104, 24, 18, 86, 21, 90, 82, 58, 98, 80, 61, 20, 31, 124, 22, 85, 11, 92, 84, 29, 112, 95, 75, 13, 97, 16, 77, 119, 78, 14, 26, 17, 81, 28, 126, 15, 103, 23, 87, 83, 9, 33, 110, 6, 27, 25, 121, 111, 51, 89, 44, 118, 50, 115, 73, 79, 60, 19, 12, 37, 39, 108, 55, 76, 47, 59, 63, 0, 116, 123, 35, 101, 91, 32, 64, 72, 36, 71, 96, 7, 120, 125, 65, 117, 100, 30, 114, 2, 56, 1, 122, 66, 42, 54, 3, 94, 8, 5, 46, 4, 67, 52, 99, 106, 68, 69, 70, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 109, 127, 43, 107, 38, 53, 40, 102, 34, 24, 88, 93, 104, 58, 18, 82, 86, 98, 90, 11, 31, 80, 21, 20, 85, 124, 75, 92, 77, 112, 97, 22, 16, 84, 95, 13, 29, 61, 78, 81, 14, 28, 119, 103, 26, 17, 25, 9, 6, 126, 87, 73, 111, 79, 23, 121, 83, 15, 19, 33, 27, 89, 50, 115, 37, 118, 47, 110, 116, 55, 108, 60, 76, 123, 12, 44, 59, 39, 63, 117, 7, 72, 35, 101, 51, 56, 64, 125, 96, 91, 71, 114, 8, 36, 120, 0, 30, 65, 2, 68, 32, 46, 66, 3, 99, 54, 4, 52, 70, 100, 94, 122, 1, 69, 5, 42, 106, 67, 74, 10], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 34, 93, 53, 88, 24, 18, 86, 104, 21, 90, 58, 82, 22, 20, 98, 16, 124, 61, 29, 31, 11, 97, 80, 112, 84, 75, 85, 92, 95, 77, 13, 28, 78, 119, 14, 103, 81, 26, 17, 25, 121, 73, 87, 23, 89, 9, 83, 27, 79, 12, 33, 108, 6, 126, 15, 50, 44, 19, 39, 111, 37, 118, 60, 76, 51, 110, 116, 117, 35, 55, 59, 101, 47, 7, 56, 63, 0, 123, 125, 115, 8, 122, 120, 64, 1, 96, 30, 70, 72, 2, 114, 71, 52, 91, 54, 46, 3, 36, 100, 32, 66, 42, 5, 67, 94, 106, 65, 99, 68, 69, 74, 4, 10], [49, 62, 41, 105, 57, 113, 48, 109, 127, 45, 43, 107, 38, 40, 34, 102, 88, 53, 93, 104, 24, 90, 18, 86, 22, 82, 58, 20, 21, 11, 31, 98, 92, 85, 112, 97, 75, 80, 16, 84, 124, 29, 61, 77, 78, 95, 13, 119, 28, 81, 14, 17, 26, 15, 25, 103, 19, 73, 23, 27, 83, 9, 89, 87, 79, 33, 126, 111, 50, 121, 60, 12, 51, 118, 37, 125, 110, 44, 70, 76, 55, 108, 6, 8, 115, 7, 39, 116, 35, 0, 101, 47, 63, 64, 59, 123, 56, 114, 117, 122, 120, 42, 71, 1, 30, 91, 96, 65, 100, 54, 52, 69, 2, 32, 67, 36, 106, 5, 94, 66, 3, 72, 46, 68, 10, 74, 4, 99], [49, 62, 41, 105, 57, 113, 48, 109, 127, 45, 43, 107, 38, 53, 102, 40, 88, 104, 24, 34, 93, 90, 18, 86, 82, 58, 98, 22, 124, 20, 29, 112, 21, 61, 85, 80, 92, 75, 31, 11, 16, 84, 77, 95, 78, 81, 97, 17, 126, 119, 13, 14, 25, 103, 28, 26, 87, 23, 83, 33, 27, 9, 73, 110, 121, 115, 79, 125, 15, 51, 89, 44, 12, 50, 108, 37, 55, 39, 60, 8, 111, 70, 47, 76, 35, 36, 19, 118, 116, 59, 117, 7, 101, 63, 6, 91, 0, 114, 100, 56, 65, 64, 42, 122, 66, 2, 32, 71, 94, 96, 67, 120, 123, 30, 54, 68, 1, 52, 69, 3, 46, 5, 106, 4, 10, 99, 72, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 40, 102, 88, 24, 34, 18, 104, 93, 82, 90, 86, 75, 58, 11, 80, 21, 98, 124, 20, 85, 16, 61, 31, 22, 77, 92, 78, 97, 84, 13, 112, 29, 95, 81, 17, 14, 119, 70, 9, 103, 28, 126, 83, 25, 26, 73, 64, 87, 79, 121, 23, 115, 15, 27, 19, 33, 8, 89, 76, 12, 7, 118, 44, 111, 51, 55, 0, 116, 50, 2, 110, 66, 101, 60, 59, 125, 108, 117, 47, 68, 67, 65, 37, 56, 6, 39, 71, 123, 63, 3, 1, 91, 5, 52, 35, 96, 122, 36, 114, 100, 30, 46, 4, 69, 54, 120, 94, 72, 10, 106, 32, 42, 74, 99], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 34, 93, 88, 58, 24, 104, 21, 124, 11, 86, 90, 18, 98, 75, 84, 82, 29, 61, 22, 31, 95, 92, 112, 80, 20, 85, 78, 97, 77, 16, 13, 26, 17, 14, 103, 119, 81, 23, 121, 25, 70, 15, 73, 28, 27, 9, 126, 8, 87, 89, 37, 19, 50, 12, 44, 47, 33, 51, 125, 83, 116, 115, 7, 110, 118, 60, 108, 79, 111, 59, 0, 39, 35, 55, 101, 76, 63, 64, 56, 91, 114, 2, 96, 1, 117, 30, 67, 100, 36, 71, 66, 52, 46, 122, 3, 123, 120, 4, 32, 69, 65, 54, 6, 5, 42, 94, 68, 99, 106, 72, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 40, 24, 53, 88, 93, 34, 18, 90, 82, 86, 11, 58, 75, 21, 20, 104, 98, 22, 16, 80, 124, 85, 31, 92, 29, 77, 13, 84, 97, 112, 78, 61, 95, 14, 17, 81, 19, 119, 25, 103, 9, 121, 26, 87, 73, 15, 28, 70, 126, 23, 79, 89, 37, 50, 33, 44, 8, 27, 12, 116, 111, 83, 76, 101, 60, 115, 0, 47, 118, 125, 55, 7, 59, 56, 117, 64, 51, 108, 66, 96, 39, 110, 65, 63, 2, 123, 35, 71, 94, 5, 114, 1, 91, 67, 30, 6, 3, 54, 46, 120, 122, 68, 69, 74, 72, 4, 100, 42, 32, 36, 99, 52, 106, 10], [49, 62, 41, 105, 113, 57, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 88, 93, 24, 58, 34, 82, 90, 18, 104, 124, 21, 20, 112, 11, 86, 85, 16, 22, 77, 97, 75, 98, 80, 92, 84, 95, 13, 61, 31, 14, 103, 29, 78, 17, 81, 119, 15, 28, 19, 23, 121, 115, 25, 9, 83, 126, 27, 73, 79, 70, 87, 26, 116, 37, 12, 44, 89, 33, 50, 59, 51, 39, 60, 108, 35, 111, 101, 118, 125, 76, 55, 7, 8, 110, 64, 6, 47, 117, 63, 114, 0, 56, 123, 91, 2, 96, 122, 46, 65, 69, 54, 71, 30, 72, 100, 1, 52, 66, 68, 36, 67, 3, 4, 42, 120, 94, 5, 32, 99, 106, 10, 74], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 88, 34, 24, 93, 82, 18, 90, 86, 75, 104, 21, 77, 11, 20, 58, 22, 85, 124, 16, 98, 31, 84, 13, 97, 95, 92, 112, 80, 61, 29, 14, 78, 17, 81, 28, 26, 119, 79, 87, 9, 15, 19, 115, 103, 73, 83, 126, 25, 12, 37, 23, 27, 33, 121, 50, 108, 89, 70, 76, 7, 118, 111, 8, 6, 44, 51, 60, 63, 116, 59, 47, 72, 39, 110, 71, 96, 0, 101, 123, 91, 55, 117, 35, 64, 68, 56, 125, 114, 2, 1, 100, 66, 46, 42, 36, 5, 122, 65, 67, 94, 10, 120, 30, 54, 4, 74, 52, 32, 69, 99, 3, 106]], "model.layers.13.self_attn.q_proj": [[115, 103, 62, 124, 126, 50, 93, 54, 90, 20, 89, 13, 122, 74, 83, 23, 127, 33, 7, 82, 61, 39, 48, 112, 80, 56, 22, 25, 2, 91, 92, 29, 31, 0, 53, 52, 67, 117, 116, 119, 51, 120, 77, 4, 65, 121, 6, 5, 87, 69, 125, 21, 15, 12, 100, 114, 64, 118, 60, 59, 66, 49, 3, 16, 95, 34, 55, 37, 28, 19, 111, 17, 18, 68, 58, 99, 85, 72, 63, 32, 57, 81, 44, 9, 70, 30, 84, 36, 88, 101, 96, 24, 113, 45, 11, 109, 35, 94, 102, 98, 26, 123, 38, 27, 14, 76, 78, 73, 8, 47, 107, 86, 43, 46, 41, 40, 42, 105, 110, 108, 79, 71, 75, 104, 97, 1, 106, 10], [115, 62, 103, 124, 50, 54, 126, 23, 74, 67, 61, 90, 122, 127, 56, 112, 7, 53, 93, 82, 6, 39, 77, 116, 48, 2, 68, 117, 52, 0, 89, 33, 121, 114, 51, 120, 85, 15, 31, 25, 119, 49, 125, 60, 11, 111, 22, 118, 55, 63, 59, 37, 69, 58, 113, 57, 95, 44, 109, 45, 9, 110, 46, 107, 38, 35, 43, 83, 13, 65, 123, 108, 102, 47, 20, 41, 34, 42, 104, 105, 106, 100, 92, 64, 5, 98, 73, 28, 40, 96, 101, 66, 32, 99, 79, 36, 87, 18, 72, 14, 30, 29, 88, 24, 94, 21, 10, 80, 26, 16, 19, 86, 4, 70, 84, 12, 3, 91, 81, 97, 27, 76, 1, 78, 17, 75, 8, 71], [62, 115, 103, 39, 124, 127, 126, 50, 90, 61, 54, 112, 122, 23, 11, 7, 15, 20, 82, 52, 74, 53, 89, 116, 51, 119, 6, 56, 14, 48, 0, 117, 68, 67, 79, 120, 84, 121, 125, 114, 59, 80, 55, 111, 100, 118, 49, 95, 60, 72, 22, 66, 57, 33, 63, 102, 58, 113, 2, 107, 73, 43, 109, 110, 93, 34, 37, 40, 101, 31, 46, 92, 45, 35, 13, 18, 44, 28, 106, 25, 98, 81, 38, 108, 96, 123, 47, 32, 69, 41, 42, 105, 104, 99, 87, 88, 16, 24, 94, 85, 77, 83, 30, 29, 91, 26, 8, 27, 9, 86, 75, 36, 70, 12, 76, 78, 65, 97, 5, 19, 64, 10, 17, 21, 71, 4, 1, 3], [115, 62, 103, 50, 126, 124, 54, 90, 61, 122, 127, 74, 112, 67, 82, 48, 56, 7, 39, 117, 77, 52, 83, 116, 114, 93, 2, 53, 120, 125, 51, 119, 121, 18, 92, 13, 6, 111, 0, 89, 49, 59, 60, 88, 109, 55, 63, 58, 118, 33, 45, 57, 23, 11, 108, 113, 47, 21, 43, 25, 102, 107, 40, 46, 65, 44, 69, 123, 66, 64, 87, 110, 37, 15, 42, 20, 105, 35, 104, 41, 106, 5, 100, 19, 101, 68, 98, 17, 73, 38, 91, 3, 95, 96, 29, 34, 9, 85, 36, 99, 10, 31, 32, 70, 4, 28, 22, 30, 12, 16, 94, 72, 26, 27, 81, 24, 1, 97, 84, 76, 86, 80, 14, 78, 71, 79, 8, 75], [123, 127, 60, 62, 38, 121, 53, 56, 61, 122, 110, 120, 59, 57, 54, 52, 44, 114, 119, 51, 125, 118, 116, 91, 117, 124, 126, 55, 115, 48, 58, 47, 113, 49, 90, 50, 111, 63, 102, 82, 46, 108, 109, 32, 112, 14, 104, 45, 29, 100, 84, 34, 103, 99, 107, 37, 86, 43, 97, 105, 39, 22, 42, 106, 36, 24, 40, 33, 89, 98, 41, 94, 101, 27, 30, 23, 75, 20, 88, 35, 96, 18, 25, 26, 95, 93, 80, 31, 78, 19, 28, 8, 9, 21, 87, 11, 81, 83, 64, 68, 85, 92, 13, 66, 17, 1, 12, 76, 77, 16, 79, 70, 71, 5, 15, 72, 3, 7, 10, 65, 6, 67, 73, 4, 0, 69, 2, 74], [127, 123, 38, 121, 62, 60, 37, 53, 56, 61, 122, 57, 120, 54, 116, 51, 59, 114, 52, 58, 44, 32, 119, 125, 36, 118, 124, 117, 49, 126, 115, 90, 110, 55, 108, 47, 50, 111, 113, 63, 89, 107, 82, 109, 91, 112, 106, 100, 48, 87, 14, 46, 84, 102, 42, 96, 45, 103, 34, 27, 25, 20, 22, 104, 40, 78, 8, 97, 94, 39, 31, 43, 95, 98, 29, 105, 101, 41, 30, 33, 99, 86, 1, 35, 75, 9, 81, 18, 26, 19, 93, 92, 24, 28, 5, 12, 68, 3, 71, 76, 64, 0, 79, 13, 88, 70, 21, 72, 23, 17, 85, 10, 15, 66, 16, 67, 83, 73, 7, 6, 80, 11, 77, 4, 2, 69, 65, 74], [123, 127, 62, 60, 121, 1, 53, 61, 56, 122, 57, 115, 120, 54, 52, 63, 51, 58, 59, 32, 114, 116, 124, 113, 125, 118, 119, 117, 49, 110, 126, 107, 55, 87, 37, 0, 111, 47, 50, 48, 112, 20, 109, 106, 104, 91, 108, 64, 46, 38, 44, 45, 67, 75, 66, 41, 42, 39, 43, 100, 40, 105, 78, 82, 103, 3, 36, 26, 71, 101, 34, 22, 68, 99, 18, 95, 72, 14, 5, 98, 88, 13, 33, 28, 79, 30, 94, 35, 102, 27, 96, 16, 97, 92, 9, 8, 93, 10, 76, 21, 17, 11, 6, 83, 2, 70, 15, 29, 23, 85, 84, 86, 31, 12, 24, 90, 25, 19, 73, 65, 7, 81, 89, 4, 69, 80, 77, 74], [127, 123, 38, 24, 83, 17, 28, 79, 90, 26, 10, 97, 13, 60, 31, 86, 104, 121, 84, 76, 4, 42, 7, 29, 62, 74, 77, 85, 40, 88, 19, 2, 53, 46, 70, 92, 15, 49, 81, 89, 57, 25, 20, 91, 9, 6, 5, 71, 64, 58, 101, 21, 116, 12, 56, 33, 18, 112, 78, 80, 99, 27, 82, 16, 87, 54, 23, 1, 65, 0, 95, 102, 14, 75, 93, 32, 51, 109, 69, 67, 115, 73, 52, 61, 94, 11, 108, 96, 30, 36, 22, 68, 59, 125, 48, 122, 72, 8, 37, 34, 55, 126, 114, 124, 66, 63, 50, 120, 43, 45, 117, 119, 103, 100, 118, 44, 3, 110, 113, 47, 98, 106, 35, 41, 107, 111, 105, 39], [121, 60, 39, 123, 120, 89, 61, 54, 49, 119, 65, 51, 122, 116, 62, 78, 127, 71, 126, 59, 112, 58, 117, 92, 110, 48, 118, 11, 50, 57, 124, 80, 53, 4, 55, 18, 27, 56, 44, 47, 68, 115, 125, 23, 63, 43, 111, 96, 69, 105, 46, 90, 99, 103, 52, 113, 106, 45, 109, 9, 114, 70, 107, 0, 101, 28, 84, 104, 91, 3, 42, 108, 73, 66, 38, 40, 82, 41, 102, 22, 95, 2, 93, 94, 36, 100, 25, 37, 10, 30, 34, 97, 98, 86, 79, 16, 33, 64, 8, 5, 29, 87, 19, 14, 20, 74, 31, 7, 35, 83, 15, 12, 72, 26, 81, 88, 17, 21, 32, 75, 77, 85, 24, 1, 67, 13, 76, 6], [60, 121, 39, 89, 123, 120, 54, 61, 116, 62, 49, 119, 23, 96, 18, 126, 84, 51, 80, 122, 117, 16, 55, 27, 127, 57, 124, 48, 9, 25, 95, 86, 92, 59, 58, 118, 28, 22, 112, 53, 50, 56, 47, 15, 65, 13, 99, 78, 69, 110, 63, 115, 52, 106, 7, 34, 94, 30, 103, 19, 111, 75, 82, 125, 105, 11, 97, 32, 44, 43, 114, 10, 46, 45, 108, 113, 21, 104, 71, 20, 77, 109, 107, 42, 38, 40, 85, 35, 66, 41, 37, 73, 93, 87, 81, 0, 91, 100, 36, 90, 31, 98, 12, 102, 33, 26, 29, 101, 70, 2, 68, 74, 83, 24, 79, 3, 17, 88, 14, 4, 67, 76, 72, 64, 5, 8, 6, 1], [121, 39, 60, 123, 120, 89, 54, 116, 23, 61, 84, 27, 28, 11, 92, 96, 51, 49, 18, 80, 99, 16, 122, 48, 62, 75, 126, 119, 86, 124, 127, 118, 58, 59, 117, 50, 55, 112, 78, 57, 22, 56, 30, 25, 47, 110, 53, 94, 95, 82, 115, 106, 111, 29, 7, 9, 44, 114, 45, 91, 15, 109, 90, 103, 46, 63, 113, 52, 33, 105, 2, 107, 4, 20, 42, 65, 102, 81, 71, 40, 104, 101, 87, 70, 43, 19, 108, 93, 125, 77, 85, 36, 37, 98, 35, 41, 79, 3, 97, 21, 6, 32, 31, 100, 83, 0, 38, 34, 68, 69, 26, 17, 14, 73, 88, 12, 64, 24, 67, 76, 13, 10, 8, 74, 72, 5, 66, 1], [60, 121, 39, 123, 54, 84, 120, 116, 89, 61, 49, 51, 18, 16, 92, 117, 99, 28, 122, 27, 126, 82, 112, 119, 62, 127, 103, 23, 124, 48, 96, 118, 58, 59, 57, 94, 55, 53, 86, 50, 9, 88, 47, 95, 56, 52, 110, 11, 25, 115, 22, 65, 30, 71, 69, 15, 42, 114, 111, 75, 45, 46, 125, 33, 106, 102, 63, 113, 43, 78, 91, 107, 90, 101, 104, 34, 41, 44, 77, 109, 40, 38, 87, 105, 0, 13, 36, 29, 85, 31, 108, 26, 20, 7, 98, 68, 14, 37, 93, 97, 100, 24, 19, 66, 32, 35, 81, 21, 80, 17, 83, 70, 2, 79, 76, 12, 74, 73, 8, 64, 5, 72, 67, 3, 4, 10, 6, 1], [51, 118, 102, 48, 90, 24, 97, 53, 126, 61, 58, 117, 18, 121, 29, 62, 63, 54, 123, 86, 50, 19, 9, 80, 124, 26, 47, 127, 111, 93, 120, 112, 125, 38, 115, 23, 59, 116, 119, 27, 60, 91, 73, 17, 46, 55, 78, 108, 20, 113, 12, 104, 103, 114, 52, 101, 57, 71, 45, 56, 39, 89, 110, 82, 11, 87, 88, 30, 107, 36, 5, 77, 109, 66, 49, 42, 13, 44, 99, 16, 14, 33, 43, 41, 98, 83, 100, 28, 74, 122, 94, 105, 31, 34, 40, 25, 106, 37, 75, 32, 35, 96, 21, 81, 95, 85, 22, 84, 92, 15, 72, 3, 7, 68, 70, 79, 76, 64, 65, 69, 2, 1, 0, 8, 10, 4, 6, 67], [118, 51, 102, 48, 61, 126, 58, 123, 53, 124, 127, 120, 62, 117, 63, 90, 54, 60, 19, 125, 121, 119, 97, 116, 59, 41, 113, 50, 3, 56, 52, 55, 29, 112, 47, 73, 57, 0, 86, 115, 38, 93, 78, 24, 26, 111, 92, 81, 122, 46, 9, 5, 1, 109, 114, 43, 69, 87, 22, 89, 103, 94, 30, 45, 110, 108, 49, 7, 44, 66, 105, 39, 98, 107, 99, 104, 106, 91, 42, 83, 28, 2, 75, 14, 18, 4, 40, 25, 76, 37, 100, 6, 84, 65, 12, 32, 35, 96, 17, 16, 33, 82, 101, 67, 21, 13, 36, 34, 15, 8, 85, 77, 11, 64, 31, 20, 88, 80, 79, 95, 71, 23, 10, 74, 27, 70, 72, 68], [51, 102, 118, 48, 24, 97, 53, 90, 19, 58, 126, 47, 63, 61, 86, 81, 38, 18, 121, 117, 93, 123, 29, 62, 60, 124, 80, 9, 50, 99, 89, 59, 112, 54, 127, 120, 45, 94, 125, 108, 119, 13, 75, 100, 11, 39, 57, 111, 113, 114, 91, 104, 88, 77, 115, 26, 116, 3, 5, 109, 73, 36, 56, 37, 98, 49, 52, 110, 103, 78, 105, 107, 42, 55, 28, 66, 41, 122, 46, 30, 71, 44, 40, 83, 35, 43, 31, 34, 27, 95, 82, 22, 87, 17, 106, 85, 92, 20, 32, 101, 23, 21, 84, 69, 15, 96, 25, 16, 7, 74, 33, 14, 72, 0, 70, 67, 10, 1, 79, 64, 12, 8, 76, 65, 68, 6, 4, 2], [51, 102, 118, 48, 24, 97, 61, 86, 53, 29, 126, 19, 90, 63, 124, 115, 78, 75, 18, 58, 80, 81, 50, 123, 93, 66, 87, 62, 89, 30, 77, 39, 117, 16, 34, 99, 54, 9, 88, 32, 47, 116, 91, 45, 120, 37, 36, 125, 96, 83, 28, 111, 31, 44, 127, 3, 59, 100, 26, 121, 11, 95, 60, 103, 38, 52, 119, 23, 109, 71, 35, 112, 92, 17, 113, 5, 94, 20, 105, 27, 110, 57, 98, 101, 106, 56, 74, 114, 41, 25, 107, 70, 108, 104, 69, 46, 33, 0, 82, 55, 21, 1, 14, 72, 49, 40, 10, 85, 43, 64, 13, 73, 122, 42, 22, 7, 15, 84, 6, 68, 12, 79, 4, 76, 8, 65, 67, 2], [47, 125, 111, 56, 14, 124, 60, 24, 120, 127, 117, 54, 118, 49, 123, 5, 57, 91, 61, 75, 2, 21, 96, 59, 122, 1, 62, 94, 55, 7, 126, 63, 26, 58, 50, 121, 48, 103, 39, 53, 9, 3, 119, 64, 116, 104, 101, 98, 40, 113, 23, 4, 20, 100, 83, 42, 114, 52, 115, 112, 37, 36, 13, 82, 18, 97, 10, 17, 85, 72, 108, 44, 22, 110, 51, 89, 73, 46, 25, 27, 6, 43, 35, 19, 78, 29, 15, 107, 41, 71, 31, 45, 81, 32, 16, 79, 106, 105, 8, 28, 34, 99, 93, 109, 87, 74, 33, 88, 102, 70, 38, 92, 95, 76, 86, 80, 11, 77, 90, 30, 65, 67, 84, 0, 12, 69, 66, 68], [47, 111, 125, 56, 98, 26, 124, 2, 5, 60, 117, 103, 85, 64, 39, 36, 61, 54, 120, 127, 92, 37, 123, 49, 4, 87, 91, 97, 126, 122, 118, 1, 59, 57, 62, 20, 14, 6, 89, 48, 7, 3, 55, 24, 9, 50, 63, 96, 121, 113, 53, 65, 29, 58, 75, 13, 27, 114, 51, 101, 10, 41, 52, 81, 83, 84, 116, 94, 35, 42, 71, 72, 46, 119, 38, 112, 44, 8, 22, 115, 106, 28, 90, 100, 16, 31, 105, 99, 107, 43, 34, 109, 0, 23, 45, 95, 110, 11, 104, 108, 68, 76, 73, 86, 102, 25, 40, 32, 79, 15, 17, 66, 21, 33, 74, 80, 30, 19, 82, 93, 67, 70, 69, 77, 18, 88, 78, 12], [125, 47, 111, 56, 60, 124, 14, 39, 117, 26, 127, 118, 120, 54, 61, 49, 123, 5, 57, 103, 126, 62, 94, 59, 122, 81, 55, 63, 98, 50, 48, 23, 121, 19, 53, 20, 46, 58, 113, 116, 76, 114, 2, 8, 1, 119, 97, 112, 64, 4, 10, 75, 115, 107, 51, 52, 38, 44, 21, 72, 87, 65, 73, 16, 104, 100, 41, 3, 90, 42, 43, 45, 71, 96, 108, 110, 106, 105, 37, 99, 86, 6, 109, 17, 24, 36, 40, 22, 79, 80, 91, 28, 9, 7, 83, 101, 85, 13, 74, 34, 102, 33, 92, 18, 35, 82, 27, 95, 31, 29, 11, 78, 84, 88, 68, 30, 0, 25, 89, 77, 12, 32, 69, 93, 15, 70, 66, 67], [47, 125, 111, 36, 56, 24, 96, 124, 91, 75, 25, 120, 100, 7, 60, 49, 83, 29, 16, 99, 84, 21, 2, 54, 94, 1, 14, 127, 117, 5, 61, 57, 80, 26, 103, 40, 118, 123, 22, 97, 64, 33, 4, 46, 35, 44, 48, 59, 110, 85, 126, 62, 104, 13, 122, 9, 27, 28, 58, 89, 3, 20, 6, 31, 116, 90, 77, 53, 108, 15, 43, 93, 121, 101, 42, 39, 119, 87, 50, 55, 17, 79, 63, 88, 98, 67, 112, 95, 82, 106, 18, 92, 23, 113, 41, 34, 30, 52, 37, 38, 102, 114, 19, 71, 32, 45, 115, 105, 74, 109, 51, 107, 11, 70, 76, 78, 8, 10, 68, 12, 65, 72, 73, 81, 86, 0, 69, 66], [123, 63, 39, 114, 57, 115, 61, 127, 121, 124, 59, 62, 120, 54, 49, 93, 116, 125, 110, 113, 97, 50, 25, 53, 58, 112, 56, 55, 118, 51, 117, 92, 52, 80, 126, 60, 42, 122, 45, 48, 22, 111, 44, 28, 108, 119, 102, 46, 107, 43, 95, 41, 40, 47, 26, 103, 100, 88, 106, 19, 37, 104, 105, 34, 109, 35, 96, 17, 86, 31, 87, 36, 99, 32, 101, 94, 38, 24, 9, 29, 6, 91, 16, 3, 98, 76, 12, 72, 30, 90, 78, 33, 83, 73, 27, 68, 0, 67, 23, 18, 81, 20, 89, 85, 77, 15, 65, 2, 1, 5, 84, 79, 21, 69, 82, 11, 8, 14, 75, 70, 13, 4, 74, 66, 71, 10, 64, 7], [63, 123, 114, 39, 57, 115, 61, 59, 127, 124, 121, 62, 54, 125, 120, 116, 113, 49, 60, 117, 58, 53, 122, 88, 56, 50, 52, 48, 19, 55, 126, 51, 28, 118, 112, 119, 43, 42, 93, 45, 97, 38, 110, 46, 6, 0, 41, 85, 104, 95, 86, 102, 107, 111, 92, 47, 87, 105, 16, 106, 103, 109, 44, 76, 25, 37, 108, 81, 40, 34, 96, 35, 73, 80, 15, 101, 22, 99, 100, 36, 26, 67, 3, 83, 94, 32, 29, 90, 98, 23, 69, 24, 33, 17, 27, 30, 72, 31, 91, 5, 2, 12, 78, 65, 84, 70, 79, 9, 89, 75, 1, 20, 4, 68, 21, 14, 18, 77, 82, 74, 11, 8, 64, 10, 13, 66, 71, 7], [39, 63, 123, 25, 82, 97, 87, 13, 115, 22, 19, 114, 74, 80, 70, 57, 6, 77, 3, 66, 83, 7, 18, 64, 84, 91, 31, 4, 12, 94, 29, 27, 10, 9, 79, 93, 92, 16, 95, 24, 75, 30, 21, 90, 86, 5, 59, 85, 81, 76, 58, 104, 78, 23, 15, 20, 89, 17, 1, 127, 71, 120, 14, 73, 61, 54, 8, 88, 124, 62, 113, 116, 43, 72, 68, 109, 49, 96, 118, 99, 32, 2, 65, 28, 11, 55, 100, 34, 35, 98, 121, 48, 107, 37, 69, 45, 110, 40, 26, 41, 50, 51, 53, 122, 67, 102, 42, 126, 0, 60, 125, 46, 56, 105, 52, 36, 38, 119, 111, 112, 117, 33, 101, 108, 44, 103, 106, 47], [63, 123, 39, 115, 57, 28, 114, 80, 97, 19, 25, 22, 127, 61, 124, 87, 9, 62, 59, 96, 88, 120, 92, 58, 54, 113, 116, 104, 121, 49, 111, 3, 12, 125, 117, 53, 51, 24, 107, 95, 20, 60, 50, 31, 126, 52, 48, 15, 56, 72, 21, 77, 112, 55, 34, 122, 41, 46, 82, 118, 40, 119, 13, 42, 94, 102, 43, 37, 30, 45, 110, 99, 23, 16, 83, 93, 84, 70, 85, 29, 90, 100, 27, 91, 44, 105, 109, 78, 38, 108, 65, 68, 106, 98, 26, 35, 47, 36, 86, 76, 81, 17, 33, 8, 101, 5, 14, 79, 32, 74, 2, 89, 73, 64, 1, 7, 75, 18, 4, 103, 6, 10, 0, 69, 67, 11, 71, 66], [38, 47, 62, 111, 93, 83, 16, 26, 33, 76, 24, 9, 23, 86, 78, 92, 81, 118, 90, 68, 1, 70, 4, 34, 73, 74, 80, 29, 84, 20, 12, 87, 14, 19, 101, 2, 77, 75, 21, 71, 51, 64, 22, 85, 15, 63, 5, 10, 91, 66, 112, 95, 17, 11, 28, 82, 67, 88, 98, 36, 27, 13, 79, 7, 69, 65, 0, 53, 32, 30, 119, 18, 89, 56, 31, 52, 6, 3, 100, 8, 25, 72, 120, 104, 61, 122, 123, 114, 102, 96, 127, 110, 117, 49, 124, 55, 94, 57, 48, 115, 60, 109, 116, 50, 99, 35, 121, 106, 59, 58, 126, 46, 42, 40, 97, 125, 54, 105, 103, 44, 37, 39, 108, 113, 45, 41, 43, 107], [38, 47, 62, 111, 33, 24, 83, 93, 79, 81, 76, 100, 86, 118, 78, 29, 67, 26, 28, 16, 74, 6, 112, 32, 87, 90, 84, 119, 53, 92, 77, 34, 63, 18, 61, 120, 70, 85, 122, 56, 71, 114, 52, 13, 25, 98, 82, 23, 3, 88, 27, 19, 75, 51, 2, 20, 127, 95, 123, 22, 9, 60, 55, 121, 99, 117, 10, 17, 5, 49, 80, 115, 48, 94, 124, 0, 1, 50, 58, 113, 91, 4, 57, 89, 30, 8, 21, 15, 31, 12, 125, 68, 101, 116, 110, 14, 104, 106, 107, 59, 11, 96, 126, 73, 105, 72, 54, 37, 46, 35, 40, 7, 36, 108, 102, 45, 43, 109, 44, 66, 39, 41, 42, 103, 97, 65, 69, 64], [62, 111, 47, 38, 118, 120, 51, 63, 56, 53, 119, 100, 117, 58, 48, 122, 55, 124, 60, 123, 113, 52, 108, 116, 115, 127, 57, 112, 114, 121, 61, 49, 50, 126, 54, 18, 104, 59, 91, 125, 106, 109, 107, 110, 46, 27, 101, 28, 41, 84, 45, 44, 95, 35, 37, 105, 40, 43, 42, 103, 96, 92, 99, 36, 39, 24, 98, 33, 32, 87, 23, 21, 34, 31, 30, 93, 86, 94, 29, 89, 26, 97, 102, 82, 25, 20, 22, 90, 88, 81, 85, 79, 83, 15, 13, 77, 17, 8, 74, 67, 10, 72, 78, 64, 16, 3, 75, 19, 7, 76, 70, 69, 71, 11, 80, 14, 6, 5, 66, 0, 12, 4, 2, 65, 1, 9, 68, 73], [62, 111, 47, 118, 120, 100, 38, 56, 119, 52, 53, 112, 58, 63, 48, 57, 123, 113, 122, 55, 125, 117, 60, 115, 54, 51, 121, 114, 50, 124, 127, 61, 49, 126, 59, 116, 109, 110, 107, 36, 46, 104, 105, 106, 44, 108, 28, 45, 43, 27, 42, 41, 35, 103, 39, 91, 40, 82, 101, 84, 95, 33, 34, 92, 23, 37, 99, 102, 31, 18, 98, 93, 87, 32, 21, 30, 29, 96, 94, 86, 24, 97, 26, 22, 25, 89, 90, 67, 81, 20, 79, 6, 85, 10, 15, 12, 88, 8, 64, 72, 78, 70, 83, 19, 14, 17, 13, 66, 3, 7, 69, 77, 74, 11, 80, 16, 65, 75, 76, 0, 73, 71, 5, 2, 4, 68, 9, 1], [42, 43, 33, 72, 38, 1, 5, 91, 113, 106, 0, 21, 114, 12, 79, 24, 30, 67, 81, 19, 119, 61, 46, 124, 82, 49, 2, 107, 74, 53, 121, 122, 69, 66, 75, 71, 13, 25, 47, 112, 28, 126, 86, 77, 94, 87, 125, 45, 4, 117, 23, 3, 123, 70, 88, 50, 20, 80, 48, 65, 68, 89, 76, 63, 101, 100, 116, 8, 15, 103, 59, 27, 17, 110, 99, 7, 60, 14, 118, 11, 62, 104, 41, 64, 58, 31, 6, 115, 102, 57, 52, 92, 37, 127, 109, 51, 9, 78, 32, 22, 111, 90, 83, 56, 54, 16, 73, 26, 35, 39, 96, 55, 105, 95, 85, 84, 120, 10, 97, 98, 18, 44, 29, 36, 40, 108, 93, 34], [42, 38, 91, 33, 117, 43, 21, 82, 79, 74, 24, 107, 12, 20, 78, 10, 70, 14, 121, 89, 112, 30, 119, 81, 106, 27, 86, 19, 18, 84, 8, 88, 92, 28, 114, 6, 3, 113, 16, 94, 61, 67, 15, 87, 64, 90, 122, 80, 96, 5, 22, 23, 123, 57, 69, 83, 9, 76, 85, 25, 37, 102, 56, 40, 72, 75, 36, 93, 73, 51, 45, 13, 29, 116, 39, 32, 65, 63, 17, 77, 4, 46, 41, 100, 71, 26, 47, 35, 31, 95, 53, 124, 101, 11, 62, 105, 127, 125, 49, 7, 110, 34, 68, 99, 52, 98, 50, 126, 103, 66, 109, 59, 104, 60, 2, 48, 58, 118, 115, 44, 111, 54, 120, 55, 1, 108, 97, 0], [117, 43, 38, 42, 113, 33, 91, 107, 30, 114, 47, 123, 41, 46, 125, 62, 24, 26, 112, 39, 106, 59, 111, 60, 53, 116, 18, 115, 40, 50, 63, 87, 56, 118, 121, 119, 80, 108, 57, 89, 48, 124, 110, 51, 86, 61, 32, 109, 45, 127, 82, 52, 54, 126, 120, 55, 31, 58, 49, 28, 34, 44, 36, 81, 101, 104, 78, 95, 122, 94, 35, 96, 98, 88, 99, 19, 14, 21, 105, 37, 84, 93, 100, 29, 90, 103, 25, 20, 79, 11, 97, 23, 22, 16, 77, 27, 92, 13, 73, 83, 102, 85, 74, 17, 9, 70, 12, 76, 75, 2, 66, 8, 67, 6, 4, 71, 72, 7, 5, 0, 68, 15, 3, 10, 1, 64, 65, 69], [43, 38, 107, 33, 114, 91, 46, 47, 26, 24, 42, 117, 62, 30, 53, 56, 123, 119, 125, 39, 116, 111, 41, 59, 82, 115, 52, 90, 113, 48, 84, 118, 49, 55, 126, 50, 124, 51, 112, 122, 60, 57, 106, 121, 105, 63, 109, 120, 61, 110, 93, 44, 45, 127, 40, 54, 77, 108, 101, 34, 58, 35, 27, 88, 18, 100, 20, 37, 32, 104, 99, 98, 103, 29, 36, 31, 25, 81, 94, 86, 96, 102, 95, 28, 89, 87, 78, 22, 92, 14, 23, 76, 80, 17, 97, 75, 83, 19, 21, 16, 13, 12, 8, 79, 73, 11, 9, 6, 70, 4, 71, 67, 74, 5, 66, 72, 68, 15, 7, 0, 3, 64, 1, 69, 10, 2, 85, 65]], "model.layers.13.self_attn.k_proj": [[115, 62, 39, 124, 127, 50, 36, 61, 54, 22, 122, 112, 126, 97, 116, 52, 53, 56, 117, 51, 121, 48, 120, 119, 49, 114, 125, 55, 118, 111, 100, 57, 60, 59, 29, 58, 63, 89, 113, 91, 46, 107, 92, 109, 47, 44, 123, 45, 110, 42, 23, 105, 95, 43, 33, 108, 106, 41, 40, 37, 82, 104, 99, 101, 38, 30, 102, 20, 87, 19, 80, 78, 31, 103, 90, 88, 98, 34, 94, 35, 24, 96, 21, 1, 81, 79, 28, 32, 8, 75, 93, 26, 86, 73, 27, 76, 17, 13, 69, 10, 16, 71, 83, 14, 12, 68, 25, 84, 85, 11, 67, 15, 5, 18, 6, 66, 70, 74, 77, 4, 72, 9, 0, 7, 3, 2, 64, 65], [127, 123, 102, 86, 35, 60, 121, 58, 28, 63, 31, 51, 57, 83, 48, 99, 115, 110, 53, 24, 108, 46, 56, 17, 62, 79, 61, 125, 49, 52, 33, 116, 122, 50, 117, 114, 90, 112, 84, 54, 42, 55, 126, 40, 25, 59, 124, 29, 119, 107, 47, 44, 45, 120, 113, 109, 43, 16, 97, 111, 104, 118, 106, 34, 13, 41, 105, 76, 10, 39, 103, 38, 101, 37, 98, 14, 100, 91, 36, 32, 82, 95, 72, 96, 71, 68, 21, 81, 94, 22, 93, 30, 87, 19, 9, 20, 80, 77, 23, 92, 26, 88, 67, 11, 1, 66, 85, 89, 27, 15, 73, 12, 74, 18, 70, 75, 5, 69, 78, 6, 2, 0, 7, 8, 4, 64, 3, 65], [121, 60, 103, 123, 86, 54, 49, 35, 120, 61, 116, 32, 51, 119, 62, 122, 127, 55, 126, 118, 53, 28, 57, 59, 48, 124, 112, 50, 58, 115, 56, 110, 47, 99, 117, 63, 42, 44, 111, 114, 46, 125, 45, 52, 109, 106, 22, 113, 100, 105, 107, 30, 43, 23, 77, 108, 41, 40, 25, 84, 18, 101, 80, 96, 102, 104, 37, 91, 89, 98, 94, 90, 75, 38, 31, 36, 19, 97, 33, 15, 29, 93, 5, 16, 95, 34, 26, 88, 21, 81, 12, 17, 87, 3, 92, 13, 24, 76, 78, 6, 79, 27, 39, 85, 64, 66, 73, 8, 0, 9, 70, 83, 11, 82, 72, 74, 20, 71, 10, 14, 4, 67, 2, 7, 1, 65, 68, 69], [51, 118, 38, 86, 33, 48, 61, 53, 93, 123, 54, 120, 58, 124, 125, 117, 62, 126, 127, 59, 60, 115, 63, 47, 121, 119, 116, 55, 56, 50, 52, 57, 24, 113, 41, 112, 99, 26, 46, 91, 122, 81, 39, 104, 43, 103, 18, 111, 80, 45, 42, 11, 114, 109, 40, 19, 49, 110, 44, 105, 101, 107, 106, 20, 108, 79, 77, 36, 102, 89, 30, 90, 84, 97, 34, 21, 25, 35, 78, 27, 37, 15, 28, 6, 95, 4, 100, 98, 23, 67, 32, 88, 17, 96, 8, 9, 74, 16, 82, 72, 1, 22, 92, 31, 94, 75, 85, 87, 12, 10, 14, 65, 5, 76, 13, 64, 29, 69, 71, 70, 7, 2, 83, 66, 0, 73, 68, 3], [111, 125, 100, 47, 22, 120, 127, 118, 56, 54, 123, 60, 124, 59, 62, 117, 61, 57, 55, 122, 53, 63, 48, 126, 29, 121, 50, 49, 32, 108, 107, 119, 116, 42, 58, 113, 114, 36, 43, 115, 52, 105, 112, 51, 40, 39, 106, 41, 44, 46, 91, 110, 24, 103, 83, 45, 102, 104, 109, 25, 80, 26, 38, 79, 84, 99, 97, 92, 101, 35, 72, 37, 95, 76, 81, 18, 30, 34, 98, 33, 10, 31, 90, 4, 93, 21, 78, 77, 11, 89, 88, 86, 17, 20, 28, 71, 85, 94, 0, 82, 96, 16, 66, 13, 3, 12, 23, 27, 15, 73, 87, 65, 14, 70, 74, 6, 19, 2, 69, 75, 5, 9, 64, 7, 1, 8, 67, 68], [103, 123, 63, 22, 57, 115, 33, 114, 54, 13, 124, 127, 82, 25, 61, 59, 62, 28, 116, 120, 87, 121, 107, 125, 58, 27, 53, 60, 49, 55, 117, 80, 84, 56, 126, 113, 19, 74, 51, 118, 50, 48, 45, 109, 38, 99, 71, 119, 52, 7, 111, 97, 26, 122, 47, 12, 46, 37, 9, 30, 112, 93, 101, 106, 108, 31, 110, 85, 44, 104, 14, 42, 29, 105, 41, 100, 75, 98, 43, 102, 70, 89, 95, 86, 2, 17, 34, 15, 4, 79, 35, 32, 36, 20, 68, 1, 67, 40, 96, 0, 88, 18, 81, 94, 5, 73, 91, 11, 21, 72, 65, 10, 24, 3, 90, 23, 77, 64, 66, 78, 8, 76, 83, 39, 92, 6, 69, 16], [111, 62, 102, 86, 97, 118, 26, 29, 47, 83, 16, 81, 76, 78, 112, 9, 24, 119, 75, 53, 127, 122, 123, 114, 51, 55, 120, 96, 52, 63, 56, 50, 4, 71, 60, 124, 117, 121, 23, 61, 115, 116, 58, 49, 79, 74, 54, 36, 126, 57, 41, 59, 84, 42, 106, 46, 48, 35, 70, 110, 125, 37, 66, 105, 103, 109, 108, 45, 98, 40, 34, 43, 107, 65, 113, 77, 69, 1, 64, 44, 28, 99, 104, 93, 39, 101, 32, 3, 30, 21, 31, 38, 91, 80, 72, 11, 25, 100, 94, 7, 95, 89, 92, 5, 33, 0, 73, 27, 82, 15, 14, 13, 12, 20, 85, 17, 19, 88, 90, 22, 18, 2, 87, 67, 10, 8, 6, 68], [106, 107, 97, 102, 122, 86, 94, 121, 117, 49, 21, 50, 36, 48, 114, 42, 79, 103, 119, 27, 63, 113, 74, 92, 126, 116, 56, 61, 109, 91, 64, 123, 70, 62, 67, 127, 51, 24, 52, 105, 40, 19, 111, 57, 5, 12, 82, 112, 110, 59, 47, 45, 77, 44, 65, 18, 53, 115, 124, 81, 84, 60, 118, 125, 99, 98, 32, 88, 55, 54, 104, 71, 90, 66, 58, 37, 72, 78, 26, 8, 108, 120, 35, 101, 4, 46, 100, 34, 96, 1, 89, 11, 80, 93, 31, 95, 43, 23, 39, 25, 29, 41, 85, 87, 75, 9, 13, 0, 38, 15, 10, 14, 73, 28, 83, 30, 68, 22, 17, 33, 7, 76, 16, 20, 2, 6, 69, 3]], "model.layers.13.self_attn.qk_proj": [[123, 62, 111, 127, 121, 60, 51, 118, 115, 63, 47, 125, 53, 124, 61, 120, 106, 56, 57, 117, 59, 48, 116, 114, 119, 107, 122, 54, 50, 126, 42, 52, 102, 22, 38, 49, 58, 112, 86, 103, 113, 39, 55, 43, 110, 33, 91, 97, 100, 46, 88, 93, 92, 45, 108, 90, 44, 19, 24, 109, 27, 29, 36, 89, 83, 99, 82, 105, 18, 104, 26, 21, 16, 25, 40, 30, 32, 35, 80, 28, 87, 17, 41, 84, 12, 77, 101, 15, 20, 79, 23, 74, 85, 37, 76, 13, 94, 81, 98, 96, 78, 14, 95, 31, 75, 10, 9, 34, 73, 65, 70, 4, 11, 64, 3, 6, 7, 8, 68, 71, 5, 0, 72, 67, 69, 2, 66, 1], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 53, 61, 56, 59, 54, 117, 57, 124, 106, 107, 120, 48, 114, 116, 119, 50, 102, 42, 122, 22, 126, 38, 58, 49, 52, 112, 103, 113, 86, 39, 43, 55, 110, 44, 93, 100, 33, 97, 90, 46, 92, 91, 45, 108, 88, 27, 36, 89, 40, 24, 29, 26, 105, 19, 99, 82, 35, 83, 25, 16, 109, 30, 32, 21, 80, 104, 37, 84, 28, 87, 18, 41, 20, 12, 85, 15, 76, 96, 17, 23, 31, 94, 81, 101, 77, 74, 78, 98, 10, 34, 75, 13, 70, 14, 95, 9, 79, 11, 71, 4, 73, 3, 7, 1, 0, 69, 65, 72, 64, 8, 5, 68, 2, 6, 67, 66], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 61, 56, 117, 124, 53, 59, 106, 57, 120, 54, 48, 114, 42, 107, 50, 126, 116, 119, 122, 38, 102, 103, 112, 49, 113, 39, 58, 52, 22, 86, 55, 43, 33, 93, 97, 110, 91, 100, 46, 108, 45, 44, 92, 88, 90, 105, 27, 36, 29, 99, 40, 24, 109, 104, 32, 26, 89, 35, 83, 25, 41, 28, 37, 85, 30, 87, 101, 21, 16, 19, 18, 84, 98, 82, 23, 80, 94, 20, 79, 12, 34, 77, 31, 17, 96, 15, 95, 70, 74, 76, 10, 81, 14, 78, 75, 13, 9, 72, 64, 65, 1, 69, 0, 11, 71, 3, 67, 73, 66, 2, 4, 5, 68, 7, 8, 6], [123, 62, 111, 127, 121, 51, 118, 63, 60, 115, 47, 125, 61, 56, 53, 54, 124, 117, 106, 120, 126, 116, 57, 119, 59, 107, 58, 122, 38, 48, 49, 42, 50, 114, 102, 103, 112, 52, 22, 55, 43, 113, 39, 86, 110, 33, 44, 46, 92, 97, 93, 91, 88, 100, 45, 90, 40, 36, 29, 27, 108, 26, 89, 24, 32, 109, 83, 99, 105, 28, 35, 82, 19, 41, 87, 25, 37, 16, 18, 85, 23, 21, 17, 104, 79, 30, 31, 101, 12, 80, 34, 96, 20, 76, 77, 84, 98, 94, 70, 74, 81, 95, 14, 10, 75, 13, 15, 9, 78, 1, 64, 71, 3, 72, 68, 0, 11, 65, 67, 66, 73, 7, 69, 4, 5, 2, 6, 8], [123, 111, 62, 127, 121, 51, 115, 118, 63, 60, 47, 125, 61, 56, 53, 106, 124, 54, 57, 126, 117, 116, 114, 48, 119, 59, 50, 122, 42, 107, 120, 58, 102, 38, 52, 112, 49, 103, 55, 22, 113, 86, 39, 110, 44, 43, 46, 33, 91, 93, 92, 97, 88, 100, 90, 108, 24, 99, 36, 45, 27, 29, 40, 83, 89, 105, 19, 87, 26, 25, 28, 35, 32, 104, 109, 82, 41, 18, 15, 77, 16, 17, 12, 21, 76, 80, 23, 85, 81, 79, 94, 101, 20, 84, 10, 78, 74, 96, 30, 70, 98, 13, 95, 31, 37, 9, 75, 34, 14, 64, 7, 11, 68, 65, 72, 71, 73, 1, 66, 69, 3, 0, 4, 67, 6, 2, 5, 8], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 53, 61, 56, 124, 106, 117, 54, 126, 116, 57, 119, 107, 59, 120, 122, 58, 42, 50, 55, 102, 38, 22, 48, 114, 112, 86, 103, 39, 52, 49, 113, 43, 44, 110, 46, 91, 88, 93, 92, 97, 33, 90, 100, 29, 83, 108, 45, 36, 104, 19, 27, 26, 24, 99, 82, 16, 105, 87, 89, 80, 17, 18, 109, 28, 41, 40, 25, 35, 76, 79, 21, 13, 77, 84, 23, 32, 81, 30, 15, 20, 12, 74, 10, 14, 34, 101, 78, 75, 96, 9, 98, 85, 31, 94, 37, 11, 7, 70, 95, 71, 72, 6, 73, 0, 65, 3, 67, 1, 66, 4, 69, 5, 68, 64, 8, 2], [123, 111, 62, 127, 121, 51, 115, 60, 63, 118, 47, 125, 61, 124, 53, 56, 126, 54, 117, 106, 122, 59, 107, 120, 119, 57, 116, 58, 114, 49, 42, 102, 48, 103, 38, 50, 55, 22, 112, 86, 39, 52, 43, 113, 88, 44, 46, 93, 110, 91, 92, 90, 97, 100, 45, 19, 33, 83, 24, 108, 27, 99, 40, 89, 36, 104, 29, 105, 82, 18, 26, 21, 80, 17, 87, 109, 35, 85, 76, 28, 25, 23, 16, 79, 77, 12, 32, 84, 37, 15, 30, 41, 94, 20, 13, 81, 78, 96, 10, 98, 14, 31, 34, 74, 9, 101, 75, 95, 72, 7, 73, 11, 6, 70, 71, 69, 67, 0, 5, 65, 1, 66, 64, 3, 8, 68, 4, 2], [123, 62, 111, 127, 121, 51, 115, 60, 63, 47, 118, 125, 61, 124, 53, 56, 106, 54, 117, 120, 48, 126, 114, 107, 122, 59, 57, 119, 103, 116, 42, 38, 102, 58, 22, 49, 86, 50, 39, 55, 112, 52, 113, 43, 44, 91, 93, 110, 90, 97, 92, 100, 33, 88, 46, 27, 24, 29, 83, 19, 89, 36, 108, 35, 40, 104, 45, 26, 87, 16, 28, 32, 23, 82, 105, 99, 109, 80, 21, 25, 18, 84, 85, 41, 37, 76, 17, 31, 30, 15, 13, 12, 96, 20, 101, 77, 81, 14, 78, 94, 74, 98, 79, 10, 9, 34, 95, 75, 6, 72, 11, 73, 1, 0, 65, 64, 7, 3, 67, 69, 5, 2, 70, 4, 68, 71, 8, 66], [123, 62, 111, 127, 51, 121, 63, 60, 115, 118, 47, 125, 61, 53, 54, 117, 124, 56, 106, 126, 119, 59, 107, 120, 48, 57, 114, 102, 50, 42, 122, 38, 116, 113, 103, 58, 22, 49, 39, 112, 55, 52, 86, 43, 93, 90, 91, 110, 44, 92, 33, 97, 100, 46, 26, 88, 45, 27, 108, 29, 89, 83, 36, 82, 24, 19, 35, 99, 104, 25, 109, 28, 32, 40, 80, 87, 85, 41, 105, 21, 23, 16, 37, 76, 18, 84, 30, 17, 98, 31, 81, 78, 96, 14, 12, 13, 94, 79, 15, 20, 101, 74, 77, 10, 34, 95, 73, 75, 6, 67, 11, 9, 3, 65, 1, 69, 8, 64, 7, 72, 0, 71, 70, 5, 66, 4, 2, 68], [62, 123, 111, 127, 121, 51, 63, 115, 60, 118, 47, 125, 53, 61, 124, 117, 59, 106, 56, 54, 48, 126, 114, 119, 57, 107, 116, 120, 38, 58, 50, 102, 42, 103, 52, 22, 112, 49, 122, 86, 39, 55, 43, 113, 44, 33, 91, 46, 93, 97, 100, 92, 110, 90, 88, 108, 24, 83, 27, 45, 89, 19, 109, 29, 36, 26, 40, 105, 104, 28, 18, 25, 99, 82, 16, 35, 41, 32, 80, 76, 37, 13, 87, 96, 30, 31, 79, 12, 23, 17, 77, 20, 85, 81, 78, 84, 98, 15, 74, 94, 101, 21, 14, 95, 10, 34, 75, 9, 6, 11, 73, 67, 8, 7, 71, 1, 68, 5, 70, 64, 65, 0, 2, 3, 4, 69, 72, 66], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 124, 53, 61, 117, 54, 120, 106, 126, 114, 116, 56, 57, 59, 119, 48, 107, 50, 122, 42, 38, 102, 49, 22, 52, 58, 39, 55, 113, 86, 103, 43, 112, 91, 33, 46, 97, 93, 100, 92, 44, 88, 90, 110, 24, 29, 27, 40, 45, 99, 89, 19, 26, 83, 82, 36, 105, 108, 25, 109, 17, 37, 32, 18, 80, 16, 35, 87, 77, 12, 30, 94, 76, 84, 28, 104, 79, 15, 31, 74, 41, 21, 13, 96, 81, 10, 6, 20, 23, 101, 73, 98, 78, 14, 95, 75, 34, 85, 71, 8, 11, 7, 3, 1, 9, 67, 68, 70, 4, 65, 64, 0, 5, 66, 69, 72, 2], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 61, 53, 54, 117, 106, 57, 126, 124, 119, 120, 116, 114, 59, 56, 122, 49, 42, 107, 50, 48, 52, 102, 22, 103, 55, 38, 58, 86, 113, 112, 39, 43, 46, 91, 97, 110, 93, 33, 90, 88, 44, 92, 100, 24, 19, 45, 27, 83, 99, 29, 36, 108, 21, 26, 89, 82, 87, 35, 109, 40, 32, 76, 80, 15, 16, 105, 77, 17, 10, 41, 18, 79, 104, 28, 13, 81, 12, 25, 85, 31, 94, 37, 96, 84, 30, 74, 20, 23, 34, 14, 78, 101, 98, 65, 11, 8, 9, 75, 73, 71, 95, 6, 70, 0, 68, 69, 67, 7, 3, 5, 64, 66, 4, 1, 72, 2], [123, 62, 111, 127, 121, 51, 115, 60, 63, 118, 47, 125, 61, 124, 53, 117, 126, 114, 54, 59, 56, 106, 120, 107, 57, 119, 50, 48, 38, 102, 116, 42, 49, 22, 103, 55, 122, 112, 52, 43, 86, 58, 113, 39, 46, 91, 110, 93, 97, 44, 92, 90, 100, 45, 33, 27, 89, 24, 29, 40, 108, 88, 104, 99, 19, 26, 25, 36, 82, 41, 109, 83, 35, 32, 28, 87, 105, 80, 18, 85, 16, 37, 76, 96, 23, 20, 17, 30, 31, 21, 84, 15, 94, 13, 79, 77, 12, 78, 81, 98, 10, 34, 95, 101, 14, 11, 75, 74, 73, 8, 70, 71, 0, 3, 65, 9, 5, 67, 66, 69, 4, 68, 6, 1, 2, 64, 7, 72], [123, 62, 111, 127, 121, 63, 60, 51, 118, 115, 47, 125, 61, 124, 53, 117, 59, 106, 119, 126, 57, 107, 56, 54, 120, 114, 50, 122, 38, 58, 55, 116, 49, 48, 22, 52, 102, 42, 86, 112, 43, 39, 103, 113, 46, 110, 44, 91, 92, 33, 97, 100, 93, 108, 88, 90, 109, 24, 45, 82, 19, 29, 83, 36, 105, 89, 80, 27, 35, 18, 104, 26, 25, 16, 13, 28, 40, 99, 17, 41, 30, 76, 87, 84, 78, 94, 21, 32, 79, 37, 23, 12, 14, 20, 96, 10, 85, 81, 98, 95, 31, 11, 74, 75, 101, 70, 77, 73, 15, 71, 9, 7, 8, 34, 67, 64, 0, 68, 65, 3, 4, 1, 6, 69, 66, 5, 72, 2], [123, 62, 111, 127, 121, 60, 51, 115, 118, 63, 47, 125, 124, 53, 61, 106, 56, 117, 119, 57, 54, 114, 120, 126, 59, 107, 50, 116, 122, 55, 42, 48, 38, 49, 52, 58, 102, 22, 86, 112, 103, 43, 39, 113, 110, 46, 91, 44, 93, 97, 100, 33, 92, 108, 24, 27, 45, 109, 88, 90, 29, 19, 82, 83, 36, 26, 40, 99, 32, 25, 104, 28, 80, 35, 18, 21, 16, 41, 89, 105, 17, 76, 20, 31, 23, 13, 37, 87, 94, 84, 12, 30, 81, 78, 96, 15, 98, 85, 74, 77, 10, 79, 14, 95, 75, 70, 34, 101, 73, 11, 71, 8, 68, 7, 9, 65, 64, 4, 3, 1, 67, 0, 5, 69, 72, 6, 2, 66], [123, 62, 111, 127, 121, 51, 60, 115, 118, 63, 47, 125, 124, 61, 53, 117, 106, 114, 54, 59, 120, 126, 57, 56, 119, 107, 116, 122, 48, 55, 42, 58, 49, 50, 102, 38, 22, 52, 112, 113, 86, 39, 103, 43, 46, 44, 93, 100, 110, 97, 33, 91, 45, 92, 27, 90, 24, 88, 108, 36, 109, 83, 82, 40, 26, 29, 21, 99, 19, 41, 104, 89, 18, 32, 15, 25, 12, 16, 28, 87, 10, 30, 79, 17, 31, 35, 13, 81, 76, 105, 80, 37, 96, 23, 84, 85, 98, 94, 74, 34, 77, 73, 20, 14, 78, 11, 101, 8, 75, 95, 70, 65, 69, 7, 71, 9, 64, 6, 1, 4, 0, 67, 68, 3, 72, 2, 5, 66], [123, 62, 111, 127, 121, 115, 51, 60, 63, 118, 47, 125, 61, 124, 53, 59, 57, 117, 126, 119, 54, 120, 106, 56, 107, 122, 38, 114, 52, 55, 58, 48, 49, 116, 42, 22, 103, 102, 86, 50, 39, 112, 113, 46, 43, 44, 91, 100, 33, 97, 93, 110, 92, 27, 90, 88, 45, 109, 108, 99, 24, 29, 83, 89, 26, 36, 104, 40, 19, 35, 32, 87, 80, 25, 105, 41, 82, 84, 28, 16, 18, 79, 85, 81, 13, 23, 37, 21, 30, 12, 17, 34, 101, 98, 94, 31, 76, 20, 15, 77, 96, 10, 95, 78, 73, 11, 75, 14, 74, 6, 7, 9, 71, 0, 64, 65, 8, 70, 3, 5, 1, 69, 72, 67, 66, 2, 4, 68], [123, 62, 111, 127, 121, 51, 115, 118, 63, 60, 47, 125, 53, 61, 124, 119, 126, 106, 117, 56, 57, 48, 107, 120, 54, 59, 38, 50, 122, 114, 55, 49, 42, 116, 22, 102, 58, 103, 86, 52, 113, 112, 39, 43, 46, 44, 97, 91, 33, 93, 100, 90, 110, 92, 45, 27, 108, 36, 88, 29, 109, 24, 40, 25, 105, 104, 19, 83, 87, 82, 35, 32, 26, 16, 89, 28, 99, 80, 18, 41, 13, 84, 34, 21, 37, 20, 17, 96, 30, 12, 81, 79, 23, 85, 14, 94, 76, 15, 78, 98, 101, 77, 31, 10, 74, 11, 95, 73, 9, 67, 75, 72, 6, 71, 70, 69, 68, 64, 7, 1, 0, 3, 65, 2, 4, 66, 5, 8], [123, 111, 62, 127, 121, 115, 51, 118, 63, 60, 47, 125, 53, 61, 117, 124, 56, 106, 120, 57, 119, 54, 58, 107, 59, 38, 126, 42, 116, 55, 102, 122, 114, 103, 50, 48, 113, 49, 39, 22, 112, 43, 86, 52, 46, 100, 97, 91, 33, 44, 93, 108, 92, 110, 45, 88, 104, 24, 36, 90, 29, 109, 40, 99, 27, 83, 25, 32, 19, 89, 82, 35, 105, 87, 26, 37, 18, 84, 21, 16, 101, 41, 30, 28, 98, 80, 96, 34, 23, 85, 94, 20, 31, 15, 81, 14, 17, 79, 13, 95, 12, 10, 77, 76, 6, 78, 9, 11, 67, 74, 75, 73, 4, 65, 0, 72, 71, 64, 7, 1, 5, 68, 3, 69, 66, 2, 8, 70], [123, 62, 111, 127, 121, 51, 63, 115, 60, 118, 47, 125, 53, 61, 124, 117, 106, 120, 59, 107, 56, 119, 57, 126, 54, 48, 116, 58, 114, 122, 38, 50, 22, 102, 42, 49, 39, 55, 86, 103, 112, 113, 52, 43, 44, 46, 97, 33, 100, 92, 93, 91, 90, 45, 27, 24, 110, 88, 29, 36, 82, 104, 19, 89, 35, 99, 26, 109, 108, 32, 28, 40, 41, 25, 83, 16, 18, 84, 87, 80, 21, 96, 23, 20, 105, 12, 30, 37, 98, 31, 13, 76, 94, 77, 81, 15, 101, 34, 17, 11, 79, 10, 14, 85, 74, 6, 95, 78, 9, 73, 75, 72, 71, 7, 3, 1, 4, 69, 64, 65, 70, 67, 5, 0, 66, 8, 68, 2], [123, 62, 111, 127, 121, 51, 115, 63, 60, 118, 47, 125, 53, 124, 61, 54, 117, 106, 119, 59, 126, 120, 122, 114, 107, 57, 50, 55, 116, 56, 49, 48, 22, 58, 38, 42, 86, 43, 102, 39, 52, 113, 103, 112, 46, 93, 91, 44, 33, 90, 92, 97, 27, 100, 45, 24, 89, 36, 88, 110, 29, 26, 108, 82, 109, 19, 40, 32, 35, 83, 28, 41, 104, 84, 18, 99, 21, 80, 16, 25, 23, 87, 17, 20, 12, 37, 13, 30, 96, 31, 98, 81, 79, 34, 76, 10, 15, 105, 94, 101, 77, 74, 85, 14, 78, 73, 7, 11, 75, 9, 3, 95, 1, 72, 6, 71, 0, 4, 69, 64, 68, 67, 65, 5, 70, 8, 2, 66], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 53, 124, 61, 120, 106, 126, 59, 119, 117, 116, 54, 56, 122, 107, 58, 49, 38, 57, 114, 48, 55, 42, 103, 102, 39, 50, 113, 43, 22, 112, 52, 86, 46, 91, 93, 44, 90, 100, 97, 110, 33, 92, 24, 45, 108, 27, 36, 88, 29, 26, 32, 109, 89, 19, 104, 35, 99, 41, 83, 82, 40, 18, 28, 87, 25, 23, 16, 30, 85, 84, 96, 80, 31, 10, 21, 37, 15, 105, 17, 101, 13, 12, 94, 78, 20, 76, 98, 34, 77, 81, 79, 95, 14, 73, 75, 72, 74, 3, 11, 9, 71, 70, 64, 65, 1, 7, 0, 6, 68, 66, 67, 5, 69, 4, 8, 2], [123, 62, 111, 127, 121, 51, 60, 63, 115, 118, 47, 125, 53, 124, 117, 54, 61, 106, 59, 114, 120, 107, 126, 122, 56, 57, 119, 116, 58, 48, 39, 103, 102, 55, 38, 22, 42, 49, 50, 112, 52, 86, 113, 43, 93, 97, 100, 110, 46, 33, 44, 90, 91, 92, 45, 27, 36, 24, 108, 88, 89, 104, 40, 29, 26, 109, 99, 32, 83, 82, 25, 87, 35, 19, 28, 84, 30, 21, 16, 41, 80, 18, 96, 23, 105, 37, 101, 31, 78, 85, 15, 12, 20, 13, 94, 76, 10, 34, 95, 81, 14, 98, 79, 17, 77, 73, 11, 70, 74, 9, 4, 1, 65, 71, 0, 67, 75, 72, 3, 64, 69, 7, 2, 68, 66, 6, 5, 8], [123, 62, 111, 127, 121, 51, 60, 63, 118, 115, 47, 125, 53, 61, 124, 59, 106, 54, 117, 56, 114, 107, 126, 119, 58, 116, 57, 48, 120, 122, 49, 38, 22, 42, 55, 103, 50, 102, 52, 39, 86, 43, 113, 112, 46, 93, 92, 90, 33, 97, 91, 100, 27, 88, 44, 24, 45, 110, 36, 29, 26, 82, 89, 109, 25, 99, 83, 28, 18, 41, 80, 108, 19, 87, 32, 20, 104, 105, 40, 35, 96, 31, 16, 30, 79, 37, 84, 81, 21, 94, 12, 73, 14, 17, 76, 85, 13, 23, 77, 10, 34, 15, 74, 78, 101, 11, 98, 70, 7, 95, 75, 71, 9, 0, 3, 67, 72, 69, 4, 65, 5, 1, 66, 6, 68, 8, 64, 2], [123, 62, 111, 127, 121, 51, 118, 60, 115, 63, 47, 125, 61, 53, 124, 57, 117, 56, 106, 120, 122, 54, 126, 119, 59, 107, 48, 50, 116, 58, 22, 49, 114, 42, 38, 86, 103, 102, 55, 52, 113, 39, 112, 43, 46, 33, 110, 45, 97, 91, 100, 93, 90, 44, 92, 108, 27, 109, 88, 24, 83, 19, 99, 26, 29, 40, 89, 36, 18, 25, 82, 35, 41, 104, 21, 30, 16, 28, 87, 23, 32, 84, 12, 31, 80, 105, 94, 77, 101, 20, 13, 37, 79, 85, 17, 15, 81, 14, 96, 98, 10, 74, 78, 76, 73, 34, 70, 95, 7, 9, 11, 75, 64, 71, 1, 0, 72, 4, 3, 65, 69, 67, 66, 68, 5, 8, 2, 6], [123, 62, 111, 127, 121, 118, 51, 115, 60, 63, 47, 125, 61, 53, 124, 117, 54, 106, 57, 59, 56, 107, 120, 126, 58, 119, 122, 48, 50, 116, 49, 112, 114, 103, 38, 102, 22, 55, 42, 86, 39, 113, 52, 43, 46, 93, 44, 97, 45, 100, 92, 33, 110, 91, 90, 24, 27, 108, 88, 40, 19, 29, 109, 89, 26, 25, 99, 36, 83, 82, 28, 35, 87, 18, 21, 84, 41, 16, 105, 30, 31, 104, 32, 80, 85, 94, 17, 96, 15, 12, 98, 81, 20, 101, 23, 77, 37, 13, 76, 10, 79, 34, 95, 78, 74, 14, 11, 9, 70, 75, 73, 7, 8, 68, 71, 3, 1, 65, 64, 0, 6, 72, 67, 5, 69, 2, 4, 66], [123, 62, 111, 127, 121, 51, 118, 115, 60, 63, 47, 125, 61, 53, 124, 120, 117, 59, 106, 54, 56, 57, 114, 107, 122, 50, 48, 116, 126, 38, 58, 49, 119, 102, 22, 42, 55, 103, 113, 86, 52, 112, 39, 43, 46, 110, 97, 91, 93, 45, 44, 33, 108, 100, 92, 99, 24, 88, 90, 19, 36, 27, 40, 26, 89, 29, 109, 35, 28, 105, 104, 18, 25, 30, 83, 80, 31, 82, 101, 87, 16, 23, 21, 20, 32, 37, 85, 84, 41, 12, 94, 13, 79, 17, 78, 10, 76, 34, 77, 96, 98, 81, 15, 95, 74, 9, 11, 14, 73, 75, 7, 70, 8, 67, 6, 71, 3, 1, 64, 0, 69, 5, 4, 68, 65, 66, 2, 72], [123, 62, 111, 127, 121, 51, 60, 115, 118, 63, 47, 125, 61, 53, 124, 106, 54, 117, 120, 56, 50, 114, 107, 59, 57, 119, 48, 126, 22, 122, 58, 116, 42, 49, 38, 86, 113, 39, 102, 103, 52, 112, 55, 43, 97, 33, 92, 44, 91, 93, 110, 90, 45, 88, 46, 100, 108, 27, 82, 19, 29, 26, 36, 24, 89, 83, 109, 40, 18, 99, 16, 25, 28, 87, 80, 20, 32, 104, 23, 35, 105, 13, 17, 21, 30, 94, 84, 41, 37, 85, 31, 12, 10, 81, 78, 76, 79, 9, 96, 101, 77, 15, 34, 95, 74, 98, 6, 14, 11, 67, 75, 8, 7, 73, 0, 65, 71, 1, 3, 4, 70, 5, 64, 68, 66, 2, 69, 72], [123, 111, 62, 127, 51, 121, 60, 115, 118, 63, 47, 125, 61, 124, 53, 56, 106, 54, 117, 48, 116, 57, 114, 50, 126, 119, 107, 42, 59, 120, 122, 58, 113, 38, 102, 22, 49, 86, 112, 52, 103, 39, 55, 43, 110, 33, 97, 91, 46, 100, 44, 93, 92, 88, 90, 27, 24, 36, 45, 29, 99, 40, 89, 108, 109, 32, 26, 104, 18, 41, 19, 83, 25, 82, 28, 21, 35, 80, 94, 96, 12, 15, 105, 23, 30, 101, 87, 17, 37, 81, 16, 13, 85, 20, 31, 84, 98, 76, 74, 77, 10, 95, 78, 34, 9, 14, 6, 79, 64, 73, 11, 65, 7, 75, 3, 4, 1, 67, 70, 71, 68, 5, 8, 66, 69, 2, 0, 72], [123, 111, 62, 127, 121, 51, 115, 60, 118, 63, 47, 125, 53, 61, 124, 106, 56, 48, 57, 120, 126, 117, 50, 119, 59, 54, 107, 114, 42, 116, 22, 122, 58, 103, 86, 38, 102, 49, 55, 39, 112, 113, 52, 43, 44, 33, 110, 93, 46, 100, 97, 92, 88, 91, 27, 90, 108, 36, 24, 19, 45, 29, 40, 89, 26, 109, 32, 82, 99, 83, 104, 18, 28, 87, 25, 16, 35, 85, 76, 105, 12, 15, 84, 80, 21, 30, 96, 41, 37, 79, 81, 77, 17, 98, 23, 10, 74, 13, 14, 20, 94, 101, 95, 73, 78, 9, 6, 31, 11, 34, 75, 8, 71, 1, 67, 4, 66, 64, 7, 65, 3, 68, 5, 0, 70, 69, 72, 2], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 53, 61, 124, 56, 106, 117, 48, 57, 126, 120, 119, 59, 114, 54, 50, 107, 116, 122, 49, 42, 38, 22, 58, 112, 86, 103, 102, 113, 55, 52, 43, 39, 33, 100, 46, 110, 91, 97, 93, 108, 90, 44, 92, 27, 88, 109, 45, 36, 29, 40, 83, 82, 26, 89, 105, 19, 32, 104, 99, 18, 16, 24, 80, 87, 41, 35, 25, 94, 30, 28, 77, 84, 37, 21, 101, 17, 15, 74, 98, 81, 79, 12, 6, 76, 96, 85, 14, 20, 31, 23, 10, 75, 13, 78, 9, 71, 11, 95, 73, 34, 7, 8, 0, 3, 68, 67, 1, 65, 4, 70, 64, 2, 69, 66, 72, 5], [123, 62, 111, 127, 121, 60, 51, 115, 118, 63, 47, 125, 61, 53, 124, 120, 106, 57, 117, 48, 59, 56, 54, 126, 116, 107, 114, 122, 119, 22, 42, 38, 103, 86, 102, 39, 58, 50, 49, 112, 52, 55, 113, 100, 43, 46, 110, 33, 97, 91, 93, 92, 44, 88, 90, 108, 27, 45, 89, 19, 24, 29, 26, 99, 36, 83, 16, 25, 105, 32, 104, 28, 109, 18, 30, 80, 87, 84, 82, 35, 40, 76, 77, 20, 12, 81, 85, 79, 21, 17, 15, 37, 23, 41, 94, 10, 14, 74, 13, 73, 101, 96, 75, 78, 31, 34, 98, 11, 6, 9, 7, 8, 71, 95, 5, 70, 67, 3, 4, 1, 72, 69, 64, 65, 0, 68, 66, 2]], "model.layers.14.self_attn.q_proj": [[60, 120, 62, 37, 51, 63, 33, 53, 90, 101, 30, 118, 117, 87, 19, 86, 57, 125, 58, 85, 74, 93, 123, 111, 109, 115, 119, 59, 54, 92, 21, 116, 121, 12, 26, 50, 52, 114, 42, 44, 29, 126, 61, 91, 127, 122, 45, 55, 56, 48, 39, 105, 124, 6, 110, 15, 49, 4, 81, 43, 97, 112, 46, 36, 113, 25, 108, 20, 104, 94, 38, 47, 17, 64, 106, 88, 107, 71, 41, 18, 13, 103, 78, 23, 99, 34, 102, 22, 16, 2, 98, 40, 32, 65, 28, 10, 89, 72, 100, 95, 31, 1, 27, 35, 14, 79, 96, 68, 83, 24, 84, 11, 82, 66, 76, 80, 70, 69, 77, 75, 7, 8, 3, 0, 73, 67, 5, 9], [60, 120, 62, 37, 33, 90, 30, 118, 19, 101, 87, 117, 51, 53, 86, 59, 115, 57, 119, 92, 58, 111, 121, 126, 85, 54, 12, 15, 63, 116, 123, 26, 127, 48, 114, 56, 122, 74, 125, 46, 105, 71, 52, 43, 81, 38, 97, 110, 61, 42, 124, 83, 41, 55, 44, 45, 93, 50, 94, 39, 88, 69, 108, 112, 102, 40, 109, 103, 106, 113, 47, 20, 6, 29, 13, 49, 107, 1, 79, 28, 2, 99, 22, 17, 25, 104, 31, 95, 34, 14, 91, 23, 8, 66, 36, 21, 24, 89, 7, 72, 64, 32, 82, 100, 76, 16, 84, 98, 35, 96, 80, 75, 4, 18, 78, 11, 10, 70, 27, 65, 67, 9, 77, 5, 73, 3, 68, 0], [60, 120, 37, 62, 90, 118, 51, 101, 33, 19, 53, 87, 117, 30, 86, 42, 74, 4, 12, 59, 93, 111, 21, 114, 58, 39, 116, 63, 46, 92, 113, 54, 57, 126, 50, 121, 115, 26, 52, 105, 2, 49, 125, 15, 123, 25, 119, 127, 45, 48, 43, 122, 94, 91, 88, 61, 17, 110, 83, 103, 112, 56, 71, 55, 124, 85, 109, 38, 41, 81, 98, 47, 106, 104, 40, 14, 108, 102, 44, 6, 8, 107, 78, 29, 34, 99, 79, 76, 28, 23, 64, 72, 31, 69, 22, 89, 18, 65, 35, 66, 84, 36, 16, 20, 97, 1, 32, 68, 100, 24, 27, 77, 96, 95, 7, 82, 73, 80, 13, 11, 10, 75, 5, 70, 0, 67, 9, 3], [120, 60, 37, 62, 118, 90, 33, 117, 30, 51, 101, 53, 111, 59, 19, 87, 63, 86, 58, 57, 85, 105, 121, 48, 115, 92, 54, 42, 12, 126, 116, 55, 122, 119, 123, 46, 125, 52, 56, 26, 93, 74, 61, 127, 110, 108, 50, 112, 124, 114, 88, 41, 69, 109, 49, 113, 47, 91, 15, 82, 4, 21, 79, 44, 107, 43, 106, 81, 45, 39, 2, 25, 97, 104, 103, 94, 28, 38, 64, 71, 22, 36, 83, 78, 29, 31, 17, 6, 13, 40, 1, 100, 66, 99, 76, 102, 35, 34, 95, 32, 24, 72, 98, 84, 23, 96, 73, 18, 16, 89, 75, 68, 10, 65, 8, 27, 20, 0, 14, 11, 80, 70, 5, 67, 9, 77, 7, 3], [52, 102, 51, 124, 125, 29, 33, 60, 116, 84, 61, 123, 26, 86, 87, 88, 56, 119, 62, 93, 55, 38, 107, 120, 118, 106, 114, 115, 81, 39, 57, 122, 113, 109, 127, 37, 45, 94, 50, 53, 110, 58, 121, 28, 20, 54, 42, 63, 101, 105, 111, 41, 103, 112, 99, 35, 49, 108, 36, 40, 104, 97, 48, 22, 44, 43, 117, 34, 90, 59, 46, 47, 85, 25, 15, 100, 126, 98, 95, 73, 14, 32, 24, 21, 96, 82, 30, 31, 23, 92, 17, 18, 91, 11, 27, 71, 89, 80, 83, 74, 19, 16, 10, 7, 13, 5, 67, 78, 79, 9, 77, 8, 76, 12, 72, 75, 69, 66, 68, 65, 6, 64, 1, 4, 2, 70, 3, 0], [52, 102, 124, 33, 29, 125, 87, 60, 61, 116, 88, 123, 56, 38, 55, 119, 51, 73, 84, 45, 101, 62, 107, 57, 86, 28, 112, 106, 115, 39, 113, 110, 50, 53, 105, 127, 63, 67, 26, 118, 120, 42, 36, 54, 121, 114, 49, 58, 108, 43, 117, 109, 64, 47, 46, 126, 48, 40, 94, 122, 111, 44, 41, 59, 15, 93, 32, 103, 104, 97, 24, 22, 83, 99, 11, 100, 91, 20, 98, 90, 37, 35, 65, 81, 95, 34, 82, 31, 3, 25, 66, 96, 69, 27, 30, 23, 92, 71, 21, 89, 13, 5, 7, 9, 17, 79, 77, 19, 1, 6, 76, 18, 85, 10, 80, 78, 75, 14, 72, 74, 4, 0, 16, 12, 70, 68, 2, 8], [52, 102, 33, 124, 86, 29, 116, 88, 125, 61, 84, 82, 60, 73, 26, 87, 15, 56, 38, 66, 24, 93, 123, 51, 81, 14, 119, 90, 105, 55, 20, 62, 28, 19, 50, 110, 9, 112, 57, 113, 18, 10, 95, 13, 120, 43, 91, 54, 5, 106, 21, 45, 79, 99, 2, 22, 53, 63, 121, 49, 77, 108, 115, 85, 31, 101, 30, 118, 127, 114, 83, 4, 107, 23, 34, 78, 39, 75, 48, 17, 11, 25, 40, 117, 64, 98, 58, 27, 122, 12, 109, 0, 71, 44, 7, 72, 37, 94, 42, 70, 92, 111, 126, 16, 59, 103, 97, 69, 35, 46, 47, 1, 76, 89, 36, 41, 104, 80, 96, 32, 3, 100, 8, 6, 74, 68, 67, 65], [52, 102, 33, 29, 125, 51, 61, 26, 86, 124, 56, 116, 88, 82, 60, 84, 81, 87, 93, 15, 71, 123, 20, 119, 85, 11, 120, 55, 38, 73, 110, 113, 97, 22, 32, 62, 17, 75, 28, 90, 103, 14, 57, 24, 13, 50, 19, 76, 5, 54, 106, 109, 115, 127, 10, 112, 67, 25, 18, 42, 63, 118, 77, 48, 122, 45, 49, 79, 121, 107, 27, 58, 23, 3, 111, 108, 16, 31, 37, 104, 46, 89, 114, 43, 39, 105, 101, 53, 40, 36, 98, 94, 30, 117, 64, 41, 7, 126, 35, 21, 47, 83, 6, 80, 9, 91, 92, 1, 44, 100, 65, 99, 59, 34, 74, 70, 96, 95, 78, 12, 72, 69, 68, 4, 8, 66, 2, 0], [103, 97, 53, 117, 23, 2, 80, 11, 9, 85, 65, 0, 69, 81, 83, 67, 51, 14, 4, 3, 12, 71, 116, 70, 1, 7, 124, 61, 127, 112, 64, 87, 118, 60, 106, 6, 122, 73, 50, 5, 89, 113, 126, 66, 74, 13, 55, 114, 25, 68, 121, 54, 120, 24, 88, 8, 39, 63, 30, 91, 10, 49, 76, 15, 78, 28, 75, 17, 59, 108, 29, 45, 119, 104, 82, 58, 42, 79, 109, 31, 43, 52, 105, 110, 47, 77, 44, 46, 20, 19, 27, 123, 90, 72, 56, 34, 40, 57, 22, 125, 26, 16, 21, 98, 32, 102, 100, 99, 86, 37, 35, 107, 101, 84, 41, 62, 95, 36, 93, 18, 38, 48, 92, 33, 111, 94, 115, 96], [103, 97, 53, 117, 106, 81, 80, 87, 14, 12, 85, 23, 74, 4, 2, 112, 51, 83, 7, 11, 127, 116, 9, 124, 118, 69, 0, 54, 60, 61, 3, 25, 15, 24, 113, 50, 55, 70, 65, 126, 88, 122, 64, 30, 45, 121, 120, 114, 63, 6, 1, 76, 18, 75, 29, 72, 73, 5, 71, 68, 28, 89, 91, 39, 49, 26, 78, 58, 16, 8, 10, 67, 59, 44, 40, 19, 109, 110, 21, 52, 105, 17, 82, 98, 92, 66, 108, 42, 86, 46, 32, 31, 79, 27, 41, 13, 77, 36, 119, 95, 90, 125, 43, 84, 47, 100, 48, 34, 57, 107, 62, 102, 38, 22, 104, 99, 37, 101, 20, 35, 115, 93, 56, 94, 96, 111, 123, 33], [103, 97, 117, 53, 85, 87, 80, 51, 58, 14, 83, 12, 106, 11, 27, 16, 10, 23, 24, 5, 127, 124, 49, 81, 113, 104, 76, 43, 116, 118, 112, 45, 54, 71, 57, 91, 25, 88, 120, 86, 108, 122, 17, 109, 60, 50, 66, 47, 78, 55, 121, 110, 92, 63, 114, 52, 6, 21, 61, 44, 99, 125, 119, 62, 40, 105, 42, 123, 38, 37, 111, 89, 126, 56, 41, 19, 101, 102, 75, 98, 72, 34, 59, 9, 29, 115, 67, 100, 46, 68, 20, 22, 4, 30, 48, 36, 107, 26, 32, 1, 94, 95, 28, 82, 35, 18, 93, 90, 77, 96, 64, 31, 39, 2, 15, 73, 69, 79, 8, 84, 13, 7, 70, 74, 33, 65, 3, 0], [103, 97, 53, 117, 81, 11, 85, 14, 87, 106, 23, 69, 80, 12, 9, 2, 88, 60, 127, 7, 5, 124, 3, 51, 74, 112, 116, 0, 65, 70, 61, 4, 28, 126, 24, 118, 16, 29, 121, 83, 15, 122, 22, 45, 54, 44, 30, 120, 6, 25, 105, 114, 21, 79, 49, 71, 78, 50, 75, 67, 10, 17, 89, 63, 76, 55, 73, 64, 113, 40, 8, 1, 19, 98, 66, 86, 77, 58, 46, 109, 27, 82, 42, 108, 72, 13, 92, 104, 91, 34, 110, 84, 59, 62, 20, 39, 31, 125, 52, 18, 123, 90, 68, 41, 26, 32, 93, 94, 96, 107, 95, 48, 57, 99, 43, 119, 101, 38, 111, 37, 36, 35, 100, 47, 102, 56, 115, 33], [56, 102, 127, 24, 17, 93, 44, 60, 78, 11, 61, 33, 86, 88, 29, 116, 113, 108, 90, 50, 35, 114, 115, 47, 19, 59, 6, 62, 92, 123, 31, 40, 26, 72, 63, 119, 54, 20, 46, 120, 118, 122, 112, 126, 49, 104, 51, 111, 53, 124, 99, 121, 117, 23, 125, 37, 48, 55, 85, 57, 110, 36, 109, 42, 106, 32, 34, 43, 94, 45, 58, 30, 103, 39, 107, 52, 38, 8, 16, 105, 41, 100, 83, 89, 87, 96, 91, 95, 81, 98, 79, 28, 101, 82, 70, 25, 14, 27, 74, 21, 4, 10, 22, 3, 84, 77, 97, 75, 71, 76, 13, 18, 12, 80, 2, 15, 66, 69, 64, 67, 9, 7, 5, 0, 73, 68, 1, 65], [56, 102, 127, 24, 61, 113, 93, 60, 116, 114, 50, 108, 115, 63, 62, 59, 47, 126, 122, 88, 123, 119, 90, 38, 120, 112, 49, 53, 125, 46, 51, 121, 55, 124, 111, 117, 54, 42, 33, 48, 44, 39, 110, 57, 52, 58, 118, 109, 17, 103, 45, 30, 105, 43, 32, 81, 104, 40, 36, 78, 29, 75, 107, 106, 37, 84, 35, 41, 96, 34, 22, 92, 86, 19, 66, 94, 99, 101, 26, 100, 87, 2, 8, 80, 83, 98, 20, 11, 95, 91, 31, 85, 28, 97, 27, 79, 14, 73, 89, 15, 16, 6, 72, 3, 23, 64, 70, 77, 25, 21, 76, 5, 68, 82, 4, 13, 74, 9, 0, 18, 1, 67, 10, 12, 7, 65, 71, 69], [56, 108, 102, 127, 93, 9, 33, 24, 17, 61, 90, 94, 113, 60, 103, 1, 116, 88, 86, 39, 115, 65, 2, 50, 30, 114, 47, 59, 120, 62, 34, 23, 85, 123, 122, 126, 119, 69, 63, 87, 44, 55, 46, 73, 111, 104, 3, 0, 121, 18, 20, 64, 112, 31, 38, 54, 53, 49, 96, 68, 91, 51, 109, 98, 124, 36, 79, 95, 5, 48, 11, 117, 26, 37, 13, 57, 32, 77, 28, 35, 125, 42, 58, 52, 21, 7, 110, 118, 8, 27, 97, 10, 74, 4, 106, 105, 100, 16, 83, 72, 99, 66, 107, 15, 25, 40, 29, 19, 92, 45, 89, 82, 70, 71, 101, 84, 78, 6, 41, 67, 80, 22, 43, 76, 12, 81, 75, 14], [56, 102, 33, 108, 127, 93, 17, 24, 88, 113, 61, 90, 11, 86, 60, 82, 116, 78, 35, 59, 29, 20, 6, 47, 114, 50, 26, 44, 38, 123, 62, 122, 115, 120, 16, 106, 37, 21, 34, 63, 95, 119, 30, 126, 112, 87, 76, 121, 55, 49, 94, 51, 27, 53, 54, 28, 36, 83, 85, 80, 9, 75, 72, 89, 117, 124, 25, 110, 48, 91, 111, 125, 81, 79, 46, 42, 22, 104, 45, 103, 118, 105, 43, 57, 58, 96, 109, 98, 39, 52, 40, 99, 18, 32, 12, 15, 97, 23, 107, 31, 92, 84, 5, 7, 19, 100, 101, 74, 41, 77, 10, 70, 13, 73, 64, 14, 4, 3, 0, 67, 71, 2, 66, 69, 8, 68, 1, 65], [123, 39, 113, 69, 3, 64, 112, 1, 77, 71, 49, 63, 29, 13, 48, 111, 91, 61, 73, 98, 122, 54, 60, 119, 120, 4, 11, 72, 124, 88, 82, 24, 53, 115, 101, 56, 78, 22, 66, 62, 95, 17, 121, 59, 57, 81, 125, 55, 65, 83, 40, 118, 43, 127, 114, 93, 51, 8, 94, 70, 68, 46, 92, 15, 116, 106, 96, 36, 89, 47, 50, 35, 76, 2, 99, 52, 110, 97, 109, 102, 108, 44, 80, 45, 117, 104, 107, 126, 38, 41, 28, 9, 105, 58, 74, 37, 32, 26, 23, 33, 10, 100, 21, 42, 0, 25, 67, 12, 87, 19, 6, 31, 30, 90, 16, 7, 18, 20, 27, 84, 103, 5, 34, 75, 85, 79, 14, 86], [113, 123, 39, 13, 73, 71, 17, 83, 115, 74, 92, 78, 76, 59, 60, 55, 126, 63, 114, 57, 120, 127, 47, 116, 52, 62, 95, 119, 118, 117, 53, 121, 54, 69, 112, 82, 51, 56, 99, 46, 122, 109, 50, 124, 3, 58, 111, 48, 6, 100, 16, 125, 49, 110, 61, 108, 45, 88, 11, 44, 75, 106, 37, 107, 42, 43, 31, 86, 28, 72, 21, 98, 94, 15, 84, 41, 105, 67, 104, 93, 5, 36, 68, 85, 65, 38, 102, 91, 101, 40, 35, 97, 30, 25, 1, 64, 90, 22, 103, 33, 96, 87, 20, 32, 23, 29, 34, 2, 80, 8, 27, 89, 18, 81, 24, 14, 4, 26, 0, 77, 12, 19, 79, 10, 70, 66, 7, 9], [123, 39, 113, 112, 49, 32, 54, 95, 60, 99, 63, 115, 120, 90, 83, 124, 122, 48, 43, 13, 121, 94, 87, 125, 119, 61, 98, 56, 53, 111, 57, 85, 40, 127, 17, 38, 59, 62, 118, 36, 55, 114, 102, 105, 37, 110, 97, 92, 101, 81, 26, 47, 104, 109, 28, 35, 51, 45, 46, 96, 108, 116, 107, 50, 30, 41, 100, 106, 74, 52, 117, 126, 31, 44, 23, 42, 58, 82, 93, 33, 73, 25, 91, 29, 22, 84, 6, 21, 78, 18, 79, 86, 89, 34, 88, 11, 27, 24, 14, 15, 19, 76, 16, 75, 103, 20, 69, 10, 71, 8, 80, 72, 66, 77, 7, 3, 4, 70, 68, 12, 67, 64, 2, 1, 65, 9, 0, 5], [113, 123, 39, 119, 52, 120, 63, 126, 121, 118, 53, 57, 115, 106, 114, 47, 56, 13, 59, 116, 55, 51, 60, 58, 122, 127, 117, 111, 46, 50, 54, 92, 48, 109, 110, 62, 45, 43, 124, 108, 107, 61, 44, 74, 42, 112, 105, 125, 73, 49, 69, 101, 82, 41, 76, 64, 83, 99, 21, 17, 102, 104, 78, 5, 38, 71, 91, 1, 6, 40, 3, 66, 70, 72, 37, 100, 90, 30, 97, 8, 25, 81, 36, 15, 4, 89, 93, 26, 75, 35, 16, 96, 80, 19, 2, 11, 88, 33, 68, 14, 9, 84, 79, 98, 27, 87, 86, 32, 31, 67, 24, 18, 103, 10, 95, 20, 94, 12, 28, 65, 29, 85, 34, 23, 22, 7, 77, 0], [62, 63, 39, 53, 127, 32, 57, 36, 54, 121, 107, 47, 116, 46, 115, 59, 52, 55, 50, 51, 41, 105, 58, 61, 123, 96, 60, 125, 91, 27, 113, 122, 120, 106, 48, 124, 44, 103, 117, 126, 114, 98, 49, 119, 56, 118, 110, 45, 85, 43, 35, 38, 89, 112, 109, 101, 104, 40, 111, 108, 81, 37, 25, 42, 99, 22, 100, 34, 102, 92, 94, 87, 19, 33, 31, 23, 95, 0, 93, 21, 86, 26, 20, 97, 78, 17, 10, 80, 12, 29, 28, 83, 90, 66, 74, 65, 77, 82, 15, 30, 64, 24, 71, 72, 18, 69, 9, 84, 67, 76, 6, 1, 3, 68, 13, 11, 5, 4, 8, 14, 16, 88, 2, 75, 79, 70, 7, 73], [39, 63, 62, 24, 84, 94, 33, 18, 53, 86, 26, 88, 92, 80, 32, 78, 90, 127, 14, 29, 75, 73, 23, 30, 16, 121, 67, 17, 57, 82, 70, 54, 9, 95, 107, 1, 98, 116, 58, 47, 115, 15, 50, 27, 59, 52, 21, 22, 74, 60, 20, 55, 46, 83, 11, 51, 93, 8, 13, 85, 61, 31, 34, 125, 96, 123, 124, 91, 28, 79, 117, 113, 120, 4, 64, 77, 41, 76, 81, 5, 89, 25, 6, 122, 42, 72, 48, 12, 2, 126, 36, 35, 65, 118, 49, 102, 38, 87, 110, 101, 103, 114, 56, 19, 71, 119, 106, 108, 45, 37, 7, 97, 104, 100, 109, 111, 44, 112, 43, 105, 99, 3, 10, 40, 69, 68, 66, 0], [63, 62, 39, 53, 127, 57, 32, 54, 116, 121, 52, 59, 55, 47, 50, 115, 125, 61, 46, 51, 58, 60, 96, 123, 120, 122, 90, 113, 91, 86, 117, 124, 41, 48, 126, 34, 119, 118, 56, 106, 114, 92, 27, 109, 107, 101, 49, 110, 26, 35, 33, 89, 45, 21, 111, 102, 105, 108, 100, 36, 85, 112, 93, 44, 42, 43, 104, 99, 40, 98, 22, 29, 81, 88, 37, 10, 94, 64, 38, 103, 95, 16, 66, 65, 31, 19, 0, 97, 14, 25, 30, 67, 83, 72, 17, 24, 76, 87, 20, 6, 23, 74, 28, 78, 68, 69, 9, 12, 8, 15, 11, 84, 1, 5, 7, 79, 77, 82, 80, 4, 71, 18, 70, 73, 2, 3, 13, 75], [39, 62, 63, 94, 84, 24, 88, 53, 86, 21, 32, 74, 14, 10, 33, 26, 83, 90, 18, 127, 8, 92, 20, 69, 30, 23, 36, 57, 16, 7, 76, 66, 68, 121, 67, 54, 96, 27, 116, 89, 73, 87, 13, 52, 47, 82, 55, 28, 72, 115, 59, 17, 50, 6, 85, 77, 60, 70, 58, 34, 22, 80, 46, 95, 5, 81, 65, 61, 107, 125, 29, 51, 98, 49, 123, 93, 91, 9, 117, 124, 101, 108, 19, 120, 79, 25, 11, 113, 78, 15, 31, 38, 35, 64, 12, 103, 105, 99, 118, 41, 110, 106, 48, 122, 56, 100, 45, 97, 126, 37, 119, 104, 114, 40, 71, 43, 109, 4, 44, 102, 42, 111, 112, 1, 75, 0, 3, 2], [113, 48, 101, 112, 26, 94, 87, 18, 123, 20, 49, 86, 122, 59, 120, 79, 57, 62, 124, 58, 55, 60, 54, 115, 76, 43, 125, 37, 53, 119, 6, 39, 45, 51, 117, 126, 52, 114, 63, 61, 56, 50, 118, 106, 30, 75, 98, 34, 88, 32, 104, 107, 111, 127, 47, 110, 41, 116, 121, 92, 82, 108, 33, 44, 29, 102, 40, 42, 15, 103, 89, 90, 96, 105, 109, 99, 46, 23, 36, 35, 24, 100, 17, 22, 13, 31, 19, 97, 28, 70, 25, 91, 16, 95, 85, 27, 84, 11, 78, 21, 38, 83, 93, 12, 81, 14, 77, 73, 67, 8, 74, 9, 72, 71, 80, 3, 10, 7, 5, 4, 65, 68, 69, 66, 1, 2, 0, 64], [48, 113, 101, 112, 26, 122, 18, 94, 59, 87, 20, 43, 123, 57, 76, 79, 30, 63, 92, 98, 49, 53, 61, 124, 51, 88, 120, 86, 58, 117, 119, 39, 60, 121, 55, 125, 62, 115, 6, 50, 52, 89, 54, 56, 126, 96, 107, 114, 45, 102, 110, 29, 118, 47, 82, 35, 127, 34, 40, 46, 106, 24, 111, 32, 42, 100, 116, 108, 105, 37, 109, 44, 75, 90, 103, 104, 93, 36, 97, 25, 41, 31, 13, 27, 22, 28, 16, 95, 99, 38, 19, 91, 83, 12, 23, 85, 81, 33, 15, 17, 84, 21, 14, 11, 9, 78, 70, 8, 72, 74, 73, 77, 80, 67, 3, 71, 10, 7, 5, 65, 68, 4, 69, 1, 66, 2, 0, 64], [48, 113, 74, 16, 101, 7, 13, 68, 64, 69, 4, 112, 2, 122, 0, 87, 5, 1, 66, 65, 94, 67, 86, 3, 20, 58, 71, 53, 54, 98, 125, 55, 107, 57, 70, 6, 60, 49, 56, 120, 119, 52, 114, 121, 32, 26, 18, 62, 117, 124, 50, 118, 10, 11, 75, 110, 127, 30, 63, 47, 89, 51, 45, 82, 105, 104, 115, 61, 80, 106, 8, 103, 28, 76, 44, 77, 73, 96, 72, 108, 12, 126, 111, 79, 59, 42, 9, 109, 123, 24, 90, 23, 46, 85, 84, 93, 81, 37, 15, 14, 22, 78, 29, 19, 116, 17, 40, 34, 102, 83, 27, 35, 43, 21, 91, 33, 88, 92, 31, 25, 97, 95, 41, 38, 39, 100, 99, 36], [48, 113, 101, 122, 94, 112, 87, 26, 18, 59, 20, 123, 86, 49, 43, 79, 98, 57, 19, 120, 124, 30, 89, 32, 8, 115, 51, 76, 107, 58, 39, 21, 97, 82, 61, 63, 60, 62, 88, 29, 13, 52, 42, 6, 125, 34, 53, 117, 56, 55, 37, 119, 111, 96, 54, 28, 106, 44, 100, 114, 16, 75, 108, 104, 102, 85, 118, 126, 50, 105, 17, 45, 36, 27, 103, 116, 127, 31, 40, 91, 110, 92, 25, 121, 35, 47, 41, 22, 23, 109, 38, 33, 83, 46, 90, 95, 99, 74, 84, 24, 93, 12, 14, 78, 81, 73, 15, 11, 9, 72, 70, 3, 67, 77, 7, 80, 71, 10, 69, 65, 4, 5, 68, 1, 66, 2, 0, 64], [56, 113, 63, 102, 117, 49, 124, 68, 53, 76, 59, 85, 55, 5, 114, 60, 116, 58, 94, 121, 61, 89, 16, 7, 2, 66, 72, 50, 62, 8, 9, 12, 111, 51, 80, 115, 123, 118, 6, 126, 122, 48, 77, 54, 120, 52, 101, 40, 125, 69, 87, 91, 46, 119, 21, 27, 1, 3, 4, 29, 30, 96, 127, 42, 112, 83, 105, 92, 43, 44, 57, 35, 39, 36, 75, 22, 45, 109, 17, 24, 107, 110, 14, 104, 108, 88, 103, 47, 18, 79, 25, 23, 78, 41, 64, 15, 28, 97, 37, 93, 106, 20, 99, 98, 95, 81, 19, 26, 33, 82, 90, 100, 10, 38, 70, 31, 11, 84, 34, 73, 32, 13, 67, 65, 74, 86, 71, 0], [113, 56, 63, 124, 53, 59, 117, 55, 9, 49, 40, 116, 94, 58, 81, 69, 121, 61, 77, 62, 50, 114, 48, 118, 54, 60, 64, 111, 125, 51, 115, 123, 122, 126, 2, 120, 1, 24, 43, 57, 127, 68, 8, 52, 12, 39, 46, 119, 21, 70, 101, 112, 3, 45, 105, 107, 104, 109, 110, 47, 106, 44, 102, 108, 96, 30, 79, 6, 42, 92, 85, 72, 41, 103, 26, 36, 33, 7, 37, 15, 98, 100, 66, 16, 35, 25, 65, 83, 74, 97, 34, 93, 73, 95, 99, 90, 4, 17, 31, 28, 88, 91, 75, 38, 89, 67, 19, 23, 10, 87, 0, 5, 29, 13, 20, 32, 71, 78, 84, 27, 18, 14, 11, 82, 80, 76, 86, 22], [113, 56, 63, 94, 124, 49, 102, 117, 24, 96, 53, 38, 59, 104, 19, 55, 92, 43, 40, 58, 26, 91, 89, 35, 9, 116, 48, 114, 118, 37, 121, 22, 62, 51, 61, 50, 15, 60, 30, 122, 97, 126, 87, 46, 111, 54, 69, 29, 123, 52, 83, 120, 1, 85, 33, 125, 127, 17, 36, 77, 98, 12, 64, 81, 68, 119, 115, 3, 73, 105, 57, 75, 90, 7, 112, 86, 45, 79, 39, 109, 2, 8, 108, 44, 99, 6, 110, 34, 95, 25, 28, 76, 100, 10, 106, 27, 16, 93, 101, 18, 82, 78, 14, 47, 42, 41, 103, 88, 70, 107, 23, 31, 13, 21, 72, 20, 4, 32, 84, 80, 11, 74, 66, 67, 71, 0, 65, 5], [113, 56, 63, 124, 49, 117, 53, 40, 59, 114, 55, 102, 58, 116, 48, 24, 50, 94, 26, 121, 51, 54, 96, 60, 62, 118, 61, 87, 89, 126, 35, 43, 120, 77, 122, 123, 81, 111, 125, 57, 109, 52, 105, 9, 46, 17, 115, 104, 29, 119, 127, 92, 97, 30, 112, 99, 44, 22, 70, 45, 91, 110, 1, 37, 42, 108, 107, 103, 100, 101, 47, 38, 36, 15, 39, 6, 41, 27, 85, 106, 3, 20, 31, 90, 98, 33, 34, 88, 14, 73, 28, 23, 95, 19, 72, 64, 68, 12, 93, 79, 18, 13, 16, 86, 32, 82, 21, 25, 8, 69, 83, 78, 74, 11, 84, 80, 76, 10, 2, 75, 7, 65, 66, 71, 5, 0, 4, 67]], "model.layers.14.self_attn.k_proj": [[60, 101, 97, 62, 86, 117, 120, 94, 51, 59, 53, 90, 118, 58, 126, 57, 48, 111, 123, 121, 115, 54, 116, 87, 61, 81, 114, 125, 119, 56, 52, 127, 124, 55, 107, 19, 110, 122, 105, 50, 63, 106, 112, 89, 37, 40, 46, 44, 109, 45, 47, 43, 49, 102, 88, 108, 113, 12, 103, 21, 92, 104, 78, 15, 25, 42, 41, 32, 38, 39, 16, 93, 1, 34, 74, 67, 100, 27, 36, 71, 70, 31, 95, 13, 91, 98, 35, 68, 0, 20, 83, 73, 24, 33, 99, 96, 14, 79, 84, 72, 22, 28, 29, 11, 85, 80, 26, 77, 30, 66, 18, 8, 23, 17, 76, 82, 10, 9, 75, 5, 69, 2, 7, 65, 64, 3, 6, 4], [52, 38, 97, 86, 60, 93, 124, 56, 119, 123, 125, 61, 50, 57, 26, 113, 110, 62, 121, 127, 55, 81, 58, 116, 63, 109, 112, 53, 45, 115, 117, 108, 42, 49, 15, 46, 54, 106, 48, 111, 122, 118, 120, 88, 47, 84, 33, 114, 126, 82, 107, 103, 43, 39, 59, 22, 40, 102, 41, 87, 37, 105, 85, 36, 44, 75, 83, 100, 96, 77, 35, 13, 101, 104, 9, 99, 0, 98, 31, 16, 27, 30, 92, 34, 25, 32, 95, 51, 80, 17, 89, 28, 94, 10, 76, 69, 65, 29, 19, 14, 21, 73, 70, 91, 2, 24, 78, 72, 12, 71, 7, 4, 18, 90, 67, 68, 23, 8, 20, 11, 79, 5, 3, 74, 1, 66, 6, 64], [53, 39, 33, 87, 117, 14, 81, 80, 7, 9, 12, 74, 0, 42, 11, 65, 85, 2, 3, 60, 4, 48, 69, 70, 114, 83, 116, 61, 51, 64, 127, 54, 66, 67, 122, 79, 126, 113, 55, 118, 63, 124, 52, 5, 72, 120, 68, 88, 91, 109, 89, 25, 115, 112, 121, 108, 6, 45, 103, 59, 40, 73, 71, 22, 49, 1, 41, 34, 105, 77, 104, 23, 26, 10, 21, 43, 106, 44, 47, 96, 13, 50, 30, 92, 16, 110, 119, 18, 93, 123, 28, 76, 20, 58, 125, 94, 75, 8, 78, 29, 102, 36, 24, 46, 101, 90, 111, 56, 97, 57, 107, 31, 95, 35, 99, 19, 37, 27, 98, 38, 84, 100, 62, 82, 17, 86, 32, 15], [56, 38, 127, 86, 97, 44, 60, 61, 119, 114, 50, 113, 122, 116, 123, 62, 110, 63, 29, 115, 126, 47, 112, 59, 120, 51, 90, 55, 121, 39, 53, 49, 57, 124, 24, 117, 58, 45, 30, 78, 48, 118, 54, 125, 6, 109, 111, 17, 11, 1, 103, 52, 46, 107, 101, 42, 0, 41, 102, 105, 106, 28, 23, 108, 89, 20, 40, 104, 16, 43, 33, 35, 100, 98, 76, 25, 96, 82, 68, 15, 84, 85, 99, 32, 77, 37, 18, 74, 95, 36, 2, 34, 91, 19, 7, 64, 21, 22, 12, 31, 80, 93, 27, 3, 92, 87, 10, 67, 13, 79, 72, 94, 9, 26, 71, 69, 5, 73, 8, 66, 81, 14, 75, 4, 65, 83, 70, 88], [123, 113, 103, 22, 34, 60, 122, 54, 124, 61, 53, 125, 111, 56, 57, 49, 62, 119, 121, 55, 18, 47, 63, 114, 127, 115, 51, 40, 59, 96, 112, 93, 117, 116, 52, 126, 120, 46, 58, 118, 50, 44, 48, 109, 45, 107, 42, 104, 27, 98, 110, 43, 75, 41, 108, 106, 14, 65, 38, 100, 7, 35, 105, 92, 68, 84, 102, 37, 101, 0, 26, 67, 80, 36, 15, 9, 32, 31, 5, 33, 24, 30, 8, 87, 97, 88, 77, 94, 99, 25, 95, 6, 79, 20, 10, 39, 28, 81, 72, 91, 23, 76, 83, 70, 2, 90, 89, 12, 29, 66, 78, 21, 82, 17, 16, 11, 85, 74, 64, 19, 71, 4, 69, 86, 3, 1, 73, 13], [63, 62, 103, 86, 127, 53, 57, 121, 54, 116, 55, 30, 59, 50, 52, 58, 115, 125, 123, 97, 120, 61, 92, 60, 46, 84, 18, 51, 113, 26, 117, 124, 96, 118, 47, 126, 48, 88, 49, 24, 110, 114, 122, 100, 56, 119, 80, 15, 108, 106, 91, 111, 45, 107, 42, 14, 112, 109, 101, 38, 41, 43, 102, 40, 33, 75, 99, 44, 9, 105, 37, 31, 77, 83, 104, 93, 35, 94, 21, 17, 16, 25, 13, 98, 85, 36, 72, 82, 23, 34, 11, 19, 79, 29, 6, 66, 78, 32, 68, 0, 12, 95, 69, 22, 27, 81, 89, 28, 10, 87, 71, 90, 20, 39, 74, 76, 65, 3, 7, 73, 70, 67, 8, 1, 2, 4, 5, 64], [112, 64, 37, 113, 48, 49, 1, 69, 16, 74, 30, 68, 86, 87, 7, 123, 13, 2, 53, 20, 122, 34, 43, 58, 57, 96, 121, 118, 59, 120, 66, 125, 62, 56, 119, 54, 60, 75, 3, 26, 18, 55, 4, 52, 6, 115, 50, 114, 124, 111, 117, 126, 63, 79, 61, 0, 51, 47, 76, 127, 67, 110, 103, 116, 41, 109, 46, 106, 108, 93, 45, 104, 89, 38, 44, 36, 105, 8, 29, 35, 39, 42, 100, 71, 81, 102, 28, 15, 33, 21, 31, 90, 40, 27, 107, 25, 94, 19, 78, 24, 84, 99, 92, 88, 70, 73, 97, 85, 22, 72, 80, 5, 9, 91, 98, 95, 14, 77, 17, 32, 23, 83, 65, 12, 10, 82, 101, 11], [56, 113, 63, 38, 99, 124, 117, 59, 53, 22, 58, 116, 114, 55, 62, 122, 60, 32, 61, 50, 126, 51, 123, 111, 48, 121, 54, 127, 115, 118, 52, 46, 120, 125, 119, 57, 108, 112, 49, 45, 110, 109, 27, 40, 47, 35, 43, 104, 44, 89, 107, 97, 41, 42, 101, 106, 103, 105, 39, 92, 30, 98, 93, 33, 36, 87, 100, 102, 37, 19, 96, 18, 31, 13, 95, 34, 24, 17, 26, 91, 86, 20, 15, 29, 94, 28, 88, 85, 65, 2, 23, 0, 70, 80, 90, 7, 83, 10, 78, 74, 82, 25, 84, 14, 73, 8, 11, 3, 67, 81, 79, 75, 68, 16, 69, 21, 5, 71, 76, 4, 9, 12, 77, 72, 66, 64, 1, 6]], "model.layers.14.self_attn.qk_proj": [[56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 58, 120, 121, 124, 122, 57, 125, 127, 61, 103, 115, 55, 51, 59, 22, 97, 50, 119, 114, 38, 102, 49, 54, 39, 37, 118, 23, 126, 101, 47, 87, 111, 94, 86, 108, 16, 80, 33, 46, 0, 42, 110, 17, 26, 64, 109, 43, 90, 24, 106, 45, 88, 81, 93, 78, 14, 44, 107, 10, 69, 105, 29, 7, 5, 83, 74, 71, 12, 84, 11, 30, 40, 34, 19, 13, 4, 77, 2, 85, 9, 96, 73, 20, 21, 1, 66, 76, 41, 75, 67, 68, 28, 104, 99, 15, 65, 18, 82, 3, 36, 89, 35, 79, 92, 25, 98, 32, 70, 6, 100, 8, 31, 91, 95, 27, 72], [113, 56, 53, 52, 60, 123, 63, 62, 48, 112, 117, 116, 58, 124, 120, 127, 61, 125, 121, 122, 55, 103, 57, 115, 51, 59, 97, 102, 39, 22, 49, 38, 114, 54, 50, 23, 118, 37, 101, 119, 126, 94, 111, 47, 87, 109, 33, 86, 16, 64, 108, 43, 80, 0, 24, 17, 106, 90, 42, 46, 110, 45, 10, 26, 88, 14, 107, 78, 81, 93, 44, 29, 74, 4, 68, 71, 11, 85, 5, 105, 83, 69, 12, 9, 99, 75, 41, 34, 96, 19, 7, 21, 28, 77, 84, 40, 104, 65, 13, 20, 18, 1, 30, 76, 6, 92, 2, 73, 66, 67, 32, 36, 79, 25, 82, 70, 15, 98, 100, 35, 31, 89, 91, 8, 3, 27, 95, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 58, 127, 124, 61, 120, 122, 103, 125, 121, 59, 115, 57, 39, 54, 55, 97, 51, 49, 38, 50, 102, 114, 118, 47, 101, 119, 126, 37, 111, 46, 23, 22, 87, 94, 43, 108, 86, 0, 33, 64, 16, 110, 106, 109, 107, 80, 45, 88, 44, 90, 24, 42, 26, 30, 17, 93, 14, 29, 71, 10, 41, 105, 81, 78, 85, 69, 83, 34, 5, 99, 96, 12, 74, 7, 11, 4, 40, 65, 19, 68, 13, 66, 28, 104, 1, 9, 21, 75, 77, 76, 20, 73, 6, 84, 2, 35, 98, 32, 3, 25, 18, 67, 36, 82, 15, 92, 89, 70, 91, 31, 27, 79, 72, 100, 95, 8], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 116, 58, 125, 121, 122, 124, 61, 120, 127, 57, 49, 103, 114, 55, 39, 54, 97, 115, 59, 50, 51, 102, 111, 119, 101, 22, 38, 126, 47, 118, 23, 37, 94, 110, 46, 87, 108, 43, 0, 64, 16, 86, 80, 45, 109, 33, 90, 106, 24, 107, 17, 44, 93, 26, 42, 78, 88, 29, 81, 41, 69, 74, 2, 19, 7, 10, 104, 14, 105, 71, 30, 85, 96, 83, 13, 34, 12, 66, 76, 84, 77, 40, 65, 5, 11, 1, 9, 73, 4, 28, 3, 68, 20, 99, 75, 92, 36, 6, 32, 21, 98, 67, 18, 25, 100, 89, 15, 35, 82, 79, 70, 72, 31, 91, 95, 27, 8], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 122, 58, 121, 125, 116, 57, 61, 120, 55, 124, 103, 49, 127, 115, 114, 39, 59, 97, 118, 38, 51, 50, 22, 46, 102, 101, 47, 23, 54, 126, 94, 37, 108, 111, 110, 86, 0, 16, 119, 87, 80, 64, 33, 43, 109, 90, 106, 14, 81, 42, 44, 107, 24, 17, 26, 78, 71, 10, 30, 93, 2, 45, 66, 83, 11, 74, 69, 19, 88, 29, 105, 5, 4, 104, 68, 84, 13, 9, 12, 76, 41, 7, 40, 96, 73, 21, 1, 34, 75, 18, 85, 20, 77, 28, 65, 3, 79, 92, 82, 32, 89, 15, 99, 98, 6, 67, 35, 100, 70, 36, 25, 27, 91, 31, 72, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 121, 55, 127, 125, 58, 124, 120, 122, 103, 57, 61, 59, 49, 115, 114, 97, 22, 118, 39, 102, 51, 38, 54, 50, 119, 23, 37, 101, 109, 94, 87, 108, 126, 110, 111, 86, 47, 16, 80, 64, 17, 33, 14, 43, 90, 26, 42, 81, 0, 44, 24, 78, 46, 45, 106, 29, 11, 4, 88, 93, 10, 9, 83, 12, 74, 76, 84, 30, 75, 41, 85, 19, 7, 71, 105, 96, 18, 68, 107, 69, 66, 20, 5, 13, 73, 77, 104, 21, 32, 82, 79, 65, 2, 15, 3, 40, 34, 28, 89, 98, 92, 1, 70, 99, 36, 67, 6, 31, 25, 35, 100, 72, 91, 27, 95, 8], [56, 113, 53, 123, 52, 60, 63, 62, 48, 112, 117, 116, 121, 120, 127, 55, 58, 115, 125, 61, 49, 122, 114, 59, 57, 103, 124, 22, 118, 97, 102, 23, 119, 38, 51, 39, 37, 101, 94, 50, 54, 87, 126, 86, 109, 111, 16, 47, 110, 80, 17, 26, 43, 108, 33, 90, 14, 46, 78, 81, 42, 24, 64, 45, 106, 85, 29, 0, 88, 44, 30, 11, 10, 105, 74, 104, 9, 75, 76, 93, 84, 71, 107, 20, 73, 7, 69, 83, 12, 15, 41, 5, 77, 96, 21, 19, 28, 40, 13, 82, 18, 32, 2, 66, 34, 67, 79, 4, 99, 36, 6, 98, 92, 68, 65, 89, 1, 70, 25, 3, 35, 31, 100, 27, 72, 91, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 58, 121, 116, 59, 115, 55, 114, 124, 57, 122, 127, 103, 49, 97, 125, 38, 51, 22, 102, 61, 119, 23, 118, 54, 37, 101, 39, 126, 94, 87, 47, 50, 86, 46, 111, 109, 110, 16, 43, 108, 33, 26, 0, 64, 90, 80, 88, 78, 45, 14, 81, 17, 85, 107, 42, 29, 24, 30, 44, 84, 106, 75, 10, 96, 83, 11, 105, 93, 21, 74, 20, 5, 34, 71, 104, 65, 76, 9, 77, 7, 15, 41, 13, 18, 82, 4, 69, 1, 68, 12, 73, 2, 66, 36, 70, 19, 32, 79, 35, 28, 92, 98, 99, 89, 25, 3, 6, 91, 40, 67, 31, 100, 72, 27, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 120, 125, 58, 127, 121, 122, 49, 124, 55, 115, 57, 103, 97, 38, 54, 61, 51, 102, 114, 50, 101, 59, 39, 118, 23, 22, 119, 37, 111, 94, 87, 47, 43, 108, 86, 109, 80, 64, 0, 126, 110, 16, 33, 46, 106, 88, 78, 90, 17, 14, 26, 85, 105, 44, 45, 81, 29, 24, 10, 42, 30, 93, 4, 96, 74, 83, 84, 19, 104, 1, 71, 107, 41, 5, 34, 7, 76, 99, 21, 69, 20, 65, 75, 68, 66, 13, 28, 73, 2, 77, 11, 12, 15, 9, 32, 98, 18, 70, 25, 92, 36, 40, 35, 82, 6, 3, 79, 67, 91, 89, 100, 31, 27, 72, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 58, 120, 116, 57, 55, 125, 124, 121, 49, 122, 114, 127, 59, 51, 97, 103, 38, 115, 61, 39, 50, 54, 119, 23, 22, 111, 118, 102, 37, 101, 87, 94, 110, 126, 33, 64, 47, 0, 16, 80, 108, 86, 46, 17, 107, 14, 78, 45, 90, 42, 109, 24, 44, 81, 43, 26, 93, 10, 29, 71, 74, 7, 11, 88, 85, 99, 106, 30, 68, 69, 13, 12, 83, 34, 28, 84, 105, 5, 76, 96, 9, 66, 73, 4, 21, 18, 104, 75, 77, 20, 1, 70, 67, 65, 3, 2, 41, 15, 19, 82, 79, 40, 32, 89, 98, 6, 92, 100, 91, 35, 25, 36, 8, 31, 95, 27, 72], [56, 113, 53, 52, 60, 123, 62, 63, 48, 112, 117, 116, 58, 120, 55, 121, 124, 57, 49, 122, 127, 125, 103, 115, 114, 59, 39, 61, 51, 97, 50, 22, 102, 54, 23, 126, 118, 38, 101, 37, 87, 119, 47, 110, 0, 86, 94, 64, 16, 108, 80, 111, 17, 109, 46, 33, 90, 10, 106, 14, 74, 42, 78, 107, 29, 93, 45, 81, 26, 24, 43, 30, 5, 44, 71, 69, 105, 88, 66, 7, 12, 76, 13, 19, 68, 73, 104, 83, 2, 11, 84, 15, 65, 96, 18, 85, 41, 9, 77, 70, 4, 20, 40, 75, 1, 21, 34, 79, 3, 82, 28, 99, 67, 6, 36, 32, 89, 98, 92, 35, 91, 100, 8, 25, 27, 31, 72, 95], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 120, 58, 121, 55, 122, 127, 124, 103, 115, 114, 49, 57, 61, 125, 51, 22, 39, 118, 23, 59, 38, 102, 97, 119, 37, 54, 101, 50, 126, 94, 111, 110, 86, 87, 108, 16, 80, 109, 17, 64, 107, 0, 47, 14, 33, 42, 46, 81, 78, 26, 90, 10, 43, 74, 29, 44, 45, 106, 30, 24, 68, 105, 88, 93, 83, 69, 41, 84, 20, 4, 76, 5, 75, 85, 73, 7, 11, 19, 12, 13, 66, 2, 77, 9, 104, 28, 71, 1, 96, 32, 18, 82, 15, 21, 67, 34, 3, 70, 79, 6, 99, 40, 65, 35, 89, 25, 36, 98, 100, 8, 31, 91, 92, 27, 72, 95], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 58, 127, 115, 124, 103, 55, 57, 59, 121, 125, 114, 61, 38, 50, 49, 51, 122, 119, 39, 22, 102, 23, 97, 111, 54, 37, 118, 101, 94, 87, 108, 126, 47, 86, 110, 33, 17, 109, 46, 16, 80, 88, 14, 81, 26, 42, 64, 43, 0, 44, 106, 45, 90, 24, 107, 78, 29, 105, 93, 30, 10, 74, 104, 75, 41, 11, 21, 7, 83, 85, 68, 4, 69, 84, 99, 20, 96, 19, 28, 77, 34, 76, 5, 32, 82, 13, 18, 73, 92, 71, 66, 36, 98, 9, 1, 15, 12, 2, 40, 3, 65, 6, 31, 25, 100, 91, 89, 95, 79, 35, 70, 27, 67, 8, 72], [56, 113, 53, 52, 60, 123, 62, 63, 48, 112, 117, 120, 116, 127, 58, 59, 125, 115, 124, 49, 114, 55, 57, 51, 121, 103, 119, 39, 122, 61, 22, 50, 97, 111, 54, 118, 102, 37, 38, 87, 23, 64, 110, 94, 101, 86, 109, 108, 47, 126, 0, 17, 42, 46, 80, 14, 74, 33, 16, 44, 81, 45, 90, 43, 26, 78, 88, 24, 29, 10, 7, 71, 73, 105, 18, 75, 83, 93, 76, 30, 69, 11, 104, 77, 107, 19, 5, 13, 66, 41, 9, 6, 4, 96, 106, 1, 84, 65, 34, 20, 21, 99, 12, 85, 15, 28, 68, 40, 36, 2, 79, 32, 82, 70, 67, 3, 92, 25, 98, 35, 27, 8, 91, 89, 31, 100, 95, 72], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 121, 116, 124, 55, 127, 125, 49, 58, 114, 57, 59, 122, 103, 39, 61, 115, 50, 119, 51, 97, 22, 38, 118, 102, 23, 54, 37, 101, 94, 111, 110, 108, 126, 86, 87, 109, 16, 0, 64, 46, 47, 43, 80, 33, 42, 14, 90, 17, 74, 44, 45, 78, 106, 26, 81, 88, 107, 10, 7, 93, 24, 29, 30, 105, 69, 11, 71, 83, 75, 84, 73, 76, 104, 66, 19, 21, 18, 13, 5, 20, 34, 40, 4, 68, 28, 1, 3, 67, 12, 85, 2, 9, 77, 6, 96, 41, 15, 32, 65, 99, 82, 70, 79, 36, 92, 89, 98, 100, 25, 27, 8, 35, 31, 91, 95, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 120, 114, 55, 121, 58, 127, 124, 122, 49, 103, 125, 115, 59, 61, 57, 39, 102, 119, 51, 118, 37, 22, 54, 97, 50, 23, 38, 126, 46, 111, 94, 108, 16, 86, 110, 87, 101, 17, 64, 45, 47, 80, 44, 109, 0, 33, 43, 106, 107, 74, 14, 90, 42, 10, 105, 81, 4, 26, 83, 78, 29, 2, 93, 11, 76, 88, 24, 68, 5, 69, 7, 75, 71, 66, 41, 19, 84, 28, 21, 13, 30, 96, 9, 20, 104, 65, 40, 12, 73, 77, 18, 6, 34, 85, 98, 70, 15, 67, 79, 92, 99, 36, 32, 3, 1, 8, 89, 31, 35, 82, 100, 25, 91, 27, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 58, 116, 114, 121, 124, 127, 55, 115, 103, 59, 39, 125, 57, 49, 122, 50, 119, 61, 38, 118, 22, 51, 97, 54, 23, 102, 126, 37, 101, 94, 86, 111, 87, 110, 17, 46, 108, 44, 16, 45, 109, 80, 33, 0, 26, 47, 64, 107, 90, 106, 78, 43, 10, 42, 93, 29, 88, 81, 30, 14, 105, 24, 74, 11, 7, 28, 5, 75, 71, 41, 77, 21, 40, 34, 18, 83, 96, 68, 104, 4, 98, 9, 13, 85, 20, 12, 69, 84, 36, 66, 65, 76, 19, 99, 6, 32, 1, 2, 92, 73, 70, 79, 82, 35, 25, 15, 100, 3, 31, 91, 89, 8, 27, 67, 95, 72], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 116, 58, 121, 61, 55, 124, 115, 103, 127, 125, 59, 49, 114, 122, 39, 38, 119, 57, 118, 22, 97, 23, 54, 50, 51, 101, 37, 102, 126, 87, 86, 94, 47, 111, 46, 64, 0, 108, 16, 109, 33, 110, 44, 45, 80, 106, 17, 42, 14, 90, 107, 88, 43, 26, 78, 81, 24, 30, 93, 105, 74, 75, 7, 28, 10, 66, 71, 96, 19, 29, 69, 40, 76, 83, 11, 104, 20, 65, 34, 99, 77, 21, 84, 85, 9, 5, 18, 4, 68, 1, 82, 70, 13, 41, 73, 12, 2, 36, 67, 15, 6, 3, 92, 79, 89, 31, 35, 25, 98, 91, 32, 27, 100, 8, 95, 72], [56, 113, 53, 123, 60, 52, 63, 62, 48, 112, 117, 58, 121, 116, 120, 103, 124, 114, 122, 125, 127, 38, 57, 55, 49, 59, 115, 51, 97, 61, 54, 39, 50, 119, 101, 118, 22, 102, 23, 94, 111, 110, 37, 87, 33, 109, 47, 46, 64, 126, 108, 0, 86, 16, 44, 80, 107, 106, 90, 45, 24, 26, 43, 78, 10, 17, 81, 30, 93, 40, 105, 42, 88, 14, 28, 34, 104, 96, 74, 65, 85, 29, 99, 7, 71, 69, 19, 20, 4, 13, 5, 83, 21, 76, 98, 66, 75, 12, 84, 41, 9, 11, 18, 68, 77, 2, 92, 1, 35, 25, 3, 73, 82, 70, 32, 36, 67, 100, 91, 89, 31, 15, 79, 6, 27, 95, 72, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 120, 58, 121, 124, 127, 59, 61, 55, 125, 49, 103, 114, 39, 57, 122, 115, 97, 54, 22, 102, 119, 51, 50, 38, 118, 23, 37, 126, 94, 101, 111, 87, 108, 44, 33, 109, 86, 107, 0, 16, 46, 80, 47, 17, 106, 43, 64, 45, 90, 26, 24, 110, 93, 78, 42, 88, 14, 74, 10, 81, 105, 30, 29, 75, 4, 76, 19, 96, 7, 83, 40, 12, 11, 71, 99, 34, 85, 84, 5, 68, 20, 77, 28, 65, 9, 104, 21, 82, 73, 98, 1, 69, 18, 36, 41, 13, 70, 2, 15, 79, 66, 32, 35, 6, 25, 92, 89, 100, 91, 67, 31, 3, 27, 95, 72, 8], [56, 113, 53, 123, 52, 60, 63, 62, 48, 112, 117, 120, 121, 61, 58, 55, 116, 59, 125, 127, 122, 124, 115, 49, 114, 103, 57, 119, 22, 54, 118, 97, 39, 38, 51, 23, 37, 50, 126, 102, 94, 101, 86, 108, 46, 111, 87, 107, 44, 80, 110, 16, 43, 47, 109, 90, 0, 45, 88, 33, 64, 26, 14, 10, 106, 17, 105, 78, 74, 42, 81, 93, 24, 7, 76, 12, 69, 30, 19, 83, 11, 29, 28, 71, 4, 96, 2, 104, 84, 9, 75, 5, 99, 20, 41, 77, 18, 40, 67, 85, 34, 13, 21, 73, 66, 65, 82, 98, 79, 68, 70, 36, 92, 1, 15, 3, 32, 6, 91, 89, 72, 35, 25, 31, 100, 27, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 58, 120, 121, 61, 59, 116, 125, 55, 127, 122, 49, 103, 115, 38, 114, 54, 57, 124, 39, 51, 119, 118, 23, 97, 126, 101, 22, 102, 37, 50, 87, 86, 47, 111, 94, 80, 64, 108, 46, 90, 16, 110, 0, 33, 109, 107, 44, 45, 43, 17, 26, 88, 106, 14, 10, 105, 93, 78, 42, 81, 24, 74, 29, 19, 7, 83, 5, 76, 30, 104, 28, 71, 65, 69, 34, 40, 20, 75, 9, 84, 85, 11, 18, 77, 96, 4, 68, 12, 13, 41, 21, 73, 2, 99, 1, 82, 70, 79, 3, 98, 36, 92, 91, 66, 15, 89, 31, 25, 35, 32, 6, 72, 67, 100, 27, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 120, 59, 58, 124, 121, 103, 116, 38, 125, 127, 122, 55, 57, 61, 115, 54, 114, 97, 118, 49, 51, 22, 39, 50, 101, 23, 119, 102, 126, 37, 94, 86, 87, 109, 111, 47, 16, 108, 46, 64, 33, 44, 80, 110, 0, 90, 45, 88, 42, 106, 26, 81, 93, 105, 14, 10, 24, 43, 17, 78, 74, 107, 83, 30, 11, 104, 5, 68, 29, 19, 34, 4, 7, 96, 77, 40, 20, 99, 69, 85, 84, 71, 76, 2, 28, 65, 21, 66, 92, 13, 73, 9, 75, 12, 41, 18, 36, 82, 6, 89, 1, 15, 3, 25, 98, 79, 35, 32, 67, 100, 91, 27, 70, 95, 72, 31, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 125, 58, 120, 121, 124, 59, 55, 61, 116, 57, 127, 103, 114, 122, 38, 97, 115, 49, 22, 51, 39, 118, 54, 23, 37, 119, 50, 94, 101, 126, 87, 111, 102, 16, 86, 47, 33, 80, 90, 110, 44, 109, 64, 42, 26, 0, 17, 108, 46, 45, 78, 106, 10, 14, 29, 88, 81, 93, 74, 24, 43, 83, 30, 107, 11, 28, 105, 76, 104, 71, 75, 4, 73, 84, 68, 41, 12, 96, 6, 7, 99, 5, 85, 66, 69, 15, 40, 13, 34, 77, 9, 19, 2, 20, 21, 82, 3, 18, 36, 32, 67, 92, 1, 100, 65, 25, 70, 89, 98, 91, 79, 27, 35, 72, 31, 95, 8], [56, 113, 53, 60, 123, 52, 63, 62, 48, 112, 117, 120, 121, 58, 59, 125, 61, 55, 116, 127, 49, 103, 57, 114, 124, 22, 115, 122, 118, 39, 38, 54, 119, 23, 51, 37, 97, 50, 86, 102, 101, 87, 94, 47, 111, 80, 126, 108, 110, 109, 46, 16, 26, 43, 33, 17, 106, 45, 78, 90, 88, 0, 64, 44, 107, 81, 29, 11, 74, 42, 14, 10, 24, 105, 83, 5, 93, 85, 30, 76, 9, 71, 40, 7, 84, 104, 69, 28, 41, 20, 13, 2, 77, 82, 18, 12, 75, 15, 96, 73, 19, 34, 1, 21, 99, 65, 66, 4, 32, 98, 68, 36, 3, 6, 100, 67, 79, 25, 89, 35, 70, 91, 31, 72, 92, 27, 95, 8], [56, 113, 60, 52, 53, 123, 63, 62, 48, 112, 117, 120, 59, 58, 61, 121, 57, 127, 116, 55, 124, 125, 103, 122, 114, 115, 22, 49, 54, 38, 118, 39, 50, 119, 51, 97, 23, 102, 37, 101, 94, 126, 86, 87, 47, 109, 110, 106, 33, 80, 26, 90, 16, 42, 88, 43, 107, 81, 111, 64, 17, 44, 108, 78, 0, 10, 29, 14, 46, 93, 24, 45, 75, 40, 85, 30, 104, 83, 74, 28, 105, 84, 41, 11, 71, 21, 76, 19, 5, 20, 32, 96, 69, 12, 18, 7, 99, 9, 77, 68, 34, 4, 36, 13, 65, 6, 2, 15, 82, 25, 66, 98, 100, 79, 73, 70, 1, 92, 89, 35, 91, 31, 3, 67, 95, 72, 27, 8], [56, 113, 60, 53, 52, 123, 63, 62, 48, 112, 117, 120, 127, 124, 59, 58, 61, 57, 121, 122, 103, 115, 116, 125, 49, 50, 51, 55, 54, 114, 39, 22, 38, 97, 23, 119, 126, 101, 118, 47, 102, 86, 37, 46, 94, 87, 44, 17, 111, 80, 26, 0, 107, 109, 64, 108, 33, 16, 90, 43, 24, 42, 106, 45, 81, 88, 105, 110, 10, 14, 30, 93, 29, 74, 68, 28, 85, 76, 78, 84, 40, 71, 83, 5, 96, 11, 104, 2, 21, 69, 77, 13, 20, 4, 12, 41, 34, 19, 66, 7, 82, 9, 75, 99, 73, 65, 67, 25, 98, 1, 3, 18, 92, 6, 15, 35, 79, 32, 70, 36, 91, 89, 31, 27, 100, 95, 72, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 124, 120, 121, 58, 116, 125, 122, 61, 127, 57, 55, 59, 114, 115, 103, 97, 22, 54, 39, 49, 38, 118, 51, 50, 101, 102, 23, 126, 37, 119, 94, 87, 86, 64, 0, 80, 108, 16, 47, 43, 90, 109, 111, 33, 26, 17, 81, 45, 46, 14, 110, 88, 106, 10, 93, 78, 44, 107, 42, 29, 71, 74, 11, 76, 105, 73, 24, 83, 75, 9, 84, 30, 12, 85, 69, 4, 7, 82, 28, 66, 65, 40, 77, 96, 104, 19, 1, 21, 5, 20, 34, 41, 13, 68, 18, 67, 2, 79, 99, 3, 70, 25, 6, 91, 32, 92, 89, 15, 98, 36, 31, 27, 35, 8, 100, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 125, 57, 121, 58, 122, 127, 124, 61, 55, 103, 97, 51, 115, 49, 54, 126, 39, 114, 118, 59, 22, 23, 38, 50, 102, 101, 37, 47, 119, 46, 94, 111, 0, 108, 109, 43, 16, 80, 87, 86, 110, 33, 90, 64, 45, 81, 17, 14, 26, 106, 42, 10, 93, 88, 74, 44, 105, 78, 83, 29, 30, 24, 71, 7, 13, 69, 107, 5, 85, 40, 34, 12, 76, 9, 2, 11, 19, 28, 65, 20, 104, 96, 98, 68, 73, 70, 4, 84, 99, 66, 82, 77, 75, 41, 92, 67, 1, 21, 79, 6, 36, 89, 18, 91, 32, 35, 100, 3, 31, 8, 15, 27, 25, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 58, 116, 121, 120, 125, 61, 55, 124, 122, 59, 127, 57, 103, 22, 97, 49, 115, 54, 51, 50, 119, 114, 39, 37, 38, 126, 102, 101, 23, 47, 118, 87, 94, 86, 45, 80, 43, 109, 16, 111, 46, 0, 26, 78, 17, 81, 10, 90, 108, 33, 64, 14, 110, 106, 29, 88, 2, 24, 93, 42, 44, 107, 74, 83, 85, 69, 12, 76, 105, 104, 73, 4, 96, 19, 40, 30, 5, 7, 11, 75, 20, 71, 84, 41, 9, 34, 68, 77, 99, 28, 21, 82, 70, 66, 13, 18, 3, 79, 65, 67, 1, 15, 32, 89, 36, 98, 6, 25, 35, 92, 8, 91, 100, 31, 95, 27, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 125, 58, 120, 61, 121, 122, 57, 124, 127, 59, 97, 115, 55, 103, 114, 22, 54, 39, 51, 50, 49, 119, 102, 37, 23, 126, 86, 38, 118, 111, 87, 94, 101, 47, 110, 16, 0, 17, 33, 64, 43, 45, 80, 78, 26, 108, 46, 109, 29, 42, 90, 44, 81, 10, 24, 88, 41, 83, 106, 107, 74, 93, 68, 71, 4, 30, 7, 5, 12, 14, 69, 75, 9, 99, 96, 105, 11, 19, 73, 76, 34, 18, 2, 13, 104, 70, 77, 66, 85, 40, 28, 21, 67, 65, 1, 84, 20, 79, 32, 89, 25, 98, 82, 15, 36, 35, 31, 6, 91, 92, 3, 8, 100, 27, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 121, 125, 58, 122, 61, 55, 59, 124, 103, 22, 127, 57, 97, 114, 51, 115, 54, 23, 38, 50, 49, 126, 102, 39, 119, 101, 37, 118, 86, 94, 87, 111, 47, 17, 78, 108, 109, 80, 26, 16, 33, 43, 42, 45, 88, 90, 46, 24, 10, 0, 11, 29, 110, 44, 107, 14, 64, 106, 74, 93, 81, 30, 83, 12, 5, 105, 75, 9, 71, 40, 13, 99, 76, 7, 73, 84, 19, 66, 85, 96, 4, 20, 18, 104, 34, 69, 70, 77, 41, 79, 15, 28, 21, 65, 25, 68, 82, 89, 32, 6, 3, 2, 92, 1, 67, 98, 36, 35, 91, 8, 100, 31, 27, 95, 72]], "model.layers.15.self_attn.q_proj": [[103, 62, 113, 33, 14, 21, 8, 11, 16, 18, 49, 89, 60, 67, 91, 19, 57, 69, 79, 26, 2, 22, 24, 59, 84, 46, 3, 95, 75, 70, 87, 65, 28, 66, 13, 6, 101, 27, 72, 80, 10, 25, 0, 73, 5, 78, 44, 88, 74, 83, 71, 56, 117, 77, 81, 76, 82, 85, 94, 20, 4, 64, 9, 92, 7, 17, 12, 119, 29, 15, 90, 23, 1, 43, 86, 116, 61, 68, 123, 110, 96, 104, 30, 40, 37, 98, 108, 112, 127, 125, 93, 106, 102, 51, 50, 122, 32, 45, 118, 120, 42, 124, 55, 111, 31, 34, 36, 54, 53, 100, 63, 99, 126, 121, 114, 58, 35, 38, 97, 47, 109, 41, 105, 48, 52, 107, 115, 39], [62, 113, 57, 60, 123, 59, 61, 58, 114, 121, 126, 56, 124, 127, 116, 120, 50, 63, 125, 122, 53, 54, 119, 55, 117, 51, 118, 48, 111, 46, 112, 52, 47, 109, 115, 49, 110, 45, 44, 42, 43, 108, 107, 35, 101, 41, 106, 105, 38, 37, 40, 104, 103, 102, 97, 36, 100, 88, 34, 99, 39, 98, 33, 96, 84, 23, 20, 30, 32, 95, 29, 31, 17, 19, 93, 90, 86, 22, 28, 94, 76, 92, 91, 26, 79, 81, 24, 21, 89, 12, 87, 14, 16, 77, 13, 83, 10, 73, 18, 27, 85, 25, 70, 82, 9, 68, 1, 4, 74, 7, 8, 2, 11, 15, 5, 67, 66, 64, 71, 80, 65, 78, 69, 6, 0, 3, 72, 75], [113, 62, 57, 60, 121, 59, 119, 126, 116, 61, 120, 123, 58, 127, 56, 114, 63, 54, 53, 125, 118, 55, 124, 122, 109, 112, 49, 51, 117, 52, 48, 50, 103, 111, 47, 115, 99, 38, 43, 106, 44, 108, 41, 110, 45, 107, 42, 46, 35, 105, 40, 104, 88, 100, 37, 101, 102, 20, 96, 36, 97, 29, 90, 39, 34, 98, 95, 17, 93, 33, 87, 32, 31, 23, 92, 24, 13, 30, 94, 28, 84, 26, 91, 22, 86, 81, 76, 77, 89, 74, 27, 79, 21, 83, 73, 18, 85, 19, 9, 10, 25, 68, 12, 15, 82, 69, 14, 16, 70, 8, 4, 5, 78, 1, 71, 6, 11, 80, 0, 64, 65, 7, 66, 2, 67, 72, 75, 3], [62, 103, 113, 57, 60, 59, 61, 58, 123, 56, 36, 126, 98, 121, 54, 116, 49, 120, 114, 117, 51, 55, 101, 125, 124, 119, 63, 53, 38, 108, 112, 122, 118, 46, 22, 48, 44, 52, 50, 32, 100, 45, 106, 34, 127, 33, 111, 99, 109, 42, 43, 47, 105, 107, 115, 35, 110, 104, 41, 89, 97, 40, 95, 93, 96, 102, 87, 31, 37, 94, 29, 17, 26, 28, 30, 86, 92, 19, 27, 23, 39, 91, 79, 24, 21, 83, 81, 88, 20, 25, 13, 18, 84, 90, 16, 15, 9, 80, 82, 85, 76, 77, 12, 10, 14, 73, 74, 5, 4, 75, 78, 11, 72, 2, 71, 6, 68, 70, 7, 8, 66, 1, 69, 65, 67, 3, 64, 0], [39, 112, 97, 45, 48, 125, 19, 21, 119, 115, 89, 14, 17, 95, 118, 93, 79, 25, 120, 12, 27, 8, 53, 22, 23, 111, 77, 40, 86, 57, 7, 127, 5, 28, 42, 55, 99, 124, 122, 10, 78, 49, 29, 85, 46, 47, 62, 9, 59, 91, 114, 116, 54, 90, 58, 108, 88, 63, 123, 24, 51, 110, 107, 117, 106, 94, 104, 113, 101, 32, 96, 82, 16, 52, 71, 100, 121, 37, 60, 84, 56, 73, 44, 38, 43, 30, 11, 34, 87, 67, 18, 126, 92, 35, 98, 41, 81, 26, 61, 102, 31, 69, 66, 50, 105, 80, 36, 20, 109, 3, 13, 83, 103, 72, 33, 76, 15, 64, 74, 1, 2, 0, 70, 6, 75, 65, 4, 68], [39, 112, 45, 97, 48, 115, 89, 19, 27, 14, 21, 118, 119, 125, 120, 53, 57, 46, 122, 99, 24, 95, 55, 108, 59, 17, 22, 28, 121, 123, 106, 58, 60, 116, 107, 124, 25, 126, 62, 63, 54, 101, 114, 88, 75, 111, 127, 34, 79, 37, 47, 113, 40, 11, 36, 30, 56, 49, 42, 50, 87, 100, 61, 71, 51, 104, 52, 109, 43, 8, 44, 110, 29, 117, 96, 85, 102, 41, 92, 35, 93, 105, 23, 91, 32, 26, 38, 98, 31, 73, 84, 12, 74, 94, 16, 78, 20, 90, 83, 80, 15, 82, 86, 81, 77, 33, 67, 103, 7, 18, 69, 5, 10, 13, 2, 76, 9, 6, 66, 72, 65, 64, 3, 70, 68, 0, 1, 4], [39, 45, 97, 48, 112, 118, 21, 19, 79, 89, 17, 9, 27, 14, 7, 12, 6, 95, 25, 115, 65, 23, 5, 122, 18, 124, 16, 127, 74, 119, 125, 8, 93, 66, 24, 3, 28, 4, 78, 22, 76, 55, 1, 15, 70, 64, 111, 51, 120, 29, 80, 10, 77, 46, 67, 117, 68, 114, 72, 109, 56, 86, 73, 30, 82, 13, 75, 57, 123, 20, 2, 121, 96, 69, 58, 116, 62, 81, 85, 71, 108, 84, 53, 43, 107, 61, 49, 42, 60, 92, 83, 100, 31, 88, 11, 40, 90, 99, 50, 98, 94, 36, 32, 44, 113, 103, 0, 38, 26, 126, 47, 41, 105, 59, 104, 37, 63, 33, 54, 35, 87, 34, 91, 52, 106, 101, 110, 102], [39, 97, 45, 112, 48, 12, 17, 19, 67, 21, 118, 25, 9, 3, 1, 7, 79, 0, 89, 95, 73, 69, 124, 5, 23, 22, 8, 122, 14, 71, 85, 66, 127, 120, 10, 28, 13, 74, 119, 114, 4, 103, 75, 93, 70, 76, 117, 51, 15, 81, 2, 24, 55, 58, 78, 29, 84, 111, 125, 72, 80, 83, 6, 115, 65, 116, 30, 18, 16, 77, 92, 82, 123, 57, 27, 11, 31, 49, 47, 108, 26, 91, 50, 99, 88, 94, 56, 41, 87, 121, 113, 53, 35, 96, 107, 46, 43, 86, 36, 90, 109, 59, 100, 62, 40, 64, 54, 37, 32, 106, 68, 63, 61, 20, 105, 102, 42, 34, 52, 60, 104, 38, 126, 101, 110, 98, 44, 33], [51, 61, 117, 36, 81, 22, 30, 27, 88, 97, 100, 10, 78, 18, 44, 96, 94, 19, 108, 2, 69, 28, 91, 80, 54, 42, 20, 68, 25, 5, 74, 104, 93, 33, 63, 11, 124, 64, 14, 115, 86, 77, 83, 71, 8, 21, 114, 12, 85, 16, 103, 118, 17, 53, 23, 119, 66, 62, 84, 120, 26, 109, 111, 24, 6, 31, 110, 76, 73, 15, 121, 70, 79, 127, 13, 34, 123, 59, 82, 38, 87, 102, 37, 126, 0, 105, 72, 50, 9, 112, 122, 1, 56, 52, 89, 107, 57, 7, 46, 43, 65, 45, 49, 29, 4, 106, 75, 55, 101, 67, 48, 39, 58, 98, 90, 95, 35, 92, 32, 113, 125, 116, 40, 47, 41, 60, 99, 3], [51, 61, 36, 117, 27, 88, 30, 96, 100, 22, 78, 19, 81, 25, 10, 94, 20, 97, 44, 91, 71, 32, 21, 62, 34, 54, 69, 53, 29, 89, 103, 80, 76, 85, 82, 33, 86, 42, 83, 11, 23, 124, 13, 119, 16, 90, 110, 5, 108, 14, 87, 98, 116, 66, 18, 93, 106, 92, 26, 28, 102, 118, 77, 111, 17, 84, 101, 45, 31, 43, 6, 40, 114, 38, 74, 99, 123, 50, 24, 35, 41, 2, 109, 95, 104, 121, 12, 122, 126, 73, 48, 39, 37, 127, 120, 79, 105, 115, 7, 9, 59, 64, 55, 63, 49, 107, 46, 113, 4, 56, 72, 57, 15, 67, 52, 75, 47, 0, 60, 70, 58, 112, 125, 8, 3, 1, 68, 65], [117, 51, 36, 100, 61, 97, 102, 91, 115, 88, 30, 18, 94, 96, 25, 44, 22, 80, 27, 93, 11, 62, 20, 33, 28, 54, 53, 78, 19, 124, 63, 81, 103, 108, 76, 119, 6, 120, 111, 42, 104, 21, 59, 3, 85, 10, 4, 32, 110, 16, 127, 82, 9, 114, 69, 15, 83, 71, 123, 48, 38, 112, 12, 65, 52, 73, 118, 26, 43, 107, 122, 35, 58, 50, 126, 56, 0, 46, 121, 40, 116, 109, 60, 14, 77, 37, 67, 57, 55, 125, 41, 47, 92, 66, 39, 98, 1, 84, 106, 45, 105, 113, 24, 75, 72, 90, 49, 86, 23, 87, 95, 79, 34, 13, 99, 89, 101, 29, 68, 2, 5, 8, 70, 31, 17, 74, 7, 64], [61, 117, 51, 36, 88, 100, 30, 108, 97, 44, 85, 115, 124, 102, 62, 27, 104, 34, 54, 120, 111, 63, 59, 94, 114, 119, 91, 127, 33, 46, 52, 123, 112, 103, 122, 126, 109, 121, 50, 78, 58, 42, 43, 56, 107, 110, 116, 80, 19, 23, 57, 55, 98, 31, 22, 25, 48, 60, 125, 113, 93, 53, 105, 118, 45, 35, 47, 21, 49, 101, 96, 106, 41, 81, 37, 40, 39, 38, 32, 99, 24, 26, 69, 10, 29, 28, 5, 90, 74, 95, 2, 83, 71, 17, 86, 92, 65, 89, 73, 7, 84, 14, 76, 3, 20, 82, 87, 72, 77, 16, 11, 0, 9, 75, 66, 64, 8, 18, 67, 15, 6, 79, 70, 1, 13, 68, 4, 12], [39, 47, 115, 34, 25, 71, 12, 10, 83, 17, 14, 4, 86, 117, 111, 79, 123, 87, 22, 18, 63, 92, 7, 89, 66, 64, 56, 9, 98, 2, 21, 5, 1, 78, 91, 24, 67, 94, 76, 62, 110, 30, 19, 68, 102, 65, 6, 15, 69, 13, 27, 51, 73, 85, 80, 57, 99, 8, 70, 88, 29, 74, 23, 84, 77, 101, 55, 96, 16, 38, 40, 116, 72, 11, 50, 26, 75, 81, 31, 93, 112, 60, 90, 3, 43, 0, 48, 82, 28, 42, 106, 95, 118, 108, 20, 37, 53, 45, 41, 36, 114, 97, 113, 124, 61, 52, 49, 103, 107, 126, 127, 109, 121, 32, 35, 46, 100, 33, 44, 105, 54, 120, 58, 125, 122, 59, 119, 104], [47, 39, 115, 34, 114, 27, 37, 43, 111, 116, 25, 117, 94, 112, 56, 83, 123, 63, 51, 79, 86, 124, 55, 30, 126, 91, 89, 16, 62, 110, 58, 60, 127, 17, 109, 15, 99, 54, 46, 118, 122, 40, 107, 57, 98, 53, 52, 59, 48, 113, 125, 29, 50, 104, 44, 10, 14, 24, 41, 108, 97, 102, 61, 23, 95, 38, 121, 88, 42, 101, 106, 120, 12, 96, 80, 21, 36, 49, 35, 119, 105, 28, 85, 22, 45, 90, 19, 33, 82, 100, 26, 93, 92, 9, 31, 32, 18, 20, 81, 8, 84, 87, 77, 75, 74, 78, 103, 13, 2, 4, 73, 11, 72, 68, 70, 76, 71, 6, 69, 67, 5, 66, 3, 0, 7, 64, 65, 1], [115, 47, 39, 43, 34, 27, 25, 60, 51, 117, 37, 112, 56, 83, 114, 116, 86, 123, 55, 63, 30, 94, 10, 127, 58, 24, 15, 16, 79, 110, 126, 89, 62, 111, 17, 118, 48, 61, 124, 57, 12, 14, 72, 122, 21, 98, 107, 54, 53, 52, 113, 125, 99, 109, 105, 120, 44, 102, 91, 41, 119, 121, 66, 38, 49, 101, 104, 46, 59, 95, 50, 4, 106, 36, 100, 68, 22, 108, 45, 96, 32, 13, 93, 19, 75, 28, 42, 40, 80, 31, 97, 26, 85, 84, 71, 18, 23, 29, 35, 33, 8, 82, 9, 103, 74, 5, 87, 92, 90, 67, 88, 78, 2, 81, 20, 70, 77, 76, 3, 6, 11, 73, 0, 64, 69, 65, 7, 1], [115, 39, 47, 114, 34, 27, 116, 55, 117, 25, 56, 123, 63, 24, 37, 112, 94, 41, 83, 58, 44, 126, 91, 89, 86, 124, 110, 62, 51, 30, 45, 60, 52, 53, 113, 15, 127, 54, 84, 118, 61, 43, 119, 111, 46, 121, 108, 125, 59, 101, 109, 107, 122, 106, 57, 42, 49, 102, 120, 104, 48, 50, 79, 29, 96, 22, 99, 36, 80, 10, 38, 105, 92, 85, 97, 20, 100, 21, 26, 17, 93, 40, 98, 16, 12, 14, 31, 33, 28, 19, 35, 103, 70, 8, 32, 95, 23, 75, 18, 88, 87, 81, 9, 82, 90, 68, 72, 13, 11, 78, 66, 74, 2, 4, 6, 77, 5, 3, 71, 67, 0, 76, 64, 69, 73, 7, 1, 65], [53, 120, 101, 127, 110, 38, 121, 25, 125, 56, 51, 100, 102, 28, 22, 40, 80, 30, 33, 97, 116, 29, 49, 104, 11, 114, 59, 113, 87, 118, 50, 32, 84, 86, 63, 35, 57, 55, 77, 61, 75, 60, 16, 119, 115, 24, 90, 79, 46, 39, 54, 124, 93, 112, 37, 98, 117, 7, 52, 96, 62, 99, 9, 48, 94, 106, 92, 108, 18, 23, 105, 45, 111, 27, 109, 34, 123, 126, 122, 103, 107, 89, 72, 58, 47, 69, 91, 83, 15, 88, 44, 26, 1, 78, 20, 42, 36, 71, 21, 31, 43, 13, 95, 41, 81, 5, 85, 19, 73, 66, 4, 14, 82, 10, 8, 17, 2, 12, 76, 67, 70, 74, 3, 68, 0, 6, 65, 64], [120, 127, 53, 51, 113, 110, 59, 104, 49, 50, 125, 56, 112, 124, 116, 114, 63, 121, 101, 44, 52, 62, 55, 61, 57, 117, 60, 118, 119, 126, 46, 115, 58, 111, 37, 123, 54, 45, 48, 122, 39, 43, 47, 27, 107, 105, 108, 109, 42, 40, 106, 38, 79, 41, 96, 103, 100, 29, 35, 33, 82, 19, 85, 87, 78, 98, 34, 99, 102, 36, 30, 88, 95, 97, 6, 31, 64, 86, 24, 66, 65, 77, 92, 91, 32, 10, 94, 9, 8, 83, 22, 75, 67, 84, 5, 20, 17, 93, 28, 4, 0, 71, 18, 90, 14, 80, 11, 25, 21, 12, 15, 68, 26, 74, 1, 81, 7, 2, 23, 69, 72, 70, 3, 76, 73, 13, 89, 16], [127, 120, 110, 53, 40, 101, 100, 121, 49, 38, 51, 25, 118, 97, 63, 56, 62, 28, 113, 117, 84, 61, 39, 125, 119, 30, 32, 124, 59, 29, 96, 52, 104, 115, 88, 105, 112, 18, 94, 50, 116, 19, 23, 99, 111, 0, 24, 87, 60, 33, 89, 37, 98, 70, 35, 108, 95, 55, 103, 48, 41, 31, 21, 122, 109, 114, 57, 126, 43, 54, 102, 14, 22, 76, 86, 44, 46, 27, 80, 47, 78, 90, 34, 10, 42, 45, 6, 93, 66, 107, 85, 17, 36, 13, 65, 12, 83, 123, 75, 26, 106, 58, 81, 3, 20, 82, 11, 74, 79, 68, 1, 91, 15, 67, 69, 72, 8, 77, 92, 9, 71, 64, 2, 5, 73, 16, 4, 7], [127, 101, 53, 120, 40, 59, 28, 84, 33, 97, 100, 30, 25, 37, 87, 51, 50, 113, 86, 63, 77, 46, 104, 112, 102, 88, 96, 29, 27, 20, 116, 35, 121, 38, 22, 83, 126, 91, 125, 56, 94, 11, 80, 98, 75, 45, 19, 34, 78, 17, 52, 103, 60, 55, 54, 62, 24, 122, 110, 61, 115, 32, 79, 81, 41, 114, 15, 93, 71, 107, 124, 48, 21, 58, 105, 99, 8, 31, 39, 7, 106, 12, 44, 10, 57, 118, 42, 76, 95, 123, 109, 111, 47, 82, 108, 119, 92, 43, 4, 26, 89, 2, 18, 49, 74, 85, 90, 36, 5, 68, 117, 72, 16, 1, 9, 14, 23, 69, 70, 66, 0, 65, 3, 6, 13, 67, 64, 73], [104, 117, 25, 95, 34, 92, 86, 52, 19, 54, 77, 97, 8, 53, 61, 51, 79, 22, 28, 17, 120, 80, 68, 59, 88, 85, 114, 45, 72, 12, 60, 121, 24, 124, 13, 40, 49, 116, 4, 122, 113, 56, 87, 55, 58, 57, 23, 73, 107, 91, 109, 46, 15, 110, 63, 127, 62, 89, 93, 65, 74, 118, 83, 48, 115, 47, 21, 31, 18, 69, 75, 90, 0, 33, 126, 100, 35, 39, 111, 125, 94, 2, 14, 43, 106, 82, 20, 112, 6, 42, 36, 3, 50, 32, 64, 44, 5, 101, 123, 105, 98, 119, 102, 76, 29, 99, 108, 84, 27, 26, 30, 9, 16, 96, 66, 103, 81, 11, 1, 38, 71, 78, 7, 70, 41, 37, 10, 67], [117, 104, 120, 25, 95, 54, 52, 34, 61, 92, 103, 86, 88, 97, 58, 51, 116, 12, 114, 124, 55, 122, 53, 60, 113, 102, 49, 121, 63, 47, 59, 85, 127, 46, 62, 106, 111, 109, 48, 56, 28, 18, 126, 118, 19, 44, 115, 8, 119, 69, 125, 45, 14, 112, 57, 91, 24, 123, 80, 43, 42, 110, 108, 50, 107, 17, 66, 20, 87, 98, 105, 22, 39, 36, 71, 33, 100, 41, 38, 32, 37, 79, 26, 35, 77, 99, 94, 93, 101, 75, 74, 40, 30, 84, 31, 82, 13, 29, 96, 16, 72, 3, 89, 78, 11, 27, 90, 23, 9, 4, 15, 65, 81, 7, 68, 21, 83, 5, 76, 2, 1, 0, 73, 6, 67, 64, 10, 70], [117, 104, 95, 25, 52, 54, 61, 34, 92, 56, 97, 86, 120, 60, 24, 12, 47, 17, 51, 116, 53, 113, 88, 20, 122, 49, 99, 124, 85, 103, 119, 80, 19, 59, 102, 77, 28, 48, 91, 63, 57, 31, 112, 106, 126, 26, 58, 18, 62, 109, 118, 111, 115, 114, 121, 127, 125, 107, 100, 8, 123, 110, 44, 45, 50, 69, 42, 55, 43, 22, 105, 36, 38, 46, 16, 37, 15, 108, 96, 78, 30, 87, 39, 75, 94, 79, 32, 33, 35, 41, 40, 82, 90, 14, 98, 84, 29, 27, 93, 83, 89, 101, 81, 21, 23, 9, 73, 76, 74, 72, 10, 11, 66, 3, 4, 7, 5, 2, 6, 13, 71, 1, 70, 0, 65, 68, 64, 67], [117, 104, 52, 25, 95, 61, 56, 86, 92, 97, 34, 54, 60, 51, 19, 53, 58, 77, 120, 17, 8, 80, 113, 114, 31, 122, 59, 66, 62, 124, 127, 116, 69, 22, 12, 49, 48, 46, 57, 88, 47, 121, 106, 55, 85, 112, 79, 115, 72, 28, 102, 14, 74, 109, 45, 63, 15, 10, 87, 110, 125, 36, 40, 111, 118, 24, 4, 43, 126, 35, 73, 94, 33, 119, 123, 27, 42, 93, 98, 30, 5, 103, 50, 100, 91, 21, 44, 3, 108, 82, 29, 32, 105, 37, 107, 26, 41, 99, 90, 78, 38, 101, 83, 20, 39, 6, 18, 96, 75, 89, 23, 11, 16, 76, 84, 71, 64, 13, 70, 81, 7, 2, 68, 9, 1, 65, 0, 67], [39, 121, 112, 33, 1, 114, 93, 23, 69, 82, 0, 21, 79, 20, 13, 14, 84, 67, 48, 2, 118, 10, 25, 90, 16, 91, 22, 5, 29, 81, 72, 17, 65, 89, 78, 124, 3, 75, 71, 66, 68, 61, 87, 80, 88, 7, 30, 56, 47, 52, 4, 6, 60, 19, 55, 120, 12, 11, 110, 119, 122, 125, 63, 116, 115, 57, 53, 42, 126, 73, 92, 49, 83, 76, 86, 9, 70, 34, 100, 64, 102, 107, 27, 59, 74, 108, 101, 98, 43, 127, 51, 111, 99, 106, 35, 85, 44, 28, 50, 8, 45, 113, 94, 24, 117, 18, 109, 58, 26, 40, 37, 95, 96, 54, 38, 41, 62, 77, 46, 36, 123, 104, 105, 32, 31, 15, 97, 103], [112, 39, 114, 93, 33, 121, 23, 20, 90, 84, 17, 110, 56, 10, 53, 61, 115, 49, 22, 91, 119, 101, 116, 108, 117, 113, 43, 50, 47, 29, 24, 55, 125, 51, 118, 16, 99, 40, 54, 82, 63, 81, 57, 59, 60, 100, 46, 42, 38, 52, 58, 124, 111, 79, 126, 45, 123, 120, 109, 6, 87, 48, 107, 13, 122, 105, 106, 62, 102, 88, 44, 3, 104, 41, 127, 95, 94, 34, 37, 98, 85, 36, 72, 83, 92, 97, 30, 27, 86, 80, 35, 32, 25, 26, 96, 11, 74, 28, 19, 103, 31, 89, 76, 21, 5, 70, 18, 15, 77, 12, 64, 71, 14, 73, 67, 78, 68, 9, 8, 7, 75, 0, 2, 4, 1, 65, 69, 66], [112, 39, 121, 114, 93, 33, 23, 61, 110, 48, 17, 120, 20, 84, 90, 47, 22, 56, 115, 91, 124, 117, 53, 45, 82, 60, 116, 49, 51, 118, 57, 119, 55, 10, 50, 29, 94, 38, 43, 24, 125, 111, 52, 30, 108, 63, 79, 126, 62, 99, 88, 59, 113, 54, 122, 109, 107, 102, 123, 44, 58, 46, 40, 104, 100, 85, 41, 6, 101, 42, 127, 106, 81, 103, 105, 13, 86, 98, 87, 25, 92, 97, 21, 74, 31, 35, 95, 76, 36, 37, 96, 26, 34, 32, 27, 15, 77, 28, 11, 8, 16, 18, 89, 3, 80, 67, 72, 83, 70, 19, 5, 12, 71, 75, 68, 78, 7, 64, 2, 73, 14, 0, 9, 1, 4, 69, 65, 66], [39, 112, 33, 93, 121, 114, 23, 120, 20, 10, 22, 17, 79, 84, 110, 96, 13, 61, 29, 72, 49, 87, 90, 21, 56, 82, 109, 6, 117, 60, 70, 47, 125, 16, 52, 26, 81, 50, 118, 92, 77, 119, 51, 57, 115, 42, 25, 53, 11, 30, 124, 34, 104, 107, 91, 116, 122, 3, 95, 98, 38, 99, 83, 100, 86, 43, 111, 41, 101, 126, 48, 74, 5, 37, 55, 88, 62, 94, 44, 27, 24, 63, 108, 113, 67, 40, 35, 59, 45, 54, 78, 123, 32, 89, 127, 14, 28, 58, 105, 46, 18, 15, 85, 106, 9, 36, 31, 12, 19, 8, 64, 80, 75, 73, 102, 4, 76, 7, 68, 97, 71, 2, 0, 1, 103, 69, 65, 66], [53, 60, 127, 124, 122, 119, 123, 120, 51, 116, 56, 55, 125, 57, 38, 126, 54, 115, 61, 63, 50, 52, 118, 49, 58, 36, 59, 114, 62, 48, 45, 121, 112, 111, 101, 39, 113, 105, 110, 47, 46, 117, 106, 42, 108, 109, 43, 104, 44, 34, 107, 40, 99, 103, 41, 91, 24, 102, 94, 95, 19, 27, 21, 97, 93, 35, 37, 31, 89, 98, 100, 29, 16, 96, 14, 23, 92, 33, 25, 90, 83, 13, 32, 85, 30, 87, 11, 88, 20, 82, 4, 80, 8, 78, 68, 72, 28, 26, 75, 77, 70, 86, 6, 17, 81, 67, 3, 79, 74, 84, 18, 15, 10, 73, 76, 9, 22, 65, 71, 1, 12, 5, 7, 64, 66, 2, 0, 69], [53, 60, 120, 124, 39, 62, 119, 36, 127, 123, 125, 56, 100, 122, 46, 104, 109, 31, 51, 37, 101, 108, 61, 110, 126, 121, 118, 117, 63, 33, 42, 115, 113, 57, 52, 54, 44, 116, 107, 106, 41, 58, 23, 43, 59, 45, 24, 50, 48, 112, 55, 47, 97, 114, 40, 95, 111, 49, 102, 105, 90, 38, 103, 92, 94, 21, 82, 32, 98, 29, 34, 86, 25, 35, 99, 85, 91, 30, 93, 14, 13, 96, 16, 20, 19, 89, 87, 88, 27, 26, 28, 22, 18, 11, 83, 78, 80, 72, 15, 84, 68, 17, 77, 81, 79, 4, 75, 8, 6, 70, 73, 10, 12, 74, 67, 9, 3, 76, 71, 7, 5, 1, 69, 2, 66, 65, 0, 64], [60, 53, 127, 56, 124, 119, 122, 51, 123, 125, 55, 52, 116, 61, 57, 114, 126, 58, 63, 54, 48, 115, 121, 59, 50, 113, 49, 118, 112, 106, 111, 62, 120, 47, 117, 38, 46, 110, 45, 109, 108, 44, 107, 43, 42, 41, 105, 40, 39, 36, 103, 100, 104, 102, 37, 97, 29, 22, 98, 101, 33, 34, 35, 99, 86, 95, 32, 90, 23, 31, 85, 96, 91, 83, 25, 87, 18, 80, 30, 94, 92, 84, 24, 75, 27, 93, 26, 89, 17, 82, 16, 21, 19, 13, 78, 81, 79, 14, 20, 68, 8, 28, 88, 76, 15, 72, 4, 11, 74, 9, 77, 6, 3, 7, 2, 70, 1, 10, 67, 12, 64, 69, 65, 0, 73, 5, 66, 71], [60, 53, 120, 20, 79, 76, 92, 36, 9, 124, 7, 82, 17, 24, 97, 2, 62, 86, 69, 71, 0, 66, 32, 125, 56, 5, 122, 100, 74, 14, 73, 12, 3, 52, 119, 34, 25, 33, 18, 11, 80, 38, 65, 94, 21, 89, 48, 28, 13, 67, 78, 15, 81, 123, 8, 84, 127, 88, 96, 19, 93, 10, 75, 99, 83, 31, 101, 16, 26, 95, 108, 23, 6, 90, 72, 77, 70, 1, 85, 87, 64, 68, 27, 43, 4, 41, 30, 110, 91, 55, 29, 44, 35, 22, 109, 59, 102, 117, 58, 112, 39, 126, 98, 104, 37, 107, 61, 103, 51, 49, 113, 47, 118, 42, 115, 105, 63, 106, 116, 45, 54, 114, 40, 121, 46, 57, 111, 50]], "model.layers.15.self_attn.k_proj": [[113, 39, 62, 22, 18, 11, 60, 14, 89, 16, 97, 8, 21, 19, 57, 59, 123, 56, 110, 92, 67, 58, 120, 70, 116, 63, 53, 127, 119, 108, 79, 10, 122, 126, 117, 61, 54, 55, 48, 109, 114, 121, 87, 51, 124, 91, 112, 107, 52, 111, 125, 101, 42, 50, 118, 30, 44, 106, 47, 36, 115, 13, 46, 76, 0, 45, 43, 41, 104, 105, 75, 37, 35, 49, 69, 38, 29, 26, 7, 9, 27, 102, 40, 71, 85, 83, 31, 100, 17, 68, 95, 96, 99, 72, 33, 2, 34, 88, 15, 73, 93, 32, 78, 80, 82, 20, 5, 90, 25, 98, 77, 23, 28, 84, 12, 65, 24, 81, 94, 66, 1, 74, 6, 86, 3, 64, 103, 4], [103, 109, 112, 33, 45, 89, 21, 118, 12, 19, 79, 17, 9, 122, 119, 14, 31, 51, 7, 125, 5, 0, 117, 3, 120, 116, 124, 55, 115, 53, 93, 27, 57, 127, 50, 123, 47, 60, 20, 114, 56, 1, 49, 40, 59, 94, 46, 61, 54, 43, 8, 58, 66, 62, 23, 63, 111, 18, 108, 25, 28, 24, 52, 35, 121, 110, 42, 48, 10, 32, 86, 30, 126, 68, 11, 70, 22, 4, 44, 87, 106, 91, 113, 105, 74, 96, 107, 104, 82, 41, 98, 100, 83, 38, 34, 99, 77, 92, 37, 102, 88, 6, 26, 81, 101, 95, 13, 36, 84, 29, 73, 65, 2, 16, 76, 78, 90, 69, 71, 39, 15, 80, 85, 75, 64, 72, 97, 67], [115, 51, 117, 61, 100, 22, 30, 33, 53, 27, 108, 19, 88, 78, 124, 81, 62, 10, 120, 54, 80, 59, 32, 114, 20, 119, 127, 118, 63, 111, 64, 44, 71, 106, 110, 4, 123, 116, 48, 112, 58, 66, 56, 125, 122, 52, 126, 55, 76, 60, 50, 109, 1, 25, 43, 39, 121, 107, 46, 17, 102, 113, 11, 38, 13, 104, 57, 47, 69, 85, 49, 24, 40, 45, 99, 15, 41, 84, 82, 36, 29, 77, 35, 92, 72, 90, 42, 105, 37, 73, 93, 28, 98, 67, 8, 86, 101, 103, 87, 89, 95, 96, 6, 26, 65, 34, 31, 94, 97, 75, 16, 18, 21, 3, 23, 70, 9, 79, 74, 91, 12, 83, 7, 0, 5, 68, 2, 14], [103, 115, 47, 111, 86, 98, 51, 25, 17, 12, 117, 83, 14, 27, 10, 30, 110, 71, 116, 56, 1, 79, 112, 126, 123, 4, 55, 63, 62, 6, 64, 9, 58, 114, 66, 127, 121, 52, 61, 113, 122, 69, 59, 43, 124, 48, 109, 54, 106, 101, 32, 125, 21, 45, 60, 41, 53, 75, 118, 34, 108, 5, 120, 2, 49, 50, 46, 13, 57, 88, 87, 105, 67, 119, 102, 29, 92, 85, 72, 40, 96, 36, 107, 44, 104, 31, 28, 37, 24, 8, 20, 82, 81, 76, 26, 100, 42, 95, 38, 99, 91, 70, 90, 18, 97, 23, 35, 15, 0, 33, 93, 78, 94, 16, 19, 84, 77, 73, 80, 3, 7, 68, 89, 11, 22, 65, 74, 39], [127, 37, 120, 53, 104, 22, 97, 102, 32, 56, 46, 110, 94, 61, 121, 125, 25, 51, 60, 28, 93, 35, 80, 89, 113, 92, 55, 116, 118, 54, 122, 119, 124, 63, 108, 117, 52, 20, 96, 57, 99, 126, 59, 112, 47, 45, 48, 115, 91, 58, 114, 41, 30, 111, 106, 50, 98, 77, 123, 33, 44, 9, 109, 49, 88, 87, 62, 107, 101, 42, 19, 39, 43, 18, 95, 4, 36, 103, 66, 86, 105, 84, 31, 65, 40, 71, 75, 17, 64, 34, 78, 12, 90, 8, 79, 10, 29, 15, 0, 85, 82, 21, 6, 16, 24, 27, 26, 13, 67, 3, 72, 69, 5, 11, 100, 23, 14, 83, 38, 81, 76, 7, 68, 73, 74, 1, 70, 2], [117, 40, 31, 98, 86, 28, 52, 61, 25, 62, 120, 60, 114, 48, 56, 122, 113, 116, 51, 124, 59, 49, 127, 115, 109, 57, 46, 125, 55, 47, 54, 58, 33, 19, 123, 121, 85, 110, 111, 12, 126, 112, 18, 119, 80, 45, 63, 118, 43, 106, 53, 24, 8, 79, 50, 108, 42, 35, 44, 39, 105, 15, 41, 36, 107, 2, 64, 87, 102, 38, 34, 101, 29, 81, 68, 100, 16, 92, 37, 77, 17, 23, 99, 74, 27, 5, 103, 104, 20, 32, 88, 96, 94, 75, 21, 91, 89, 65, 1, 30, 97, 13, 9, 67, 93, 10, 14, 11, 71, 90, 6, 84, 82, 26, 83, 78, 95, 70, 76, 73, 7, 22, 66, 72, 69, 0, 3, 4], [103, 121, 112, 97, 48, 29, 114, 22, 57, 120, 23, 84, 118, 50, 119, 61, 116, 17, 52, 56, 111, 115, 90, 49, 53, 79, 60, 10, 43, 42, 82, 64, 3, 59, 122, 47, 46, 55, 58, 72, 124, 125, 36, 123, 102, 62, 104, 117, 63, 126, 45, 13, 110, 51, 37, 106, 91, 113, 28, 127, 41, 44, 6, 38, 54, 109, 21, 68, 107, 24, 2, 25, 76, 93, 73, 98, 105, 32, 108, 89, 78, 86, 71, 101, 40, 99, 83, 35, 80, 33, 16, 26, 34, 15, 31, 11, 27, 85, 94, 77, 95, 19, 92, 96, 18, 100, 75, 14, 12, 30, 69, 39, 7, 65, 5, 88, 9, 1, 8, 20, 87, 66, 4, 0, 70, 81, 67, 74], [60, 53, 86, 100, 120, 94, 76, 125, 79, 7, 124, 20, 101, 9, 82, 92, 74, 69, 96, 17, 25, 122, 44, 62, 24, 61, 119, 52, 118, 2, 106, 56, 127, 123, 117, 57, 51, 63, 121, 19, 58, 112, 126, 48, 0, 109, 107, 54, 114, 55, 59, 16, 113, 49, 47, 46, 50, 14, 90, 1, 115, 116, 111, 104, 72, 110, 41, 34, 13, 45, 5, 6, 43, 108, 81, 97, 75, 39, 42, 3, 10, 27, 37, 40, 11, 105, 102, 103, 35, 98, 12, 15, 23, 99, 33, 21, 22, 31, 30, 71, 78, 38, 73, 26, 93, 29, 85, 28, 95, 32, 8, 84, 67, 91, 70, 77, 18, 87, 83, 65, 89, 4, 68, 66, 64, 88, 36, 80]], "model.layers.15.self_attn.qk_proj": [[117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 124, 48, 86, 39, 100, 57, 56, 125, 118, 25, 116, 103, 89, 122, 22, 110, 111, 91, 109, 46, 97, 40, 63, 59, 50, 81, 30, 54, 119, 33, 123, 104, 93, 19, 49, 108, 55, 52, 83, 17, 58, 21, 79, 85, 126, 14, 101, 94, 36, 92, 98, 15, 12, 44, 107, 76, 78, 88, 20, 28, 106, 24, 37, 43, 84, 34, 74, 42, 31, 102, 95, 87, 27, 10, 9, 29, 32, 23, 7, 71, 96, 41, 82, 80, 16, 18, 5, 73, 38, 35, 105, 72, 26, 11, 69, 77, 75, 67, 3, 64, 0, 13, 66, 99, 8, 90, 2, 68, 70, 4, 1, 6, 65], [117, 112, 53, 115, 60, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 57, 48, 39, 86, 124, 56, 103, 100, 25, 118, 125, 22, 89, 46, 109, 116, 122, 54, 63, 111, 119, 91, 97, 110, 50, 40, 55, 49, 59, 52, 30, 108, 81, 104, 33, 123, 93, 126, 101, 83, 94, 19, 17, 15, 36, 58, 21, 85, 107, 98, 14, 12, 78, 92, 79, 76, 88, 34, 24, 28, 37, 10, 20, 106, 43, 44, 102, 74, 27, 84, 7, 31, 9, 42, 95, 23, 87, 71, 18, 82, 41, 96, 80, 16, 29, 32, 26, 35, 38, 99, 72, 0, 8, 69, 73, 5, 105, 2, 66, 75, 77, 11, 64, 67, 3, 13, 90, 4, 65, 6, 70, 68, 1], [117, 112, 53, 60, 115, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 56, 48, 124, 57, 118, 86, 100, 89, 103, 122, 111, 46, 25, 116, 109, 59, 125, 22, 91, 123, 97, 104, 110, 63, 40, 54, 119, 33, 30, 108, 52, 55, 93, 36, 81, 101, 50, 94, 58, 49, 19, 83, 126, 17, 21, 15, 28, 37, 85, 12, 24, 98, 92, 107, 44, 88, 76, 78, 34, 14, 102, 43, 79, 27, 95, 42, 106, 84, 9, 10, 74, 31, 20, 87, 23, 29, 7, 71, 18, 41, 32, 38, 64, 26, 82, 16, 96, 35, 2, 5, 0, 8, 80, 66, 99, 73, 69, 105, 67, 90, 11, 75, 13, 3, 72, 77, 6, 68, 65, 1, 4, 70], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 47, 127, 121, 45, 124, 114, 48, 39, 57, 56, 118, 100, 125, 116, 89, 103, 86, 109, 111, 25, 22, 122, 97, 91, 110, 54, 63, 59, 40, 30, 55, 81, 119, 46, 104, 58, 123, 52, 50, 108, 33, 126, 93, 101, 49, 19, 94, 21, 17, 37, 107, 79, 98, 14, 85, 12, 83, 76, 28, 88, 36, 92, 24, 78, 34, 15, 102, 44, 42, 20, 95, 27, 43, 16, 106, 84, 23, 80, 96, 9, 29, 82, 74, 87, 31, 10, 38, 18, 7, 32, 0, 71, 5, 41, 69, 35, 26, 73, 99, 66, 8, 11, 67, 3, 64, 90, 77, 2, 105, 1, 75, 72, 65, 13, 6, 4, 68, 70], [117, 112, 53, 115, 60, 62, 113, 51, 120, 61, 127, 47, 121, 45, 124, 114, 118, 48, 57, 56, 39, 125, 86, 100, 103, 116, 122, 59, 63, 22, 89, 110, 25, 109, 55, 97, 111, 54, 40, 91, 46, 119, 104, 30, 108, 123, 81, 33, 52, 49, 58, 50, 17, 19, 83, 93, 101, 12, 85, 15, 14, 98, 37, 43, 78, 79, 126, 94, 76, 92, 21, 107, 24, 42, 28, 74, 106, 44, 88, 84, 34, 102, 20, 36, 87, 10, 38, 96, 27, 9, 8, 7, 95, 32, 71, 41, 69, 35, 80, 23, 82, 18, 16, 5, 31, 105, 64, 29, 73, 26, 3, 66, 2, 75, 67, 77, 11, 13, 0, 99, 72, 1, 65, 90, 4, 68, 6, 70], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 118, 124, 57, 56, 86, 39, 48, 89, 100, 25, 116, 22, 125, 63, 111, 110, 103, 59, 91, 97, 54, 30, 109, 52, 40, 122, 119, 50, 81, 55, 49, 108, 46, 33, 123, 104, 93, 17, 58, 85, 83, 14, 19, 126, 76, 21, 12, 101, 15, 79, 94, 107, 92, 78, 44, 28, 88, 98, 34, 36, 42, 84, 24, 43, 37, 20, 10, 95, 87, 106, 9, 23, 18, 82, 16, 31, 80, 74, 7, 102, 27, 96, 38, 71, 32, 41, 8, 29, 35, 13, 75, 105, 73, 5, 69, 77, 67, 11, 99, 26, 64, 2, 66, 90, 0, 3, 68, 72, 6, 4, 65, 1, 70], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 118, 114, 57, 48, 124, 86, 100, 89, 39, 22, 56, 25, 59, 125, 103, 63, 111, 110, 91, 97, 122, 54, 116, 30, 81, 40, 123, 55, 119, 52, 109, 93, 46, 19, 126, 104, 50, 83, 108, 33, 101, 21, 49, 17, 92, 85, 76, 14, 15, 12, 58, 78, 79, 107, 94, 44, 28, 98, 84, 34, 88, 36, 24, 42, 106, 27, 37, 20, 87, 18, 10, 95, 23, 74, 96, 9, 38, 102, 16, 82, 105, 32, 31, 43, 80, 41, 71, 7, 29, 8, 13, 35, 73, 69, 11, 75, 26, 5, 77, 3, 0, 2, 99, 67, 66, 64, 72, 90, 68, 1, 6, 70, 4, 65], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 57, 118, 48, 39, 89, 56, 86, 103, 100, 25, 22, 124, 122, 125, 59, 97, 116, 91, 63, 111, 46, 30, 54, 110, 119, 109, 55, 40, 123, 104, 33, 83, 19, 108, 81, 49, 93, 50, 52, 101, 126, 17, 21, 92, 15, 36, 12, 14, 79, 28, 58, 85, 98, 78, 94, 76, 107, 24, 44, 42, 87, 88, 20, 84, 37, 106, 34, 43, 27, 96, 102, 23, 38, 31, 74, 18, 10, 41, 16, 9, 82, 95, 29, 32, 7, 105, 71, 80, 26, 35, 8, 0, 73, 11, 75, 5, 13, 66, 69, 99, 2, 64, 67, 3, 77, 72, 65, 70, 90, 1, 68, 4, 6], [117, 112, 53, 60, 115, 113, 51, 62, 120, 127, 61, 121, 47, 45, 57, 48, 118, 114, 56, 124, 39, 86, 89, 100, 125, 25, 103, 22, 122, 97, 63, 109, 30, 116, 111, 91, 55, 110, 123, 50, 54, 46, 59, 104, 119, 40, 101, 33, 49, 81, 19, 83, 93, 108, 98, 17, 58, 52, 21, 126, 36, 14, 107, 12, 79, 76, 94, 92, 85, 15, 24, 78, 42, 37, 88, 44, 28, 34, 20, 95, 106, 102, 27, 43, 16, 84, 31, 41, 87, 18, 32, 23, 10, 29, 74, 96, 80, 71, 9, 38, 82, 105, 35, 7, 73, 75, 99, 8, 13, 77, 11, 5, 0, 69, 26, 2, 72, 64, 67, 1, 66, 3, 90, 70, 65, 4, 68, 6], [117, 112, 53, 60, 115, 62, 113, 51, 61, 127, 120, 47, 121, 45, 114, 48, 124, 39, 57, 56, 118, 89, 125, 86, 100, 103, 22, 122, 25, 111, 123, 97, 55, 110, 109, 63, 91, 116, 54, 52, 59, 30, 119, 58, 46, 81, 83, 40, 104, 49, 50, 108, 126, 33, 93, 17, 19, 101, 14, 15, 94, 98, 85, 21, 12, 79, 107, 76, 44, 102, 28, 92, 36, 34, 78, 74, 24, 37, 42, 20, 43, 10, 84, 27, 88, 71, 7, 87, 23, 106, 41, 95, 18, 38, 9, 31, 16, 73, 32, 80, 96, 29, 5, 69, 82, 105, 0, 8, 35, 64, 3, 99, 66, 2, 67, 72, 13, 77, 26, 75, 11, 90, 70, 4, 68, 65, 1, 6], [117, 112, 53, 60, 115, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 124, 39, 48, 118, 57, 56, 89, 86, 100, 122, 22, 25, 116, 125, 103, 110, 119, 97, 109, 46, 30, 59, 111, 54, 91, 123, 63, 40, 81, 55, 104, 52, 108, 33, 101, 93, 83, 50, 58, 49, 12, 19, 17, 14, 76, 79, 94, 21, 126, 15, 36, 78, 88, 85, 24, 28, 98, 107, 92, 10, 43, 44, 84, 34, 7, 102, 20, 87, 37, 71, 74, 18, 9, 106, 80, 5, 16, 23, 42, 29, 32, 38, 41, 95, 69, 73, 72, 75, 82, 64, 96, 27, 66, 0, 31, 35, 8, 2, 67, 105, 3, 13, 77, 11, 99, 1, 65, 90, 68, 26, 6, 4, 70], [117, 112, 53, 115, 60, 113, 62, 51, 61, 127, 120, 47, 121, 45, 114, 39, 124, 118, 48, 86, 89, 57, 25, 22, 110, 100, 116, 125, 103, 109, 56, 63, 122, 55, 54, 59, 97, 91, 46, 30, 104, 40, 119, 111, 123, 81, 50, 52, 19, 49, 93, 108, 17, 101, 21, 12, 33, 83, 76, 15, 14, 126, 79, 98, 58, 92, 44, 94, 43, 78, 107, 28, 85, 88, 20, 87, 36, 106, 34, 24, 84, 42, 10, 18, 102, 9, 37, 74, 95, 7, 23, 16, 82, 41, 71, 27, 73, 32, 75, 80, 31, 96, 29, 35, 72, 38, 5, 69, 77, 0, 13, 11, 3, 105, 8, 99, 64, 2, 1, 26, 66, 67, 70, 6, 4, 68, 90, 65], [117, 112, 53, 115, 60, 62, 51, 113, 120, 61, 127, 47, 121, 45, 114, 118, 57, 48, 39, 56, 86, 124, 89, 100, 22, 25, 103, 111, 125, 55, 97, 63, 109, 59, 91, 54, 116, 110, 119, 122, 30, 93, 46, 40, 104, 81, 52, 108, 123, 126, 33, 36, 101, 50, 83, 19, 49, 94, 107, 76, 98, 21, 79, 17, 92, 15, 58, 14, 85, 28, 44, 20, 88, 12, 37, 42, 43, 78, 34, 87, 24, 27, 41, 106, 95, 84, 74, 102, 31, 96, 18, 23, 82, 35, 29, 38, 10, 105, 80, 7, 32, 73, 16, 72, 71, 99, 26, 5, 75, 9, 13, 11, 67, 77, 90, 69, 3, 66, 1, 2, 8, 0, 68, 64, 6, 4, 70, 65], [117, 112, 53, 115, 60, 62, 113, 51, 127, 61, 47, 120, 121, 45, 48, 114, 39, 118, 57, 56, 124, 86, 125, 89, 116, 100, 22, 54, 103, 25, 122, 55, 109, 111, 91, 119, 123, 59, 63, 40, 97, 30, 46, 104, 110, 93, 81, 33, 50, 108, 52, 94, 49, 19, 58, 83, 76, 15, 107, 101, 79, 21, 14, 126, 78, 17, 98, 36, 85, 44, 12, 74, 24, 28, 92, 20, 10, 42, 88, 71, 43, 34, 73, 37, 7, 23, 29, 16, 84, 102, 87, 32, 18, 72, 95, 38, 64, 82, 31, 80, 35, 27, 41, 96, 9, 5, 69, 0, 106, 75, 2, 99, 105, 3, 26, 66, 13, 11, 67, 77, 8, 6, 1, 65, 68, 90, 4, 70], [117, 112, 53, 115, 60, 62, 113, 51, 127, 120, 61, 47, 121, 45, 114, 48, 57, 56, 118, 39, 125, 124, 86, 100, 89, 55, 103, 25, 63, 110, 22, 119, 116, 109, 111, 54, 122, 97, 59, 91, 30, 46, 40, 108, 104, 94, 50, 33, 123, 93, 19, 81, 52, 101, 43, 79, 76, 83, 36, 21, 49, 14, 126, 98, 44, 15, 42, 17, 12, 78, 85, 92, 58, 28, 10, 24, 88, 34, 84, 74, 107, 20, 95, 102, 23, 7, 73, 18, 87, 106, 27, 71, 37, 9, 41, 72, 16, 38, 32, 75, 5, 80, 31, 35, 82, 29, 96, 105, 69, 0, 64, 13, 2, 26, 66, 11, 8, 67, 77, 99, 3, 90, 68, 6, 1, 4, 70, 65], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 57, 124, 118, 48, 63, 86, 100, 116, 56, 103, 22, 89, 25, 110, 119, 125, 55, 97, 122, 54, 109, 91, 111, 40, 30, 59, 52, 108, 104, 46, 50, 81, 123, 101, 126, 93, 94, 83, 79, 19, 12, 49, 76, 17, 33, 36, 21, 14, 98, 44, 15, 85, 92, 78, 74, 107, 28, 58, 88, 42, 20, 34, 43, 24, 95, 87, 73, 37, 23, 82, 18, 106, 80, 84, 102, 16, 38, 96, 27, 9, 71, 32, 7, 41, 29, 72, 10, 99, 31, 105, 69, 5, 77, 75, 35, 67, 11, 26, 0, 13, 66, 8, 3, 64, 90, 2, 1, 6, 68, 4, 65, 70], [117, 112, 53, 115, 60, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 39, 57, 48, 89, 100, 116, 124, 86, 118, 125, 22, 25, 103, 56, 122, 54, 63, 97, 111, 109, 55, 110, 104, 91, 59, 52, 30, 119, 123, 46, 93, 49, 40, 19, 126, 50, 81, 83, 108, 101, 17, 33, 36, 21, 94, 107, 98, 85, 28, 44, 34, 15, 14, 76, 58, 79, 92, 12, 78, 24, 43, 102, 20, 87, 88, 96, 27, 74, 106, 84, 42, 37, 41, 18, 23, 29, 95, 38, 80, 10, 16, 31, 71, 32, 99, 105, 9, 82, 7, 73, 35, 26, 72, 5, 77, 69, 90, 64, 75, 66, 67, 13, 0, 11, 65, 8, 3, 2, 4, 70, 68, 1, 6], [117, 112, 53, 60, 115, 62, 113, 51, 127, 61, 47, 120, 121, 45, 114, 48, 56, 57, 39, 103, 124, 100, 86, 118, 116, 89, 25, 22, 125, 54, 59, 91, 122, 63, 109, 119, 123, 104, 97, 55, 30, 111, 40, 110, 46, 93, 49, 33, 52, 81, 126, 50, 108, 19, 101, 36, 17, 94, 83, 76, 14, 79, 15, 92, 98, 107, 28, 21, 44, 78, 85, 74, 58, 12, 37, 24, 34, 42, 88, 20, 102, 87, 106, 10, 43, 84, 27, 41, 38, 23, 32, 95, 71, 96, 7, 80, 82, 16, 18, 73, 9, 29, 105, 31, 5, 35, 0, 69, 8, 2, 26, 3, 72, 66, 99, 64, 75, 77, 13, 90, 67, 11, 1, 4, 68, 70, 65, 6], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 124, 57, 56, 48, 118, 100, 103, 63, 86, 25, 125, 22, 116, 109, 89, 122, 104, 111, 97, 110, 54, 119, 59, 123, 91, 108, 40, 30, 55, 126, 49, 33, 46, 94, 93, 50, 83, 19, 101, 17, 81, 98, 52, 76, 106, 85, 79, 44, 42, 107, 58, 78, 34, 14, 36, 21, 102, 37, 28, 92, 24, 12, 15, 10, 88, 38, 43, 74, 96, 84, 27, 32, 41, 95, 23, 87, 7, 20, 16, 35, 18, 80, 31, 9, 71, 5, 29, 82, 73, 0, 99, 69, 64, 26, 13, 67, 66, 8, 105, 72, 2, 75, 11, 3, 90, 77, 1, 70, 4, 65, 68, 6], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 124, 57, 48, 89, 118, 86, 56, 100, 116, 103, 22, 54, 122, 110, 109, 97, 25, 63, 125, 30, 91, 46, 126, 59, 111, 52, 119, 55, 50, 104, 40, 33, 108, 123, 93, 81, 36, 49, 101, 19, 83, 94, 58, 12, 79, 98, 28, 21, 44, 85, 15, 34, 76, 78, 14, 42, 92, 74, 107, 17, 102, 24, 106, 37, 87, 20, 88, 95, 84, 41, 27, 31, 23, 38, 18, 96, 43, 29, 16, 82, 7, 80, 9, 71, 10, 32, 35, 8, 99, 105, 73, 26, 75, 66, 69, 13, 5, 90, 0, 11, 64, 77, 2, 67, 72, 3, 68, 1, 70, 4, 65, 6], [117, 112, 115, 53, 60, 113, 62, 51, 61, 120, 127, 47, 121, 45, 114, 124, 48, 56, 57, 118, 86, 39, 89, 116, 100, 103, 22, 125, 63, 109, 122, 110, 54, 59, 97, 91, 30, 25, 46, 55, 111, 40, 104, 126, 52, 119, 50, 123, 93, 81, 108, 33, 17, 21, 19, 15, 83, 101, 58, 76, 14, 79, 92, 85, 44, 12, 28, 49, 24, 34, 36, 98, 88, 74, 94, 107, 43, 78, 42, 37, 87, 20, 106, 84, 23, 102, 9, 16, 31, 18, 71, 96, 29, 35, 7, 80, 27, 32, 38, 95, 41, 10, 82, 26, 105, 69, 8, 5, 77, 75, 73, 66, 11, 3, 13, 64, 0, 2, 90, 72, 99, 67, 4, 70, 65, 68, 1, 6], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 56, 124, 57, 48, 118, 89, 100, 103, 86, 39, 116, 125, 122, 22, 63, 54, 40, 97, 91, 109, 25, 111, 59, 119, 104, 126, 30, 110, 55, 123, 46, 50, 81, 17, 93, 33, 83, 52, 94, 19, 58, 108, 49, 15, 101, 21, 98, 36, 28, 79, 76, 12, 37, 92, 88, 24, 14, 85, 44, 34, 43, 78, 20, 87, 27, 74, 107, 42, 106, 84, 31, 102, 96, 18, 38, 23, 80, 95, 7, 32, 71, 29, 35, 41, 69, 16, 10, 9, 0, 82, 8, 73, 64, 105, 26, 66, 5, 67, 99, 90, 77, 11, 3, 75, 13, 2, 1, 4, 72, 65, 6, 70, 68], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 48, 124, 118, 56, 100, 86, 125, 89, 103, 57, 116, 22, 25, 97, 122, 54, 104, 63, 91, 59, 111, 109, 55, 119, 46, 110, 30, 40, 123, 108, 52, 126, 33, 81, 19, 93, 58, 50, 83, 101, 36, 15, 94, 85, 92, 17, 79, 76, 98, 28, 21, 49, 12, 14, 78, 34, 37, 43, 24, 44, 88, 87, 42, 38, 84, 102, 107, 96, 27, 20, 106, 10, 23, 32, 95, 18, 74, 31, 7, 29, 82, 71, 41, 80, 8, 35, 9, 105, 16, 99, 11, 5, 75, 69, 26, 64, 73, 66, 77, 2, 0, 67, 90, 13, 3, 72, 4, 65, 1, 6, 68, 70], [117, 112, 53, 115, 60, 113, 62, 51, 61, 120, 127, 47, 121, 45, 114, 39, 48, 56, 57, 116, 124, 100, 89, 103, 86, 118, 125, 22, 59, 91, 97, 110, 25, 122, 52, 30, 54, 109, 63, 111, 104, 46, 119, 40, 55, 126, 33, 123, 81, 108, 93, 19, 50, 49, 58, 83, 79, 17, 94, 85, 15, 101, 14, 12, 28, 44, 21, 92, 34, 76, 98, 74, 107, 24, 78, 88, 36, 37, 43, 102, 96, 42, 20, 84, 87, 106, 8, 27, 23, 10, 7, 95, 32, 31, 29, 38, 71, 35, 16, 73, 41, 82, 80, 18, 105, 9, 26, 69, 75, 5, 67, 11, 13, 99, 2, 90, 77, 64, 3, 0, 66, 72, 6, 4, 68, 1, 70, 65], [117, 112, 53, 115, 60, 113, 62, 51, 120, 127, 61, 47, 121, 45, 114, 86, 57, 118, 124, 48, 56, 125, 39, 22, 100, 103, 116, 122, 89, 59, 63, 25, 97, 109, 110, 91, 54, 40, 119, 30, 123, 111, 104, 81, 108, 55, 46, 126, 52, 50, 33, 83, 17, 14, 93, 19, 49, 76, 101, 85, 58, 21, 36, 92, 15, 44, 94, 12, 79, 107, 78, 43, 74, 28, 88, 42, 34, 98, 84, 24, 87, 106, 20, 102, 71, 23, 37, 96, 16, 27, 10, 18, 7, 95, 38, 32, 105, 82, 69, 80, 29, 41, 9, 8, 31, 73, 35, 11, 26, 77, 75, 5, 66, 3, 67, 0, 2, 72, 64, 13, 99, 90, 68, 6, 65, 1, 4, 70], [117, 112, 53, 115, 60, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 57, 56, 118, 124, 86, 48, 125, 103, 89, 22, 100, 122, 39, 25, 63, 54, 119, 59, 116, 97, 46, 91, 30, 110, 109, 126, 55, 111, 104, 40, 123, 52, 108, 49, 81, 33, 19, 93, 92, 21, 17, 50, 94, 85, 44, 83, 58, 14, 101, 36, 15, 12, 76, 79, 98, 43, 28, 37, 88, 107, 24, 34, 106, 78, 87, 84, 96, 10, 95, 102, 23, 20, 42, 74, 27, 82, 7, 32, 41, 31, 29, 38, 80, 18, 73, 16, 35, 105, 71, 9, 8, 26, 99, 69, 11, 75, 77, 5, 72, 90, 66, 13, 0, 2, 64, 3, 67, 6, 68, 65, 4, 70, 1], [117, 112, 53, 115, 60, 113, 62, 51, 61, 127, 120, 47, 121, 45, 114, 124, 56, 118, 57, 48, 86, 100, 39, 103, 89, 125, 22, 25, 122, 59, 116, 123, 40, 63, 119, 91, 109, 54, 111, 55, 30, 46, 104, 97, 110, 93, 81, 108, 52, 50, 19, 33, 101, 126, 58, 17, 92, 44, 49, 36, 83, 94, 79, 15, 21, 85, 12, 14, 28, 76, 98, 102, 24, 78, 43, 34, 88, 37, 106, 107, 96, 87, 42, 20, 16, 84, 32, 23, 82, 10, 29, 95, 38, 27, 7, 31, 71, 73, 80, 74, 18, 41, 35, 99, 9, 26, 69, 72, 105, 75, 11, 5, 64, 2, 8, 67, 0, 66, 77, 13, 65, 3, 6, 90, 68, 1, 70, 4], [117, 112, 53, 60, 115, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 124, 48, 39, 57, 118, 56, 86, 103, 89, 100, 116, 125, 22, 25, 119, 110, 63, 97, 30, 54, 40, 91, 122, 46, 52, 59, 109, 111, 55, 104, 123, 81, 58, 50, 126, 33, 19, 17, 93, 83, 49, 108, 94, 21, 85, 15, 24, 101, 36, 76, 79, 92, 12, 14, 78, 44, 37, 43, 98, 28, 84, 74, 88, 34, 10, 107, 20, 42, 106, 87, 102, 7, 16, 96, 71, 31, 23, 82, 41, 29, 27, 72, 9, 32, 18, 73, 80, 95, 105, 35, 5, 38, 26, 0, 69, 2, 77, 67, 64, 11, 66, 3, 99, 75, 13, 8, 90, 1, 68, 4, 70, 65, 6], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 39, 124, 48, 57, 118, 86, 100, 125, 116, 110, 89, 56, 22, 103, 25, 63, 119, 46, 109, 59, 40, 97, 91, 104, 52, 123, 122, 54, 30, 111, 108, 58, 101, 55, 93, 33, 81, 36, 126, 21, 50, 17, 19, 83, 94, 92, 12, 15, 98, 107, 44, 14, 43, 78, 76, 79, 85, 34, 49, 106, 42, 28, 87, 24, 88, 102, 10, 37, 84, 7, 18, 74, 41, 71, 20, 82, 32, 96, 9, 95, 31, 5, 72, 16, 38, 27, 23, 29, 69, 26, 2, 64, 11, 35, 73, 80, 0, 13, 105, 90, 66, 77, 75, 3, 99, 67, 4, 70, 8, 1, 65, 68, 6], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 39, 57, 124, 56, 86, 118, 125, 116, 100, 103, 22, 48, 89, 110, 25, 59, 54, 63, 97, 91, 122, 30, 40, 52, 109, 111, 126, 46, 123, 104, 58, 55, 119, 50, 108, 33, 17, 81, 93, 19, 83, 107, 85, 12, 14, 49, 21, 101, 15, 94, 92, 98, 36, 44, 79, 76, 78, 106, 34, 24, 42, 88, 28, 37, 43, 10, 84, 96, 20, 74, 102, 87, 27, 7, 9, 16, 23, 31, 38, 72, 95, 71, 41, 18, 32, 80, 73, 29, 82, 26, 105, 5, 77, 11, 69, 35, 66, 99, 3, 13, 67, 75, 64, 2, 90, 0, 65, 8, 68, 70, 4, 6, 1], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 124, 39, 114, 57, 125, 56, 118, 103, 48, 116, 86, 100, 89, 22, 122, 91, 46, 40, 110, 63, 109, 25, 59, 111, 55, 123, 54, 97, 52, 50, 58, 126, 108, 104, 119, 30, 33, 81, 93, 101, 83, 94, 49, 17, 21, 36, 107, 79, 12, 19, 76, 44, 14, 78, 15, 92, 42, 28, 85, 24, 34, 98, 106, 20, 10, 37, 43, 71, 88, 84, 74, 7, 96, 87, 72, 27, 102, 32, 31, 9, 73, 95, 38, 80, 23, 16, 5, 69, 29, 82, 18, 11, 41, 66, 0, 105, 26, 64, 35, 99, 77, 13, 67, 2, 3, 70, 4, 75, 90, 8, 68, 1, 6, 65], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 56, 124, 57, 86, 100, 118, 125, 89, 116, 48, 22, 103, 91, 25, 59, 111, 122, 97, 40, 109, 110, 119, 54, 123, 63, 30, 55, 104, 81, 46, 93, 126, 33, 19, 50, 52, 17, 49, 79, 83, 85, 101, 14, 94, 15, 92, 12, 21, 78, 108, 76, 58, 44, 98, 107, 36, 10, 28, 88, 74, 20, 106, 24, 43, 84, 87, 34, 37, 102, 42, 31, 72, 96, 71, 27, 95, 9, 73, 23, 7, 38, 18, 16, 32, 82, 80, 29, 35, 26, 69, 99, 5, 13, 11, 41, 75, 77, 105, 90, 8, 2, 66, 3, 70, 67, 0, 64, 6, 65, 4, 68, 1]], "model.layers.16.self_attn.q_proj": [[61, 125, 115, 121, 36, 101, 43, 62, 127, 63, 55, 92, 100, 97, 57, 40, 124, 60, 119, 54, 109, 49, 123, 126, 53, 59, 58, 113, 47, 117, 114, 118, 112, 50, 103, 51, 116, 52, 122, 45, 120, 102, 56, 41, 48, 89, 46, 110, 111, 37, 42, 107, 44, 108, 104, 32, 33, 39, 38, 105, 106, 99, 84, 35, 34, 98, 76, 10, 13, 24, 29, 72, 70, 95, 5, 16, 88, 21, 96, 2, 25, 71, 9, 4, 94, 86, 65, 0, 26, 80, 31, 23, 11, 93, 15, 30, 82, 77, 20, 3, 28, 78, 22, 67, 73, 18, 27, 90, 91, 87, 85, 6, 14, 75, 83, 79, 69, 8, 74, 19, 12, 81, 17, 68, 66, 1, 7, 64], [125, 61, 121, 115, 62, 55, 124, 101, 57, 126, 49, 52, 119, 127, 60, 112, 63, 50, 123, 120, 59, 53, 47, 117, 54, 113, 114, 118, 43, 56, 116, 122, 51, 107, 58, 48, 109, 45, 46, 111, 110, 40, 108, 44, 106, 92, 42, 41, 35, 105, 104, 39, 36, 38, 103, 37, 100, 102, 98, 99, 22, 25, 34, 97, 33, 96, 19, 32, 93, 89, 95, 18, 31, 94, 86, 15, 30, 29, 76, 78, 4, 26, 24, 23, 11, 72, 28, 16, 70, 9, 13, 77, 27, 21, 87, 83, 81, 90, 85, 17, 91, 88, 10, 71, 20, 2, 73, 84, 0, 82, 65, 74, 3, 79, 5, 80, 14, 67, 6, 12, 8, 1, 75, 64, 68, 66, 69, 7], [61, 125, 27, 121, 36, 94, 87, 100, 25, 62, 115, 35, 21, 63, 101, 20, 43, 127, 122, 60, 55, 32, 57, 49, 102, 40, 96, 74, 84, 95, 54, 126, 123, 52, 79, 53, 37, 113, 124, 118, 47, 119, 93, 114, 50, 117, 44, 120, 10, 112, 56, 51, 109, 59, 16, 48, 116, 83, 58, 45, 99, 46, 42, 19, 41, 104, 30, 31, 90, 82, 110, 29, 111, 92, 6, 107, 86, 12, 89, 105, 23, 13, 80, 15, 97, 98, 39, 91, 18, 28, 34, 108, 38, 77, 78, 33, 106, 17, 4, 24, 26, 103, 88, 73, 85, 5, 66, 68, 1, 9, 2, 76, 0, 67, 14, 81, 8, 22, 71, 7, 69, 64, 70, 72, 3, 75, 11, 65], [61, 125, 100, 96, 26, 30, 88, 122, 27, 121, 87, 77, 92, 98, 115, 32, 81, 75, 20, 43, 17, 93, 8, 94, 36, 62, 33, 29, 90, 44, 86, 99, 23, 41, 63, 34, 127, 40, 85, 49, 68, 101, 83, 57, 73, 102, 69, 19, 55, 110, 53, 56, 118, 60, 113, 48, 116, 2, 103, 16, 84, 13, 24, 12, 11, 50, 52, 47, 54, 31, 112, 51, 91, 79, 14, 109, 18, 6, 117, 126, 95, 78, 123, 119, 25, 97, 89, 21, 120, 124, 35, 111, 1, 72, 46, 42, 58, 76, 15, 114, 104, 45, 64, 59, 80, 38, 7, 28, 105, 67, 39, 107, 108, 106, 74, 82, 70, 71, 37, 66, 10, 4, 22, 9, 5, 3, 0, 65], [42, 98, 30, 124, 24, 85, 116, 106, 15, 19, 12, 17, 8, 88, 102, 78, 95, 52, 27, 10, 68, 66, 70, 94, 81, 38, 7, 119, 60, 90, 125, 22, 47, 83, 64, 120, 63, 55, 11, 75, 21, 59, 37, 92, 41, 53, 1, 26, 86, 76, 9, 111, 109, 114, 28, 44, 49, 103, 126, 23, 80, 117, 93, 121, 57, 36, 91, 108, 34, 25, 62, 123, 29, 6, 14, 48, 87, 35, 58, 3, 105, 79, 32, 45, 20, 33, 99, 118, 67, 113, 100, 18, 51, 69, 43, 115, 110, 31, 61, 46, 127, 5, 13, 122, 40, 89, 39, 72, 104, 56, 73, 54, 65, 2, 97, 82, 101, 50, 77, 107, 74, 0, 112, 16, 84, 96, 71, 4], [42, 124, 98, 63, 116, 30, 106, 38, 27, 19, 85, 88, 24, 62, 15, 35, 90, 17, 28, 10, 87, 57, 102, 99, 53, 122, 114, 47, 59, 61, 78, 109, 48, 108, 56, 60, 94, 123, 120, 125, 117, 113, 40, 12, 119, 26, 110, 43, 8, 45, 50, 126, 44, 118, 115, 31, 58, 52, 95, 46, 111, 80, 121, 93, 127, 112, 51, 54, 105, 36, 107, 86, 70, 81, 20, 49, 100, 68, 55, 37, 103, 39, 22, 96, 104, 33, 11, 41, 32, 18, 83, 21, 97, 23, 5, 84, 29, 101, 91, 34, 0, 92, 3, 6, 66, 74, 25, 89, 1, 82, 77, 71, 9, 16, 4, 13, 79, 7, 14, 75, 64, 69, 67, 76, 73, 65, 2, 72], [42, 98, 30, 124, 106, 85, 19, 63, 24, 27, 17, 59, 116, 15, 88, 12, 10, 78, 68, 8, 53, 70, 60, 94, 81, 102, 95, 28, 66, 122, 117, 52, 58, 45, 93, 125, 96, 76, 90, 64, 72, 89, 119, 38, 55, 120, 41, 103, 62, 100, 49, 47, 80, 82, 126, 114, 108, 21, 127, 83, 123, 44, 87, 79, 111, 113, 32, 37, 36, 35, 40, 14, 26, 20, 50, 43, 110, 46, 91, 121, 105, 48, 75, 25, 109, 115, 56, 92, 104, 1, 4, 71, 51, 61, 65, 101, 16, 6, 57, 11, 74, 31, 54, 29, 67, 22, 7, 3, 84, 23, 99, 33, 118, 2, 97, 39, 107, 13, 112, 86, 18, 73, 9, 77, 34, 5, 0, 69], [42, 98, 124, 52, 106, 30, 85, 15, 24, 27, 19, 88, 63, 60, 17, 45, 10, 116, 53, 95, 12, 108, 117, 59, 100, 121, 8, 113, 102, 94, 114, 119, 122, 48, 93, 47, 44, 58, 40, 62, 36, 37, 123, 78, 105, 35, 111, 49, 28, 125, 81, 90, 38, 57, 43, 104, 33, 56, 109, 46, 70, 103, 50, 41, 118, 68, 115, 23, 61, 51, 32, 54, 55, 127, 20, 126, 87, 6, 112, 39, 110, 80, 101, 120, 67, 91, 99, 25, 97, 96, 83, 21, 86, 82, 66, 92, 26, 107, 89, 31, 1, 3, 13, 71, 84, 74, 29, 22, 75, 79, 18, 11, 77, 34, 16, 64, 14, 72, 76, 69, 73, 2, 5, 9, 7, 65, 4, 0], [120, 39, 51, 48, 119, 123, 25, 112, 98, 89, 82, 53, 13, 31, 54, 44, 52, 113, 127, 111, 122, 121, 59, 58, 124, 60, 126, 117, 57, 50, 118, 116, 62, 61, 63, 125, 91, 107, 47, 114, 46, 115, 41, 49, 109, 56, 55, 20, 108, 28, 106, 45, 110, 42, 104, 43, 92, 103, 102, 1, 4, 11, 87, 105, 96, 65, 35, 36, 86, 101, 37, 72, 38, 100, 40, 33, 18, 99, 32, 66, 93, 90, 29, 95, 23, 7, 79, 97, 30, 69, 94, 8, 34, 80, 21, 77, 16, 27, 84, 15, 26, 85, 75, 14, 0, 9, 68, 19, 76, 88, 64, 22, 3, 24, 83, 78, 17, 5, 81, 10, 71, 67, 12, 2, 73, 6, 74, 70], [51, 48, 39, 120, 25, 98, 82, 123, 119, 87, 127, 91, 50, 121, 124, 54, 45, 122, 89, 106, 13, 31, 118, 53, 111, 86, 28, 52, 58, 112, 61, 113, 49, 46, 92, 126, 47, 110, 60, 57, 115, 62, 40, 117, 63, 44, 56, 108, 43, 55, 15, 41, 79, 116, 104, 59, 125, 77, 73, 36, 84, 109, 4, 38, 114, 65, 20, 23, 42, 103, 96, 7, 107, 99, 105, 27, 18, 100, 11, 102, 3, 37, 35, 68, 95, 8, 93, 101, 72, 88, 76, 34, 66, 94, 64, 22, 24, 30, 69, 85, 83, 90, 32, 33, 80, 78, 26, 75, 97, 10, 81, 17, 16, 29, 19, 74, 1, 2, 12, 70, 21, 14, 67, 9, 5, 71, 6, 0], [120, 39, 51, 25, 48, 98, 31, 89, 54, 127, 53, 123, 82, 20, 91, 11, 13, 56, 58, 118, 86, 49, 41, 57, 113, 116, 119, 112, 124, 15, 43, 4, 111, 47, 121, 60, 122, 62, 115, 28, 108, 63, 52, 107, 23, 114, 92, 50, 117, 126, 61, 59, 42, 45, 125, 7, 72, 32, 46, 55, 36, 102, 79, 88, 1, 104, 8, 109, 44, 19, 101, 93, 96, 27, 103, 87, 37, 77, 40, 110, 76, 99, 9, 33, 94, 106, 95, 17, 26, 66, 105, 69, 35, 100, 81, 30, 29, 97, 90, 85, 38, 75, 0, 84, 22, 21, 16, 80, 83, 24, 65, 10, 18, 68, 14, 34, 78, 6, 64, 12, 73, 2, 74, 70, 71, 5, 67, 3], [39, 51, 56, 120, 48, 123, 98, 119, 25, 20, 31, 89, 82, 28, 13, 50, 52, 54, 122, 43, 60, 87, 86, 41, 59, 79, 53, 91, 116, 88, 115, 4, 113, 124, 9, 92, 118, 7, 42, 58, 127, 126, 73, 112, 11, 57, 77, 114, 76, 125, 101, 109, 121, 61, 63, 1, 117, 81, 17, 111, 62, 102, 49, 10, 55, 19, 90, 108, 46, 8, 104, 45, 21, 85, 47, 93, 44, 22, 27, 74, 6, 38, 97, 23, 70, 75, 107, 14, 95, 32, 69, 15, 78, 40, 105, 103, 106, 84, 99, 33, 18, 96, 26, 16, 94, 0, 66, 36, 35, 110, 72, 67, 30, 80, 29, 12, 37, 24, 3, 5, 65, 100, 64, 71, 83, 34, 68, 2], [42, 100, 106, 90, 79, 78, 18, 20, 31, 85, 94, 75, 77, 111, 16, 86, 70, 73, 26, 4, 76, 46, 72, 126, 125, 124, 58, 3, 64, 67, 66, 56, 103, 23, 55, 117, 40, 28, 82, 21, 52, 2, 45, 65, 114, 80, 0, 11, 33, 127, 24, 10, 12, 119, 44, 71, 83, 81, 95, 88, 62, 15, 7, 97, 84, 8, 74, 5, 14, 53, 25, 93, 39, 122, 22, 109, 30, 59, 50, 61, 121, 69, 29, 6, 57, 89, 123, 9, 115, 13, 32, 113, 104, 47, 17, 91, 36, 54, 27, 41, 120, 99, 37, 51, 110, 105, 112, 34, 92, 116, 19, 108, 87, 48, 107, 43, 98, 60, 96, 35, 102, 49, 38, 68, 101, 118, 63, 1], [42, 100, 20, 106, 18, 77, 85, 90, 88, 111, 79, 75, 95, 73, 31, 70, 58, 40, 62, 23, 94, 26, 125, 52, 46, 16, 127, 68, 82, 78, 2, 89, 124, 104, 126, 45, 11, 109, 117, 72, 80, 61, 84, 56, 57, 67, 64, 1, 9, 99, 47, 114, 60, 33, 48, 96, 122, 21, 86, 103, 55, 12, 36, 14, 13, 97, 119, 3, 54, 25, 91, 15, 50, 115, 112, 101, 76, 17, 81, 4, 29, 65, 69, 39, 44, 102, 51, 32, 27, 92, 38, 8, 93, 53, 116, 105, 37, 24, 107, 22, 0, 123, 30, 49, 63, 41, 87, 108, 7, 110, 28, 5, 71, 83, 113, 74, 43, 59, 10, 118, 6, 34, 121, 120, 35, 19, 98, 66], [42, 100, 106, 20, 78, 90, 18, 85, 77, 73, 88, 111, 75, 94, 16, 70, 46, 124, 58, 26, 79, 72, 31, 125, 4, 56, 2, 23, 40, 67, 82, 52, 64, 127, 62, 33, 126, 104, 9, 117, 119, 95, 3, 103, 24, 11, 13, 45, 68, 57, 65, 55, 0, 99, 30, 86, 80, 29, 114, 44, 21, 8, 47, 48, 50, 66, 112, 22, 25, 60, 12, 91, 84, 69, 115, 7, 89, 109, 1, 81, 39, 96, 83, 116, 105, 97, 51, 36, 122, 93, 15, 123, 113, 92, 54, 32, 59, 87, 14, 28, 38, 63, 120, 110, 17, 74, 49, 53, 107, 118, 102, 61, 27, 108, 71, 10, 35, 5, 37, 34, 43, 41, 19, 76, 101, 6, 121, 98], [42, 100, 94, 88, 106, 85, 31, 78, 18, 90, 16, 111, 20, 79, 95, 70, 75, 77, 46, 86, 11, 73, 56, 124, 127, 40, 80, 72, 10, 55, 58, 45, 67, 26, 71, 21, 39, 65, 125, 104, 13, 83, 23, 119, 62, 117, 33, 82, 47, 103, 126, 5, 114, 12, 57, 91, 109, 2, 69, 0, 19, 97, 6, 74, 24, 107, 60, 1, 52, 84, 17, 61, 76, 53, 30, 81, 15, 29, 7, 102, 44, 113, 28, 115, 98, 105, 110, 4, 59, 38, 89, 120, 122, 101, 116, 99, 3, 25, 50, 32, 64, 22, 49, 93, 36, 8, 54, 123, 118, 34, 37, 112, 63, 92, 27, 108, 51, 35, 96, 87, 121, 43, 48, 41, 68, 66, 14, 9], [40, 54, 63, 36, 98, 122, 90, 20, 88, 81, 22, 123, 29, 78, 15, 93, 24, 13, 82, 33, 57, 62, 115, 126, 26, 38, 91, 68, 83, 51, 27, 76, 8, 61, 125, 120, 92, 114, 17, 127, 113, 34, 45, 124, 121, 21, 74, 19, 10, 11, 59, 100, 116, 50, 117, 119, 56, 46, 55, 97, 118, 73, 70, 16, 94, 31, 84, 53, 44, 107, 52, 28, 23, 48, 111, 106, 75, 109, 87, 18, 49, 77, 72, 80, 96, 69, 30, 35, 43, 25, 79, 89, 108, 112, 85, 0, 71, 32, 95, 101, 99, 1, 47, 39, 9, 2, 65, 58, 6, 37, 42, 4, 110, 105, 41, 103, 102, 60, 86, 14, 67, 3, 5, 104, 66, 64, 12, 7], [40, 54, 98, 63, 29, 82, 20, 76, 16, 122, 22, 71, 88, 50, 36, 3, 78, 90, 26, 93, 8, 73, 7, 83, 13, 61, 123, 15, 81, 84, 87, 5, 80, 66, 17, 64, 127, 115, 21, 77, 89, 74, 113, 65, 62, 12, 0, 125, 120, 100, 119, 86, 27, 23, 104, 97, 52, 91, 116, 121, 24, 2, 59, 19, 55, 112, 10, 57, 18, 48, 14, 51, 79, 118, 126, 9, 11, 43, 69, 53, 49, 92, 124, 45, 114, 56, 106, 6, 4, 111, 25, 33, 31, 117, 95, 70, 34, 30, 94, 67, 32, 85, 72, 46, 110, 28, 75, 37, 58, 38, 35, 103, 96, 99, 108, 42, 102, 109, 105, 101, 47, 1, 68, 60, 39, 41, 107, 44], [40, 61, 63, 122, 98, 54, 26, 123, 29, 88, 20, 24, 113, 93, 57, 22, 81, 90, 127, 125, 82, 78, 45, 76, 16, 73, 91, 92, 44, 121, 99, 32, 50, 62, 96, 19, 85, 38, 107, 15, 106, 28, 126, 47, 118, 112, 41, 120, 31, 109, 105, 119, 46, 116, 102, 21, 51, 39, 23, 74, 111, 117, 14, 55, 37, 43, 33, 89, 86, 35, 94, 53, 42, 103, 52, 5, 69, 60, 114, 110, 97, 115, 49, 80, 58, 12, 27, 17, 95, 13, 124, 108, 56, 48, 71, 59, 100, 83, 36, 104, 25, 101, 84, 87, 11, 30, 3, 10, 79, 66, 0, 64, 75, 68, 2, 34, 9, 18, 65, 8, 70, 1, 77, 6, 67, 4, 72, 7], [63, 40, 54, 122, 100, 123, 125, 127, 62, 57, 98, 113, 121, 119, 118, 59, 116, 115, 61, 51, 55, 114, 120, 53, 29, 124, 50, 52, 117, 91, 48, 56, 126, 20, 111, 26, 45, 36, 93, 88, 49, 112, 46, 87, 47, 25, 110, 27, 43, 106, 22, 81, 103, 109, 23, 108, 58, 60, 107, 99, 39, 38, 42, 44, 105, 41, 102, 96, 35, 101, 34, 33, 97, 37, 89, 104, 94, 92, 83, 21, 82, 28, 32, 79, 86, 24, 18, 90, 95, 15, 31, 16, 30, 75, 84, 13, 17, 85, 11, 80, 0, 73, 19, 74, 10, 2, 1, 72, 76, 64, 3, 69, 66, 68, 5, 8, 6, 78, 14, 71, 9, 70, 67, 77, 4, 12, 65, 7], [48, 52, 63, 116, 127, 55, 123, 61, 53, 54, 124, 117, 51, 56, 119, 62, 60, 122, 115, 126, 120, 112, 118, 59, 47, 111, 121, 49, 114, 57, 125, 58, 113, 50, 45, 46, 109, 36, 110, 101, 107, 108, 42, 44, 106, 43, 105, 98, 32, 96, 41, 38, 40, 39, 103, 100, 104, 37, 92, 102, 34, 90, 26, 99, 33, 35, 89, 93, 95, 97, 91, 31, 15, 94, 28, 20, 30, 23, 27, 29, 86, 22, 84, 17, 85, 81, 79, 87, 21, 25, 82, 76, 88, 14, 18, 74, 12, 72, 8, 10, 16, 83, 78, 24, 66, 69, 67, 5, 71, 77, 7, 3, 19, 11, 64, 2, 68, 80, 6, 4, 70, 73, 13, 75, 0, 1, 65, 9], [116, 52, 48, 101, 57, 37, 100, 97, 120, 59, 123, 94, 95, 63, 112, 114, 58, 54, 126, 38, 34, 60, 92, 50, 110, 127, 15, 125, 46, 39, 47, 113, 43, 55, 106, 61, 118, 56, 124, 111, 121, 44, 45, 99, 32, 22, 49, 105, 107, 103, 108, 119, 115, 40, 62, 51, 53, 42, 36, 109, 88, 85, 117, 98, 41, 122, 102, 35, 76, 91, 31, 23, 104, 89, 27, 28, 96, 90, 86, 30, 12, 17, 18, 20, 87, 26, 33, 29, 82, 84, 93, 25, 79, 14, 72, 21, 81, 24, 74, 83, 19, 10, 8, 78, 16, 80, 71, 5, 67, 69, 7, 3, 77, 11, 13, 73, 75, 2, 70, 9, 4, 66, 68, 6, 1, 65, 0, 64], [116, 48, 97, 101, 52, 88, 83, 77, 11, 16, 73, 4, 71, 22, 14, 70, 126, 85, 18, 61, 0, 2, 68, 42, 30, 64, 26, 27, 105, 55, 75, 9, 28, 80, 13, 93, 17, 47, 120, 65, 82, 121, 6, 24, 7, 67, 19, 41, 91, 5, 78, 118, 66, 37, 74, 127, 20, 69, 34, 21, 86, 10, 89, 23, 108, 3, 29, 31, 54, 57, 81, 53, 84, 87, 79, 114, 15, 36, 103, 72, 123, 50, 12, 8, 99, 1, 76, 45, 35, 43, 51, 33, 25, 92, 96, 98, 115, 46, 110, 94, 95, 90, 49, 112, 32, 113, 109, 102, 38, 44, 63, 39, 100, 106, 104, 40, 58, 59, 125, 60, 56, 124, 119, 62, 111, 107, 117, 122], [52, 63, 116, 48, 60, 114, 55, 51, 54, 122, 124, 53, 117, 125, 59, 127, 126, 120, 119, 47, 62, 58, 115, 123, 118, 61, 121, 111, 113, 46, 49, 101, 57, 56, 50, 109, 110, 45, 36, 112, 42, 98, 107, 32, 105, 43, 44, 97, 92, 106, 15, 108, 41, 37, 94, 90, 96, 104, 40, 103, 38, 39, 99, 100, 102, 35, 85, 34, 23, 26, 95, 33, 84, 29, 82, 87, 28, 30, 20, 27, 76, 93, 31, 91, 79, 89, 25, 17, 22, 86, 18, 81, 12, 21, 74, 8, 10, 72, 88, 5, 69, 24, 83, 14, 16, 71, 67, 78, 77, 7, 2, 3, 66, 4, 73, 9, 19, 13, 68, 11, 80, 0, 65, 75, 70, 6, 64, 1], [124, 49, 37, 55, 126, 61, 87, 118, 26, 93, 96, 121, 80, 12, 86, 119, 57, 84, 101, 50, 54, 60, 78, 123, 25, 90, 32, 67, 122, 18, 58, 85, 115, 112, 72, 29, 51, 16, 102, 120, 56, 6, 127, 59, 53, 15, 83, 52, 116, 34, 63, 92, 38, 89, 114, 62, 47, 46, 76, 79, 111, 103, 110, 104, 117, 125, 48, 98, 9, 44, 109, 36, 17, 43, 39, 45, 97, 81, 33, 21, 27, 107, 30, 10, 20, 108, 23, 106, 113, 35, 28, 31, 74, 73, 105, 42, 70, 88, 41, 22, 95, 91, 94, 11, 99, 13, 3, 19, 40, 77, 100, 24, 64, 14, 82, 75, 66, 4, 1, 69, 7, 0, 8, 65, 71, 68, 2, 5], [49, 124, 37, 55, 93, 61, 96, 126, 50, 118, 54, 57, 119, 123, 102, 121, 87, 79, 127, 112, 115, 53, 60, 122, 58, 125, 51, 120, 110, 114, 59, 85, 56, 90, 101, 26, 63, 62, 43, 116, 38, 36, 5, 46, 108, 113, 42, 52, 48, 117, 111, 44, 86, 47, 107, 41, 32, 30, 45, 109, 12, 39, 40, 104, 18, 35, 80, 105, 21, 103, 106, 24, 99, 20, 69, 100, 29, 25, 84, 33, 34, 73, 8, 92, 31, 98, 71, 89, 95, 77, 2, 94, 6, 23, 97, 91, 27, 17, 67, 28, 0, 88, 19, 11, 22, 64, 72, 15, 13, 7, 83, 16, 78, 3, 65, 81, 76, 9, 4, 70, 75, 82, 10, 74, 66, 14, 1, 68], [49, 124, 37, 55, 61, 126, 101, 118, 93, 119, 26, 87, 96, 50, 57, 123, 60, 121, 54, 21, 44, 115, 122, 58, 29, 12, 84, 85, 51, 56, 63, 90, 53, 120, 125, 47, 112, 102, 25, 59, 110, 62, 48, 116, 46, 52, 113, 114, 105, 100, 104, 45, 108, 111, 127, 43, 39, 86, 117, 103, 19, 109, 99, 27, 107, 38, 42, 28, 34, 88, 81, 80, 106, 41, 36, 72, 33, 31, 6, 40, 97, 79, 32, 35, 24, 78, 95, 98, 75, 30, 92, 67, 20, 73, 94, 17, 18, 83, 23, 82, 77, 91, 22, 10, 15, 89, 76, 14, 3, 69, 11, 8, 64, 13, 16, 70, 9, 7, 71, 0, 65, 74, 4, 5, 66, 1, 68, 2], [49, 124, 37, 55, 87, 61, 126, 118, 96, 50, 57, 123, 26, 80, 93, 101, 54, 121, 90, 86, 119, 60, 12, 122, 120, 51, 127, 115, 58, 53, 78, 84, 6, 56, 63, 114, 67, 112, 29, 52, 59, 47, 104, 85, 116, 113, 46, 25, 48, 125, 62, 102, 111, 20, 83, 65, 44, 89, 70, 16, 8, 110, 108, 107, 15, 79, 23, 21, 45, 18, 33, 99, 117, 109, 64, 92, 43, 35, 94, 72, 39, 40, 106, 32, 77, 42, 22, 2, 17, 0, 97, 71, 38, 27, 36, 103, 105, 10, 28, 76, 100, 14, 81, 30, 95, 73, 88, 34, 41, 4, 19, 9, 24, 3, 11, 13, 91, 31, 98, 69, 7, 68, 74, 82, 75, 1, 5, 66], [103, 34, 45, 28, 109, 84, 24, 38, 79, 17, 85, 86, 30, 92, 31, 12, 11, 95, 9, 22, 46, 111, 49, 23, 122, 116, 88, 14, 19, 63, 27, 36, 20, 75, 57, 44, 33, 121, 53, 107, 119, 52, 83, 115, 101, 113, 104, 124, 15, 50, 32, 48, 61, 100, 37, 97, 29, 91, 25, 94, 125, 108, 26, 93, 54, 73, 126, 106, 16, 55, 47, 56, 114, 40, 43, 42, 35, 117, 99, 112, 51, 120, 60, 21, 89, 58, 41, 123, 81, 102, 82, 62, 59, 110, 72, 105, 96, 127, 13, 90, 118, 18, 10, 87, 39, 8, 5, 98, 7, 68, 76, 78, 77, 6, 4, 80, 65, 69, 74, 67, 3, 66, 70, 1, 71, 2, 64, 0], [103, 45, 34, 28, 14, 109, 17, 12, 7, 84, 71, 66, 79, 9, 3, 5, 24, 85, 69, 44, 64, 65, 67, 31, 86, 10, 20, 96, 127, 98, 105, 81, 2, 61, 25, 76, 35, 38, 39, 51, 0, 63, 8, 124, 52, 49, 55, 72, 125, 58, 90, 23, 30, 68, 78, 19, 4, 92, 73, 75, 122, 115, 118, 57, 16, 88, 95, 111, 100, 70, 11, 77, 6, 74, 21, 46, 119, 102, 60, 89, 18, 113, 94, 13, 59, 97, 120, 83, 93, 121, 107, 126, 36, 80, 43, 53, 1, 40, 15, 116, 29, 110, 112, 54, 91, 22, 56, 123, 37, 50, 82, 48, 117, 101, 47, 41, 87, 114, 42, 99, 27, 62, 104, 33, 106, 108, 26, 32], [103, 45, 34, 38, 109, 0, 28, 79, 7, 3, 24, 84, 12, 9, 14, 2, 17, 66, 5, 31, 86, 85, 1, 67, 10, 125, 69, 122, 92, 57, 100, 25, 23, 20, 112, 64, 98, 35, 61, 58, 44, 121, 51, 26, 55, 22, 68, 21, 72, 118, 59, 49, 96, 81, 65, 105, 102, 75, 48, 82, 32, 30, 123, 70, 63, 52, 74, 90, 54, 46, 93, 13, 76, 36, 60, 111, 116, 88, 56, 124, 42, 119, 104, 6, 126, 127, 8, 120, 50, 89, 40, 110, 73, 71, 115, 18, 80, 4, 114, 94, 95, 83, 19, 91, 43, 108, 16, 62, 117, 47, 39, 29, 78, 99, 77, 15, 113, 53, 27, 107, 87, 37, 106, 97, 101, 41, 11, 33], [103, 34, 45, 28, 109, 84, 79, 24, 31, 92, 17, 86, 22, 12, 108, 63, 9, 29, 88, 30, 102, 55, 25, 89, 98, 122, 112, 58, 14, 105, 104, 53, 100, 27, 49, 20, 61, 11, 50, 57, 114, 72, 113, 33, 42, 121, 127, 81, 95, 52, 116, 38, 23, 35, 90, 83, 36, 85, 111, 124, 41, 93, 125, 18, 126, 46, 77, 39, 74, 94, 115, 123, 80, 5, 101, 59, 26, 82, 62, 119, 16, 73, 19, 13, 43, 21, 47, 118, 96, 76, 56, 32, 15, 91, 48, 51, 87, 99, 110, 106, 78, 44, 117, 97, 75, 7, 37, 8, 10, 107, 70, 54, 60, 4, 6, 120, 68, 40, 69, 3, 71, 66, 67, 65, 1, 64, 0, 2]], "model.layers.16.self_attn.k_proj": [[125, 61, 86, 36, 96, 127, 17, 115, 121, 49, 30, 62, 63, 55, 57, 113, 60, 54, 26, 122, 53, 39, 51, 118, 48, 56, 126, 107, 116, 47, 112, 59, 52, 29, 58, 108, 124, 87, 40, 114, 120, 109, 44, 119, 123, 117, 79, 50, 27, 83, 11, 20, 110, 46, 78, 18, 43, 45, 111, 13, 25, 85, 98, 105, 106, 97, 41, 24, 42, 104, 72, 16, 82, 33, 71, 38, 101, 103, 9, 34, 28, 76, 10, 37, 91, 102, 100, 89, 31, 88, 35, 4, 99, 22, 70, 80, 95, 5, 94, 92, 21, 15, 14, 2, 32, 90, 19, 93, 77, 12, 65, 84, 23, 81, 0, 7, 3, 67, 73, 75, 6, 8, 74, 1, 69, 68, 66, 64], [106, 94, 34, 124, 116, 24, 63, 19, 52, 59, 85, 17, 15, 42, 55, 60, 121, 10, 70, 78, 12, 120, 27, 49, 8, 117, 111, 58, 47, 108, 114, 53, 109, 64, 68, 18, 1, 66, 4, 126, 123, 50, 122, 22, 44, 102, 45, 91, 51, 125, 46, 127, 90, 0, 119, 62, 112, 54, 56, 11, 84, 105, 104, 48, 61, 100, 28, 57, 41, 110, 40, 113, 115, 95, 118, 14, 80, 92, 103, 37, 77, 107, 101, 76, 73, 69, 2, 16, 26, 89, 32, 99, 43, 31, 13, 35, 9, 36, 23, 33, 97, 87, 20, 25, 7, 39, 93, 65, 96, 38, 75, 71, 67, 3, 86, 29, 74, 79, 5, 88, 82, 21, 83, 30, 72, 6, 98, 81], [103, 51, 120, 34, 86, 56, 48, 95, 123, 115, 112, 119, 53, 28, 54, 122, 58, 89, 60, 126, 52, 88, 40, 82, 61, 15, 124, 118, 111, 50, 121, 63, 46, 57, 113, 47, 62, 125, 117, 55, 107, 116, 20, 26, 114, 98, 91, 108, 110, 8, 59, 45, 49, 44, 81, 11, 43, 127, 109, 13, 42, 4, 38, 105, 93, 65, 41, 73, 106, 6, 64, 30, 16, 68, 37, 24, 27, 104, 72, 23, 102, 35, 33, 36, 85, 29, 80, 100, 76, 94, 101, 99, 12, 9, 87, 97, 14, 67, 1, 21, 79, 19, 66, 83, 71, 78, 17, 32, 90, 96, 10, 7, 2, 18, 92, 0, 70, 74, 77, 25, 5, 3, 84, 39, 69, 31, 75, 22], [106, 36, 90, 18, 85, 47, 20, 42, 79, 77, 110, 16, 58, 72, 75, 124, 126, 73, 78, 70, 56, 127, 109, 117, 119, 114, 30, 95, 120, 40, 86, 4, 125, 65, 108, 76, 112, 62, 45, 52, 43, 67, 60, 99, 88, 61, 96, 51, 39, 53, 111, 0, 66, 54, 93, 97, 69, 68, 8, 100, 19, 116, 25, 55, 38, 74, 13, 103, 32, 91, 29, 115, 48, 6, 23, 57, 89, 41, 107, 64, 123, 11, 2, 31, 7, 5, 63, 101, 81, 1, 44, 94, 50, 122, 104, 92, 87, 27, 28, 17, 49, 46, 37, 105, 34, 3, 118, 12, 113, 33, 71, 83, 35, 121, 22, 80, 59, 102, 14, 9, 26, 98, 15, 82, 24, 84, 10, 21], [104, 54, 63, 34, 22, 93, 122, 120, 90, 123, 62, 119, 125, 121, 126, 127, 61, 50, 88, 59, 113, 51, 118, 81, 52, 20, 16, 116, 115, 76, 55, 64, 46, 82, 57, 53, 48, 114, 117, 56, 71, 60, 73, 91, 124, 45, 69, 111, 78, 49, 39, 109, 13, 106, 110, 43, 112, 2, 58, 44, 15, 108, 1, 87, 98, 47, 74, 11, 102, 67, 42, 80, 79, 41, 29, 100, 103, 37, 105, 96, 107, 35, 14, 92, 97, 21, 28, 25, 26, 10, 85, 95, 83, 99, 32, 19, 38, 9, 101, 8, 18, 33, 30, 70, 94, 36, 12, 72, 3, 31, 65, 68, 23, 75, 24, 40, 77, 27, 6, 89, 4, 86, 66, 84, 17, 0, 7, 5], [52, 116, 37, 48, 83, 16, 77, 33, 22, 88, 11, 112, 73, 70, 4, 71, 14, 126, 61, 127, 63, 0, 2, 50, 123, 106, 1, 121, 29, 44, 51, 55, 58, 114, 60, 111, 120, 57, 125, 93, 115, 30, 118, 56, 108, 110, 91, 122, 119, 99, 53, 49, 113, 124, 59, 54, 85, 62, 105, 117, 46, 47, 109, 107, 41, 43, 40, 102, 104, 42, 90, 103, 35, 81, 74, 101, 45, 95, 38, 75, 18, 39, 98, 8, 92, 5, 36, 9, 67, 12, 25, 32, 96, 20, 97, 34, 100, 3, 65, 94, 31, 21, 78, 89, 13, 23, 69, 24, 82, 17, 84, 87, 19, 76, 15, 64, 7, 79, 10, 28, 6, 27, 80, 72, 68, 26, 66, 86], [124, 49, 101, 113, 86, 55, 32, 126, 57, 118, 29, 54, 61, 119, 121, 50, 123, 120, 26, 58, 122, 53, 115, 51, 114, 52, 63, 47, 60, 56, 125, 116, 62, 59, 111, 112, 48, 45, 108, 80, 46, 109, 44, 103, 99, 18, 127, 87, 43, 117, 78, 110, 107, 105, 37, 42, 106, 40, 102, 98, 72, 64, 104, 12, 39, 97, 73, 3, 20, 38, 84, 41, 21, 19, 36, 100, 17, 79, 94, 92, 33, 34, 25, 30, 23, 35, 85, 89, 27, 68, 81, 6, 31, 28, 24, 10, 77, 95, 14, 91, 4, 70, 93, 83, 88, 66, 1, 16, 96, 11, 5, 71, 90, 75, 0, 22, 7, 67, 13, 65, 74, 82, 15, 9, 8, 2, 76, 69], [109, 39, 98, 84, 92, 12, 17, 64, 7, 79, 14, 9, 5, 45, 3, 66, 61, 121, 41, 49, 122, 52, 86, 95, 127, 58, 21, 119, 40, 111, 124, 55, 65, 57, 118, 63, 51, 24, 23, 126, 48, 116, 25, 102, 28, 72, 30, 60, 34, 1, 22, 44, 125, 47, 0, 106, 59, 100, 18, 120, 90, 80, 82, 117, 112, 32, 97, 46, 89, 36, 29, 104, 6, 11, 19, 4, 70, 53, 26, 74, 123, 93, 77, 35, 108, 99, 85, 68, 62, 27, 50, 16, 56, 8, 107, 31, 115, 88, 114, 78, 83, 15, 91, 38, 20, 105, 94, 37, 42, 76, 13, 54, 43, 96, 110, 10, 87, 33, 73, 101, 113, 2, 71, 81, 67, 69, 75, 103]], "model.layers.16.self_attn.qk_proj": [[106, 124, 125, 116, 61, 52, 109, 42, 49, 63, 51, 54, 120, 48, 121, 45, 53, 123, 55, 119, 127, 56, 58, 98, 126, 28, 22, 39, 34, 122, 26, 84, 20, 88, 86, 79, 57, 15, 36, 59, 112, 101, 37, 40, 100, 62, 113, 118, 103, 50, 24, 30, 47, 117, 81, 85, 78, 111, 21, 115, 104, 12, 60, 29, 82, 14, 17, 76, 93, 90, 9, 92, 114, 80, 18, 16, 73, 94, 77, 110, 11, 31, 44, 83, 95, 27, 13, 46, 19, 72, 96, 108, 71, 75, 70, 25, 107, 102, 89, 64, 91, 32, 43, 97, 3, 7, 87, 23, 8, 68, 0, 41, 4, 67, 38, 6, 69, 5, 74, 105, 33, 2, 66, 10, 35, 99, 1, 65], [106, 124, 125, 61, 116, 52, 109, 42, 63, 49, 51, 54, 120, 48, 45, 121, 56, 123, 119, 127, 98, 58, 55, 53, 28, 126, 34, 39, 20, 86, 22, 118, 36, 26, 88, 84, 37, 79, 122, 47, 15, 59, 57, 111, 113, 24, 101, 112, 40, 115, 21, 100, 62, 30, 103, 85, 117, 60, 81, 29, 78, 76, 114, 14, 50, 104, 12, 92, 17, 90, 82, 93, 73, 9, 44, 77, 94, 16, 31, 18, 108, 80, 11, 72, 27, 46, 83, 13, 95, 96, 70, 107, 75, 110, 89, 19, 25, 7, 64, 102, 0, 87, 91, 38, 67, 8, 32, 71, 97, 43, 4, 68, 23, 5, 69, 41, 3, 105, 2, 74, 66, 35, 33, 1, 99, 6, 65, 10], [106, 125, 124, 116, 61, 52, 109, 42, 49, 63, 51, 54, 120, 48, 45, 121, 119, 123, 55, 98, 126, 56, 127, 58, 53, 28, 34, 39, 88, 36, 101, 122, 86, 84, 26, 100, 22, 20, 47, 113, 118, 57, 62, 79, 24, 59, 112, 85, 15, 40, 37, 103, 81, 50, 111, 104, 21, 60, 117, 29, 115, 30, 114, 93, 14, 90, 78, 12, 94, 17, 18, 92, 9, 27, 73, 44, 82, 76, 64, 31, 102, 110, 13, 95, 80, 83, 16, 96, 91, 11, 0, 77, 46, 25, 108, 72, 70, 19, 32, 107, 38, 75, 7, 71, 87, 4, 68, 3, 89, 8, 97, 41, 23, 1, 66, 69, 105, 67, 2, 65, 33, 35, 43, 5, 74, 99, 6, 10], [106, 125, 124, 116, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 123, 119, 56, 127, 58, 98, 55, 126, 28, 39, 59, 62, 57, 22, 20, 79, 101, 118, 34, 36, 26, 84, 100, 47, 113, 122, 86, 37, 24, 115, 112, 111, 40, 117, 15, 50, 30, 88, 103, 85, 104, 81, 29, 21, 14, 78, 60, 90, 92, 114, 93, 17, 94, 76, 27, 9, 73, 18, 0, 12, 44, 82, 80, 110, 16, 83, 13, 96, 77, 46, 108, 102, 70, 31, 19, 11, 107, 95, 72, 75, 7, 25, 97, 91, 38, 67, 87, 89, 71, 68, 8, 4, 41, 43, 64, 3, 2, 32, 66, 69, 23, 1, 5, 105, 33, 65, 35, 10, 99, 74, 6], [106, 124, 125, 116, 61, 52, 63, 49, 109, 42, 51, 54, 120, 48, 45, 121, 56, 123, 119, 53, 127, 126, 58, 55, 98, 57, 20, 22, 34, 39, 86, 47, 122, 59, 101, 118, 28, 112, 79, 62, 50, 84, 15, 40, 100, 88, 26, 36, 103, 111, 24, 117, 114, 81, 37, 30, 115, 85, 104, 14, 17, 113, 18, 78, 12, 93, 29, 60, 44, 76, 27, 73, 9, 90, 21, 94, 11, 80, 92, 13, 83, 68, 82, 107, 110, 77, 16, 46, 108, 64, 19, 31, 102, 72, 95, 0, 96, 25, 7, 8, 71, 4, 70, 67, 75, 41, 3, 91, 38, 69, 87, 23, 89, 2, 43, 32, 97, 1, 5, 33, 66, 6, 105, 35, 65, 99, 74, 10], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 119, 123, 58, 127, 56, 126, 53, 98, 55, 20, 26, 22, 34, 47, 79, 57, 39, 84, 86, 62, 118, 59, 15, 103, 101, 88, 28, 36, 100, 111, 81, 24, 112, 122, 37, 50, 85, 30, 40, 60, 78, 117, 114, 14, 17, 29, 76, 104, 93, 27, 12, 21, 80, 82, 18, 73, 90, 16, 11, 94, 113, 13, 9, 92, 77, 44, 83, 115, 31, 19, 46, 110, 108, 107, 75, 96, 89, 25, 72, 8, 95, 7, 64, 23, 71, 6, 32, 91, 102, 0, 41, 97, 43, 69, 68, 87, 4, 67, 3, 70, 105, 5, 2, 35, 38, 66, 65, 33, 74, 99, 1, 10], [106, 125, 124, 116, 61, 52, 63, 109, 42, 49, 51, 120, 54, 48, 45, 119, 121, 127, 53, 56, 123, 126, 58, 98, 55, 20, 26, 57, 122, 84, 86, 22, 88, 34, 79, 118, 36, 47, 62, 28, 39, 81, 15, 24, 112, 101, 103, 37, 50, 59, 40, 30, 114, 100, 111, 85, 14, 78, 115, 90, 18, 117, 12, 17, 104, 29, 93, 27, 80, 21, 60, 113, 44, 76, 73, 94, 9, 82, 16, 83, 11, 92, 13, 77, 110, 107, 95, 19, 25, 31, 96, 108, 46, 41, 43, 75, 71, 89, 23, 102, 97, 6, 91, 32, 8, 7, 0, 72, 38, 3, 33, 67, 87, 64, 2, 69, 4, 35, 68, 74, 105, 5, 99, 66, 70, 10, 65, 1], [106, 124, 116, 125, 61, 52, 109, 42, 63, 49, 51, 54, 120, 48, 45, 119, 121, 127, 53, 56, 123, 55, 126, 98, 58, 22, 20, 26, 28, 84, 39, 86, 122, 101, 34, 57, 88, 36, 24, 100, 79, 15, 30, 81, 103, 37, 40, 112, 111, 62, 59, 85, 115, 47, 50, 118, 14, 113, 60, 117, 90, 17, 78, 93, 21, 114, 29, 104, 92, 80, 18, 12, 27, 16, 82, 73, 9, 13, 25, 76, 83, 95, 96, 44, 94, 77, 46, 107, 32, 11, 43, 91, 110, 102, 6, 19, 8, 31, 89, 0, 108, 75, 71, 64, 23, 41, 7, 97, 68, 105, 87, 33, 38, 72, 67, 3, 66, 4, 99, 35, 69, 2, 74, 5, 65, 1, 70, 10], [106, 124, 116, 125, 61, 52, 109, 42, 63, 51, 49, 54, 120, 48, 45, 127, 119, 121, 53, 98, 56, 55, 58, 34, 28, 123, 22, 20, 39, 126, 84, 101, 26, 88, 103, 86, 24, 100, 122, 36, 37, 79, 85, 112, 50, 111, 15, 30, 47, 57, 115, 113, 59, 60, 40, 62, 118, 17, 81, 78, 114, 14, 12, 21, 93, 90, 104, 117, 29, 18, 27, 76, 82, 80, 44, 108, 92, 16, 9, 73, 95, 13, 25, 77, 94, 83, 46, 96, 102, 19, 11, 8, 31, 107, 43, 75, 6, 91, 89, 0, 71, 110, 32, 23, 87, 64, 7, 97, 33, 41, 2, 38, 4, 72, 105, 67, 68, 66, 3, 35, 5, 69, 99, 1, 10, 74, 65, 70], [106, 124, 125, 61, 116, 52, 109, 42, 63, 49, 51, 54, 48, 120, 45, 121, 127, 119, 53, 56, 123, 55, 98, 58, 126, 34, 28, 101, 20, 57, 59, 84, 22, 26, 47, 115, 39, 103, 118, 36, 37, 122, 88, 50, 86, 100, 79, 111, 15, 24, 117, 113, 30, 40, 112, 17, 85, 62, 21, 12, 81, 78, 104, 60, 90, 44, 14, 9, 93, 29, 114, 18, 76, 92, 82, 16, 8, 94, 110, 27, 80, 73, 13, 83, 108, 46, 95, 96, 11, 75, 77, 6, 102, 7, 0, 64, 25, 19, 31, 32, 107, 71, 89, 3, 72, 38, 43, 4, 97, 23, 70, 91, 5, 33, 41, 68, 67, 2, 87, 66, 69, 99, 35, 74, 105, 1, 65, 10], [106, 124, 116, 125, 61, 52, 109, 42, 63, 49, 54, 51, 120, 48, 121, 45, 127, 53, 123, 119, 56, 58, 98, 55, 126, 122, 118, 57, 28, 34, 20, 15, 84, 39, 79, 22, 59, 86, 101, 26, 100, 50, 81, 88, 36, 113, 85, 37, 47, 111, 24, 40, 30, 115, 62, 60, 17, 112, 9, 12, 14, 78, 103, 117, 82, 21, 76, 73, 29, 104, 8, 18, 92, 90, 114, 93, 16, 27, 44, 94, 11, 0, 13, 80, 77, 83, 31, 64, 19, 68, 110, 7, 108, 75, 71, 70, 96, 107, 72, 25, 46, 4, 91, 67, 102, 95, 32, 89, 3, 87, 6, 2, 66, 1, 69, 5, 23, 65, 38, 97, 41, 33, 35, 105, 43, 74, 10, 99], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 127, 53, 119, 58, 56, 126, 98, 123, 122, 86, 84, 22, 34, 20, 26, 57, 15, 55, 118, 28, 39, 79, 62, 47, 100, 101, 30, 50, 40, 88, 36, 37, 85, 103, 81, 112, 111, 59, 24, 17, 76, 115, 113, 14, 21, 60, 78, 117, 90, 114, 29, 82, 104, 12, 18, 9, 73, 16, 93, 27, 92, 94, 31, 80, 83, 44, 13, 77, 0, 110, 11, 95, 8, 19, 71, 46, 70, 102, 96, 108, 75, 91, 7, 72, 89, 68, 87, 4, 32, 64, 107, 25, 38, 67, 41, 66, 3, 43, 23, 97, 5, 69, 33, 2, 99, 6, 105, 10, 65, 35, 74, 1], [106, 124, 116, 125, 61, 52, 63, 109, 49, 42, 51, 120, 54, 48, 121, 45, 119, 53, 127, 56, 58, 123, 98, 126, 20, 26, 101, 39, 34, 22, 28, 118, 86, 37, 47, 57, 122, 88, 84, 114, 112, 79, 40, 55, 103, 62, 30, 59, 15, 36, 24, 100, 85, 50, 60, 113, 21, 117, 111, 115, 90, 81, 29, 93, 78, 104, 17, 82, 94, 92, 14, 18, 76, 95, 27, 44, 46, 12, 16, 9, 25, 108, 110, 31, 96, 19, 77, 13, 73, 83, 102, 70, 80, 43, 11, 107, 75, 91, 71, 87, 32, 89, 7, 41, 38, 72, 23, 97, 8, 99, 35, 0, 33, 67, 105, 5, 3, 68, 4, 64, 69, 66, 2, 74, 1, 10, 6, 65], [106, 124, 116, 125, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 45, 121, 119, 53, 58, 123, 127, 126, 98, 56, 55, 20, 22, 28, 39, 57, 34, 101, 112, 118, 47, 122, 26, 79, 100, 37, 88, 59, 84, 86, 50, 36, 15, 62, 85, 30, 24, 115, 40, 114, 103, 81, 117, 104, 111, 60, 78, 113, 14, 12, 93, 76, 17, 82, 21, 29, 16, 73, 18, 90, 94, 9, 92, 11, 27, 70, 80, 44, 77, 64, 46, 83, 19, 72, 110, 71, 96, 31, 108, 75, 89, 0, 8, 13, 95, 25, 7, 107, 91, 97, 102, 2, 32, 43, 41, 67, 4, 87, 38, 68, 66, 23, 35, 3, 69, 65, 99, 5, 33, 10, 1, 105, 74, 6], [106, 124, 116, 125, 61, 52, 109, 63, 42, 51, 54, 49, 120, 48, 45, 121, 119, 53, 58, 127, 98, 55, 123, 126, 56, 20, 39, 22, 28, 34, 101, 15, 84, 118, 122, 47, 88, 86, 26, 79, 57, 37, 100, 30, 59, 36, 62, 24, 40, 112, 111, 115, 85, 81, 117, 50, 17, 103, 9, 78, 104, 29, 76, 93, 114, 14, 12, 113, 60, 90, 92, 94, 21, 82, 16, 73, 11, 18, 31, 80, 77, 27, 64, 46, 108, 75, 72, 19, 110, 83, 71, 44, 96, 25, 70, 102, 91, 8, 13, 41, 7, 107, 95, 4, 43, 87, 89, 67, 0, 68, 32, 3, 2, 97, 65, 38, 5, 33, 66, 69, 74, 6, 23, 35, 99, 10, 105, 1], [106, 125, 116, 124, 61, 52, 63, 109, 42, 49, 54, 51, 120, 48, 45, 121, 119, 53, 127, 56, 123, 55, 58, 98, 126, 39, 20, 118, 86, 122, 22, 28, 84, 101, 57, 26, 15, 37, 30, 79, 34, 112, 36, 47, 59, 62, 88, 50, 111, 100, 103, 81, 85, 60, 113, 24, 78, 117, 115, 40, 76, 114, 14, 12, 9, 29, 104, 93, 73, 90, 17, 18, 77, 82, 21, 94, 44, 108, 16, 31, 92, 96, 80, 110, 11, 83, 19, 46, 75, 13, 27, 72, 0, 107, 71, 7, 102, 8, 91, 70, 87, 43, 41, 64, 4, 25, 68, 23, 95, 3, 67, 97, 89, 32, 38, 6, 69, 2, 33, 5, 66, 74, 65, 105, 35, 1, 99, 10], [106, 124, 125, 116, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 119, 127, 98, 58, 126, 123, 56, 20, 26, 57, 122, 28, 101, 39, 22, 55, 36, 86, 84, 37, 34, 50, 112, 59, 15, 79, 100, 30, 114, 40, 88, 47, 115, 24, 62, 81, 118, 60, 21, 103, 85, 90, 14, 93, 104, 29, 111, 17, 117, 78, 94, 82, 16, 110, 76, 44, 113, 92, 108, 18, 9, 12, 27, 96, 77, 80, 19, 102, 25, 46, 83, 95, 73, 31, 75, 43, 13, 107, 91, 11, 72, 38, 41, 89, 32, 97, 6, 87, 64, 71, 7, 23, 35, 33, 0, 99, 66, 8, 5, 105, 67, 3, 68, 4, 69, 70, 74, 2, 65, 1, 10], [106, 124, 116, 125, 61, 52, 109, 63, 42, 51, 49, 54, 120, 48, 45, 121, 127, 119, 53, 98, 123, 58, 56, 126, 28, 39, 101, 55, 122, 26, 20, 22, 100, 86, 57, 36, 34, 59, 118, 84, 40, 15, 88, 37, 30, 117, 103, 79, 50, 24, 113, 111, 112, 62, 114, 14, 17, 60, 85, 29, 115, 81, 21, 47, 93, 44, 76, 104, 90, 95, 16, 82, 9, 27, 94, 78, 92, 73, 18, 12, 80, 108, 75, 77, 96, 31, 6, 110, 83, 72, 13, 0, 107, 46, 102, 11, 19, 91, 7, 43, 25, 87, 32, 64, 38, 97, 89, 41, 71, 23, 33, 8, 105, 2, 4, 67, 69, 3, 68, 35, 66, 74, 99, 65, 5, 1, 70, 10], [106, 124, 125, 116, 61, 52, 109, 63, 42, 49, 54, 51, 120, 48, 45, 53, 121, 119, 127, 56, 98, 58, 123, 101, 126, 34, 36, 55, 39, 118, 40, 59, 28, 20, 50, 103, 86, 115, 37, 111, 26, 100, 57, 22, 84, 24, 122, 15, 47, 88, 113, 30, 114, 62, 112, 79, 117, 104, 44, 60, 21, 81, 29, 14, 108, 78, 90, 82, 94, 76, 17, 85, 93, 92, 73, 16, 110, 18, 9, 12, 27, 102, 31, 46, 41, 77, 83, 80, 107, 75, 6, 19, 72, 95, 96, 89, 87, 8, 13, 43, 7, 11, 38, 97, 64, 0, 105, 25, 91, 23, 32, 71, 4, 33, 3, 35, 99, 69, 67, 68, 5, 66, 2, 74, 1, 65, 10, 70], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 53, 121, 119, 123, 127, 98, 58, 56, 55, 20, 39, 126, 22, 28, 57, 59, 37, 86, 34, 101, 84, 36, 118, 15, 122, 100, 30, 79, 26, 47, 103, 111, 40, 115, 24, 88, 117, 112, 113, 50, 90, 85, 81, 114, 29, 21, 14, 94, 92, 62, 93, 60, 12, 104, 78, 76, 82, 44, 27, 110, 17, 73, 80, 108, 9, 31, 77, 18, 75, 96, 83, 16, 13, 6, 95, 25, 72, 91, 11, 46, 19, 89, 102, 7, 107, 8, 0, 41, 23, 87, 32, 64, 71, 43, 105, 38, 35, 97, 33, 67, 69, 3, 68, 4, 5, 1, 99, 65, 74, 2, 66, 10, 70], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 53, 119, 123, 127, 58, 56, 98, 57, 126, 22, 86, 59, 122, 34, 28, 20, 15, 100, 39, 84, 101, 26, 55, 103, 36, 40, 118, 88, 113, 37, 79, 47, 62, 114, 24, 78, 30, 117, 111, 115, 112, 50, 85, 81, 90, 76, 29, 93, 60, 17, 27, 14, 92, 9, 94, 104, 12, 44, 18, 73, 21, 82, 80, 108, 110, 83, 31, 16, 75, 13, 95, 46, 77, 96, 91, 19, 107, 89, 72, 11, 0, 25, 43, 102, 6, 7, 8, 41, 64, 87, 71, 32, 23, 97, 38, 3, 68, 33, 5, 67, 35, 4, 66, 105, 1, 74, 70, 69, 2, 65, 99, 10], [106, 124, 125, 116, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 45, 121, 123, 53, 119, 127, 98, 58, 56, 126, 59, 28, 57, 39, 55, 34, 101, 26, 22, 20, 122, 84, 15, 36, 103, 86, 47, 88, 100, 115, 113, 118, 24, 79, 112, 62, 30, 37, 114, 111, 40, 117, 85, 21, 104, 90, 78, 76, 81, 29, 93, 14, 17, 110, 108, 50, 60, 18, 27, 94, 44, 92, 82, 9, 16, 73, 80, 83, 13, 12, 25, 0, 31, 77, 46, 102, 107, 64, 95, 19, 7, 8, 96, 75, 11, 91, 97, 38, 43, 72, 71, 87, 70, 4, 89, 67, 32, 3, 23, 6, 68, 2, 33, 66, 41, 5, 65, 1, 35, 69, 105, 99, 74, 10], [106, 124, 116, 125, 61, 52, 63, 49, 109, 42, 51, 54, 120, 48, 45, 53, 119, 121, 123, 58, 127, 98, 56, 126, 57, 122, 34, 86, 84, 101, 26, 22, 36, 28, 20, 39, 59, 55, 15, 100, 114, 103, 115, 88, 47, 37, 30, 62, 111, 113, 24, 85, 40, 79, 90, 112, 118, 29, 81, 50, 60, 117, 104, 76, 14, 78, 93, 17, 21, 94, 92, 82, 18, 44, 27, 80, 9, 110, 73, 77, 31, 16, 95, 108, 12, 11, 83, 13, 25, 46, 107, 19, 102, 64, 70, 96, 91, 8, 43, 87, 41, 7, 75, 4, 71, 32, 38, 97, 72, 68, 33, 89, 0, 35, 23, 3, 2, 1, 65, 105, 5, 69, 67, 66, 74, 99, 10, 6], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 53, 121, 119, 123, 58, 98, 127, 56, 126, 57, 59, 55, 28, 34, 26, 22, 103, 101, 86, 62, 84, 39, 118, 36, 122, 79, 88, 20, 30, 15, 117, 100, 37, 112, 40, 81, 111, 113, 114, 24, 47, 29, 115, 93, 50, 85, 90, 21, 76, 78, 14, 104, 94, 12, 92, 17, 27, 44, 82, 73, 60, 18, 9, 31, 80, 108, 70, 95, 46, 110, 16, 19, 75, 77, 8, 13, 11, 83, 96, 91, 25, 89, 102, 32, 72, 107, 7, 97, 87, 43, 38, 23, 35, 71, 41, 3, 0, 105, 33, 4, 68, 64, 74, 69, 67, 5, 66, 2, 10, 65, 1, 99, 6], [106, 125, 116, 124, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 119, 53, 58, 98, 127, 56, 123, 126, 122, 26, 34, 59, 20, 55, 28, 84, 86, 22, 39, 100, 88, 15, 101, 62, 57, 118, 36, 103, 47, 79, 50, 37, 85, 40, 30, 112, 81, 117, 24, 114, 111, 14, 115, 113, 60, 76, 104, 110, 17, 93, 18, 90, 44, 78, 21, 80, 108, 82, 73, 94, 29, 12, 31, 9, 92, 27, 70, 46, 16, 77, 95, 83, 13, 19, 11, 107, 75, 8, 96, 25, 102, 71, 7, 89, 41, 23, 0, 91, 64, 87, 32, 38, 72, 43, 33, 67, 97, 68, 3, 66, 4, 5, 105, 69, 35, 74, 1, 2, 10, 65, 99, 6], [106, 116, 124, 125, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 98, 119, 56, 127, 123, 122, 58, 55, 59, 26, 20, 126, 28, 86, 22, 101, 100, 88, 84, 34, 39, 15, 115, 36, 57, 40, 79, 62, 37, 30, 85, 113, 118, 50, 44, 117, 24, 111, 112, 103, 60, 47, 81, 114, 93, 29, 108, 21, 94, 78, 104, 76, 17, 90, 14, 110, 12, 92, 9, 18, 46, 73, 82, 80, 77, 102, 31, 95, 27, 16, 83, 11, 25, 8, 96, 13, 19, 107, 75, 91, 70, 41, 105, 71, 89, 23, 0, 7, 43, 38, 72, 87, 67, 32, 97, 64, 35, 33, 68, 4, 99, 2, 74, 5, 66, 69, 3, 1, 6, 65, 10], [106, 124, 116, 125, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 121, 45, 123, 127, 55, 53, 98, 56, 122, 119, 126, 40, 58, 26, 28, 34, 86, 60, 57, 39, 20, 88, 101, 115, 118, 36, 22, 37, 100, 59, 103, 84, 15, 47, 50, 79, 62, 117, 85, 30, 104, 111, 24, 90, 112, 114, 93, 81, 44, 78, 113, 29, 12, 76, 17, 9, 21, 18, 14, 82, 31, 102, 27, 95, 108, 16, 96, 110, 73, 92, 80, 46, 94, 77, 83, 8, 19, 75, 41, 11, 13, 25, 64, 89, 91, 23, 7, 71, 32, 38, 97, 0, 107, 43, 67, 68, 33, 72, 6, 3, 87, 105, 4, 70, 66, 2, 69, 35, 74, 65, 5, 99, 1, 10], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 54, 51, 120, 48, 45, 121, 123, 55, 56, 53, 119, 58, 98, 127, 122, 28, 126, 34, 39, 20, 57, 101, 59, 84, 118, 79, 26, 22, 88, 100, 15, 103, 86, 36, 47, 117, 37, 111, 62, 115, 112, 40, 81, 85, 113, 30, 78, 24, 17, 50, 76, 60, 93, 104, 18, 29, 9, 12, 82, 21, 14, 73, 90, 114, 110, 27, 44, 92, 94, 16, 8, 77, 19, 75, 13, 80, 0, 108, 6, 83, 11, 46, 95, 31, 7, 64, 91, 102, 71, 96, 72, 3, 25, 23, 97, 41, 107, 89, 32, 5, 4, 87, 67, 66, 38, 35, 2, 105, 65, 69, 43, 68, 33, 74, 70, 99, 1, 10], [106, 116, 125, 124, 61, 52, 109, 63, 49, 42, 54, 51, 120, 48, 45, 121, 53, 119, 123, 55, 56, 58, 98, 127, 39, 34, 126, 57, 101, 28, 118, 59, 22, 26, 115, 84, 20, 100, 37, 86, 36, 122, 88, 15, 103, 111, 47, 85, 79, 40, 117, 113, 30, 50, 62, 60, 24, 78, 112, 104, 44, 93, 29, 90, 12, 81, 110, 92, 76, 9, 17, 14, 18, 73, 21, 27, 114, 82, 94, 8, 80, 31, 19, 46, 6, 13, 64, 77, 108, 95, 83, 11, 16, 7, 75, 96, 102, 25, 72, 0, 71, 32, 89, 4, 91, 38, 41, 23, 107, 87, 2, 105, 3, 69, 1, 5, 67, 66, 97, 33, 68, 43, 65, 74, 99, 35, 10, 70], [106, 116, 125, 124, 61, 52, 109, 63, 49, 42, 51, 54, 120, 48, 121, 45, 53, 119, 55, 58, 98, 56, 123, 126, 39, 59, 20, 26, 127, 22, 34, 84, 15, 86, 28, 57, 118, 88, 101, 36, 122, 37, 79, 100, 62, 115, 117, 111, 81, 103, 50, 78, 60, 30, 47, 85, 40, 113, 12, 24, 14, 17, 104, 76, 9, 93, 73, 29, 21, 112, 44, 18, 27, 90, 80, 82, 92, 16, 6, 94, 114, 13, 46, 83, 19, 75, 11, 77, 110, 108, 8, 96, 64, 31, 95, 7, 71, 3, 87, 25, 91, 32, 72, 107, 4, 68, 102, 67, 89, 41, 5, 66, 105, 97, 23, 43, 38, 69, 33, 10, 0, 2, 74, 35, 99, 1, 65, 70], [106, 125, 124, 116, 61, 52, 49, 109, 63, 42, 51, 54, 120, 48, 53, 121, 45, 123, 58, 127, 119, 56, 98, 126, 59, 122, 39, 55, 26, 118, 28, 34, 100, 62, 57, 15, 101, 22, 20, 37, 47, 36, 86, 84, 88, 40, 79, 60, 50, 112, 115, 117, 111, 103, 81, 30, 24, 85, 113, 104, 14, 73, 78, 29, 93, 21, 12, 76, 9, 80, 44, 90, 17, 18, 94, 82, 27, 92, 114, 11, 6, 96, 46, 13, 31, 16, 72, 110, 83, 0, 19, 77, 7, 75, 8, 64, 43, 71, 108, 95, 91, 68, 89, 25, 32, 67, 4, 38, 5, 102, 3, 97, 66, 87, 23, 33, 70, 107, 2, 69, 105, 99, 41, 1, 35, 65, 10, 74], [106, 124, 125, 116, 61, 52, 42, 109, 63, 49, 51, 120, 54, 48, 121, 45, 53, 123, 126, 98, 58, 119, 86, 59, 127, 56, 57, 26, 55, 22, 84, 79, 28, 39, 20, 15, 115, 34, 101, 40, 88, 36, 62, 37, 111, 122, 118, 117, 60, 112, 24, 30, 100, 103, 47, 81, 85, 50, 14, 12, 78, 113, 29, 104, 21, 76, 93, 90, 73, 9, 17, 18, 114, 82, 92, 11, 16, 13, 94, 80, 27, 95, 77, 46, 44, 31, 19, 72, 83, 75, 110, 108, 107, 96, 89, 7, 25, 6, 43, 71, 91, 102, 32, 8, 41, 97, 70, 23, 105, 35, 68, 87, 67, 38, 3, 5, 64, 4, 0, 69, 10, 74, 33, 99, 2, 66, 1, 65]], "model.layers.17.self_attn.q_proj": [[60, 106, 114, 100, 29, 89, 115, 123, 93, 86, 27, 83, 120, 47, 80, 42, 22, 112, 11, 57, 21, 113, 122, 53, 62, 127, 50, 16, 46, 61, 118, 59, 32, 111, 126, 45, 125, 56, 14, 99, 110, 63, 119, 91, 116, 81, 69, 55, 108, 96, 54, 26, 48, 117, 44, 51, 52, 121, 124, 58, 37, 97, 49, 109, 43, 105, 24, 76, 38, 41, 88, 71, 75, 95, 5, 107, 102, 103, 39, 104, 87, 31, 20, 40, 36, 15, 9, 101, 79, 35, 34, 98, 84, 17, 28, 10, 2, 94, 19, 90, 85, 8, 3, 23, 30, 33, 0, 7, 12, 1, 25, 4, 82, 6, 64, 18, 66, 77, 13, 78, 92, 74, 65, 67, 73, 72, 70, 68], [114, 106, 100, 60, 29, 89, 93, 50, 42, 86, 120, 59, 44, 61, 83, 22, 80, 115, 87, 99, 118, 27, 32, 105, 119, 81, 47, 31, 95, 46, 43, 21, 121, 97, 17, 25, 63, 91, 48, 54, 56, 116, 113, 51, 57, 14, 24, 11, 111, 103, 117, 96, 45, 53, 107, 125, 62, 110, 76, 58, 122, 41, 19, 38, 126, 112, 101, 37, 127, 40, 94, 55, 52, 16, 124, 88, 20, 102, 104, 98, 109, 33, 85, 108, 123, 71, 8, 23, 49, 35, 39, 90, 92, 30, 82, 79, 34, 9, 26, 15, 5, 10, 28, 1, 13, 36, 65, 84, 69, 77, 18, 12, 74, 75, 0, 3, 66, 6, 67, 72, 4, 70, 64, 7, 78, 73, 68, 2], [106, 60, 114, 100, 50, 29, 89, 86, 42, 57, 62, 107, 59, 93, 22, 80, 83, 56, 116, 37, 11, 14, 63, 47, 124, 99, 115, 91, 111, 46, 51, 71, 123, 52, 69, 48, 97, 110, 44, 125, 27, 55, 61, 49, 9, 16, 24, 113, 32, 76, 53, 58, 105, 81, 104, 120, 126, 119, 54, 31, 117, 127, 118, 122, 41, 38, 40, 45, 102, 121, 20, 108, 79, 103, 43, 109, 112, 19, 101, 94, 98, 39, 28, 35, 26, 33, 95, 96, 5, 88, 23, 25, 2, 30, 34, 12, 90, 82, 92, 21, 75, 85, 67, 1, 0, 4, 36, 17, 6, 18, 84, 74, 87, 13, 77, 8, 15, 10, 7, 72, 70, 3, 66, 78, 65, 64, 73, 68], [106, 60, 100, 115, 114, 89, 29, 83, 14, 86, 93, 42, 11, 71, 80, 27, 9, 50, 22, 25, 69, 110, 2, 67, 76, 1, 91, 4, 24, 62, 0, 16, 6, 53, 61, 116, 57, 3, 111, 37, 85, 21, 26, 97, 122, 113, 125, 7, 63, 118, 51, 127, 90, 77, 119, 19, 55, 35, 47, 17, 105, 123, 117, 75, 112, 52, 81, 95, 54, 64, 43, 108, 44, 120, 48, 78, 49, 18, 87, 46, 13, 88, 59, 98, 101, 96, 38, 92, 20, 45, 34, 40, 58, 121, 104, 68, 84, 124, 23, 103, 28, 94, 79, 102, 10, 107, 31, 73, 65, 70, 32, 41, 126, 56, 39, 33, 12, 99, 72, 109, 36, 74, 30, 15, 8, 82, 66, 5], [103, 126, 29, 82, 99, 85, 15, 76, 73, 71, 112, 90, 3, 5, 24, 105, 35, 93, 96, 97, 57, 23, 120, 87, 21, 61, 54, 18, 81, 119, 84, 17, 33, 26, 80, 49, 67, 115, 106, 64, 116, 83, 88, 50, 79, 16, 111, 78, 28, 101, 19, 12, 9, 121, 13, 127, 1, 69, 114, 25, 65, 34, 91, 7, 122, 89, 118, 74, 8, 32, 53, 110, 62, 20, 86, 14, 104, 11, 92, 4, 98, 117, 63, 40, 59, 51, 102, 10, 22, 27, 55, 31, 66, 41, 38, 77, 36, 43, 56, 123, 94, 58, 125, 100, 52, 70, 47, 107, 39, 42, 95, 46, 60, 6, 44, 75, 37, 30, 124, 109, 72, 68, 48, 113, 108, 45, 2, 0], [103, 126, 29, 99, 15, 85, 82, 76, 73, 71, 112, 3, 5, 90, 96, 24, 26, 1, 16, 27, 21, 9, 114, 119, 67, 57, 64, 84, 74, 11, 19, 93, 78, 79, 81, 23, 120, 88, 65, 14, 97, 101, 61, 91, 49, 33, 18, 98, 104, 105, 116, 6, 50, 43, 87, 8, 69, 92, 55, 32, 106, 7, 2, 30, 121, 94, 117, 13, 34, 125, 12, 54, 25, 56, 28, 80, 70, 75, 41, 68, 66, 127, 83, 36, 20, 115, 110, 10, 22, 17, 102, 100, 63, 122, 4, 51, 118, 39, 86, 58, 59, 113, 0, 108, 111, 35, 62, 77, 124, 60, 37, 72, 46, 44, 45, 89, 107, 31, 95, 53, 109, 48, 42, 123, 40, 38, 47, 52], [103, 126, 35, 49, 112, 29, 90, 114, 99, 56, 26, 85, 88, 24, 120, 93, 82, 116, 84, 119, 121, 96, 115, 46, 86, 63, 55, 117, 41, 57, 15, 60, 54, 110, 43, 32, 111, 122, 106, 62, 74, 59, 127, 101, 76, 16, 125, 52, 61, 104, 50, 118, 22, 124, 102, 123, 58, 107, 105, 108, 47, 51, 53, 97, 23, 73, 71, 40, 34, 113, 109, 77, 2, 48, 27, 42, 36, 11, 83, 0, 21, 37, 19, 44, 33, 18, 45, 98, 20, 38, 14, 100, 5, 8, 3, 95, 4, 78, 25, 66, 79, 91, 75, 39, 94, 68, 92, 80, 31, 28, 89, 72, 67, 87, 30, 1, 13, 81, 10, 64, 17, 69, 7, 65, 70, 12, 9, 6], [103, 126, 29, 85, 24, 82, 76, 90, 15, 112, 93, 99, 120, 26, 73, 71, 74, 35, 88, 127, 119, 41, 98, 21, 23, 3, 115, 5, 13, 111, 116, 121, 78, 57, 54, 96, 92, 16, 62, 49, 34, 63, 61, 91, 36, 14, 114, 55, 60, 104, 84, 117, 25, 30, 56, 32, 18, 33, 118, 106, 27, 50, 83, 28, 53, 51, 31, 79, 110, 86, 67, 52, 43, 87, 97, 17, 122, 19, 105, 101, 8, 47, 125, 12, 42, 102, 81, 46, 58, 20, 22, 77, 107, 108, 7, 123, 89, 48, 1, 95, 40, 94, 59, 38, 113, 124, 11, 100, 80, 44, 37, 45, 68, 72, 9, 109, 64, 10, 65, 66, 75, 39, 4, 6, 69, 70, 0, 2], [45, 109, 55, 58, 111, 59, 57, 63, 124, 52, 47, 116, 46, 125, 115, 119, 61, 53, 114, 48, 62, 51, 113, 38, 123, 60, 126, 50, 54, 117, 121, 112, 102, 122, 106, 49, 118, 120, 101, 127, 107, 36, 110, 42, 56, 43, 108, 33, 44, 28, 105, 29, 104, 41, 40, 103, 26, 39, 92, 99, 97, 37, 96, 98, 87, 34, 31, 100, 21, 35, 86, 32, 91, 30, 23, 95, 24, 18, 94, 89, 90, 27, 12, 84, 93, 25, 85, 78, 88, 22, 20, 82, 81, 17, 80, 7, 15, 11, 76, 9, 14, 79, 73, 16, 19, 83, 71, 75, 13, 69, 4, 68, 70, 74, 5, 2, 77, 65, 66, 8, 3, 10, 6, 1, 0, 64, 67, 72], [58, 45, 38, 47, 94, 55, 59, 89, 95, 60, 124, 112, 109, 111, 33, 57, 52, 115, 28, 35, 127, 87, 46, 113, 61, 92, 123, 51, 117, 122, 49, 118, 116, 63, 53, 54, 125, 101, 96, 50, 119, 121, 62, 105, 126, 110, 48, 41, 114, 120, 107, 86, 106, 43, 42, 44, 40, 108, 104, 56, 85, 36, 16, 37, 19, 98, 82, 103, 23, 20, 39, 12, 34, 84, 18, 91, 100, 15, 93, 81, 32, 22, 27, 99, 97, 102, 9, 11, 31, 29, 24, 78, 17, 25, 90, 7, 26, 88, 21, 71, 30, 79, 75, 14, 73, 76, 4, 77, 69, 5, 2, 13, 83, 68, 74, 1, 80, 66, 65, 10, 64, 0, 6, 8, 70, 67, 3, 72], [45, 109, 47, 52, 38, 101, 53, 60, 58, 94, 61, 115, 92, 59, 62, 46, 126, 23, 34, 116, 28, 111, 51, 119, 118, 108, 87, 33, 54, 57, 50, 112, 110, 63, 49, 85, 127, 123, 113, 117, 48, 30, 42, 44, 114, 89, 21, 41, 122, 103, 36, 40, 102, 55, 121, 120, 43, 35, 39, 107, 125, 37, 106, 104, 29, 105, 124, 99, 95, 100, 56, 93, 27, 96, 26, 97, 31, 78, 12, 32, 25, 24, 98, 16, 80, 18, 9, 88, 91, 86, 73, 82, 90, 81, 7, 20, 22, 84, 17, 11, 76, 83, 79, 19, 14, 75, 15, 69, 77, 71, 6, 74, 66, 10, 68, 4, 5, 2, 70, 67, 13, 0, 3, 72, 65, 64, 1, 8], [45, 38, 58, 19, 89, 94, 13, 59, 16, 74, 109, 35, 70, 87, 8, 3, 47, 6, 33, 67, 95, 30, 52, 111, 0, 55, 64, 72, 124, 1, 85, 77, 92, 84, 17, 65, 88, 69, 15, 10, 5, 81, 23, 21, 83, 82, 80, 28, 22, 20, 27, 78, 115, 25, 4, 31, 66, 76, 9, 11, 90, 32, 101, 2, 18, 75, 79, 37, 14, 96, 57, 126, 71, 63, 73, 12, 26, 68, 7, 29, 86, 98, 34, 24, 39, 91, 93, 108, 50, 125, 122, 61, 62, 105, 44, 104, 103, 116, 60, 100, 118, 121, 36, 97, 43, 99, 112, 120, 117, 51, 54, 40, 49, 56, 107, 123, 42, 106, 41, 113, 48, 53, 110, 119, 114, 127, 46, 102], [40, 50, 109, 98, 126, 29, 51, 84, 23, 80, 13, 18, 71, 73, 53, 75, 26, 56, 47, 90, 104, 2, 87, 66, 4, 111, 72, 93, 69, 68, 112, 39, 86, 94, 70, 1, 20, 0, 120, 11, 22, 19, 65, 74, 30, 103, 101, 64, 5, 25, 96, 59, 43, 119, 60, 16, 67, 89, 34, 76, 31, 77, 8, 36, 95, 38, 78, 117, 57, 113, 108, 100, 32, 6, 62, 121, 27, 45, 105, 124, 127, 7, 102, 21, 88, 33, 55, 97, 48, 15, 85, 99, 41, 118, 44, 28, 110, 83, 91, 35, 92, 58, 63, 10, 52, 12, 61, 46, 42, 17, 24, 115, 116, 82, 3, 123, 114, 81, 37, 107, 14, 9, 54, 122, 125, 49, 106, 79], [40, 126, 29, 98, 109, 87, 47, 39, 50, 23, 93, 84, 20, 75, 36, 90, 125, 58, 52, 124, 26, 59, 13, 121, 119, 56, 54, 115, 117, 53, 81, 18, 57, 108, 118, 34, 17, 103, 31, 122, 88, 120, 104, 63, 60, 127, 61, 107, 111, 105, 97, 110, 72, 44, 21, 51, 42, 46, 48, 24, 15, 32, 116, 123, 41, 45, 94, 70, 30, 43, 114, 49, 112, 99, 55, 106, 113, 80, 37, 38, 62, 85, 35, 100, 25, 92, 22, 101, 95, 79, 96, 102, 33, 14, 77, 27, 19, 83, 11, 4, 89, 28, 12, 91, 16, 86, 6, 82, 8, 76, 78, 10, 74, 73, 66, 9, 1, 68, 2, 65, 7, 3, 71, 5, 64, 69, 0, 67], [40, 126, 98, 109, 29, 60, 84, 90, 51, 87, 93, 124, 47, 23, 80, 50, 26, 18, 75, 59, 54, 34, 117, 36, 39, 88, 20, 119, 118, 53, 121, 13, 63, 114, 58, 103, 22, 123, 125, 127, 37, 55, 44, 122, 52, 45, 46, 111, 56, 115, 105, 21, 48, 104, 110, 107, 62, 120, 94, 92, 102, 61, 96, 108, 57, 112, 30, 38, 91, 49, 113, 43, 72, 70, 24, 12, 116, 35, 41, 42, 106, 97, 89, 86, 32, 31, 99, 100, 101, 16, 17, 85, 28, 79, 19, 33, 25, 73, 81, 82, 76, 15, 27, 95, 83, 14, 78, 4, 11, 10, 9, 6, 74, 8, 3, 69, 68, 67, 7, 77, 5, 71, 65, 1, 66, 2, 0, 64], [40, 51, 98, 29, 126, 13, 80, 18, 87, 84, 72, 23, 90, 4, 75, 93, 96, 26, 70, 60, 10, 73, 88, 124, 52, 19, 94, 121, 56, 125, 71, 54, 86, 30, 119, 81, 65, 117, 20, 109, 47, 59, 122, 77, 28, 50, 53, 111, 46, 16, 3, 61, 104, 105, 127, 112, 63, 74, 108, 78, 64, 15, 101, 68, 32, 39, 7, 91, 5, 89, 95, 97, 79, 67, 82, 11, 83, 58, 8, 118, 41, 22, 123, 106, 14, 2, 115, 120, 33, 76, 6, 62, 31, 34, 48, 69, 49, 113, 55, 116, 27, 12, 38, 9, 21, 100, 36, 44, 37, 110, 25, 57, 43, 92, 45, 17, 1, 103, 99, 24, 35, 66, 85, 102, 114, 42, 0, 107], [113, 115, 105, 57, 49, 63, 124, 53, 119, 34, 30, 39, 112, 48, 56, 127, 123, 61, 58, 45, 59, 117, 60, 116, 52, 120, 122, 62, 47, 27, 38, 118, 110, 107, 37, 108, 54, 121, 55, 94, 51, 125, 46, 111, 106, 85, 126, 44, 88, 114, 99, 104, 109, 87, 101, 90, 42, 43, 89, 36, 25, 24, 19, 103, 78, 18, 28, 40, 50, 92, 23, 41, 100, 35, 80, 83, 91, 22, 98, 33, 102, 31, 93, 21, 32, 14, 97, 26, 74, 95, 96, 6, 17, 16, 82, 65, 29, 12, 76, 20, 70, 81, 66, 64, 67, 72, 8, 10, 3, 0, 69, 86, 15, 11, 1, 2, 5, 13, 7, 68, 9, 75, 77, 4, 79, 73, 71, 84], [115, 61, 105, 113, 34, 30, 50, 124, 121, 120, 37, 38, 48, 63, 57, 126, 122, 53, 55, 112, 119, 27, 123, 49, 56, 59, 92, 60, 101, 62, 58, 127, 52, 46, 47, 45, 39, 118, 90, 110, 107, 44, 85, 111, 54, 125, 94, 116, 117, 108, 109, 103, 99, 87, 25, 114, 51, 40, 106, 41, 42, 33, 43, 89, 88, 104, 93, 96, 80, 32, 23, 91, 35, 36, 19, 31, 95, 102, 100, 98, 82, 78, 29, 28, 83, 21, 26, 18, 16, 97, 24, 22, 70, 17, 74, 6, 14, 64, 76, 0, 65, 20, 66, 1, 3, 69, 12, 86, 8, 81, 67, 2, 5, 10, 13, 72, 11, 15, 7, 9, 71, 4, 79, 68, 84, 73, 77, 75], [105, 113, 61, 57, 34, 115, 22, 20, 27, 30, 55, 41, 24, 17, 63, 53, 80, 13, 37, 59, 74, 38, 82, 18, 15, 11, 124, 123, 21, 12, 19, 7, 56, 51, 97, 121, 122, 70, 88, 26, 62, 47, 49, 103, 75, 99, 117, 9, 90, 116, 106, 31, 95, 36, 52, 127, 119, 60, 23, 58, 71, 10, 118, 39, 120, 89, 100, 46, 72, 112, 33, 45, 83, 111, 2, 6, 91, 50, 44, 3, 28, 48, 29, 14, 114, 126, 109, 101, 77, 93, 40, 85, 107, 54, 104, 84, 64, 110, 102, 16, 73, 81, 96, 76, 66, 42, 32, 25, 108, 87, 86, 92, 79, 68, 78, 94, 5, 69, 43, 1, 98, 4, 125, 35, 65, 8, 0, 67], [105, 57, 115, 113, 53, 34, 27, 22, 20, 15, 30, 41, 63, 9, 17, 24, 13, 49, 127, 74, 120, 4, 94, 55, 39, 116, 61, 37, 85, 1, 68, 3, 47, 36, 11, 87, 76, 73, 29, 72, 78, 77, 28, 31, 6, 46, 118, 123, 48, 79, 88, 95, 58, 119, 93, 103, 60, 26, 8, 18, 107, 84, 54, 38, 92, 104, 124, 35, 90, 112, 52, 43, 83, 110, 56, 40, 100, 121, 45, 109, 126, 23, 42, 59, 106, 122, 10, 96, 21, 50, 44, 62, 117, 125, 108, 81, 64, 111, 32, 114, 33, 86, 89, 12, 70, 25, 80, 51, 97, 99, 5, 16, 19, 14, 75, 66, 91, 98, 69, 65, 101, 67, 82, 102, 0, 7, 71, 2], [108, 37, 126, 88, 19, 90, 78, 32, 75, 81, 44, 8, 5, 36, 93, 86, 85, 67, 21, 1, 24, 34, 61, 22, 0, 16, 15, 83, 18, 12, 11, 72, 20, 69, 119, 58, 80, 65, 17, 25, 28, 14, 89, 13, 84, 87, 103, 23, 73, 10, 2, 98, 77, 74, 79, 35, 91, 27, 29, 30, 3, 76, 9, 94, 71, 82, 46, 26, 100, 118, 127, 68, 70, 95, 97, 125, 31, 53, 6, 7, 51, 92, 106, 64, 45, 52, 40, 66, 54, 4, 120, 122, 33, 113, 107, 117, 60, 115, 63, 99, 57, 123, 59, 105, 39, 42, 96, 111, 62, 104, 110, 43, 50, 114, 116, 102, 124, 109, 55, 47, 121, 101, 38, 48, 41, 49, 56, 112], [126, 108, 37, 44, 23, 58, 120, 57, 113, 51, 48, 61, 122, 63, 114, 118, 56, 62, 115, 116, 102, 104, 59, 32, 52, 119, 93, 124, 55, 53, 46, 54, 60, 125, 121, 101, 49, 100, 117, 110, 50, 103, 47, 45, 111, 123, 41, 109, 112, 40, 127, 106, 16, 107, 27, 42, 43, 90, 105, 39, 38, 91, 36, 35, 92, 84, 98, 99, 97, 95, 87, 33, 94, 34, 30, 25, 86, 88, 96, 29, 31, 85, 81, 28, 17, 20, 80, 89, 22, 74, 10, 2, 19, 24, 12, 26, 83, 82, 15, 0, 6, 65, 4, 21, 76, 66, 18, 70, 71, 13, 79, 78, 11, 67, 64, 7, 5, 9, 77, 69, 68, 3, 1, 73, 14, 75, 8, 72], [108, 61, 126, 44, 36, 119, 90, 85, 32, 122, 82, 93, 35, 24, 88, 37, 104, 58, 7, 19, 12, 15, 81, 13, 62, 91, 29, 101, 59, 95, 28, 63, 86, 46, 51, 20, 23, 79, 27, 120, 48, 78, 53, 57, 60, 96, 109, 87, 98, 116, 97, 115, 74, 89, 45, 71, 4, 18, 118, 114, 70, 66, 99, 52, 49, 33, 124, 64, 54, 125, 25, 127, 113, 56, 107, 117, 100, 34, 55, 9, 75, 41, 121, 26, 42, 92, 47, 31, 110, 30, 94, 123, 38, 16, 39, 112, 50, 76, 40, 102, 84, 6, 111, 106, 105, 65, 43, 103, 80, 21, 77, 2, 68, 22, 8, 83, 3, 11, 17, 0, 10, 5, 1, 73, 67, 69, 14, 72], [108, 44, 37, 126, 93, 23, 90, 58, 81, 24, 32, 61, 36, 85, 120, 19, 123, 41, 56, 86, 31, 29, 96, 118, 114, 110, 16, 78, 125, 112, 54, 105, 113, 88, 33, 63, 38, 103, 122, 48, 106, 87, 51, 111, 42, 109, 62, 115, 52, 57, 30, 47, 124, 119, 50, 102, 53, 127, 97, 43, 116, 27, 60, 34, 55, 121, 35, 45, 46, 99, 107, 98, 94, 74, 117, 49, 40, 84, 59, 104, 91, 7, 28, 25, 80, 39, 95, 100, 92, 26, 75, 82, 76, 89, 101, 13, 17, 9, 22, 10, 77, 15, 18, 0, 79, 71, 2, 83, 8, 67, 21, 64, 66, 5, 20, 12, 11, 70, 4, 68, 6, 73, 65, 14, 1, 69, 3, 72], [56, 126, 101, 62, 60, 122, 37, 125, 116, 123, 88, 53, 119, 58, 55, 51, 118, 120, 115, 117, 63, 48, 50, 89, 46, 127, 114, 84, 54, 124, 121, 33, 61, 59, 57, 103, 49, 14, 52, 11, 34, 113, 47, 92, 110, 39, 95, 107, 104, 44, 112, 98, 109, 108, 45, 111, 100, 106, 41, 40, 43, 93, 42, 35, 26, 16, 20, 38, 91, 105, 36, 15, 94, 99, 102, 18, 12, 22, 31, 82, 96, 90, 28, 25, 72, 86, 80, 32, 30, 68, 24, 87, 5, 23, 29, 17, 74, 97, 2, 7, 21, 70, 27, 79, 77, 19, 83, 85, 76, 81, 6, 73, 78, 64, 65, 9, 13, 75, 3, 0, 8, 66, 10, 67, 71, 69, 4, 1], [56, 126, 101, 116, 39, 62, 37, 20, 91, 53, 84, 26, 123, 127, 14, 125, 60, 119, 114, 33, 46, 100, 103, 120, 48, 54, 122, 118, 59, 51, 124, 55, 88, 58, 121, 45, 11, 104, 115, 40, 110, 50, 41, 36, 25, 106, 107, 63, 89, 61, 47, 117, 35, 34, 108, 44, 57, 99, 111, 52, 105, 42, 43, 112, 109, 95, 113, 29, 38, 16, 98, 49, 82, 31, 102, 94, 32, 12, 87, 93, 96, 83, 90, 92, 72, 15, 27, 30, 24, 28, 80, 97, 18, 77, 74, 86, 73, 22, 5, 6, 7, 68, 81, 70, 19, 21, 23, 65, 78, 79, 66, 75, 85, 9, 76, 17, 64, 2, 13, 8, 71, 67, 0, 10, 4, 3, 69, 1], [56, 126, 101, 60, 84, 116, 26, 40, 11, 21, 93, 62, 89, 28, 2, 125, 109, 68, 122, 127, 92, 33, 53, 55, 31, 118, 37, 79, 51, 29, 88, 24, 90, 114, 119, 124, 63, 58, 47, 123, 57, 61, 80, 48, 59, 110, 117, 50, 0, 103, 75, 120, 23, 121, 91, 52, 15, 16, 41, 54, 49, 108, 18, 106, 32, 34, 8, 115, 46, 100, 113, 112, 96, 67, 111, 35, 44, 17, 20, 30, 85, 70, 95, 105, 81, 19, 94, 77, 7, 45, 43, 22, 42, 82, 38, 9, 39, 107, 36, 99, 13, 104, 74, 76, 98, 3, 69, 102, 83, 5, 6, 73, 14, 87, 1, 65, 25, 27, 12, 72, 86, 10, 78, 4, 71, 97, 66, 64], [126, 56, 101, 62, 37, 88, 122, 53, 123, 15, 103, 125, 116, 24, 60, 91, 119, 120, 118, 23, 124, 92, 114, 48, 58, 127, 51, 117, 55, 63, 89, 54, 12, 33, 46, 121, 94, 50, 25, 61, 84, 59, 14, 6, 39, 49, 115, 57, 113, 52, 44, 47, 100, 11, 90, 45, 110, 18, 34, 112, 111, 20, 9, 99, 107, 17, 36, 105, 87, 7, 109, 40, 108, 95, 81, 98, 72, 22, 96, 104, 42, 102, 26, 35, 3, 28, 38, 106, 31, 32, 5, 86, 93, 68, 41, 79, 30, 82, 43, 80, 27, 16, 70, 97, 29, 74, 19, 64, 65, 76, 21, 66, 85, 1, 67, 8, 78, 2, 77, 73, 10, 4, 83, 75, 0, 71, 13, 69], [120, 109, 104, 124, 34, 123, 127, 59, 27, 89, 58, 60, 30, 52, 56, 125, 114, 86, 91, 54, 126, 118, 78, 121, 61, 113, 53, 62, 29, 45, 51, 50, 88, 119, 63, 40, 35, 110, 122, 84, 116, 94, 117, 19, 25, 49, 42, 55, 57, 112, 48, 43, 46, 107, 111, 47, 39, 115, 37, 106, 100, 108, 105, 41, 36, 10, 101, 103, 96, 44, 102, 99, 33, 21, 38, 18, 97, 26, 32, 31, 92, 28, 98, 87, 90, 95, 15, 22, 93, 17, 20, 16, 82, 14, 23, 85, 83, 9, 69, 80, 79, 24, 12, 76, 74, 13, 81, 73, 64, 5, 65, 77, 1, 6, 11, 7, 75, 4, 71, 68, 70, 72, 2, 67, 0, 8, 66, 3], [120, 109, 104, 123, 52, 58, 30, 127, 121, 59, 54, 124, 60, 122, 126, 61, 114, 112, 53, 56, 125, 117, 118, 45, 62, 51, 116, 113, 63, 110, 89, 107, 50, 49, 119, 57, 48, 78, 55, 115, 111, 46, 106, 47, 43, 42, 105, 34, 38, 108, 102, 25, 44, 18, 87, 41, 40, 101, 39, 100, 99, 103, 37, 94, 36, 98, 91, 35, 97, 27, 96, 88, 10, 33, 86, 32, 93, 28, 31, 95, 19, 85, 29, 82, 92, 90, 16, 84, 22, 26, 23, 74, 14, 21, 64, 20, 65, 83, 69, 1, 79, 15, 80, 76, 6, 17, 0, 5, 9, 24, 12, 81, 4, 3, 70, 66, 2, 67, 13, 11, 7, 71, 68, 73, 8, 75, 72, 77], [104, 120, 34, 88, 109, 17, 13, 75, 15, 7, 9, 86, 84, 92, 66, 71, 91, 2, 124, 11, 85, 19, 30, 69, 5, 4, 27, 40, 45, 22, 74, 0, 98, 68, 20, 64, 81, 82, 77, 24, 83, 79, 18, 80, 87, 29, 73, 26, 21, 76, 70, 107, 1, 14, 23, 94, 3, 72, 93, 90, 35, 8, 25, 123, 16, 47, 12, 6, 89, 56, 65, 116, 67, 78, 10, 59, 127, 46, 63, 31, 96, 54, 28, 95, 32, 97, 113, 106, 58, 52, 42, 99, 100, 101, 126, 55, 118, 43, 33, 37, 36, 112, 60, 105, 122, 125, 121, 114, 62, 117, 115, 49, 38, 110, 108, 61, 50, 39, 57, 53, 48, 102, 51, 103, 44, 119, 41, 111], [109, 104, 120, 34, 123, 78, 27, 59, 88, 116, 30, 112, 54, 86, 84, 52, 127, 125, 10, 121, 89, 40, 29, 18, 58, 126, 124, 60, 91, 114, 55, 56, 46, 113, 50, 61, 87, 122, 53, 62, 25, 110, 115, 26, 98, 45, 118, 63, 117, 119, 49, 19, 43, 48, 15, 51, 39, 42, 57, 100, 47, 36, 97, 90, 35, 106, 108, 38, 32, 28, 96, 107, 99, 101, 14, 31, 44, 102, 33, 111, 94, 17, 16, 9, 37, 105, 41, 92, 95, 23, 93, 103, 83, 82, 13, 22, 21, 76, 85, 20, 24, 80, 4, 6, 74, 79, 69, 12, 0, 2, 75, 73, 72, 11, 7, 81, 5, 71, 64, 1, 77, 70, 68, 65, 67, 8, 66, 3]], "model.layers.17.self_attn.k_proj": [[42, 36, 114, 60, 86, 93, 91, 80, 83, 89, 11, 63, 14, 19, 124, 119, 0, 53, 125, 46, 9, 69, 113, 111, 59, 25, 56, 54, 120, 126, 76, 50, 47, 44, 57, 115, 123, 118, 127, 116, 117, 32, 45, 43, 15, 2, 33, 71, 62, 51, 48, 122, 101, 121, 110, 49, 4, 55, 108, 41, 52, 24, 103, 104, 66, 106, 40, 105, 112, 58, 102, 90, 74, 98, 107, 1, 109, 39, 97, 18, 30, 77, 61, 5, 35, 88, 12, 34, 31, 100, 87, 95, 38, 81, 37, 20, 23, 17, 99, 28, 21, 6, 96, 72, 82, 70, 8, 92, 27, 26, 84, 94, 10, 79, 85, 65, 78, 7, 67, 13, 68, 29, 3, 64, 22, 16, 73, 75], [126, 39, 93, 85, 15, 82, 24, 1, 76, 5, 73, 71, 64, 48, 112, 74, 26, 50, 3, 120, 116, 96, 35, 121, 119, 117, 77, 118, 57, 115, 110, 105, 41, 16, 113, 114, 63, 60, 97, 54, 55, 42, 125, 46, 56, 52, 43, 49, 6, 29, 84, 34, 4, 40, 59, 122, 66, 61, 32, 27, 123, 62, 53, 90, 47, 75, 124, 99, 19, 51, 107, 127, 72, 87, 78, 38, 86, 109, 44, 111, 58, 37, 70, 100, 108, 23, 45, 104, 101, 25, 14, 83, 22, 11, 36, 106, 89, 13, 68, 95, 28, 69, 33, 81, 94, 2, 17, 102, 98, 31, 91, 92, 0, 30, 80, 20, 10, 8, 88, 9, 65, 67, 18, 21, 7, 103, 12, 79], [109, 58, 102, 30, 59, 8, 45, 13, 74, 19, 25, 86, 16, 70, 3, 97, 89, 65, 99, 28, 23, 0, 46, 32, 49, 47, 69, 36, 127, 121, 12, 126, 94, 115, 18, 15, 62, 91, 51, 84, 118, 53, 61, 120, 111, 107, 63, 116, 35, 77, 117, 48, 125, 11, 114, 83, 43, 101, 20, 87, 9, 110, 24, 27, 57, 106, 50, 52, 54, 68, 78, 4, 98, 123, 112, 113, 124, 122, 56, 29, 105, 17, 44, 37, 55, 119, 85, 31, 42, 60, 34, 7, 108, 104, 71, 64, 41, 39, 103, 40, 2, 100, 80, 90, 81, 6, 66, 26, 93, 82, 21, 5, 10, 96, 14, 88, 95, 75, 33, 79, 72, 92, 67, 73, 22, 76, 1, 38], [104, 126, 34, 50, 51, 115, 93, 23, 84, 26, 114, 13, 80, 111, 18, 45, 75, 56, 124, 59, 53, 70, 105, 72, 48, 127, 65, 119, 4, 58, 81, 73, 122, 64, 55, 57, 125, 103, 118, 94, 123, 3, 49, 60, 121, 99, 63, 2, 30, 117, 61, 10, 109, 110, 106, 32, 120, 22, 46, 14, 29, 24, 108, 113, 7, 38, 62, 36, 12, 107, 87, 116, 92, 112, 95, 42, 52, 102, 41, 28, 44, 54, 90, 25, 39, 101, 83, 19, 31, 71, 100, 47, 86, 43, 91, 37, 21, 96, 35, 15, 27, 89, 76, 33, 98, 5, 97, 85, 78, 88, 17, 79, 9, 67, 69, 82, 1, 6, 0, 74, 77, 16, 8, 20, 68, 11, 66, 40], [41, 98, 57, 115, 113, 94, 24, 51, 11, 20, 15, 91, 13, 22, 17, 61, 27, 55, 49, 9, 63, 68, 62, 59, 85, 26, 25, 72, 127, 56, 53, 123, 19, 116, 6, 114, 101, 35, 99, 107, 7, 18, 102, 48, 80, 108, 28, 122, 109, 65, 104, 54, 87, 125, 16, 60, 118, 8, 84, 117, 66, 74, 47, 103, 82, 97, 43, 121, 111, 93, 120, 78, 45, 33, 119, 42, 52, 58, 44, 105, 110, 124, 112, 96, 36, 50, 40, 46, 100, 76, 126, 3, 32, 73, 95, 5, 106, 29, 77, 31, 64, 39, 92, 90, 0, 67, 37, 79, 75, 89, 21, 23, 69, 70, 12, 83, 88, 71, 38, 81, 34, 2, 86, 10, 30, 14, 4, 1], [44, 126, 86, 101, 108, 100, 96, 61, 29, 58, 78, 90, 57, 19, 8, 51, 63, 119, 75, 120, 62, 81, 55, 125, 5, 124, 114, 47, 117, 116, 59, 122, 118, 123, 53, 16, 67, 60, 46, 74, 111, 56, 52, 112, 115, 110, 54, 0, 50, 121, 49, 88, 106, 127, 48, 109, 113, 23, 34, 65, 104, 42, 85, 99, 105, 39, 45, 98, 9, 107, 43, 41, 27, 66, 83, 40, 103, 82, 25, 38, 4, 70, 35, 13, 36, 102, 1, 94, 92, 28, 30, 73, 31, 26, 20, 93, 33, 95, 77, 97, 2, 68, 89, 15, 24, 7, 3, 17, 91, 12, 21, 79, 76, 84, 64, 18, 6, 72, 71, 32, 22, 80, 37, 11, 10, 87, 14, 69], [126, 37, 56, 22, 97, 62, 51, 125, 122, 123, 59, 119, 61, 60, 53, 116, 127, 58, 118, 50, 63, 48, 121, 120, 117, 124, 52, 49, 57, 112, 114, 55, 113, 54, 92, 47, 104, 115, 46, 94, 111, 27, 109, 110, 44, 42, 108, 45, 107, 103, 38, 93, 33, 43, 106, 39, 105, 41, 89, 82, 40, 35, 102, 101, 80, 84, 36, 98, 96, 99, 88, 86, 28, 34, 19, 77, 79, 100, 91, 32, 95, 30, 74, 90, 25, 3, 29, 9, 31, 7, 5, 85, 87, 26, 17, 23, 15, 81, 64, 16, 83, 21, 76, 24, 14, 65, 75, 72, 12, 6, 18, 78, 13, 2, 68, 4, 73, 71, 20, 11, 8, 70, 10, 66, 67, 1, 69, 0], [120, 40, 45, 86, 98, 123, 54, 27, 109, 60, 17, 59, 127, 113, 94, 58, 7, 88, 61, 13, 126, 52, 56, 117, 114, 62, 63, 9, 15, 53, 121, 118, 119, 122, 48, 75, 34, 116, 110, 66, 57, 55, 49, 124, 50, 51, 115, 112, 111, 46, 43, 47, 125, 84, 19, 103, 42, 78, 44, 26, 10, 107, 18, 5, 39, 106, 89, 102, 65, 108, 64, 36, 31, 105, 41, 93, 16, 101, 38, 87, 21, 95, 69, 8, 6, 99, 67, 4, 35, 30, 33, 37, 72, 90, 28, 97, 96, 24, 100, 76, 20, 32, 11, 12, 29, 104, 85, 92, 23, 68, 80, 83, 3, 71, 70, 81, 91, 0, 25, 82, 77, 79, 14, 1, 74, 73, 22, 2]], "model.layers.17.self_attn.qk_proj": [[126, 120, 109, 58, 44, 114, 56, 60, 115, 45, 113, 108, 104, 29, 57, 42, 61, 51, 50, 59, 41, 40, 106, 55, 119, 22, 118, 123, 49, 34, 105, 124, 125, 116, 93, 87, 112, 86, 127, 62, 117, 90, 111, 24, 94, 98, 53, 63, 27, 80, 37, 16, 52, 47, 30, 89, 23, 100, 25, 26, 54, 121, 39, 88, 18, 91, 83, 13, 85, 82, 75, 101, 122, 19, 48, 21, 103, 77, 110, 35, 84, 11, 20, 15, 36, 73, 79, 17, 99, 102, 32, 96, 43, 107, 9, 46, 78, 81, 14, 92, 76, 72, 38, 12, 97, 69, 7, 8, 5, 74, 71, 3, 28, 6, 64, 67, 33, 10, 65, 1, 70, 0, 4, 95, 68, 66, 31, 2], [126, 120, 109, 56, 44, 58, 114, 60, 115, 45, 113, 108, 29, 57, 104, 42, 61, 59, 50, 51, 40, 106, 123, 41, 22, 119, 55, 118, 49, 105, 124, 125, 34, 86, 93, 53, 112, 47, 98, 87, 116, 37, 127, 111, 63, 90, 24, 94, 54, 52, 27, 101, 39, 62, 30, 26, 16, 25, 121, 80, 48, 23, 91, 18, 83, 103, 110, 100, 82, 35, 89, 13, 117, 21, 88, 85, 79, 19, 122, 20, 77, 11, 75, 36, 32, 84, 46, 102, 96, 99, 73, 78, 15, 9, 107, 76, 17, 92, 81, 97, 14, 43, 12, 72, 69, 5, 38, 74, 65, 71, 7, 10, 67, 8, 28, 0, 70, 33, 64, 1, 68, 3, 6, 31, 66, 4, 95, 2], [126, 120, 109, 56, 58, 44, 114, 60, 115, 45, 113, 108, 29, 104, 61, 42, 57, 51, 40, 59, 50, 106, 41, 55, 22, 123, 105, 124, 118, 119, 34, 86, 93, 90, 49, 125, 112, 98, 116, 127, 62, 39, 52, 26, 87, 27, 16, 121, 63, 47, 24, 100, 37, 101, 25, 94, 117, 30, 111, 53, 80, 91, 18, 122, 103, 48, 89, 54, 88, 82, 23, 32, 19, 83, 99, 85, 36, 15, 102, 35, 20, 21, 13, 77, 84, 79, 11, 73, 75, 110, 43, 96, 92, 46, 65, 9, 1, 17, 64, 5, 3, 78, 107, 70, 76, 14, 81, 97, 10, 8, 67, 74, 0, 33, 7, 38, 69, 12, 72, 71, 28, 4, 2, 6, 31, 68, 66, 95], [126, 120, 109, 58, 56, 44, 114, 60, 113, 115, 45, 108, 104, 29, 61, 57, 42, 51, 50, 40, 59, 41, 119, 106, 55, 49, 123, 105, 125, 118, 22, 34, 116, 124, 62, 52, 90, 93, 63, 53, 98, 39, 122, 86, 127, 94, 47, 87, 112, 16, 117, 121, 100, 27, 30, 37, 111, 85, 101, 25, 48, 80, 24, 88, 26, 54, 91, 18, 23, 15, 103, 89, 36, 19, 21, 110, 82, 35, 46, 102, 77, 20, 96, 84, 75, 79, 83, 13, 32, 99, 73, 43, 11, 67, 92, 17, 10, 65, 76, 70, 5, 78, 81, 14, 74, 9, 3, 1, 12, 64, 107, 97, 8, 0, 69, 38, 33, 7, 72, 6, 68, 28, 71, 4, 66, 2, 31, 95], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 61, 57, 104, 42, 50, 59, 51, 29, 41, 40, 106, 123, 119, 105, 55, 124, 116, 62, 22, 49, 118, 34, 90, 47, 125, 86, 93, 98, 127, 112, 111, 87, 122, 39, 94, 63, 27, 16, 80, 117, 24, 100, 53, 88, 37, 89, 18, 121, 91, 52, 23, 48, 19, 83, 26, 82, 30, 25, 15, 54, 85, 77, 75, 20, 13, 35, 103, 110, 101, 102, 46, 84, 79, 21, 11, 36, 43, 9, 73, 3, 96, 78, 14, 17, 76, 81, 70, 32, 99, 8, 1, 10, 67, 92, 107, 7, 5, 69, 12, 97, 74, 0, 65, 6, 72, 64, 71, 28, 33, 68, 4, 38, 66, 31, 2, 95], [126, 120, 109, 56, 58, 44, 114, 60, 115, 113, 45, 108, 104, 29, 61, 57, 50, 42, 51, 59, 41, 40, 55, 22, 106, 123, 105, 49, 119, 124, 34, 116, 125, 118, 62, 86, 93, 87, 127, 63, 16, 98, 90, 27, 94, 24, 37, 80, 117, 47, 18, 112, 39, 111, 53, 88, 91, 85, 30, 13, 89, 19, 82, 26, 48, 122, 83, 121, 23, 54, 25, 84, 35, 15, 100, 79, 110, 75, 103, 77, 20, 21, 52, 101, 11, 46, 17, 102, 99, 96, 36, 9, 81, 97, 14, 12, 78, 73, 92, 32, 8, 10, 74, 107, 76, 43, 70, 5, 69, 7, 72, 71, 33, 65, 67, 28, 38, 1, 3, 64, 0, 6, 31, 95, 4, 68, 2, 66], [126, 120, 109, 58, 56, 44, 60, 114, 115, 113, 45, 108, 29, 57, 42, 104, 51, 61, 50, 106, 40, 59, 41, 123, 22, 55, 119, 124, 49, 105, 118, 34, 127, 125, 86, 87, 93, 24, 27, 53, 94, 47, 116, 90, 80, 37, 88, 62, 89, 111, 16, 98, 25, 63, 83, 121, 52, 19, 18, 84, 112, 117, 91, 122, 15, 54, 30, 26, 77, 39, 100, 23, 48, 75, 21, 82, 85, 110, 101, 13, 46, 35, 20, 36, 103, 102, 11, 99, 79, 96, 17, 14, 81, 78, 9, 12, 32, 73, 92, 10, 107, 76, 97, 43, 8, 33, 7, 28, 38, 69, 74, 72, 5, 3, 71, 1, 0, 31, 70, 67, 65, 68, 6, 64, 4, 95, 66, 2], [126, 120, 109, 58, 44, 56, 60, 114, 115, 113, 108, 45, 57, 29, 42, 61, 51, 104, 59, 50, 22, 123, 106, 41, 40, 55, 118, 119, 125, 53, 49, 86, 124, 90, 105, 93, 34, 62, 87, 27, 127, 94, 24, 25, 89, 88, 98, 116, 37, 112, 80, 47, 63, 101, 16, 52, 39, 111, 18, 91, 26, 100, 83, 23, 30, 117, 121, 19, 54, 122, 99, 15, 103, 85, 21, 13, 82, 84, 35, 77, 102, 20, 46, 48, 75, 36, 32, 110, 97, 79, 96, 81, 92, 73, 17, 11, 33, 14, 78, 9, 43, 28, 12, 107, 8, 69, 5, 71, 6, 10, 76, 7, 65, 72, 31, 0, 3, 74, 64, 67, 1, 38, 4, 95, 68, 66, 70, 2], [126, 120, 109, 58, 56, 44, 60, 114, 113, 115, 45, 108, 57, 29, 104, 42, 61, 51, 50, 59, 40, 106, 123, 41, 55, 22, 119, 86, 49, 34, 93, 105, 118, 127, 37, 47, 124, 122, 90, 53, 116, 94, 62, 25, 89, 87, 26, 111, 27, 63, 125, 24, 30, 98, 52, 121, 16, 88, 80, 18, 101, 112, 83, 48, 54, 23, 35, 46, 100, 91, 82, 39, 85, 15, 117, 103, 84, 21, 13, 20, 110, 36, 19, 102, 32, 11, 99, 77, 43, 75, 79, 78, 96, 17, 81, 97, 73, 92, 107, 14, 74, 9, 12, 10, 33, 76, 28, 6, 69, 71, 8, 38, 7, 5, 72, 64, 65, 1, 31, 0, 68, 67, 3, 95, 4, 66, 2, 70], [126, 120, 109, 58, 56, 44, 114, 60, 113, 115, 45, 108, 57, 29, 61, 104, 42, 51, 50, 40, 123, 41, 106, 62, 59, 119, 49, 22, 55, 86, 93, 127, 34, 52, 118, 124, 105, 37, 125, 121, 47, 116, 53, 94, 90, 87, 63, 54, 98, 30, 122, 111, 89, 101, 16, 80, 25, 39, 85, 112, 24, 88, 27, 117, 83, 100, 23, 26, 46, 82, 91, 13, 18, 11, 35, 77, 36, 48, 75, 79, 110, 21, 103, 20, 19, 84, 102, 99, 76, 81, 15, 96, 9, 73, 17, 78, 32, 43, 92, 107, 5, 8, 7, 97, 71, 69, 6, 72, 12, 14, 74, 28, 3, 65, 1, 10, 38, 0, 33, 67, 64, 4, 70, 95, 68, 66, 2, 31], [126, 120, 109, 58, 56, 44, 114, 60, 115, 45, 113, 108, 104, 29, 57, 42, 61, 51, 123, 41, 40, 59, 50, 55, 106, 125, 105, 34, 49, 22, 119, 118, 62, 124, 86, 53, 93, 80, 116, 112, 98, 90, 16, 47, 52, 127, 94, 37, 122, 111, 87, 89, 27, 63, 13, 24, 30, 18, 117, 39, 85, 54, 23, 103, 25, 88, 26, 15, 100, 91, 84, 19, 11, 75, 77, 83, 82, 121, 79, 20, 48, 101, 73, 21, 96, 110, 36, 9, 35, 46, 17, 5, 67, 6, 14, 81, 72, 92, 78, 32, 43, 102, 99, 1, 69, 71, 76, 7, 10, 12, 97, 74, 65, 3, 8, 0, 64, 70, 68, 38, 33, 107, 4, 28, 2, 66, 95, 31], [126, 120, 109, 58, 56, 44, 114, 60, 115, 108, 113, 45, 104, 29, 42, 57, 61, 50, 51, 40, 59, 106, 41, 123, 22, 55, 49, 34, 86, 125, 105, 118, 124, 119, 47, 62, 127, 52, 98, 80, 90, 53, 24, 93, 27, 87, 16, 116, 122, 117, 37, 89, 94, 26, 39, 121, 112, 88, 91, 13, 63, 18, 30, 111, 83, 103, 48, 19, 82, 85, 54, 11, 23, 101, 84, 100, 25, 79, 75, 21, 20, 15, 77, 35, 46, 36, 9, 110, 78, 14, 73, 81, 99, 102, 96, 12, 32, 43, 17, 71, 97, 92, 72, 33, 107, 10, 69, 7, 5, 76, 6, 74, 3, 67, 1, 8, 0, 38, 65, 70, 64, 28, 31, 4, 68, 2, 66, 95], [126, 120, 109, 56, 114, 58, 60, 44, 115, 113, 108, 45, 29, 57, 61, 42, 123, 51, 104, 50, 40, 106, 41, 59, 124, 125, 55, 119, 118, 22, 86, 105, 49, 93, 62, 122, 52, 37, 34, 90, 53, 94, 116, 87, 47, 127, 27, 24, 88, 98, 101, 89, 111, 30, 121, 112, 103, 54, 25, 82, 91, 117, 16, 80, 48, 46, 26, 100, 83, 18, 35, 110, 21, 85, 39, 19, 63, 96, 102, 99, 23, 13, 20, 36, 84, 32, 79, 97, 11, 81, 15, 77, 43, 75, 78, 92, 107, 12, 17, 73, 14, 33, 9, 10, 71, 28, 72, 5, 38, 74, 76, 70, 69, 8, 7, 31, 68, 67, 6, 95, 1, 65, 3, 0, 4, 66, 64, 2], [126, 120, 109, 56, 114, 58, 44, 60, 115, 108, 113, 45, 57, 29, 104, 51, 42, 61, 123, 50, 106, 40, 59, 119, 41, 118, 55, 22, 124, 86, 105, 125, 49, 34, 62, 127, 90, 116, 37, 93, 47, 53, 63, 54, 98, 52, 87, 30, 16, 111, 24, 117, 27, 94, 122, 80, 101, 100, 26, 25, 89, 121, 91, 112, 103, 88, 83, 23, 35, 39, 84, 13, 85, 48, 46, 19, 21, 18, 82, 11, 75, 79, 81, 99, 107, 9, 36, 102, 20, 32, 15, 77, 110, 43, 97, 78, 96, 73, 14, 71, 17, 5, 76, 70, 92, 10, 72, 64, 33, 12, 7, 65, 69, 0, 1, 74, 8, 3, 38, 67, 28, 4, 95, 31, 66, 68, 2, 6], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 108, 45, 57, 29, 104, 42, 61, 51, 50, 123, 59, 106, 40, 41, 22, 55, 119, 49, 93, 105, 34, 118, 124, 98, 86, 87, 62, 125, 37, 80, 90, 47, 16, 52, 89, 53, 94, 111, 127, 122, 27, 39, 116, 24, 30, 54, 101, 46, 117, 112, 18, 100, 26, 21, 19, 91, 13, 82, 83, 11, 85, 103, 25, 75, 23, 84, 15, 63, 36, 81, 77, 20, 88, 107, 110, 73, 79, 121, 9, 35, 102, 17, 48, 78, 96, 99, 14, 12, 76, 92, 7, 32, 70, 10, 5, 97, 72, 71, 67, 43, 69, 8, 3, 64, 74, 33, 1, 0, 65, 38, 28, 4, 6, 68, 31, 2, 95, 66], [126, 120, 109, 56, 58, 44, 60, 114, 115, 113, 45, 108, 29, 57, 42, 61, 104, 50, 51, 59, 40, 123, 41, 55, 49, 119, 106, 105, 22, 34, 90, 124, 47, 62, 98, 116, 118, 125, 86, 117, 53, 111, 122, 37, 52, 127, 103, 30, 112, 24, 93, 94, 87, 26, 46, 54, 80, 39, 101, 16, 27, 63, 82, 25, 85, 13, 88, 19, 110, 91, 83, 89, 100, 11, 121, 18, 15, 75, 73, 21, 84, 36, 35, 23, 48, 77, 20, 79, 9, 102, 99, 96, 107, 81, 92, 78, 10, 12, 32, 14, 72, 76, 70, 17, 33, 69, 97, 43, 7, 67, 3, 8, 71, 65, 74, 38, 0, 1, 5, 64, 6, 28, 68, 4, 66, 95, 31, 2], [126, 120, 109, 114, 56, 44, 58, 60, 115, 108, 113, 45, 29, 57, 61, 42, 104, 51, 50, 40, 106, 123, 41, 59, 55, 49, 119, 118, 22, 86, 34, 105, 124, 62, 94, 52, 93, 116, 98, 125, 111, 90, 37, 47, 30, 53, 27, 117, 87, 127, 101, 39, 24, 122, 54, 63, 80, 112, 88, 25, 26, 91, 100, 89, 19, 82, 46, 48, 16, 21, 13, 85, 83, 103, 84, 35, 23, 121, 18, 96, 79, 102, 15, 36, 11, 110, 107, 99, 92, 20, 32, 75, 17, 12, 77, 81, 33, 43, 73, 78, 97, 9, 14, 10, 64, 69, 28, 71, 38, 76, 72, 74, 8, 5, 7, 65, 70, 67, 1, 0, 3, 95, 31, 4, 6, 2, 68, 66], [126, 120, 109, 58, 114, 44, 56, 60, 115, 113, 108, 45, 29, 57, 104, 42, 61, 51, 50, 106, 40, 123, 41, 59, 119, 55, 22, 49, 34, 105, 118, 93, 86, 62, 124, 125, 47, 111, 98, 90, 94, 52, 122, 26, 37, 116, 117, 87, 24, 63, 27, 30, 39, 53, 80, 112, 88, 85, 16, 89, 101, 127, 25, 91, 121, 100, 82, 18, 54, 83, 13, 110, 103, 46, 19, 23, 36, 11, 20, 102, 21, 35, 15, 32, 48, 99, 96, 12, 75, 79, 84, 77, 9, 92, 73, 17, 107, 81, 97, 78, 14, 43, 71, 33, 7, 69, 5, 74, 8, 72, 28, 38, 64, 67, 10, 1, 3, 0, 76, 65, 6, 70, 31, 68, 4, 66, 2, 95], [126, 120, 109, 58, 44, 60, 56, 114, 115, 113, 45, 108, 57, 104, 29, 61, 42, 51, 50, 40, 59, 123, 41, 124, 106, 119, 62, 22, 105, 49, 34, 118, 55, 37, 93, 47, 116, 52, 94, 98, 111, 90, 125, 87, 53, 122, 39, 100, 86, 127, 30, 112, 54, 121, 27, 80, 25, 26, 101, 117, 24, 18, 103, 35, 63, 16, 88, 110, 36, 89, 46, 23, 21, 83, 82, 91, 102, 85, 96, 19, 15, 75, 11, 48, 84, 13, 20, 99, 107, 79, 81, 73, 77, 32, 38, 92, 17, 97, 9, 12, 43, 76, 14, 78, 8, 1, 74, 71, 67, 6, 10, 5, 64, 7, 3, 69, 72, 28, 33, 65, 0, 70, 95, 68, 66, 31, 4, 2], [126, 120, 109, 58, 56, 114, 44, 60, 115, 113, 45, 108, 29, 57, 61, 104, 42, 51, 50, 40, 59, 41, 123, 22, 119, 106, 124, 49, 34, 55, 105, 118, 62, 125, 93, 98, 116, 52, 86, 94, 90, 37, 111, 112, 63, 47, 53, 16, 101, 87, 121, 127, 30, 25, 24, 27, 54, 26, 91, 80, 100, 103, 23, 122, 82, 89, 110, 18, 39, 48, 11, 85, 88, 83, 21, 19, 96, 46, 13, 117, 35, 84, 102, 15, 75, 107, 77, 99, 20, 36, 79, 92, 73, 81, 32, 14, 17, 9, 78, 97, 74, 76, 33, 10, 12, 8, 6, 5, 71, 43, 1, 7, 72, 38, 67, 28, 69, 65, 64, 3, 0, 95, 4, 70, 68, 31, 66, 2], [126, 120, 109, 58, 56, 114, 44, 60, 115, 113, 45, 108, 29, 104, 61, 42, 57, 51, 40, 50, 59, 123, 106, 49, 41, 55, 22, 62, 119, 124, 34, 118, 86, 105, 52, 125, 90, 93, 94, 116, 37, 63, 98, 53, 16, 47, 24, 87, 80, 117, 112, 111, 39, 30, 54, 26, 27, 91, 89, 88, 101, 82, 25, 23, 121, 48, 103, 13, 21, 85, 83, 127, 18, 100, 84, 20, 19, 110, 75, 11, 77, 36, 96, 79, 122, 102, 107, 46, 32, 15, 99, 35, 9, 17, 81, 92, 73, 12, 33, 43, 78, 14, 76, 10, 6, 71, 8, 97, 74, 7, 5, 72, 3, 28, 69, 0, 65, 1, 67, 38, 64, 31, 95, 70, 4, 68, 66, 2], [126, 120, 109, 58, 56, 114, 44, 60, 113, 115, 45, 108, 104, 29, 51, 61, 57, 42, 40, 106, 59, 50, 41, 62, 123, 119, 105, 124, 55, 118, 34, 49, 125, 86, 52, 22, 37, 87, 98, 93, 116, 47, 90, 121, 53, 63, 112, 111, 94, 117, 89, 16, 39, 24, 27, 127, 26, 88, 30, 110, 54, 122, 18, 80, 91, 101, 23, 83, 85, 48, 102, 82, 25, 21, 100, 20, 96, 19, 35, 13, 77, 46, 11, 36, 84, 107, 99, 17, 15, 79, 75, 92, 103, 12, 32, 33, 73, 81, 43, 0, 9, 78, 14, 67, 65, 64, 1, 5, 8, 10, 69, 72, 97, 6, 74, 76, 38, 71, 7, 28, 3, 4, 70, 31, 2, 68, 66, 95], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 61, 29, 104, 57, 51, 42, 40, 50, 59, 123, 106, 41, 118, 62, 55, 22, 124, 119, 105, 49, 116, 86, 34, 93, 37, 98, 111, 94, 87, 90, 125, 53, 30, 117, 47, 112, 16, 121, 110, 52, 39, 54, 127, 27, 100, 103, 122, 91, 88, 63, 26, 101, 18, 89, 80, 24, 48, 19, 83, 20, 25, 82, 23, 21, 35, 36, 85, 107, 46, 77, 13, 99, 84, 102, 11, 32, 79, 96, 92, 75, 15, 17, 9, 12, 73, 33, 78, 81, 14, 43, 69, 3, 8, 97, 76, 7, 74, 10, 28, 38, 1, 65, 5, 6, 64, 68, 70, 71, 72, 0, 67, 31, 66, 95, 4, 2], [126, 120, 109, 56, 58, 114, 44, 60, 115, 113, 108, 45, 29, 104, 61, 42, 57, 51, 123, 50, 40, 106, 59, 41, 62, 22, 49, 118, 119, 55, 105, 124, 34, 116, 93, 47, 125, 86, 87, 98, 37, 63, 90, 52, 94, 111, 53, 30, 80, 91, 117, 24, 121, 88, 39, 16, 26, 18, 122, 127, 112, 100, 48, 27, 89, 19, 54, 101, 46, 85, 25, 110, 82, 20, 11, 83, 103, 15, 75, 23, 21, 36, 102, 35, 84, 77, 107, 96, 13, 79, 32, 73, 17, 92, 14, 76, 9, 99, 7, 12, 97, 81, 78, 8, 69, 43, 10, 38, 70, 33, 74, 5, 28, 71, 72, 67, 1, 0, 3, 65, 64, 68, 6, 95, 4, 31, 66, 2], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 57, 29, 104, 42, 61, 51, 40, 123, 50, 106, 59, 41, 22, 55, 119, 105, 49, 118, 62, 34, 124, 93, 125, 86, 87, 121, 52, 47, 98, 39, 90, 116, 94, 37, 26, 111, 24, 63, 89, 127, 112, 117, 16, 54, 27, 18, 80, 30, 91, 82, 53, 88, 83, 122, 101, 100, 48, 23, 19, 21, 85, 77, 15, 20, 110, 75, 11, 13, 25, 84, 103, 35, 96, 36, 73, 17, 46, 79, 102, 9, 81, 76, 107, 92, 78, 32, 99, 74, 14, 12, 70, 43, 71, 7, 5, 72, 8, 69, 97, 10, 3, 38, 28, 67, 65, 33, 1, 64, 0, 4, 6, 31, 68, 95, 2, 66], [126, 120, 109, 56, 58, 44, 114, 60, 115, 113, 45, 108, 29, 104, 57, 42, 61, 123, 51, 50, 59, 40, 106, 119, 41, 49, 22, 118, 124, 105, 55, 34, 63, 62, 125, 121, 86, 116, 52, 90, 98, 93, 53, 87, 47, 24, 94, 54, 37, 39, 117, 101, 27, 26, 30, 127, 89, 111, 122, 112, 18, 48, 80, 16, 88, 91, 100, 25, 83, 35, 84, 23, 82, 19, 96, 36, 46, 110, 13, 21, 75, 85, 9, 102, 107, 103, 15, 11, 99, 92, 20, 77, 79, 73, 43, 76, 32, 97, 14, 17, 33, 78, 81, 74, 28, 38, 7, 10, 71, 5, 72, 70, 69, 12, 8, 67, 1, 65, 64, 0, 31, 95, 3, 68, 6, 66, 4, 2], [126, 120, 109, 58, 44, 56, 114, 60, 115, 108, 45, 113, 29, 104, 42, 57, 61, 51, 123, 50, 59, 40, 106, 119, 41, 124, 55, 118, 62, 125, 34, 22, 105, 90, 86, 53, 49, 116, 93, 94, 127, 52, 122, 37, 98, 47, 63, 117, 39, 112, 87, 111, 27, 54, 103, 30, 121, 101, 100, 48, 25, 88, 89, 18, 23, 24, 16, 26, 80, 83, 110, 91, 36, 84, 35, 99, 75, 82, 20, 21, 19, 46, 107, 13, 96, 43, 77, 102, 15, 85, 9, 11, 32, 17, 7, 79, 97, 14, 92, 78, 81, 73, 76, 33, 38, 72, 67, 74, 12, 69, 5, 70, 28, 3, 71, 31, 8, 64, 65, 10, 0, 1, 6, 68, 4, 95, 2, 66], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 29, 104, 57, 42, 61, 51, 50, 40, 59, 106, 123, 41, 62, 22, 124, 118, 119, 49, 105, 55, 116, 87, 86, 47, 93, 34, 125, 122, 94, 121, 111, 98, 117, 63, 52, 37, 112, 90, 127, 30, 39, 80, 18, 24, 53, 16, 27, 89, 54, 26, 25, 101, 91, 83, 100, 13, 75, 84, 23, 19, 88, 82, 21, 48, 96, 110, 11, 35, 15, 79, 20, 36, 85, 77, 102, 107, 73, 99, 32, 9, 17, 81, 103, 46, 12, 14, 76, 78, 74, 10, 7, 5, 92, 97, 72, 1, 43, 69, 67, 71, 8, 64, 33, 6, 3, 65, 38, 70, 0, 4, 28, 68, 66, 95, 31, 2], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 29, 61, 104, 57, 42, 40, 50, 51, 59, 123, 41, 106, 22, 49, 124, 105, 62, 55, 119, 118, 34, 86, 125, 63, 98, 47, 93, 116, 94, 111, 87, 30, 37, 90, 122, 16, 127, 112, 39, 27, 52, 100, 24, 117, 54, 80, 96, 91, 82, 101, 25, 26, 121, 89, 88, 18, 53, 110, 75, 46, 103, 83, 13, 85, 79, 48, 23, 19, 21, 35, 84, 36, 77, 11, 15, 102, 20, 73, 9, 43, 107, 14, 99, 81, 72, 76, 5, 92, 17, 32, 1, 74, 78, 97, 64, 65, 12, 10, 8, 0, 7, 3, 69, 6, 71, 38, 67, 33, 70, 28, 4, 66, 68, 95, 31, 2], [126, 120, 109, 56, 58, 44, 114, 115, 60, 113, 45, 108, 29, 104, 61, 51, 42, 57, 59, 50, 123, 40, 41, 106, 22, 118, 105, 49, 124, 119, 55, 34, 125, 86, 116, 93, 98, 87, 94, 47, 63, 53, 90, 127, 52, 37, 30, 117, 80, 24, 111, 112, 39, 62, 16, 54, 27, 121, 26, 82, 88, 25, 75, 83, 91, 13, 21, 100, 23, 89, 101, 18, 35, 48, 79, 96, 85, 77, 110, 122, 84, 46, 20, 11, 15, 19, 73, 103, 99, 36, 92, 107, 76, 14, 9, 78, 81, 43, 102, 97, 12, 17, 32, 72, 5, 6, 7, 10, 33, 74, 28, 69, 71, 8, 67, 3, 38, 1, 64, 65, 0, 70, 31, 4, 68, 2, 95, 66], [126, 120, 109, 56, 114, 58, 44, 115, 60, 113, 45, 108, 104, 29, 61, 42, 50, 57, 51, 123, 59, 40, 41, 106, 49, 118, 119, 55, 124, 105, 116, 22, 34, 125, 86, 52, 63, 112, 98, 62, 47, 30, 94, 87, 111, 127, 93, 39, 90, 37, 54, 101, 121, 27, 24, 26, 16, 53, 80, 100, 117, 122, 103, 91, 82, 23, 89, 48, 18, 96, 88, 85, 25, 75, 35, 13, 84, 15, 110, 19, 21, 9, 36, 73, 77, 83, 20, 79, 7, 11, 107, 99, 46, 14, 5, 6, 76, 102, 81, 32, 92, 78, 43, 12, 69, 17, 10, 97, 8, 74, 67, 72, 64, 0, 33, 65, 1, 71, 38, 3, 28, 4, 95, 68, 70, 31, 66, 2], [126, 120, 109, 44, 56, 114, 58, 60, 115, 108, 113, 45, 29, 104, 57, 61, 42, 51, 50, 59, 40, 41, 106, 22, 123, 55, 118, 116, 119, 125, 124, 34, 86, 105, 49, 62, 87, 93, 47, 98, 94, 37, 127, 111, 90, 27, 39, 53, 30, 80, 100, 24, 16, 52, 63, 117, 121, 112, 122, 26, 25, 19, 101, 91, 88, 54, 13, 23, 82, 75, 18, 48, 35, 89, 83, 85, 21, 15, 110, 84, 77, 79, 96, 20, 11, 103, 9, 73, 14, 46, 76, 36, 102, 107, 97, 43, 32, 99, 81, 92, 78, 7, 17, 6, 69, 12, 72, 8, 74, 28, 33, 38, 71, 5, 10, 67, 31, 1, 95, 68, 3, 4, 65, 64, 0, 70, 66, 2]], "model.layers.18.self_attn.q_proj": [[39, 52, 112, 113, 32, 26, 87, 85, 121, 49, 29, 90, 74, 79, 116, 70, 77, 20, 53, 93, 82, 80, 46, 44, 23, 125, 119, 0, 75, 57, 98, 124, 18, 55, 61, 68, 111, 3, 48, 60, 54, 123, 115, 110, 62, 106, 65, 126, 19, 7, 117, 107, 84, 99, 118, 38, 56, 96, 120, 42, 47, 43, 9, 50, 109, 8, 2, 40, 51, 58, 11, 105, 21, 17, 25, 97, 41, 101, 59, 102, 122, 35, 24, 45, 6, 127, 63, 89, 104, 94, 33, 37, 30, 114, 100, 28, 108, 66, 22, 34, 14, 86, 36, 31, 88, 27, 92, 83, 103, 15, 69, 91, 10, 95, 81, 5, 16, 76, 71, 72, 1, 78, 12, 73, 67, 13, 4, 64], [39, 113, 52, 110, 112, 49, 29, 46, 87, 85, 32, 116, 90, 53, 26, 80, 93, 60, 83, 92, 77, 125, 115, 108, 23, 44, 123, 79, 21, 31, 121, 98, 119, 56, 111, 88, 58, 8, 48, 124, 107, 61, 105, 122, 42, 101, 18, 74, 118, 126, 62, 100, 25, 57, 55, 33, 63, 104, 106, 97, 45, 43, 24, 70, 117, 54, 99, 35, 38, 41, 82, 89, 47, 114, 120, 59, 50, 40, 109, 51, 127, 102, 36, 86, 68, 12, 11, 28, 17, 78, 91, 96, 94, 37, 15, 22, 34, 84, 9, 95, 30, 20, 7, 76, 19, 13, 27, 72, 75, 6, 16, 81, 73, 14, 2, 4, 64, 71, 65, 3, 69, 10, 0, 103, 1, 67, 66, 5], [39, 113, 52, 49, 29, 85, 32, 87, 116, 90, 26, 121, 79, 60, 125, 110, 46, 93, 112, 119, 53, 44, 21, 77, 124, 111, 122, 59, 61, 96, 109, 118, 123, 80, 98, 84, 56, 115, 54, 82, 47, 55, 57, 18, 23, 8, 74, 108, 83, 101, 120, 50, 92, 43, 22, 88, 89, 37, 31, 48, 97, 127, 42, 41, 51, 11, 105, 38, 35, 91, 106, 45, 62, 58, 75, 107, 36, 104, 78, 70, 117, 33, 63, 28, 114, 126, 94, 15, 19, 86, 95, 102, 40, 30, 27, 34, 9, 99, 17, 100, 24, 16, 76, 68, 25, 20, 71, 7, 81, 65, 3, 14, 12, 103, 2, 13, 1, 0, 72, 73, 4, 67, 64, 10, 6, 5, 69, 66], [39, 113, 52, 112, 110, 49, 29, 32, 85, 116, 77, 122, 87, 26, 31, 98, 46, 93, 80, 48, 8, 119, 50, 83, 70, 79, 53, 18, 123, 120, 74, 44, 125, 111, 114, 60, 42, 58, 124, 88, 57, 100, 92, 21, 117, 118, 90, 56, 115, 23, 121, 61, 35, 7, 108, 45, 27, 109, 63, 101, 25, 37, 62, 43, 0, 59, 82, 51, 54, 28, 6, 86, 68, 99, 102, 36, 104, 107, 41, 97, 38, 47, 55, 106, 105, 126, 127, 69, 40, 33, 34, 12, 3, 94, 95, 13, 91, 11, 30, 78, 19, 24, 84, 22, 15, 96, 81, 89, 20, 65, 67, 66, 9, 2, 73, 75, 4, 71, 76, 72, 16, 14, 17, 10, 1, 64, 5, 103], [124, 63, 56, 100, 115, 36, 39, 110, 102, 122, 116, 111, 59, 54, 105, 58, 103, 57, 121, 120, 60, 49, 62, 127, 50, 45, 25, 117, 125, 47, 112, 123, 55, 48, 114, 53, 109, 93, 126, 35, 52, 42, 40, 113, 51, 101, 41, 34, 119, 46, 87, 118, 107, 108, 61, 32, 44, 99, 104, 43, 38, 106, 37, 80, 90, 96, 94, 20, 30, 98, 23, 24, 97, 33, 84, 27, 31, 95, 29, 73, 85, 91, 89, 19, 28, 22, 21, 92, 78, 26, 88, 11, 75, 16, 86, 18, 69, 83, 77, 82, 13, 7, 9, 14, 1, 71, 5, 6, 66, 17, 3, 2, 79, 81, 67, 4, 15, 70, 12, 72, 74, 10, 65, 76, 64, 0, 8, 68], [63, 124, 30, 39, 115, 100, 19, 36, 25, 96, 22, 12, 18, 17, 15, 99, 27, 89, 32, 13, 72, 24, 5, 103, 35, 74, 92, 6, 123, 122, 56, 45, 2, 54, 4, 1, 90, 120, 126, 28, 33, 71, 21, 121, 42, 85, 14, 113, 26, 98, 83, 125, 73, 84, 75, 11, 94, 108, 86, 117, 23, 69, 52, 9, 10, 0, 81, 20, 97, 7, 67, 101, 38, 112, 87, 118, 93, 60, 43, 29, 16, 57, 34, 47, 37, 78, 91, 107, 80, 82, 88, 3, 58, 105, 79, 44, 65, 127, 119, 48, 70, 59, 62, 95, 53, 40, 109, 51, 66, 76, 31, 102, 111, 104, 49, 77, 106, 110, 114, 50, 68, 64, 61, 55, 116, 41, 46, 8], [124, 63, 100, 36, 120, 98, 103, 38, 25, 96, 32, 84, 19, 87, 97, 45, 94, 101, 126, 39, 22, 56, 51, 89, 117, 59, 121, 58, 113, 27, 21, 17, 62, 34, 54, 31, 23, 107, 40, 86, 52, 33, 13, 47, 111, 48, 46, 112, 102, 90, 122, 127, 99, 15, 60, 24, 57, 43, 91, 125, 83, 116, 29, 118, 44, 26, 28, 119, 110, 114, 30, 88, 109, 106, 49, 61, 93, 123, 105, 115, 35, 55, 53, 108, 12, 80, 85, 37, 78, 42, 20, 11, 104, 41, 81, 18, 95, 92, 50, 7, 14, 16, 82, 5, 77, 6, 74, 79, 75, 73, 72, 69, 76, 4, 2, 71, 9, 64, 68, 70, 8, 10, 66, 65, 1, 67, 3, 0], [124, 63, 24, 100, 21, 30, 18, 17, 12, 22, 15, 39, 115, 8, 74, 72, 27, 4, 90, 96, 79, 84, 13, 92, 76, 36, 1, 25, 94, 122, 68, 64, 75, 26, 77, 51, 121, 0, 6, 7, 3, 83, 2, 33, 85, 69, 81, 31, 10, 28, 70, 19, 86, 89, 46, 14, 5, 78, 67, 11, 82, 66, 20, 16, 71, 23, 80, 102, 88, 65, 73, 87, 93, 29, 43, 111, 103, 95, 32, 9, 127, 97, 35, 91, 98, 114, 126, 42, 125, 44, 37, 99, 56, 40, 61, 45, 120, 52, 58, 34, 104, 105, 119, 50, 101, 118, 38, 53, 59, 108, 107, 123, 55, 106, 112, 41, 49, 116, 60, 57, 109, 110, 113, 48, 54, 117, 47, 62], [55, 102, 118, 123, 46, 97, 61, 62, 53, 116, 106, 21, 120, 27, 57, 51, 115, 112, 126, 58, 122, 119, 48, 49, 63, 114, 91, 124, 50, 45, 31, 109, 52, 117, 24, 99, 110, 127, 60, 95, 41, 125, 121, 54, 113, 56, 103, 111, 59, 14, 47, 108, 104, 44, 38, 87, 88, 40, 18, 28, 19, 78, 70, 6, 42, 39, 26, 105, 107, 92, 86, 43, 101, 36, 37, 100, 93, 98, 96, 35, 34, 32, 65, 33, 82, 23, 94, 25, 30, 29, 0, 89, 16, 84, 90, 85, 22, 76, 80, 64, 17, 1, 3, 68, 20, 4, 83, 67, 11, 12, 8, 15, 66, 81, 2, 71, 72, 5, 9, 75, 7, 69, 74, 79, 77, 73, 13, 10], [118, 102, 55, 123, 48, 97, 53, 62, 61, 27, 46, 24, 57, 91, 51, 21, 116, 120, 31, 38, 122, 126, 115, 50, 49, 119, 124, 112, 58, 114, 63, 110, 40, 117, 95, 106, 109, 52, 54, 113, 99, 121, 18, 127, 56, 86, 88, 125, 45, 47, 111, 41, 60, 59, 104, 14, 44, 39, 43, 103, 108, 100, 107, 42, 92, 33, 105, 26, 101, 94, 98, 37, 36, 32, 78, 19, 35, 70, 34, 96, 29, 93, 30, 25, 28, 22, 17, 23, 84, 87, 82, 90, 6, 16, 85, 89, 80, 83, 20, 76, 12, 65, 15, 81, 3, 0, 1, 8, 75, 9, 71, 73, 11, 68, 79, 67, 64, 5, 4, 66, 77, 7, 2, 69, 74, 72, 10, 13], [102, 55, 118, 97, 123, 86, 24, 84, 46, 15, 27, 82, 17, 88, 77, 90, 74, 18, 48, 11, 72, 76, 30, 38, 120, 91, 54, 8, 95, 69, 92, 106, 93, 62, 71, 104, 22, 5, 16, 3, 20, 9, 81, 29, 14, 119, 75, 10, 31, 7, 116, 13, 19, 79, 21, 89, 45, 25, 85, 26, 126, 83, 28, 80, 23, 12, 67, 122, 101, 94, 40, 1, 87, 33, 108, 98, 70, 124, 49, 78, 109, 61, 96, 73, 36, 64, 65, 53, 68, 6, 114, 32, 103, 34, 0, 66, 57, 4, 37, 35, 60, 2, 107, 99, 100, 59, 110, 44, 125, 58, 41, 113, 105, 42, 47, 39, 112, 43, 50, 111, 51, 117, 56, 63, 52, 127, 121, 115], [102, 118, 55, 97, 123, 84, 24, 86, 17, 46, 27, 15, 31, 62, 91, 14, 76, 90, 18, 38, 88, 73, 95, 22, 79, 101, 82, 120, 81, 53, 61, 48, 11, 54, 12, 104, 75, 80, 19, 74, 49, 106, 28, 83, 20, 71, 29, 119, 122, 6, 116, 23, 85, 51, 45, 30, 77, 10, 25, 126, 69, 96, 87, 21, 9, 89, 112, 16, 26, 2, 108, 92, 93, 7, 32, 57, 5, 13, 94, 109, 124, 67, 65, 35, 58, 98, 115, 125, 37, 114, 78, 72, 103, 8, 34, 63, 50, 59, 52, 110, 44, 99, 100, 111, 40, 60, 113, 127, 121, 47, 3, 56, 39, 66, 117, 36, 0, 42, 68, 41, 105, 70, 107, 43, 64, 1, 33, 4], [43, 36, 54, 96, 63, 60, 89, 20, 107, 82, 126, 50, 87, 115, 80, 62, 25, 21, 9, 52, 16, 61, 32, 79, 13, 48, 101, 112, 47, 75, 53, 39, 57, 29, 85, 41, 69, 71, 103, 122, 119, 111, 19, 106, 105, 35, 104, 45, 18, 84, 55, 86, 23, 40, 127, 94, 46, 59, 125, 99, 88, 44, 93, 121, 28, 26, 90, 38, 100, 58, 92, 97, 51, 78, 6, 98, 110, 108, 120, 33, 10, 37, 114, 15, 74, 27, 123, 124, 7, 116, 56, 17, 102, 83, 113, 118, 24, 49, 67, 77, 95, 12, 8, 11, 109, 34, 31, 14, 22, 72, 30, 91, 117, 81, 76, 42, 66, 73, 4, 1, 5, 2, 70, 64, 3, 0, 65, 68], [43, 60, 96, 36, 63, 107, 89, 48, 123, 20, 126, 82, 21, 119, 46, 111, 25, 19, 85, 44, 80, 26, 42, 55, 112, 23, 13, 51, 122, 79, 9, 45, 61, 54, 18, 75, 41, 53, 59, 87, 16, 62, 50, 39, 105, 69, 52, 57, 103, 38, 40, 106, 116, 27, 121, 113, 101, 71, 114, 92, 109, 84, 49, 32, 58, 115, 35, 104, 95, 117, 124, 108, 125, 56, 86, 33, 11, 90, 127, 37, 81, 110, 88, 98, 29, 99, 83, 118, 34, 100, 30, 102, 24, 7, 120, 93, 97, 17, 91, 47, 94, 77, 1, 31, 22, 28, 8, 15, 0, 14, 72, 3, 73, 78, 67, 12, 10, 6, 2, 74, 5, 68, 76, 70, 66, 4, 64, 65], [43, 96, 54, 36, 63, 47, 20, 89, 107, 25, 60, 119, 82, 9, 80, 16, 87, 21, 69, 79, 18, 111, 28, 71, 13, 48, 55, 112, 75, 120, 53, 38, 32, 126, 19, 61, 123, 41, 62, 50, 115, 51, 84, 85, 57, 110, 104, 40, 103, 39, 45, 29, 67, 106, 122, 125, 1, 118, 59, 95, 26, 37, 116, 44, 64, 121, 56, 127, 76, 58, 100, 113, 108, 49, 7, 114, 102, 65, 105, 52, 83, 46, 94, 109, 124, 117, 91, 86, 90, 101, 99, 34, 92, 23, 30, 35, 22, 42, 73, 98, 27, 93, 4, 97, 33, 6, 14, 66, 0, 72, 81, 74, 78, 15, 88, 31, 11, 5, 24, 10, 3, 68, 2, 17, 8, 77, 12, 70], [43, 54, 96, 36, 60, 89, 107, 82, 20, 87, 75, 63, 25, 80, 79, 71, 111, 13, 9, 47, 29, 69, 85, 16, 21, 77, 126, 97, 40, 32, 41, 38, 92, 106, 18, 119, 115, 28, 34, 117, 53, 121, 67, 19, 44, 27, 46, 57, 109, 58, 50, 103, 14, 48, 101, 10, 84, 35, 55, 122, 45, 93, 65, 95, 1, 114, 91, 23, 118, 51, 12, 37, 2, 83, 112, 15, 104, 88, 94, 31, 78, 123, 62, 30, 66, 59, 116, 99, 90, 11, 100, 64, 26, 61, 42, 127, 22, 56, 113, 33, 49, 7, 120, 24, 74, 81, 76, 73, 39, 110, 125, 98, 8, 124, 52, 17, 108, 5, 86, 4, 102, 105, 6, 70, 72, 68, 0, 3], [120, 104, 59, 113, 109, 51, 62, 110, 22, 56, 88, 57, 119, 112, 90, 52, 83, 41, 21, 114, 36, 98, 55, 116, 107, 50, 103, 29, 115, 123, 93, 54, 95, 43, 38, 42, 124, 49, 60, 122, 99, 45, 117, 105, 126, 102, 61, 48, 106, 58, 33, 30, 127, 108, 39, 118, 47, 125, 44, 46, 111, 53, 27, 121, 63, 101, 37, 97, 35, 28, 100, 92, 31, 96, 80, 10, 32, 34, 79, 86, 40, 74, 17, 25, 91, 94, 81, 19, 76, 23, 85, 77, 84, 26, 13, 24, 89, 14, 15, 20, 71, 87, 72, 82, 16, 18, 7, 12, 11, 67, 69, 3, 9, 78, 5, 6, 75, 8, 1, 70, 65, 68, 73, 0, 66, 4, 2, 64], [104, 120, 59, 113, 98, 88, 18, 80, 65, 22, 6, 76, 93, 9, 110, 15, 14, 62, 68, 40, 112, 119, 72, 57, 83, 84, 97, 32, 51, 28, 36, 100, 26, 123, 29, 1, 67, 82, 74, 52, 11, 5, 8, 24, 70, 81, 99, 61, 4, 27, 86, 21, 91, 90, 46, 19, 107, 75, 66, 117, 34, 55, 116, 33, 58, 77, 16, 124, 54, 37, 49, 114, 118, 50, 31, 23, 25, 60, 13, 42, 109, 127, 115, 53, 78, 38, 3, 87, 106, 17, 126, 89, 122, 30, 69, 101, 12, 20, 92, 108, 64, 85, 105, 71, 41, 56, 45, 95, 96, 35, 0, 7, 102, 94, 43, 125, 48, 44, 103, 10, 111, 47, 2, 63, 79, 73, 121, 39], [113, 104, 120, 59, 88, 98, 22, 93, 52, 51, 83, 57, 29, 21, 112, 80, 90, 119, 62, 110, 124, 114, 76, 28, 116, 81, 55, 123, 50, 11, 95, 18, 122, 117, 118, 15, 108, 54, 49, 42, 96, 115, 58, 61, 41, 85, 100, 39, 14, 56, 53, 47, 84, 46, 109, 60, 127, 44, 125, 91, 27, 92, 31, 36, 86, 48, 43, 103, 35, 26, 101, 126, 105, 97, 45, 40, 121, 106, 38, 82, 6, 99, 72, 111, 19, 17, 102, 65, 63, 13, 30, 79, 25, 74, 24, 107, 23, 33, 37, 94, 87, 10, 77, 32, 89, 9, 34, 12, 20, 5, 75, 68, 3, 8, 71, 66, 16, 69, 7, 67, 78, 0, 2, 1, 4, 64, 70, 73], [59, 104, 120, 113, 112, 57, 62, 51, 83, 36, 43, 55, 105, 89, 119, 52, 50, 98, 95, 60, 110, 124, 49, 126, 115, 90, 35, 121, 117, 122, 88, 45, 97, 92, 56, 109, 123, 21, 61, 118, 102, 54, 116, 58, 42, 46, 103, 127, 17, 107, 114, 47, 48, 44, 15, 125, 23, 108, 41, 106, 63, 74, 34, 53, 100, 39, 101, 32, 99, 37, 111, 38, 19, 10, 26, 29, 30, 96, 13, 93, 28, 24, 33, 25, 31, 81, 94, 84, 79, 91, 5, 67, 40, 85, 87, 3, 71, 20, 0, 77, 22, 65, 69, 27, 14, 64, 18, 7, 76, 11, 75, 66, 86, 1, 68, 9, 72, 6, 82, 80, 2, 8, 4, 12, 78, 73, 70, 16], [127, 54, 122, 125, 100, 62, 126, 119, 120, 41, 40, 36, 42, 28, 117, 106, 118, 56, 83, 115, 92, 112, 58, 124, 59, 113, 39, 55, 49, 46, 57, 111, 53, 123, 63, 86, 52, 116, 99, 61, 114, 121, 48, 104, 47, 60, 51, 50, 105, 107, 37, 103, 108, 82, 110, 43, 45, 44, 38, 109, 35, 15, 74, 94, 101, 33, 25, 102, 98, 29, 34, 31, 80, 20, 96, 89, 90, 71, 95, 77, 24, 79, 97, 32, 88, 67, 26, 22, 75, 30, 91, 93, 65, 19, 27, 18, 23, 21, 6, 14, 85, 87, 10, 7, 68, 66, 84, 0, 13, 76, 73, 17, 78, 4, 1, 64, 8, 81, 16, 2, 12, 11, 70, 9, 72, 3, 69, 5], [122, 54, 127, 40, 100, 125, 24, 90, 33, 31, 62, 120, 105, 99, 19, 36, 86, 26, 126, 28, 106, 119, 53, 25, 43, 81, 102, 41, 98, 20, 88, 87, 49, 69, 35, 94, 115, 85, 59, 76, 66, 73, 61, 32, 7, 104, 56, 91, 27, 52, 34, 15, 110, 23, 8, 57, 38, 83, 72, 22, 75, 114, 29, 58, 63, 12, 78, 5, 101, 3, 109, 113, 92, 117, 48, 123, 42, 60, 111, 51, 17, 124, 16, 116, 37, 70, 103, 74, 112, 93, 55, 121, 96, 47, 82, 10, 21, 13, 50, 11, 9, 14, 30, 95, 46, 118, 39, 1, 89, 4, 80, 108, 107, 45, 77, 6, 44, 67, 18, 79, 84, 2, 71, 64, 65, 68, 0, 97], [54, 122, 127, 100, 41, 40, 92, 33, 106, 120, 125, 126, 36, 24, 103, 28, 83, 86, 119, 117, 94, 32, 26, 99, 115, 15, 42, 62, 49, 31, 59, 90, 29, 21, 105, 57, 96, 88, 53, 104, 82, 124, 113, 56, 39, 58, 95, 63, 74, 112, 35, 38, 34, 16, 123, 114, 55, 118, 48, 80, 61, 50, 98, 52, 37, 22, 51, 121, 46, 89, 101, 27, 102, 23, 47, 97, 85, 19, 77, 116, 111, 60, 107, 20, 43, 30, 110, 25, 87, 91, 13, 79, 109, 81, 44, 93, 75, 108, 76, 18, 12, 45, 71, 78, 14, 11, 17, 84, 8, 6, 10, 73, 9, 66, 4, 67, 72, 65, 7, 0, 69, 68, 70, 1, 64, 3, 5, 2], [127, 122, 54, 100, 41, 40, 126, 86, 26, 33, 28, 120, 15, 36, 42, 43, 117, 119, 96, 125, 83, 106, 49, 53, 57, 59, 21, 92, 118, 39, 124, 38, 115, 88, 103, 62, 107, 56, 24, 58, 35, 19, 114, 46, 108, 95, 112, 61, 37, 48, 105, 110, 81, 52, 123, 47, 63, 55, 113, 102, 17, 104, 50, 60, 111, 97, 116, 31, 99, 34, 98, 121, 51, 101, 44, 30, 22, 45, 13, 84, 109, 82, 32, 74, 77, 10, 94, 29, 11, 18, 93, 90, 89, 91, 87, 78, 27, 23, 14, 71, 20, 7, 85, 25, 79, 80, 75, 16, 67, 76, 72, 73, 8, 6, 4, 66, 1, 65, 12, 68, 9, 0, 64, 3, 70, 69, 2, 5], [105, 98, 117, 86, 96, 28, 53, 20, 52, 116, 25, 41, 94, 38, 80, 126, 109, 56, 63, 119, 123, 45, 111, 82, 19, 14, 44, 83, 62, 118, 71, 75, 120, 89, 77, 87, 115, 54, 78, 8, 49, 21, 112, 79, 23, 48, 51, 10, 50, 58, 57, 102, 61, 30, 46, 55, 59, 37, 39, 124, 27, 114, 125, 6, 107, 100, 127, 16, 43, 40, 93, 31, 35, 101, 104, 29, 60, 103, 121, 26, 42, 97, 110, 2, 85, 122, 4, 95, 113, 106, 17, 33, 36, 99, 47, 108, 9, 90, 34, 24, 22, 91, 88, 84, 18, 7, 32, 64, 81, 1, 92, 76, 12, 11, 15, 67, 68, 13, 73, 70, 0, 74, 66, 3, 65, 72, 5, 69], [105, 98, 96, 86, 20, 117, 82, 53, 75, 25, 41, 123, 79, 80, 52, 77, 63, 8, 116, 28, 94, 6, 126, 10, 16, 118, 89, 2, 119, 112, 4, 59, 62, 56, 45, 115, 44, 91, 111, 3, 109, 11, 57, 120, 13, 30, 88, 84, 26, 23, 54, 48, 97, 7, 51, 38, 17, 93, 50, 22, 74, 127, 67, 29, 114, 83, 18, 0, 58, 14, 31, 95, 102, 21, 61, 85, 78, 70, 39, 76, 104, 81, 19, 40, 107, 55, 92, 36, 15, 5, 24, 60, 64, 101, 125, 35, 121, 124, 106, 27, 113, 68, 87, 32, 1, 37, 90, 33, 72, 43, 49, 47, 110, 73, 34, 69, 99, 66, 65, 9, 103, 46, 122, 12, 42, 100, 108, 71], [105, 98, 86, 96, 117, 82, 8, 53, 79, 25, 63, 20, 75, 80, 4, 77, 41, 94, 2, 64, 123, 62, 6, 89, 48, 116, 118, 0, 115, 52, 72, 28, 66, 1, 29, 68, 13, 126, 16, 71, 69, 70, 87, 44, 85, 10, 112, 30, 65, 51, 109, 45, 11, 59, 119, 56, 57, 50, 88, 5, 18, 7, 24, 26, 93, 19, 111, 120, 35, 22, 78, 17, 15, 127, 21, 84, 3, 74, 76, 58, 39, 114, 12, 9, 100, 92, 60, 106, 23, 73, 14, 83, 102, 99, 67, 27, 40, 38, 121, 55, 31, 95, 91, 97, 61, 49, 101, 113, 122, 36, 124, 34, 81, 47, 33, 37, 90, 46, 54, 110, 32, 125, 107, 42, 103, 104, 43, 108], [105, 98, 20, 28, 96, 25, 86, 82, 41, 117, 53, 123, 80, 94, 63, 116, 77, 75, 52, 118, 79, 126, 115, 16, 59, 56, 44, 8, 89, 111, 62, 119, 10, 50, 57, 17, 120, 48, 109, 112, 51, 71, 45, 127, 84, 18, 12, 13, 30, 38, 22, 23, 6, 91, 95, 31, 55, 83, 78, 81, 76, 88, 54, 27, 61, 60, 19, 39, 74, 93, 46, 33, 124, 24, 104, 106, 7, 49, 34, 37, 26, 114, 58, 4, 102, 43, 107, 122, 11, 99, 15, 9, 2, 125, 87, 97, 14, 85, 92, 36, 90, 29, 21, 110, 101, 72, 100, 32, 35, 121, 40, 42, 68, 103, 70, 113, 69, 108, 47, 3, 1, 0, 64, 66, 5, 73, 65, 67], [112, 105, 113, 125, 110, 41, 114, 89, 126, 104, 117, 54, 61, 27, 84, 127, 57, 59, 58, 56, 107, 123, 30, 91, 99, 48, 52, 51, 46, 111, 49, 108, 47, 55, 120, 92, 42, 115, 101, 119, 53, 94, 122, 43, 109, 23, 118, 44, 116, 62, 86, 25, 121, 124, 106, 50, 63, 60, 34, 102, 36, 45, 39, 37, 100, 40, 98, 33, 38, 35, 103, 90, 21, 97, 32, 15, 96, 28, 82, 95, 88, 31, 79, 18, 80, 29, 93, 81, 26, 87, 85, 17, 20, 22, 24, 83, 13, 12, 19, 78, 3, 14, 76, 69, 74, 75, 65, 77, 16, 11, 10, 9, 8, 71, 72, 73, 6, 5, 70, 7, 67, 68, 66, 1, 2, 4, 64, 0], [105, 112, 86, 30, 41, 84, 91, 80, 82, 78, 27, 76, 98, 99, 89, 94, 126, 113, 9, 10, 7, 110, 61, 114, 74, 34, 25, 52, 71, 35, 127, 14, 58, 125, 20, 49, 23, 109, 88, 67, 54, 69, 57, 59, 3, 123, 56, 117, 19, 47, 13, 46, 62, 5, 22, 111, 18, 108, 81, 48, 51, 118, 55, 77, 116, 17, 21, 85, 16, 107, 43, 120, 115, 119, 37, 79, 40, 32, 73, 50, 122, 92, 11, 60, 93, 24, 102, 83, 53, 124, 65, 63, 28, 121, 15, 106, 8, 4, 104, 97, 36, 44, 87, 64, 42, 45, 90, 12, 95, 38, 39, 29, 26, 103, 33, 75, 31, 101, 96, 72, 100, 2, 70, 66, 0, 1, 6, 68], [112, 105, 86, 91, 41, 30, 84, 25, 27, 58, 126, 89, 15, 62, 80, 113, 106, 82, 33, 111, 114, 110, 17, 127, 116, 104, 52, 124, 78, 94, 83, 117, 76, 49, 61, 46, 48, 39, 125, 59, 92, 57, 11, 56, 107, 10, 85, 47, 98, 36, 99, 54, 13, 51, 35, 123, 45, 7, 87, 60, 101, 23, 32, 119, 63, 108, 18, 22, 29, 6, 121, 120, 115, 44, 12, 109, 71, 55, 93, 21, 97, 40, 43, 96, 118, 20, 95, 72, 34, 16, 37, 31, 53, 122, 28, 42, 88, 50, 26, 38, 103, 90, 69, 2, 9, 79, 24, 102, 81, 100, 64, 14, 19, 67, 68, 75, 74, 5, 77, 8, 73, 4, 66, 1, 70, 65, 0, 3], [105, 112, 30, 84, 86, 89, 41, 98, 126, 82, 48, 113, 118, 123, 110, 25, 80, 35, 85, 45, 31, 91, 99, 52, 54, 94, 88, 61, 125, 78, 27, 58, 115, 9, 59, 127, 92, 116, 109, 117, 39, 114, 33, 17, 57, 12, 60, 32, 49, 121, 95, 76, 108, 6, 40, 73, 56, 111, 47, 74, 120, 18, 103, 51, 62, 26, 97, 79, 55, 53, 122, 75, 20, 106, 63, 19, 22, 107, 104, 93, 90, 119, 13, 42, 28, 124, 102, 46, 21, 3, 100, 101, 43, 50, 44, 83, 23, 37, 4, 36, 10, 87, 96, 72, 38, 24, 15, 14, 29, 34, 11, 16, 81, 77, 68, 1, 71, 69, 70, 67, 5, 8, 7, 65, 0, 66, 2, 64]], "model.layers.18.self_attn.k_proj": [[113, 52, 103, 110, 96, 93, 90, 87, 85, 18, 80, 112, 77, 49, 55, 44, 124, 74, 48, 119, 54, 79, 118, 53, 20, 70, 121, 57, 46, 62, 105, 61, 86, 56, 125, 51, 43, 34, 111, 116, 8, 126, 127, 109, 120, 115, 64, 123, 50, 47, 108, 117, 41, 58, 0, 68, 65, 59, 107, 102, 63, 45, 122, 42, 114, 9, 60, 40, 106, 104, 38, 19, 7, 83, 99, 37, 101, 2, 3, 30, 28, 76, 36, 98, 95, 1, 4, 92, 78, 81, 88, 97, 35, 33, 75, 25, 5, 11, 94, 100, 12, 24, 91, 89, 27, 73, 31, 66, 17, 69, 14, 72, 84, 16, 22, 15, 29, 67, 13, 71, 23, 21, 26, 82, 32, 6, 39, 10], [63, 124, 36, 22, 120, 15, 30, 17, 74, 103, 12, 56, 72, 18, 13, 54, 4, 32, 27, 24, 117, 62, 60, 51, 122, 64, 107, 46, 121, 2, 44, 58, 126, 21, 47, 118, 59, 109, 28, 61, 69, 57, 114, 45, 10, 39, 38, 119, 113, 79, 50, 53, 49, 55, 43, 48, 116, 40, 19, 125, 115, 108, 42, 123, 127, 106, 105, 110, 104, 8, 112, 52, 111, 41, 102, 1, 25, 100, 37, 97, 67, 26, 101, 34, 7, 76, 29, 33, 35, 96, 95, 98, 90, 99, 71, 6, 85, 93, 91, 78, 94, 73, 31, 75, 82, 92, 23, 84, 88, 80, 3, 20, 87, 11, 5, 81, 14, 9, 83, 86, 16, 0, 89, 66, 77, 65, 68, 70], [55, 118, 38, 123, 33, 86, 91, 24, 61, 62, 48, 15, 116, 53, 18, 95, 119, 126, 120, 122, 17, 112, 102, 115, 109, 77, 49, 51, 57, 84, 124, 117, 63, 114, 110, 21, 11, 45, 52, 90, 31, 46, 50, 74, 104, 19, 44, 113, 54, 58, 121, 76, 125, 111, 127, 60, 37, 59, 56, 105, 108, 7, 3, 16, 103, 30, 5, 107, 42, 23, 78, 35, 47, 8, 106, 43, 41, 1, 39, 94, 27, 40, 9, 100, 99, 36, 32, 6, 14, 93, 98, 64, 20, 89, 101, 29, 72, 82, 34, 87, 12, 28, 96, 88, 25, 75, 26, 92, 85, 79, 66, 71, 83, 13, 80, 22, 10, 69, 4, 73, 67, 81, 68, 97, 2, 70, 65, 0], [107, 60, 100, 32, 89, 54, 47, 87, 20, 80, 63, 82, 75, 21, 9, 43, 53, 79, 126, 13, 69, 71, 1, 117, 111, 118, 29, 51, 64, 121, 61, 2, 57, 67, 52, 0, 50, 62, 114, 102, 115, 127, 124, 92, 48, 122, 125, 49, 56, 34, 103, 7, 41, 40, 77, 105, 119, 46, 108, 58, 45, 110, 120, 116, 94, 123, 42, 112, 37, 86, 113, 39, 44, 30, 3, 27, 109, 59, 38, 101, 81, 55, 104, 91, 19, 28, 31, 90, 106, 11, 22, 26, 78, 99, 85, 33, 35, 98, 72, 83, 95, 93, 4, 14, 70, 68, 8, 74, 97, 17, 6, 76, 15, 10, 66, 88, 25, 24, 12, 5, 18, 16, 23, 73, 65, 36, 96, 84], [40, 120, 59, 34, 22, 113, 29, 49, 90, 14, 80, 119, 31, 51, 9, 88, 56, 112, 57, 18, 76, 52, 6, 83, 118, 123, 110, 117, 68, 124, 109, 61, 116, 50, 58, 62, 43, 60, 98, 115, 54, 122, 55, 114, 102, 72, 53, 46, 127, 95, 125, 45, 65, 101, 0, 108, 126, 121, 63, 47, 28, 105, 111, 48, 99, 107, 82, 91, 81, 44, 16, 39, 106, 42, 79, 41, 103, 36, 89, 30, 2, 21, 66, 23, 38, 37, 32, 27, 11, 96, 75, 13, 17, 97, 35, 86, 100, 67, 71, 33, 92, 94, 74, 24, 85, 69, 3, 25, 73, 84, 5, 20, 78, 15, 104, 64, 87, 26, 10, 93, 77, 7, 70, 12, 1, 8, 4, 19], [122, 54, 127, 36, 97, 120, 125, 124, 86, 53, 62, 119, 126, 57, 118, 104, 121, 49, 58, 56, 117, 113, 48, 112, 50, 51, 52, 123, 59, 42, 26, 60, 114, 115, 55, 43, 116, 61, 63, 39, 47, 105, 106, 41, 111, 46, 94, 81, 110, 93, 28, 109, 38, 29, 107, 40, 19, 45, 100, 108, 24, 44, 30, 15, 83, 73, 103, 69, 27, 33, 67, 21, 82, 99, 102, 75, 87, 101, 76, 4, 25, 6, 8, 23, 88, 65, 34, 37, 91, 85, 98, 32, 78, 16, 64, 31, 35, 90, 12, 92, 20, 13, 0, 17, 95, 96, 10, 89, 66, 71, 84, 14, 22, 80, 79, 77, 18, 7, 2, 11, 72, 74, 1, 5, 70, 9, 68, 3], [41, 53, 34, 25, 63, 30, 52, 79, 86, 80, 117, 126, 20, 75, 32, 8, 48, 10, 77, 51, 82, 108, 116, 47, 45, 59, 123, 109, 118, 6, 4, 2, 56, 50, 64, 120, 119, 62, 1, 105, 113, 9, 40, 57, 71, 55, 111, 106, 61, 83, 28, 87, 18, 54, 14, 29, 114, 31, 67, 69, 70, 110, 127, 103, 23, 92, 44, 46, 58, 35, 36, 124, 74, 21, 0, 94, 122, 102, 37, 66, 115, 90, 121, 3, 33, 81, 107, 88, 24, 93, 39, 49, 112, 101, 97, 100, 125, 78, 60, 104, 91, 12, 38, 13, 5, 85, 42, 22, 43, 17, 73, 27, 26, 76, 19, 98, 15, 11, 84, 95, 96, 72, 99, 65, 7, 68, 16, 89], [41, 112, 48, 94, 86, 113, 35, 27, 82, 110, 80, 89, 126, 84, 78, 49, 105, 61, 52, 125, 58, 59, 76, 9, 114, 115, 111, 123, 54, 50, 34, 57, 127, 117, 51, 91, 109, 60, 108, 62, 98, 10, 45, 7, 116, 47, 118, 56, 55, 5, 63, 119, 120, 53, 96, 46, 121, 124, 107, 1, 81, 37, 43, 40, 36, 44, 88, 122, 102, 101, 16, 39, 106, 79, 12, 100, 30, 67, 97, 104, 21, 83, 42, 38, 31, 103, 32, 33, 13, 93, 29, 95, 8, 87, 99, 24, 26, 28, 17, 69, 23, 75, 15, 11, 14, 64, 90, 92, 74, 19, 20, 6, 18, 85, 71, 72, 25, 77, 68, 66, 22, 70, 73, 4, 2, 3, 65, 0]], "model.layers.18.self_attn.qk_proj": [[113, 112, 63, 124, 55, 118, 41, 54, 122, 52, 120, 59, 60, 105, 107, 127, 53, 43, 123, 36, 86, 48, 22, 102, 110, 117, 126, 25, 62, 49, 30, 84, 100, 89, 29, 51, 119, 82, 40, 18, 20, 16, 80, 96, 38, 57, 116, 15, 61, 115, 56, 46, 79, 109, 77, 34, 125, 94, 47, 23, 58, 50, 98, 27, 26, 114, 24, 45, 104, 39, 103, 121, 111, 85, 32, 21, 92, 88, 87, 93, 13, 42, 91, 8, 10, 75, 11, 90, 81, 72, 76, 74, 19, 31, 44, 97, 9, 17, 33, 95, 37, 14, 35, 108, 12, 83, 106, 28, 78, 73, 7, 71, 6, 101, 0, 99, 64, 69, 5, 2, 70, 65, 67, 68, 1, 3, 4, 66], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 86, 102, 36, 22, 117, 110, 48, 49, 25, 126, 100, 89, 30, 29, 84, 62, 18, 116, 82, 16, 15, 40, 119, 94, 46, 20, 57, 96, 56, 51, 79, 47, 61, 38, 80, 77, 104, 58, 125, 103, 50, 39, 34, 26, 27, 23, 24, 115, 87, 109, 98, 114, 85, 21, 88, 92, 32, 13, 42, 8, 10, 93, 45, 121, 90, 111, 81, 97, 74, 75, 11, 91, 35, 28, 83, 44, 9, 12, 76, 19, 95, 31, 106, 108, 72, 17, 6, 69, 37, 78, 14, 33, 99, 0, 101, 71, 64, 7, 67, 73, 65, 66, 68, 1, 4, 70, 2, 5, 3], [113, 112, 63, 124, 55, 118, 54, 41, 122, 52, 120, 59, 105, 60, 107, 127, 43, 53, 123, 22, 86, 36, 48, 102, 126, 110, 25, 117, 49, 30, 29, 100, 89, 84, 51, 94, 62, 56, 82, 40, 104, 116, 47, 18, 20, 119, 79, 109, 38, 96, 58, 39, 57, 16, 27, 125, 15, 80, 26, 61, 103, 114, 23, 98, 115, 46, 50, 34, 45, 24, 85, 87, 77, 13, 88, 121, 90, 91, 8, 32, 21, 42, 92, 111, 75, 93, 11, 10, 19, 6, 33, 81, 106, 44, 64, 28, 31, 97, 108, 83, 35, 12, 95, 76, 72, 9, 14, 74, 0, 37, 2, 101, 7, 69, 65, 17, 78, 67, 73, 3, 1, 66, 5, 4, 68, 71, 70, 99], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 36, 117, 48, 22, 102, 126, 110, 49, 86, 25, 62, 51, 116, 89, 47, 82, 58, 100, 56, 30, 125, 119, 96, 46, 29, 20, 61, 18, 15, 94, 79, 57, 84, 39, 16, 115, 40, 103, 80, 38, 114, 50, 104, 109, 34, 27, 26, 24, 13, 23, 98, 75, 111, 77, 45, 121, 87, 85, 21, 93, 32, 97, 106, 8, 91, 90, 42, 92, 10, 88, 44, 76, 74, 81, 35, 11, 0, 6, 108, 83, 37, 72, 95, 17, 64, 31, 28, 3, 12, 14, 66, 2, 78, 1, 73, 19, 33, 9, 7, 70, 71, 4, 69, 101, 5, 68, 65, 99, 67], [113, 112, 63, 124, 55, 54, 118, 41, 52, 122, 120, 60, 59, 105, 107, 127, 53, 43, 123, 48, 22, 110, 117, 36, 86, 126, 25, 62, 49, 116, 102, 125, 82, 51, 89, 30, 20, 56, 47, 57, 84, 16, 79, 100, 18, 119, 109, 29, 58, 61, 15, 40, 94, 115, 38, 96, 104, 98, 111, 46, 80, 26, 103, 50, 23, 114, 34, 13, 39, 77, 45, 121, 21, 8, 27, 24, 88, 85, 106, 10, 93, 75, 90, 32, 87, 44, 42, 81, 92, 12, 97, 91, 11, 108, 19, 28, 74, 76, 78, 72, 14, 33, 31, 17, 83, 95, 73, 35, 37, 6, 101, 70, 9, 0, 7, 66, 64, 3, 69, 65, 2, 4, 71, 68, 5, 1, 99, 67], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 53, 123, 22, 48, 86, 110, 117, 36, 126, 102, 25, 51, 49, 15, 89, 30, 82, 18, 62, 116, 79, 20, 100, 57, 96, 40, 80, 125, 16, 84, 109, 119, 47, 29, 58, 50, 39, 115, 56, 34, 94, 98, 103, 61, 104, 46, 23, 114, 77, 38, 13, 24, 21, 26, 27, 111, 85, 88, 8, 32, 87, 42, 93, 75, 10, 11, 97, 90, 45, 92, 81, 78, 12, 83, 74, 31, 17, 121, 28, 76, 106, 91, 44, 19, 72, 95, 14, 9, 35, 33, 108, 71, 37, 73, 101, 99, 70, 64, 0, 6, 69, 7, 2, 1, 66, 65, 5, 67, 4, 68, 3], [113, 63, 112, 124, 55, 118, 41, 54, 122, 52, 120, 60, 59, 105, 107, 43, 127, 53, 123, 22, 86, 48, 36, 126, 49, 25, 110, 117, 30, 102, 82, 18, 62, 89, 96, 84, 15, 29, 51, 47, 16, 100, 80, 20, 79, 40, 58, 56, 57, 119, 94, 77, 104, 23, 98, 34, 38, 115, 27, 61, 116, 125, 26, 32, 109, 24, 88, 50, 46, 114, 103, 13, 85, 91, 39, 21, 75, 93, 111, 87, 92, 121, 8, 10, 74, 97, 11, 90, 78, 31, 12, 81, 35, 33, 95, 76, 42, 44, 19, 17, 14, 101, 45, 106, 83, 9, 108, 73, 72, 28, 99, 70, 71, 37, 64, 7, 5, 65, 69, 2, 66, 0, 1, 68, 67, 6, 4, 3], [113, 63, 112, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 22, 53, 123, 36, 86, 102, 126, 48, 117, 25, 30, 110, 49, 119, 62, 29, 84, 89, 82, 16, 20, 56, 18, 96, 100, 40, 47, 80, 51, 94, 57, 79, 15, 98, 116, 38, 125, 46, 23, 26, 61, 27, 104, 50, 115, 24, 58, 39, 85, 88, 34, 91, 13, 32, 114, 109, 77, 21, 87, 103, 121, 93, 90, 111, 92, 33, 75, 19, 31, 17, 97, 10, 42, 95, 11, 81, 8, 45, 44, 14, 108, 78, 9, 83, 76, 106, 12, 70, 35, 74, 72, 28, 99, 37, 73, 64, 101, 5, 71, 7, 1, 0, 68, 2, 69, 3, 4, 66, 65, 67, 6], [113, 63, 112, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 53, 43, 22, 123, 36, 102, 110, 86, 126, 48, 117, 25, 89, 49, 30, 84, 57, 119, 96, 29, 20, 18, 79, 82, 40, 38, 62, 94, 116, 16, 100, 56, 47, 46, 125, 80, 15, 51, 115, 27, 58, 61, 104, 24, 98, 39, 34, 88, 26, 32, 87, 13, 114, 21, 23, 103, 77, 85, 93, 50, 92, 90, 97, 11, 109, 111, 75, 28, 106, 108, 81, 91, 72, 10, 19, 121, 95, 17, 31, 12, 42, 35, 74, 33, 76, 45, 8, 44, 83, 78, 70, 14, 101, 37, 99, 73, 9, 7, 5, 71, 64, 68, 0, 69, 65, 3, 4, 67, 1, 2, 66, 6], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 60, 59, 105, 107, 127, 123, 53, 43, 48, 102, 117, 36, 22, 86, 126, 25, 110, 62, 49, 57, 100, 116, 125, 18, 51, 82, 30, 47, 89, 40, 119, 20, 56, 29, 58, 84, 79, 61, 80, 38, 16, 96, 15, 46, 27, 104, 26, 94, 34, 115, 39, 98, 50, 24, 85, 103, 77, 21, 32, 88, 72, 13, 109, 75, 23, 10, 93, 45, 87, 106, 111, 121, 92, 114, 97, 31, 74, 90, 76, 81, 108, 35, 95, 19, 91, 44, 28, 11, 8, 42, 83, 78, 12, 14, 17, 9, 70, 71, 37, 73, 7, 101, 64, 33, 5, 99, 67, 68, 3, 0, 65, 4, 69, 6, 2, 1, 66], [113, 112, 63, 124, 55, 54, 41, 118, 120, 122, 52, 60, 59, 105, 107, 127, 53, 43, 123, 117, 48, 22, 126, 36, 102, 86, 110, 116, 62, 25, 51, 18, 49, 56, 82, 89, 20, 100, 61, 79, 96, 47, 57, 30, 80, 125, 15, 84, 40, 119, 16, 46, 58, 38, 29, 104, 77, 50, 98, 115, 39, 34, 72, 94, 13, 23, 109, 24, 114, 103, 27, 26, 45, 75, 85, 87, 32, 111, 93, 121, 21, 10, 92, 11, 88, 90, 8, 28, 81, 106, 76, 12, 74, 17, 97, 14, 83, 78, 6, 35, 91, 42, 73, 44, 64, 71, 37, 31, 2, 108, 19, 9, 69, 0, 95, 7, 33, 101, 68, 70, 1, 65, 5, 4, 67, 66, 3, 99], [113, 63, 112, 124, 55, 41, 118, 54, 52, 120, 122, 60, 59, 105, 107, 127, 43, 53, 123, 22, 86, 48, 102, 36, 117, 126, 25, 89, 84, 18, 110, 82, 30, 100, 49, 96, 80, 15, 20, 79, 29, 62, 51, 40, 56, 16, 116, 98, 47, 46, 94, 57, 115, 119, 77, 85, 23, 38, 58, 125, 104, 61, 34, 13, 24, 26, 72, 27, 75, 87, 103, 39, 109, 21, 32, 50, 93, 11, 88, 111, 74, 10, 90, 114, 97, 92, 19, 81, 45, 121, 31, 91, 17, 78, 14, 95, 12, 106, 76, 6, 44, 33, 108, 8, 9, 35, 83, 42, 73, 28, 99, 101, 71, 7, 37, 64, 68, 5, 2, 70, 1, 66, 3, 69, 0, 4, 67, 65], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 123, 53, 36, 48, 102, 86, 22, 117, 25, 110, 126, 30, 100, 62, 29, 96, 49, 89, 38, 18, 82, 40, 51, 20, 94, 84, 47, 80, 79, 58, 61, 16, 15, 56, 125, 116, 115, 57, 119, 98, 46, 23, 24, 39, 27, 88, 26, 104, 21, 85, 34, 32, 103, 50, 87, 90, 97, 109, 121, 13, 93, 77, 111, 91, 31, 72, 92, 75, 19, 45, 114, 10, 95, 11, 106, 35, 17, 83, 44, 28, 42, 81, 33, 74, 78, 14, 76, 12, 37, 73, 108, 6, 101, 99, 8, 9, 7, 71, 69, 64, 66, 4, 1, 65, 5, 0, 67, 2, 68, 70, 3], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 59, 60, 107, 105, 127, 43, 53, 123, 22, 86, 48, 62, 117, 36, 102, 110, 126, 49, 25, 89, 100, 30, 82, 61, 51, 15, 18, 125, 96, 47, 80, 29, 20, 119, 58, 40, 115, 98, 50, 94, 79, 57, 38, 84, 116, 46, 56, 16, 39, 24, 34, 104, 121, 27, 21, 23, 85, 72, 13, 103, 77, 114, 88, 26, 75, 10, 32, 109, 111, 87, 93, 74, 97, 92, 11, 12, 35, 90, 106, 45, 33, 83, 91, 28, 31, 81, 19, 8, 95, 6, 42, 17, 0, 73, 76, 14, 7, 108, 44, 78, 65, 71, 101, 69, 64, 9, 67, 99, 37, 5, 4, 3, 66, 1, 68, 2, 70], [113, 63, 112, 124, 55, 41, 54, 118, 52, 120, 122, 59, 60, 105, 107, 43, 127, 53, 123, 22, 48, 117, 102, 86, 36, 110, 126, 25, 100, 62, 89, 51, 82, 15, 30, 40, 80, 20, 96, 18, 61, 79, 49, 115, 84, 125, 29, 47, 56, 57, 94, 46, 23, 16, 116, 98, 50, 34, 58, 38, 85, 72, 39, 13, 27, 119, 77, 26, 21, 104, 103, 45, 88, 111, 93, 32, 24, 10, 11, 75, 109, 74, 121, 87, 8, 114, 83, 12, 91, 81, 97, 90, 28, 17, 31, 106, 76, 44, 92, 33, 78, 95, 19, 35, 42, 6, 73, 14, 7, 101, 64, 69, 9, 108, 37, 4, 71, 0, 66, 3, 1, 68, 67, 65, 70, 2, 99, 5], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 22, 117, 48, 86, 36, 102, 110, 62, 126, 100, 25, 89, 82, 116, 96, 30, 15, 49, 20, 56, 51, 46, 84, 18, 40, 79, 80, 16, 47, 61, 57, 125, 94, 29, 115, 98, 77, 119, 23, 50, 58, 38, 39, 85, 34, 24, 13, 111, 72, 103, 121, 104, 114, 75, 27, 10, 26, 11, 88, 109, 32, 87, 74, 42, 45, 97, 106, 8, 21, 81, 93, 92, 12, 76, 90, 19, 95, 83, 91, 44, 108, 17, 31, 14, 9, 78, 6, 35, 33, 28, 101, 73, 7, 37, 4, 70, 64, 0, 5, 2, 71, 1, 65, 99, 67, 69, 68, 66, 3], [113, 63, 112, 124, 55, 118, 41, 52, 54, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 48, 86, 36, 22, 126, 102, 117, 25, 30, 82, 110, 62, 49, 89, 18, 96, 80, 29, 20, 40, 47, 94, 84, 100, 16, 15, 104, 46, 57, 125, 61, 38, 79, 51, 98, 115, 39, 119, 56, 116, 23, 26, 58, 24, 27, 103, 34, 50, 21, 111, 85, 13, 109, 32, 88, 77, 97, 114, 87, 121, 91, 93, 75, 92, 42, 90, 74, 8, 19, 11, 31, 106, 10, 95, 76, 28, 108, 44, 17, 33, 72, 81, 35, 12, 45, 83, 78, 37, 14, 70, 101, 73, 69, 9, 64, 71, 65, 2, 99, 7, 6, 0, 4, 68, 1, 66, 67, 3, 5], [113, 63, 112, 124, 55, 54, 118, 41, 52, 120, 122, 60, 105, 59, 107, 43, 123, 127, 53, 48, 36, 86, 102, 22, 117, 126, 25, 89, 49, 110, 30, 96, 29, 94, 62, 40, 18, 82, 20, 84, 116, 56, 100, 47, 80, 38, 15, 79, 104, 16, 119, 46, 58, 57, 23, 115, 27, 77, 125, 98, 39, 34, 24, 61, 26, 51, 85, 21, 13, 88, 32, 111, 103, 50, 114, 11, 87, 74, 90, 92, 121, 109, 8, 97, 93, 91, 75, 10, 95, 19, 81, 106, 72, 12, 70, 31, 14, 42, 28, 35, 33, 108, 83, 76, 78, 44, 17, 71, 0, 45, 9, 1, 69, 3, 73, 64, 37, 68, 7, 66, 101, 65, 99, 6, 4, 2, 5, 67], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 53, 127, 43, 123, 36, 48, 117, 22, 102, 86, 126, 62, 116, 56, 110, 25, 57, 47, 100, 125, 30, 49, 46, 115, 89, 20, 18, 96, 40, 38, 82, 94, 61, 51, 84, 80, 29, 119, 34, 15, 104, 79, 16, 114, 58, 121, 39, 98, 32, 23, 27, 77, 50, 103, 111, 8, 88, 21, 24, 109, 74, 87, 26, 106, 85, 13, 45, 91, 11, 92, 90, 97, 95, 93, 10, 75, 35, 31, 42, 17, 76, 37, 19, 81, 72, 108, 28, 44, 83, 70, 12, 33, 14, 9, 99, 71, 0, 64, 101, 73, 69, 1, 78, 3, 4, 7, 65, 2, 66, 67, 68, 5, 6], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 53, 43, 48, 123, 117, 102, 36, 22, 86, 126, 110, 56, 62, 100, 25, 30, 49, 57, 116, 47, 15, 96, 119, 51, 125, 89, 61, 18, 94, 29, 84, 46, 115, 39, 20, 40, 82, 79, 16, 38, 80, 23, 98, 34, 27, 13, 103, 21, 58, 104, 26, 50, 8, 24, 109, 121, 88, 87, 114, 32, 77, 45, 111, 97, 85, 93, 92, 74, 11, 10, 95, 90, 91, 31, 108, 28, 76, 75, 106, 33, 83, 42, 12, 35, 19, 17, 81, 78, 72, 44, 37, 14, 70, 9, 64, 73, 65, 71, 5, 101, 0, 4, 7, 99, 2, 69, 1, 67, 68, 3, 66, 6], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 117, 36, 48, 22, 102, 86, 126, 25, 62, 89, 96, 110, 51, 56, 57, 30, 119, 100, 116, 47, 49, 84, 94, 38, 20, 79, 61, 15, 29, 98, 18, 115, 80, 40, 82, 125, 46, 23, 34, 58, 16, 121, 26, 104, 111, 50, 77, 27, 24, 39, 88, 103, 13, 85, 32, 8, 97, 21, 114, 109, 75, 87, 93, 92, 91, 90, 19, 10, 106, 11, 95, 78, 81, 76, 31, 74, 17, 108, 12, 35, 83, 42, 28, 33, 72, 44, 45, 14, 37, 70, 71, 73, 9, 101, 99, 64, 65, 7, 0, 1, 3, 67, 4, 6, 5, 68, 66, 69, 2], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 117, 48, 36, 126, 86, 62, 22, 25, 102, 116, 110, 56, 89, 30, 61, 51, 57, 100, 125, 47, 49, 119, 94, 38, 84, 96, 20, 82, 40, 80, 79, 115, 15, 29, 18, 98, 46, 58, 16, 104, 34, 50, 103, 121, 27, 23, 26, 77, 111, 88, 32, 24, 85, 114, 39, 21, 13, 90, 91, 8, 93, 87, 11, 92, 109, 106, 75, 97, 74, 19, 95, 28, 10, 35, 31, 17, 83, 42, 108, 44, 12, 72, 76, 45, 0, 81, 14, 64, 33, 6, 78, 9, 71, 70, 37, 2, 7, 73, 99, 1, 101, 65, 69, 68, 67, 5, 3, 66, 4], [113, 63, 112, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 123, 53, 48, 117, 36, 102, 22, 86, 25, 126, 110, 62, 57, 30, 116, 51, 119, 100, 40, 96, 47, 61, 84, 89, 38, 56, 29, 18, 94, 20, 82, 15, 16, 49, 46, 125, 79, 98, 23, 104, 50, 34, 80, 26, 24, 115, 58, 103, 111, 39, 27, 121, 87, 77, 32, 85, 88, 21, 109, 93, 114, 97, 35, 13, 31, 28, 45, 90, 106, 92, 91, 8, 42, 83, 75, 74, 108, 95, 19, 44, 11, 81, 76, 10, 6, 33, 12, 72, 17, 78, 9, 14, 101, 37, 0, 64, 7, 65, 71, 73, 1, 99, 5, 4, 69, 66, 2, 67, 3, 70, 68], [113, 63, 112, 124, 55, 118, 54, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 48, 22, 36, 117, 86, 102, 126, 116, 25, 62, 110, 96, 51, 30, 100, 57, 18, 119, 16, 89, 61, 49, 47, 15, 94, 84, 82, 29, 20, 79, 40, 38, 125, 34, 98, 39, 46, 23, 104, 26, 50, 56, 80, 103, 77, 115, 58, 24, 27, 121, 13, 32, 21, 85, 109, 8, 74, 88, 92, 114, 90, 75, 111, 93, 97, 87, 72, 10, 91, 95, 81, 11, 19, 45, 106, 76, 83, 28, 42, 12, 6, 17, 31, 35, 14, 78, 108, 44, 7, 73, 37, 9, 33, 71, 99, 101, 0, 4, 5, 68, 67, 69, 1, 3, 64, 66, 65, 70, 2], [113, 112, 63, 124, 55, 118, 54, 41, 52, 122, 120, 59, 60, 105, 107, 127, 43, 123, 53, 22, 117, 86, 48, 36, 102, 25, 62, 126, 89, 116, 30, 110, 20, 16, 51, 84, 18, 115, 82, 61, 100, 96, 40, 119, 79, 57, 47, 15, 49, 38, 80, 29, 94, 23, 98, 46, 109, 58, 125, 34, 56, 50, 104, 26, 88, 103, 27, 114, 77, 13, 111, 85, 32, 75, 21, 121, 39, 24, 10, 87, 90, 42, 72, 11, 93, 31, 8, 92, 97, 74, 28, 91, 76, 45, 81, 19, 83, 95, 6, 106, 35, 17, 78, 12, 108, 33, 44, 14, 73, 101, 9, 64, 7, 99, 71, 4, 65, 0, 37, 69, 66, 2, 5, 3, 1, 68, 67, 70], [113, 63, 112, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 123, 53, 48, 86, 36, 22, 117, 102, 62, 126, 25, 30, 110, 100, 89, 116, 29, 51, 96, 94, 20, 119, 40, 61, 56, 57, 18, 16, 82, 47, 84, 15, 38, 49, 125, 46, 109, 115, 23, 98, 58, 80, 50, 104, 24, 79, 77, 26, 34, 121, 85, 103, 27, 21, 13, 88, 92, 39, 87, 11, 93, 111, 32, 114, 72, 91, 74, 97, 42, 90, 75, 10, 31, 95, 81, 19, 45, 106, 33, 8, 12, 28, 76, 108, 17, 83, 44, 78, 35, 73, 14, 6, 9, 37, 0, 101, 7, 99, 65, 69, 2, 3, 71, 64, 67, 5, 68, 70, 66, 4, 1], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 43, 123, 53, 36, 117, 48, 102, 62, 22, 110, 126, 86, 25, 61, 116, 89, 49, 30, 18, 100, 51, 56, 96, 40, 119, 57, 84, 47, 29, 125, 20, 121, 23, 58, 39, 16, 38, 98, 104, 94, 82, 46, 50, 115, 79, 34, 15, 80, 109, 114, 26, 111, 77, 103, 27, 24, 13, 88, 32, 85, 97, 21, 42, 72, 87, 93, 92, 90, 75, 74, 91, 106, 35, 11, 45, 108, 10, 44, 19, 17, 12, 31, 81, 83, 76, 28, 95, 37, 33, 101, 8, 78, 14, 73, 99, 6, 9, 0, 7, 70, 64, 69, 5, 71, 4, 68, 67, 66, 3, 2, 1, 65], [113, 112, 63, 124, 55, 54, 118, 41, 120, 52, 122, 60, 59, 105, 107, 127, 43, 53, 123, 22, 48, 36, 117, 126, 102, 86, 110, 62, 25, 61, 20, 89, 18, 47, 100, 82, 30, 51, 125, 96, 49, 56, 116, 79, 40, 80, 15, 84, 16, 119, 29, 46, 57, 23, 58, 38, 94, 50, 104, 121, 88, 77, 34, 115, 27, 26, 103, 72, 13, 39, 85, 111, 24, 114, 32, 21, 93, 92, 98, 11, 109, 74, 75, 91, 90, 45, 97, 87, 10, 19, 12, 28, 106, 17, 76, 81, 42, 8, 78, 31, 35, 95, 70, 83, 7, 9, 44, 0, 73, 14, 108, 69, 99, 3, 71, 65, 37, 64, 101, 4, 33, 5, 2, 1, 68, 6, 67, 66], [113, 63, 112, 124, 55, 118, 41, 54, 120, 52, 122, 60, 59, 105, 107, 127, 123, 53, 43, 48, 117, 36, 62, 22, 86, 110, 51, 102, 126, 119, 49, 30, 25, 125, 100, 96, 89, 47, 61, 40, 18, 115, 57, 56, 20, 15, 29, 84, 46, 116, 16, 79, 82, 94, 39, 34, 23, 104, 58, 38, 50, 103, 77, 121, 80, 26, 98, 72, 109, 45, 85, 32, 27, 92, 13, 93, 111, 21, 114, 106, 74, 10, 42, 90, 11, 24, 75, 88, 97, 87, 81, 108, 95, 35, 76, 70, 91, 28, 78, 31, 44, 12, 8, 19, 17, 33, 9, 5, 0, 64, 83, 14, 37, 67, 7, 65, 1, 73, 71, 101, 66, 69, 3, 4, 2, 99, 68, 6], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 53, 43, 123, 48, 117, 22, 36, 86, 110, 102, 126, 62, 51, 119, 25, 49, 89, 18, 30, 100, 82, 79, 56, 46, 125, 20, 84, 116, 115, 15, 40, 29, 61, 96, 94, 16, 47, 80, 23, 109, 77, 98, 57, 39, 38, 34, 103, 24, 58, 13, 114, 50, 72, 121, 32, 42, 21, 88, 85, 104, 45, 111, 92, 27, 93, 87, 11, 76, 10, 97, 75, 74, 26, 106, 17, 90, 108, 91, 81, 14, 19, 8, 44, 95, 70, 83, 31, 28, 35, 12, 78, 9, 33, 7, 73, 71, 5, 101, 64, 67, 37, 69, 4, 99, 65, 3, 2, 0, 1, 68, 6, 66], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 48, 117, 102, 86, 36, 62, 22, 51, 110, 126, 25, 116, 30, 89, 100, 49, 125, 82, 96, 29, 61, 47, 16, 40, 20, 115, 119, 94, 39, 79, 18, 15, 84, 23, 58, 121, 56, 57, 109, 77, 50, 26, 34, 38, 104, 80, 98, 46, 114, 103, 27, 72, 85, 74, 88, 10, 24, 21, 87, 42, 32, 92, 111, 45, 97, 93, 75, 13, 76, 91, 8, 11, 90, 70, 14, 81, 31, 7, 19, 83, 12, 33, 5, 78, 95, 35, 17, 28, 106, 73, 108, 71, 44, 9, 0, 64, 69, 3, 4, 1, 65, 68, 67, 101, 66, 2, 37, 6, 99], [113, 112, 63, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 53, 123, 86, 36, 102, 48, 22, 117, 25, 126, 62, 110, 100, 82, 29, 30, 51, 15, 49, 84, 89, 18, 16, 96, 61, 79, 116, 20, 47, 40, 119, 80, 94, 125, 38, 34, 56, 77, 57, 26, 98, 115, 23, 39, 27, 109, 46, 50, 88, 13, 24, 114, 58, 85, 21, 103, 92, 121, 32, 10, 72, 104, 74, 75, 42, 111, 45, 91, 97, 76, 87, 93, 11, 8, 95, 90, 106, 81, 83, 14, 19, 12, 78, 31, 33, 44, 28, 17, 73, 35, 108, 70, 9, 7, 71, 101, 37, 69, 99, 5, 67, 64, 6, 68, 4, 65, 66, 0, 1, 3, 2]], "model.layers.19.self_attn.q_proj": [[124, 110, 63, 115, 58, 119, 101, 120, 57, 46, 112, 52, 60, 123, 127, 109, 53, 59, 56, 116, 54, 88, 126, 50, 108, 61, 122, 55, 94, 111, 121, 114, 104, 51, 37, 47, 62, 97, 48, 40, 44, 49, 113, 117, 125, 45, 27, 19, 24, 118, 90, 34, 41, 105, 42, 81, 98, 106, 43, 107, 96, 31, 22, 39, 89, 102, 26, 16, 92, 9, 100, 38, 103, 36, 35, 78, 91, 99, 93, 25, 18, 30, 80, 29, 32, 73, 17, 20, 95, 23, 67, 85, 15, 28, 21, 69, 77, 84, 33, 12, 6, 14, 87, 83, 86, 10, 70, 79, 75, 13, 4, 1, 82, 68, 72, 0, 74, 5, 11, 64, 3, 65, 66, 8, 7, 76, 71, 2], [63, 57, 46, 101, 123, 112, 59, 110, 124, 120, 47, 97, 53, 121, 119, 81, 109, 114, 90, 24, 94, 111, 34, 105, 60, 19, 88, 87, 122, 126, 31, 85, 55, 104, 27, 91, 117, 58, 45, 52, 61, 25, 56, 106, 20, 39, 22, 116, 32, 28, 78, 42, 16, 38, 10, 107, 62, 44, 41, 54, 83, 127, 113, 50, 108, 14, 102, 100, 17, 80, 37, 49, 35, 51, 118, 40, 26, 12, 103, 36, 95, 96, 125, 43, 115, 89, 21, 48, 84, 98, 99, 93, 74, 30, 23, 29, 86, 92, 15, 11, 76, 77, 75, 6, 67, 7, 72, 18, 70, 13, 9, 73, 82, 68, 33, 79, 8, 69, 66, 1, 71, 4, 0, 2, 3, 64, 65, 5], [63, 124, 110, 57, 115, 101, 58, 46, 50, 122, 97, 88, 52, 60, 27, 120, 49, 55, 116, 81, 56, 59, 112, 123, 94, 119, 42, 121, 109, 111, 54, 126, 24, 125, 105, 127, 62, 117, 114, 44, 51, 93, 113, 37, 53, 84, 61, 90, 47, 31, 104, 20, 85, 108, 45, 6, 103, 118, 75, 73, 9, 41, 40, 67, 106, 48, 65, 43, 36, 38, 16, 4, 39, 72, 22, 14, 107, 0, 17, 100, 68, 12, 83, 80, 69, 30, 102, 87, 71, 91, 95, 7, 26, 15, 99, 19, 96, 11, 2, 35, 10, 74, 25, 3, 29, 34, 66, 77, 64, 32, 28, 92, 98, 13, 23, 21, 82, 70, 1, 8, 89, 18, 5, 79, 86, 76, 78, 33], [124, 63, 110, 101, 54, 46, 122, 119, 58, 59, 28, 112, 61, 55, 62, 120, 94, 56, 114, 111, 57, 90, 50, 53, 113, 116, 126, 123, 127, 121, 60, 125, 118, 97, 82, 22, 117, 48, 37, 47, 52, 115, 18, 49, 108, 109, 51, 41, 45, 44, 43, 39, 107, 32, 19, 106, 86, 102, 100, 42, 27, 92, 36, 30, 104, 40, 105, 25, 38, 103, 99, 34, 14, 35, 98, 96, 78, 20, 15, 33, 83, 88, 31, 77, 12, 26, 10, 29, 93, 24, 95, 89, 87, 23, 84, 74, 91, 13, 85, 76, 79, 21, 81, 17, 8, 75, 7, 72, 11, 80, 71, 9, 16, 0, 70, 68, 73, 67, 3, 2, 6, 66, 4, 5, 65, 69, 64, 1], [59, 118, 39, 46, 105, 127, 63, 34, 113, 112, 60, 119, 53, 54, 111, 50, 48, 126, 61, 122, 30, 125, 109, 49, 123, 89, 58, 97, 57, 121, 115, 56, 110, 43, 52, 25, 47, 104, 116, 55, 51, 107, 31, 108, 45, 62, 114, 36, 44, 41, 42, 120, 117, 102, 23, 33, 106, 28, 40, 82, 37, 38, 22, 94, 90, 84, 27, 98, 91, 18, 80, 101, 78, 87, 86, 99, 100, 26, 35, 32, 93, 103, 95, 96, 29, 14, 92, 21, 24, 19, 75, 20, 8, 124, 0, 76, 16, 12, 7, 88, 67, 66, 9, 2, 68, 70, 71, 73, 79, 6, 85, 65, 81, 3, 69, 4, 64, 83, 11, 1, 72, 17, 13, 5, 15, 10, 77, 74], [39, 59, 118, 34, 124, 90, 46, 84, 43, 79, 75, 54, 60, 122, 98, 30, 112, 49, 109, 127, 21, 19, 88, 111, 25, 94, 115, 76, 119, 23, 63, 89, 105, 22, 113, 9, 38, 87, 56, 8, 41, 106, 57, 31, 44, 81, 110, 33, 50, 70, 92, 48, 82, 116, 125, 100, 114, 53, 27, 62, 120, 99, 107, 42, 40, 73, 108, 58, 45, 123, 93, 104, 55, 95, 126, 121, 35, 91, 20, 18, 37, 52, 51, 61, 14, 72, 97, 28, 47, 6, 16, 86, 13, 24, 36, 96, 3, 102, 29, 11, 12, 101, 32, 26, 117, 71, 80, 66, 85, 78, 15, 7, 83, 2, 67, 68, 17, 10, 5, 69, 77, 4, 65, 103, 64, 0, 1, 74], [118, 39, 59, 34, 124, 127, 46, 105, 54, 63, 43, 55, 89, 112, 90, 84, 50, 120, 47, 57, 49, 113, 30, 115, 122, 53, 51, 125, 107, 60, 119, 80, 35, 75, 126, 45, 21, 19, 116, 123, 110, 25, 8, 111, 109, 22, 82, 88, 48, 52, 104, 27, 94, 33, 56, 121, 44, 61, 108, 62, 98, 92, 117, 58, 106, 37, 78, 79, 31, 38, 23, 40, 101, 103, 86, 99, 91, 42, 95, 24, 114, 97, 93, 12, 87, 102, 81, 32, 26, 100, 41, 20, 29, 18, 76, 14, 7, 36, 73, 28, 71, 96, 16, 13, 3, 67, 11, 70, 2, 69, 66, 68, 6, 10, 9, 0, 72, 4, 64, 77, 65, 85, 1, 83, 15, 5, 17, 74], [39, 59, 118, 34, 81, 90, 21, 13, 19, 79, 10, 46, 69, 54, 100, 65, 8, 68, 74, 115, 77, 33, 3, 11, 75, 95, 49, 55, 38, 120, 122, 71, 85, 30, 94, 0, 32, 88, 87, 18, 17, 107, 56, 112, 23, 44, 57, 63, 25, 126, 86, 67, 24, 83, 58, 119, 113, 5, 76, 72, 15, 41, 127, 89, 7, 111, 4, 12, 16, 26, 20, 43, 29, 80, 91, 14, 64, 110, 99, 22, 78, 84, 60, 123, 66, 92, 70, 37, 6, 105, 31, 1, 27, 36, 82, 96, 73, 98, 50, 35, 124, 93, 9, 47, 28, 45, 2, 53, 101, 125, 52, 114, 102, 121, 103, 109, 48, 62, 61, 104, 116, 97, 51, 40, 42, 117, 108, 106], [105, 112, 34, 84, 18, 12, 15, 27, 22, 5, 71, 48, 9, 41, 96, 2, 64, 89, 53, 82, 91, 58, 13, 4, 122, 30, 83, 1, 20, 25, 79, 63, 86, 118, 76, 87, 7, 74, 31, 78, 80, 50, 19, 85, 69, 119, 120, 70, 75, 94, 127, 16, 73, 11, 14, 81, 10, 92, 77, 46, 98, 17, 68, 6, 103, 60, 90, 28, 21, 24, 37, 72, 93, 23, 65, 107, 113, 61, 3, 26, 67, 59, 8, 56, 52, 114, 32, 115, 66, 57, 88, 110, 29, 95, 0, 123, 33, 62, 49, 55, 35, 125, 44, 36, 126, 101, 42, 99, 116, 51, 97, 47, 54, 104, 100, 102, 124, 109, 111, 43, 39, 117, 45, 38, 106, 40, 108, 121], [112, 105, 53, 25, 127, 58, 118, 89, 56, 34, 114, 54, 101, 83, 125, 96, 32, 49, 48, 120, 124, 116, 55, 52, 57, 93, 59, 33, 41, 113, 119, 62, 117, 115, 126, 61, 60, 107, 50, 106, 121, 122, 104, 45, 111, 110, 98, 51, 63, 44, 109, 47, 42, 46, 94, 123, 43, 108, 102, 37, 103, 100, 27, 38, 36, 39, 23, 99, 35, 40, 81, 87, 18, 95, 30, 22, 97, 77, 29, 31, 13, 6, 28, 8, 92, 10, 90, 68, 24, 91, 84, 16, 65, 26, 88, 86, 70, 15, 4, 19, 14, 85, 17, 80, 82, 11, 67, 21, 1, 74, 72, 75, 64, 3, 7, 78, 12, 0, 20, 66, 73, 71, 79, 5, 9, 2, 76, 69], [105, 112, 34, 15, 22, 12, 84, 18, 9, 27, 41, 89, 48, 71, 53, 5, 96, 4, 91, 58, 25, 13, 85, 2, 78, 81, 75, 64, 122, 19, 50, 67, 30, 1, 80, 31, 86, 20, 77, 82, 63, 79, 119, 72, 59, 83, 95, 88, 69, 73, 118, 21, 37, 76, 74, 7, 46, 11, 57, 3, 60, 26, 14, 6, 65, 23, 92, 28, 17, 127, 70, 62, 93, 87, 8, 24, 115, 98, 56, 61, 52, 68, 107, 125, 90, 33, 16, 120, 114, 110, 36, 10, 66, 47, 94, 35, 29, 126, 55, 116, 51, 113, 0, 97, 32, 42, 103, 123, 49, 100, 124, 102, 43, 54, 109, 117, 39, 101, 38, 99, 44, 111, 45, 104, 121, 106, 108, 40], [105, 53, 112, 34, 41, 48, 88, 25, 89, 94, 27, 33, 98, 13, 22, 84, 58, 56, 114, 118, 101, 96, 125, 120, 59, 19, 18, 78, 57, 80, 91, 124, 116, 117, 55, 62, 127, 107, 119, 49, 93, 68, 113, 6, 75, 115, 60, 50, 122, 70, 100, 15, 110, 52, 63, 54, 126, 83, 103, 47, 61, 51, 111, 4, 37, 29, 121, 92, 12, 104, 45, 97, 109, 30, 31, 42, 71, 38, 46, 123, 65, 36, 23, 44, 102, 43, 67, 74, 21, 99, 108, 10, 26, 106, 1, 24, 35, 0, 95, 39, 11, 3, 86, 81, 77, 16, 32, 40, 82, 14, 90, 9, 28, 72, 85, 17, 8, 87, 2, 20, 66, 64, 5, 7, 79, 73, 69, 76], [120, 121, 53, 57, 118, 126, 63, 113, 124, 51, 59, 60, 58, 123, 125, 115, 49, 116, 122, 119, 117, 62, 127, 56, 112, 54, 47, 55, 99, 50, 61, 114, 48, 111, 83, 44, 108, 110, 52, 13, 45, 103, 109, 46, 43, 41, 107, 40, 31, 32, 90, 106, 42, 105, 39, 38, 102, 100, 35, 9, 69, 34, 3, 97, 104, 101, 21, 82, 71, 96, 37, 28, 2, 36, 70, 98, 18, 76, 85, 68, 77, 23, 33, 30, 66, 11, 81, 92, 25, 26, 72, 0, 10, 93, 15, 29, 94, 84, 24, 86, 88, 22, 20, 16, 1, 89, 91, 75, 4, 79, 27, 78, 12, 5, 95, 65, 80, 17, 14, 19, 7, 64, 87, 73, 8, 74, 67, 6], [121, 120, 53, 57, 113, 63, 58, 118, 60, 125, 124, 126, 59, 56, 117, 119, 51, 116, 123, 55, 115, 48, 114, 99, 49, 62, 50, 61, 112, 122, 54, 52, 47, 43, 44, 110, 45, 108, 127, 111, 109, 46, 39, 41, 40, 42, 102, 106, 107, 105, 89, 90, 100, 83, 103, 35, 26, 104, 38, 32, 3, 101, 96, 37, 31, 2, 71, 72, 36, 82, 70, 34, 98, 33, 0, 13, 4, 69, 16, 76, 11, 91, 9, 92, 21, 27, 29, 22, 97, 95, 1, 94, 93, 77, 85, 8, 87, 30, 23, 15, 18, 86, 28, 88, 10, 68, 14, 25, 64, 19, 80, 12, 20, 78, 65, 84, 75, 17, 74, 79, 24, 66, 6, 7, 81, 67, 5, 73], [121, 120, 126, 40, 53, 99, 36, 63, 58, 124, 39, 51, 42, 106, 117, 88, 49, 113, 89, 59, 57, 102, 26, 122, 96, 31, 60, 123, 56, 28, 50, 46, 118, 119, 24, 105, 16, 116, 87, 112, 110, 55, 125, 32, 90, 98, 34, 82, 52, 103, 127, 35, 10, 115, 30, 54, 48, 95, 47, 100, 25, 45, 37, 38, 111, 104, 44, 41, 109, 61, 62, 33, 114, 22, 83, 43, 80, 21, 107, 27, 91, 97, 101, 29, 108, 69, 72, 92, 93, 19, 94, 15, 9, 70, 13, 71, 23, 11, 77, 84, 3, 86, 14, 85, 76, 20, 81, 79, 18, 17, 68, 74, 2, 0, 1, 12, 7, 65, 75, 78, 8, 4, 6, 73, 67, 64, 66, 5], [121, 120, 118, 39, 63, 117, 126, 124, 53, 116, 57, 51, 102, 125, 56, 60, 54, 119, 123, 58, 113, 49, 35, 99, 62, 83, 105, 112, 15, 40, 36, 44, 127, 114, 61, 48, 52, 59, 122, 42, 45, 111, 50, 47, 55, 38, 108, 107, 115, 101, 32, 106, 103, 110, 21, 46, 24, 13, 96, 88, 41, 109, 90, 26, 81, 16, 31, 43, 100, 37, 75, 34, 104, 91, 33, 25, 20, 28, 84, 69, 92, 97, 79, 98, 9, 27, 71, 89, 85, 74, 86, 87, 29, 94, 23, 22, 30, 77, 19, 93, 12, 76, 10, 11, 14, 17, 80, 5, 7, 3, 73, 95, 82, 18, 68, 78, 70, 8, 67, 4, 0, 72, 2, 65, 66, 6, 64, 1], [55, 105, 101, 120, 116, 49, 32, 112, 25, 87, 46, 57, 110, 81, 91, 109, 27, 48, 47, 119, 28, 60, 96, 94, 39, 113, 126, 44, 54, 59, 115, 63, 89, 107, 118, 84, 127, 76, 78, 85, 83, 11, 50, 106, 37, 20, 14, 122, 104, 31, 123, 58, 52, 111, 125, 61, 124, 108, 71, 88, 99, 51, 121, 19, 92, 114, 56, 45, 42, 103, 38, 43, 117, 26, 97, 62, 30, 35, 95, 34, 98, 21, 7, 86, 53, 22, 29, 73, 100, 18, 24, 102, 93, 90, 79, 15, 17, 33, 40, 5, 16, 12, 80, 70, 36, 82, 68, 41, 10, 75, 3, 23, 8, 9, 74, 2, 4, 77, 13, 72, 6, 69, 67, 65, 66, 1, 64, 0], [105, 55, 109, 116, 101, 120, 32, 112, 57, 46, 87, 25, 113, 28, 60, 110, 81, 119, 91, 104, 49, 35, 89, 84, 48, 121, 85, 39, 88, 111, 92, 42, 108, 37, 78, 27, 58, 114, 11, 107, 97, 117, 53, 31, 96, 95, 127, 125, 118, 56, 61, 94, 124, 126, 63, 62, 38, 44, 115, 99, 59, 93, 30, 54, 14, 45, 26, 50, 52, 29, 51, 98, 122, 76, 100, 106, 20, 41, 123, 33, 17, 73, 36, 79, 47, 103, 40, 12, 23, 43, 102, 86, 83, 16, 21, 24, 22, 34, 7, 19, 15, 71, 80, 75, 82, 18, 90, 10, 74, 5, 70, 6, 72, 77, 9, 3, 8, 4, 13, 2, 67, 68, 69, 65, 1, 0, 64, 66], [55, 105, 101, 120, 46, 112, 116, 57, 32, 87, 25, 39, 81, 28, 126, 47, 119, 48, 94, 84, 27, 89, 50, 56, 113, 34, 92, 118, 14, 122, 91, 60, 124, 125, 127, 38, 11, 111, 104, 78, 115, 45, 83, 110, 58, 36, 114, 108, 96, 54, 121, 109, 52, 117, 20, 31, 103, 106, 95, 42, 44, 49, 30, 107, 63, 123, 33, 37, 73, 29, 59, 79, 61, 70, 43, 51, 85, 62, 72, 99, 53, 19, 100, 102, 22, 97, 86, 21, 93, 35, 98, 17, 8, 41, 76, 40, 24, 12, 82, 90, 26, 88, 16, 75, 80, 18, 10, 15, 5, 7, 3, 6, 23, 4, 68, 9, 71, 77, 69, 66, 74, 2, 67, 13, 65, 64, 0, 1], [105, 57, 120, 116, 46, 112, 32, 55, 101, 119, 39, 113, 127, 87, 92, 94, 28, 53, 62, 104, 50, 115, 25, 126, 60, 114, 124, 38, 108, 58, 52, 27, 121, 56, 54, 111, 123, 84, 107, 98, 110, 61, 48, 122, 117, 44, 36, 106, 96, 59, 42, 51, 100, 118, 47, 63, 43, 125, 45, 109, 89, 91, 83, 79, 41, 49, 85, 33, 99, 88, 81, 35, 86, 29, 37, 95, 22, 20, 77, 30, 34, 97, 21, 18, 26, 31, 103, 40, 93, 102, 14, 78, 74, 82, 24, 90, 73, 23, 13, 75, 80, 70, 11, 76, 16, 5, 15, 64, 19, 17, 12, 7, 66, 0, 8, 2, 67, 3, 65, 72, 1, 10, 69, 68, 71, 4, 6, 9], [102, 62, 56, 55, 97, 22, 29, 19, 89, 35, 15, 88, 93, 38, 31, 76, 45, 84, 27, 60, 17, 90, 25, 33, 21, 12, 86, 46, 96, 30, 81, 95, 43, 14, 98, 78, 50, 100, 7, 26, 9, 24, 44, 61, 101, 91, 40, 87, 79, 94, 57, 122, 116, 23, 67, 71, 83, 59, 10, 34, 20, 39, 127, 85, 32, 92, 53, 28, 121, 54, 103, 73, 36, 99, 82, 18, 115, 105, 114, 117, 63, 113, 13, 104, 111, 11, 49, 124, 16, 112, 70, 41, 58, 80, 37, 125, 48, 109, 42, 8, 119, 106, 68, 51, 110, 2, 107, 123, 52, 108, 120, 75, 47, 1, 69, 126, 118, 74, 77, 0, 3, 5, 65, 72, 4, 64, 66, 6], [56, 102, 97, 55, 62, 29, 15, 45, 19, 89, 22, 60, 76, 88, 38, 93, 7, 69, 17, 82, 10, 74, 2, 59, 32, 49, 31, 81, 3, 54, 33, 40, 46, 86, 58, 25, 91, 9, 98, 68, 112, 21, 104, 43, 50, 122, 28, 27, 64, 26, 11, 13, 110, 120, 108, 90, 44, 78, 84, 73, 57, 14, 23, 117, 121, 53, 124, 61, 80, 100, 116, 101, 83, 1, 37, 115, 126, 51, 6, 12, 48, 20, 47, 114, 52, 96, 36, 87, 30, 107, 63, 119, 41, 35, 127, 118, 125, 92, 123, 75, 109, 85, 113, 67, 99, 39, 42, 106, 103, 4, 8, 94, 71, 16, 24, 77, 18, 111, 70, 105, 34, 79, 95, 72, 5, 0, 66, 65], [55, 102, 62, 56, 97, 29, 22, 38, 93, 19, 60, 15, 76, 74, 88, 12, 69, 27, 28, 13, 45, 17, 90, 81, 84, 25, 118, 32, 104, 10, 23, 82, 73, 89, 21, 31, 49, 37, 79, 42, 11, 35, 58, 16, 30, 53, 50, 101, 61, 92, 46, 109, 98, 94, 86, 7, 71, 107, 117, 26, 91, 96, 44, 83, 59, 57, 106, 114, 121, 68, 24, 43, 2, 100, 110, 124, 126, 87, 48, 47, 54, 20, 39, 36, 112, 52, 85, 125, 113, 63, 33, 95, 99, 108, 116, 119, 78, 123, 75, 9, 3, 105, 14, 127, 115, 41, 122, 51, 80, 111, 120, 34, 40, 5, 103, 77, 18, 67, 72, 64, 6, 4, 8, 70, 66, 0, 1, 65], [102, 62, 56, 97, 55, 50, 29, 43, 40, 22, 61, 45, 25, 38, 89, 19, 126, 27, 17, 42, 127, 116, 18, 88, 46, 91, 57, 15, 59, 24, 53, 13, 114, 123, 93, 8, 54, 39, 109, 60, 113, 76, 125, 81, 117, 121, 111, 104, 124, 52, 115, 119, 47, 58, 122, 120, 63, 107, 112, 110, 6, 10, 87, 48, 77, 51, 49, 106, 31, 100, 20, 44, 86, 108, 103, 26, 118, 101, 65, 84, 30, 33, 105, 78, 35, 94, 11, 95, 41, 9, 37, 23, 72, 5, 99, 36, 14, 98, 69, 28, 96, 7, 74, 73, 90, 3, 75, 83, 68, 34, 66, 82, 32, 92, 85, 80, 21, 70, 16, 2, 1, 4, 67, 12, 79, 64, 71, 0], [41, 120, 47, 34, 57, 105, 22, 93, 84, 80, 109, 60, 78, 76, 88, 117, 45, 26, 103, 61, 118, 127, 125, 74, 126, 110, 108, 8, 90, 52, 113, 86, 114, 59, 124, 58, 119, 75, 35, 49, 89, 48, 116, 99, 111, 20, 1, 123, 101, 53, 81, 51, 62, 54, 104, 24, 28, 3, 43, 29, 7, 56, 19, 31, 112, 87, 2, 69, 92, 70, 107, 121, 17, 4, 25, 95, 18, 63, 50, 16, 55, 73, 115, 122, 27, 44, 102, 13, 21, 42, 94, 32, 91, 100, 106, 96, 11, 40, 46, 33, 71, 85, 36, 79, 39, 14, 23, 37, 10, 83, 6, 97, 30, 82, 77, 15, 38, 12, 98, 5, 9, 72, 65, 66, 68, 67, 0, 64], [41, 120, 57, 34, 84, 22, 88, 109, 93, 60, 110, 116, 29, 108, 90, 80, 49, 95, 42, 105, 76, 52, 123, 24, 26, 35, 74, 39, 127, 126, 113, 104, 78, 81, 25, 44, 82, 61, 40, 53, 124, 101, 112, 19, 43, 54, 119, 111, 48, 45, 122, 125, 7, 59, 121, 86, 55, 107, 58, 98, 63, 97, 115, 32, 62, 38, 106, 56, 37, 51, 96, 27, 18, 28, 118, 103, 79, 85, 36, 117, 114, 92, 83, 20, 102, 47, 16, 91, 50, 17, 71, 94, 46, 15, 89, 21, 23, 31, 100, 30, 99, 70, 33, 14, 87, 77, 13, 75, 8, 4, 69, 66, 68, 11, 73, 1, 12, 9, 65, 10, 5, 2, 72, 3, 6, 67, 64, 0], [41, 120, 34, 47, 70, 57, 64, 8, 76, 78, 93, 67, 66, 80, 87, 105, 7, 22, 84, 74, 1, 124, 0, 60, 3, 19, 11, 88, 13, 65, 18, 86, 117, 101, 126, 61, 20, 83, 69, 59, 26, 5, 72, 110, 73, 90, 82, 68, 121, 108, 16, 89, 31, 37, 25, 9, 2, 109, 79, 10, 75, 17, 111, 14, 4, 122, 24, 12, 114, 92, 32, 27, 116, 112, 6, 23, 77, 58, 85, 71, 81, 15, 95, 127, 100, 91, 102, 21, 115, 94, 62, 113, 43, 119, 28, 30, 107, 125, 103, 53, 48, 104, 99, 55, 33, 97, 50, 45, 96, 35, 36, 51, 63, 49, 118, 38, 39, 44, 46, 106, 123, 56, 40, 42, 54, 52, 29, 98], [120, 41, 34, 109, 22, 88, 48, 84, 47, 60, 113, 93, 117, 26, 110, 49, 119, 51, 124, 43, 61, 111, 29, 58, 24, 125, 50, 55, 80, 115, 81, 62, 63, 127, 116, 57, 114, 123, 108, 112, 54, 42, 126, 56, 78, 105, 90, 118, 36, 53, 40, 121, 102, 74, 44, 46, 35, 52, 28, 59, 107, 37, 122, 76, 106, 7, 95, 86, 39, 19, 45, 101, 38, 87, 99, 103, 18, 104, 100, 97, 27, 32, 83, 20, 31, 25, 33, 30, 94, 92, 96, 91, 17, 89, 70, 98, 11, 79, 73, 68, 8, 21, 1, 82, 23, 75, 15, 16, 77, 66, 9, 85, 13, 14, 71, 10, 12, 67, 5, 69, 65, 4, 6, 2, 64, 72, 3, 0], [38, 48, 112, 24, 125, 84, 17, 109, 27, 22, 91, 97, 78, 47, 18, 10, 107, 35, 95, 81, 121, 43, 11, 98, 118, 77, 115, 46, 52, 49, 119, 86, 102, 20, 68, 13, 50, 26, 120, 90, 54, 59, 92, 40, 34, 29, 88, 53, 7, 14, 15, 56, 108, 89, 82, 55, 30, 9, 37, 96, 103, 42, 117, 51, 39, 60, 64, 106, 110, 28, 87, 94, 32, 69, 126, 61, 113, 83, 123, 105, 41, 45, 25, 63, 122, 111, 85, 19, 93, 44, 33, 116, 114, 127, 8, 80, 72, 124, 23, 104, 31, 58, 21, 100, 73, 57, 79, 16, 76, 36, 99, 6, 101, 2, 12, 75, 74, 62, 5, 71, 70, 67, 65, 4, 1, 3, 66, 0], [48, 38, 112, 24, 22, 93, 107, 84, 123, 17, 52, 50, 27, 91, 95, 118, 53, 30, 20, 98, 94, 115, 78, 125, 111, 99, 113, 49, 117, 55, 126, 45, 97, 81, 19, 119, 104, 77, 58, 110, 92, 41, 109, 28, 47, 102, 10, 36, 13, 116, 86, 105, 101, 32, 88, 35, 57, 18, 54, 80, 90, 122, 61, 121, 62, 120, 108, 34, 106, 39, 33, 40, 44, 56, 59, 43, 42, 31, 100, 124, 63, 60, 103, 127, 51, 37, 72, 96, 114, 8, 26, 46, 25, 29, 65, 23, 87, 11, 16, 68, 9, 71, 85, 89, 21, 14, 12, 75, 82, 83, 15, 7, 6, 76, 64, 79, 74, 5, 2, 73, 67, 69, 70, 4, 66, 3, 1, 0], [38, 48, 112, 24, 22, 125, 84, 107, 17, 93, 19, 80, 95, 10, 20, 78, 109, 50, 62, 13, 77, 33, 18, 27, 47, 98, 97, 68, 94, 111, 28, 92, 72, 81, 34, 35, 75, 56, 119, 6, 52, 88, 96, 8, 86, 90, 26, 9, 30, 91, 82, 49, 116, 14, 32, 31, 69, 99, 64, 43, 67, 12, 54, 53, 41, 117, 55, 29, 126, 89, 16, 15, 45, 120, 21, 36, 83, 61, 85, 114, 0, 79, 73, 39, 74, 1, 121, 57, 23, 7, 118, 59, 51, 87, 63, 25, 46, 100, 44, 110, 76, 40, 124, 58, 123, 2, 106, 5, 42, 66, 104, 37, 115, 101, 103, 65, 102, 113, 127, 11, 108, 71, 70, 105, 60, 122, 4, 3], [38, 48, 112, 24, 22, 107, 84, 93, 17, 62, 123, 97, 78, 10, 121, 30, 20, 110, 125, 18, 98, 49, 27, 96, 95, 115, 94, 50, 33, 35, 127, 116, 19, 34, 13, 45, 124, 91, 120, 72, 43, 77, 59, 86, 126, 101, 105, 51, 14, 53, 92, 111, 119, 102, 113, 41, 52, 28, 118, 80, 57, 7, 68, 109, 103, 106, 32, 55, 81, 88, 89, 40, 36, 47, 54, 117, 8, 104, 114, 42, 60, 5, 21, 99, 63, 58, 122, 26, 61, 44, 56, 39, 74, 12, 31, 100, 9, 108, 46, 25, 37, 29, 87, 82, 64, 3, 11, 65, 90, 75, 15, 16, 83, 2, 71, 66, 67, 79, 76, 85, 23, 70, 6, 69, 73, 4, 1, 0]], "model.layers.19.self_attn.k_proj": [[63, 124, 110, 37, 33, 22, 53, 55, 112, 113, 59, 122, 114, 121, 120, 125, 54, 116, 61, 126, 62, 127, 119, 50, 49, 117, 56, 111, 47, 57, 123, 118, 58, 60, 30, 48, 109, 92, 51, 105, 43, 52, 14, 108, 44, 90, 45, 15, 104, 36, 69, 35, 12, 81, 71, 101, 107, 115, 3, 9, 46, 85, 88, 99, 74, 106, 40, 39, 42, 96, 41, 102, 38, 86, 103, 91, 20, 1, 77, 82, 64, 34, 19, 72, 100, 24, 11, 25, 98, 78, 32, 75, 70, 29, 18, 87, 66, 31, 26, 93, 95, 68, 89, 94, 76, 23, 16, 28, 27, 21, 80, 83, 84, 97, 79, 0, 13, 4, 10, 2, 8, 7, 73, 5, 67, 17, 65, 6], [118, 59, 103, 98, 94, 10, 79, 81, 13, 21, 69, 19, 75, 110, 25, 8, 97, 90, 80, 78, 0, 122, 44, 43, 57, 49, 23, 48, 108, 84, 120, 87, 77, 22, 71, 82, 26, 115, 35, 74, 76, 42, 2, 106, 54, 107, 119, 95, 1, 123, 52, 102, 68, 91, 41, 113, 9, 7, 121, 112, 55, 53, 63, 50, 88, 127, 14, 27, 67, 24, 65, 101, 93, 58, 100, 62, 99, 56, 104, 60, 117, 126, 92, 105, 111, 51, 114, 38, 116, 125, 28, 61, 47, 109, 36, 17, 96, 5, 32, 124, 73, 40, 37, 4, 83, 18, 70, 11, 12, 31, 46, 33, 45, 86, 72, 66, 6, 89, 29, 85, 3, 16, 30, 39, 20, 15, 34, 64], [41, 112, 98, 22, 48, 9, 27, 64, 15, 12, 71, 84, 2, 63, 5, 25, 114, 18, 53, 62, 122, 125, 56, 113, 94, 117, 59, 124, 1, 60, 120, 58, 119, 57, 83, 118, 116, 13, 51, 50, 126, 55, 49, 4, 46, 47, 105, 115, 54, 45, 32, 111, 74, 97, 29, 123, 110, 108, 61, 70, 52, 43, 127, 121, 78, 75, 107, 80, 67, 42, 37, 85, 88, 44, 87, 109, 103, 106, 104, 102, 101, 100, 40, 19, 93, 28, 21, 39, 8, 66, 95, 35, 81, 38, 30, 36, 72, 6, 11, 99, 10, 14, 16, 90, 92, 31, 26, 33, 17, 96, 91, 0, 24, 73, 76, 65, 34, 69, 3, 89, 77, 23, 86, 68, 79, 7, 20, 82], [121, 120, 35, 22, 104, 126, 124, 95, 57, 49, 60, 117, 59, 46, 112, 63, 50, 38, 58, 51, 119, 92, 127, 106, 111, 48, 123, 118, 125, 56, 122, 113, 61, 115, 116, 91, 41, 52, 54, 103, 45, 36, 47, 28, 109, 110, 62, 18, 42, 108, 55, 32, 114, 102, 43, 12, 44, 20, 101, 53, 14, 40, 81, 39, 107, 105, 26, 94, 37, 34, 79, 29, 80, 88, 74, 97, 99, 100, 27, 96, 33, 70, 13, 23, 98, 9, 8, 71, 31, 24, 30, 21, 89, 93, 83, 87, 25, 90, 3, 4, 10, 11, 86, 84, 16, 15, 17, 1, 69, 75, 85, 2, 19, 73, 7, 72, 82, 78, 5, 76, 68, 64, 77, 6, 67, 0, 66, 65], [55, 37, 110, 116, 120, 96, 112, 41, 30, 91, 89, 28, 57, 81, 45, 124, 87, 23, 78, 103, 56, 109, 83, 102, 84, 52, 49, 36, 60, 11, 123, 31, 40, 25, 34, 76, 85, 94, 121, 117, 62, 99, 8, 71, 73, 111, 39, 97, 54, 126, 58, 14, 44, 12, 59, 125, 115, 114, 50, 122, 22, 17, 61, 24, 119, 35, 118, 51, 127, 107, 43, 93, 79, 33, 18, 48, 95, 9, 47, 63, 21, 108, 42, 106, 113, 68, 6, 5, 2, 32, 70, 27, 26, 88, 15, 90, 92, 98, 104, 3, 29, 80, 100, 86, 53, 19, 16, 7, 38, 46, 82, 67, 20, 74, 75, 13, 10, 72, 1, 65, 105, 64, 0, 77, 69, 4, 101, 66], [62, 56, 38, 55, 33, 22, 93, 15, 19, 89, 17, 76, 81, 59, 7, 109, 29, 120, 102, 78, 110, 57, 90, 53, 121, 88, 74, 27, 12, 60, 119, 10, 69, 117, 124, 103, 63, 2, 113, 125, 104, 54, 45, 9, 31, 61, 77, 58, 116, 47, 0, 49, 107, 126, 118, 114, 122, 21, 112, 52, 39, 44, 127, 115, 34, 111, 46, 48, 13, 87, 123, 84, 108, 41, 3, 37, 99, 51, 23, 18, 106, 80, 96, 105, 71, 73, 36, 24, 79, 30, 95, 94, 50, 4, 92, 16, 14, 91, 85, 83, 32, 40, 42, 28, 67, 65, 6, 26, 75, 100, 20, 68, 101, 11, 8, 64, 98, 82, 43, 35, 5, 25, 72, 86, 70, 97, 66, 1], [105, 120, 98, 47, 29, 22, 57, 64, 84, 8, 80, 76, 111, 78, 67, 88, 74, 56, 70, 48, 66, 69, 60, 50, 26, 19, 119, 46, 117, 126, 124, 61, 82, 114, 44, 115, 53, 116, 125, 63, 62, 122, 43, 45, 113, 95, 59, 7, 23, 118, 41, 99, 112, 51, 55, 65, 58, 127, 81, 107, 110, 52, 49, 18, 33, 54, 89, 71, 123, 106, 27, 109, 42, 38, 121, 4, 101, 3, 104, 87, 15, 37, 83, 39, 77, 79, 40, 13, 103, 108, 28, 102, 96, 25, 30, 31, 2, 1, 91, 94, 32, 34, 16, 35, 85, 92, 36, 100, 12, 97, 21, 73, 11, 75, 5, 9, 24, 90, 20, 68, 17, 6, 93, 72, 0, 10, 14, 86], [112, 48, 102, 22, 84, 24, 78, 107, 18, 17, 10, 13, 33, 68, 93, 72, 31, 92, 9, 109, 125, 94, 65, 19, 64, 12, 98, 6, 27, 30, 62, 80, 71, 2, 67, 77, 52, 91, 85, 8, 119, 5, 32, 35, 121, 113, 123, 11, 7, 105, 34, 36, 28, 118, 43, 61, 26, 111, 87, 106, 56, 75, 69, 120, 50, 115, 122, 83, 47, 89, 126, 23, 63, 79, 58, 49, 97, 40, 16, 90, 51, 53, 99, 45, 29, 46, 104, 76, 103, 117, 41, 25, 44, 101, 54, 100, 21, 60, 70, 124, 15, 57, 42, 20, 37, 55, 95, 127, 108, 96, 110, 116, 73, 74, 114, 59, 39, 38, 82, 14, 88, 3, 86, 81, 4, 66, 1, 0]], "model.layers.19.self_attn.qk_proj": [[112, 120, 55, 48, 62, 59, 56, 63, 118, 121, 124, 41, 110, 57, 105, 116, 86, 38, 102, 47, 98, 22, 53, 125, 50, 20, 111, 46, 58, 29, 60, 122, 93, 119, 84, 126, 113, 37, 25, 127, 17, 91, 51, 24, 89, 27, 109, 79, 61, 88, 12, 49, 76, 81, 83, 19, 90, 115, 97, 78, 54, 101, 14, 117, 114, 94, 123, 34, 82, 15, 52, 33, 35, 39, 32, 26, 45, 107, 87, 10, 30, 13, 18, 43, 103, 74, 92, 31, 44, 16, 77, 106, 72, 23, 108, 21, 95, 28, 96, 42, 9, 73, 11, 104, 71, 64, 80, 5, 40, 99, 75, 85, 36, 6, 7, 69, 0, 8, 100, 2, 70, 66, 67, 3, 4, 68, 65, 1], [112, 120, 55, 48, 59, 62, 56, 121, 63, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 53, 98, 58, 22, 125, 119, 50, 84, 91, 37, 111, 113, 20, 89, 126, 127, 93, 29, 46, 60, 25, 76, 24, 83, 12, 17, 49, 51, 109, 88, 79, 78, 114, 117, 81, 123, 34, 27, 61, 122, 90, 35, 19, 107, 33, 94, 97, 54, 115, 15, 18, 32, 39, 43, 103, 14, 92, 52, 26, 101, 30, 87, 10, 82, 74, 13, 45, 104, 96, 23, 44, 16, 77, 108, 31, 95, 40, 7, 9, 42, 21, 106, 28, 80, 73, 69, 99, 5, 0, 75, 71, 85, 11, 8, 64, 100, 6, 72, 70, 36, 2, 68, 66, 67, 4, 1, 65, 3], [112, 120, 48, 55, 59, 62, 56, 121, 63, 118, 41, 124, 105, 110, 57, 116, 86, 38, 102, 47, 22, 53, 58, 98, 29, 125, 93, 126, 20, 84, 46, 119, 49, 113, 127, 60, 122, 91, 89, 25, 50, 24, 81, 37, 111, 109, 88, 27, 19, 90, 34, 54, 94, 79, 51, 61, 12, 15, 76, 17, 35, 33, 97, 18, 117, 115, 107, 78, 26, 14, 83, 32, 39, 103, 101, 123, 52, 30, 114, 43, 23, 108, 45, 87, 8, 92, 13, 10, 82, 77, 96, 74, 95, 16, 42, 64, 31, 0, 11, 104, 44, 99, 9, 7, 21, 69, 5, 71, 80, 70, 28, 106, 36, 73, 100, 75, 66, 2, 3, 85, 40, 68, 4, 1, 67, 72, 6, 65], [112, 120, 48, 55, 62, 59, 63, 56, 118, 121, 124, 41, 110, 105, 57, 116, 38, 86, 102, 47, 126, 58, 53, 22, 29, 98, 84, 93, 89, 50, 46, 27, 91, 25, 20, 113, 123, 119, 49, 127, 51, 122, 125, 34, 81, 94, 15, 61, 37, 19, 76, 117, 24, 12, 115, 109, 78, 17, 88, 54, 79, 14, 60, 90, 107, 39, 83, 114, 97, 32, 111, 101, 35, 30, 33, 52, 18, 82, 45, 43, 23, 92, 87, 103, 8, 108, 104, 10, 26, 96, 95, 74, 13, 44, 106, 77, 0, 70, 69, 16, 64, 31, 71, 99, 5, 73, 42, 9, 80, 11, 7, 21, 75, 2, 40, 85, 66, 36, 3, 100, 68, 28, 6, 67, 4, 72, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 63, 118, 124, 41, 110, 105, 57, 116, 38, 86, 47, 22, 102, 53, 98, 126, 20, 58, 29, 125, 84, 61, 25, 60, 51, 27, 46, 93, 122, 127, 119, 78, 17, 81, 19, 76, 49, 111, 37, 117, 12, 91, 89, 88, 50, 113, 79, 34, 24, 94, 15, 115, 90, 123, 14, 54, 97, 103, 18, 83, 101, 109, 30, 45, 33, 8, 43, 82, 106, 52, 32, 39, 13, 107, 74, 35, 23, 10, 114, 77, 73, 104, 96, 87, 108, 44, 31, 16, 11, 26, 92, 28, 71, 5, 64, 95, 7, 99, 9, 69, 70, 100, 85, 0, 42, 36, 21, 75, 40, 68, 66, 80, 2, 67, 3, 4, 72, 6, 65, 1], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 57, 105, 116, 86, 38, 102, 47, 22, 98, 84, 25, 29, 20, 126, 119, 53, 58, 27, 17, 113, 46, 49, 37, 81, 89, 76, 12, 88, 24, 78, 60, 93, 15, 83, 19, 122, 125, 127, 52, 50, 54, 14, 117, 123, 79, 111, 51, 91, 61, 90, 94, 115, 109, 97, 82, 34, 33, 13, 18, 39, 32, 43, 74, 101, 107, 10, 30, 8, 114, 103, 35, 96, 92, 44, 23, 45, 87, 16, 106, 104, 21, 108, 73, 11, 31, 28, 5, 80, 71, 77, 85, 26, 0, 42, 9, 75, 95, 69, 7, 99, 70, 64, 2, 66, 40, 6, 36, 100, 67, 3, 68, 72, 4, 1, 65], [112, 120, 55, 48, 62, 59, 121, 56, 63, 118, 41, 124, 57, 110, 105, 116, 86, 102, 38, 47, 22, 29, 20, 84, 98, 58, 17, 125, 53, 119, 24, 60, 25, 93, 111, 126, 27, 109, 12, 46, 19, 50, 49, 89, 76, 90, 14, 37, 81, 52, 91, 122, 78, 79, 88, 39, 117, 113, 83, 127, 15, 94, 61, 34, 97, 123, 82, 114, 32, 51, 18, 45, 33, 101, 54, 115, 92, 30, 8, 13, 96, 23, 10, 43, 103, 74, 104, 26, 35, 77, 87, 16, 31, 80, 107, 21, 85, 28, 9, 11, 44, 42, 108, 71, 106, 5, 7, 95, 99, 73, 100, 69, 6, 75, 64, 70, 40, 36, 0, 2, 68, 3, 67, 66, 1, 4, 72, 65], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 41, 124, 105, 57, 110, 116, 86, 102, 38, 47, 53, 22, 29, 84, 98, 93, 24, 58, 126, 119, 20, 60, 111, 109, 27, 46, 37, 17, 89, 122, 25, 91, 50, 117, 81, 76, 88, 125, 12, 49, 54, 19, 90, 15, 107, 14, 83, 79, 94, 114, 78, 39, 113, 34, 123, 32, 115, 61, 97, 33, 101, 127, 51, 18, 35, 82, 92, 52, 26, 43, 103, 96, 23, 45, 77, 10, 30, 42, 104, 87, 8, 16, 74, 31, 44, 28, 13, 80, 21, 99, 7, 9, 71, 108, 5, 11, 36, 40, 85, 6, 106, 95, 64, 73, 0, 69, 75, 100, 2, 66, 72, 3, 4, 68, 70, 67, 1, 65], [112, 120, 55, 48, 62, 59, 56, 63, 118, 121, 41, 124, 105, 110, 57, 116, 86, 102, 47, 38, 53, 22, 29, 126, 84, 98, 119, 37, 93, 60, 122, 25, 58, 27, 91, 17, 20, 88, 24, 125, 109, 81, 114, 54, 94, 46, 50, 111, 113, 89, 12, 15, 49, 19, 127, 83, 33, 76, 123, 14, 117, 79, 107, 39, 61, 90, 97, 51, 34, 32, 35, 115, 52, 101, 18, 78, 103, 43, 13, 30, 82, 77, 92, 104, 26, 42, 87, 74, 23, 45, 10, 28, 31, 108, 44, 96, 16, 7, 99, 21, 6, 80, 36, 9, 8, 40, 73, 95, 11, 106, 5, 71, 85, 75, 64, 69, 100, 72, 0, 2, 70, 3, 67, 4, 66, 68, 1, 65], [112, 120, 48, 55, 62, 59, 56, 118, 63, 121, 41, 124, 57, 105, 110, 116, 102, 38, 86, 47, 22, 53, 98, 84, 58, 29, 125, 60, 126, 25, 93, 20, 37, 49, 12, 46, 119, 27, 88, 89, 122, 113, 50, 91, 81, 117, 61, 127, 109, 114, 19, 123, 76, 54, 94, 111, 24, 17, 97, 83, 79, 34, 107, 15, 115, 39, 32, 51, 14, 30, 10, 78, 103, 33, 101, 90, 92, 35, 104, 31, 82, 45, 74, 43, 77, 18, 13, 44, 42, 52, 23, 108, 26, 96, 9, 87, 106, 6, 16, 7, 99, 80, 95, 5, 0, 71, 72, 73, 69, 11, 28, 8, 64, 36, 40, 75, 21, 100, 85, 68, 67, 70, 66, 2, 4, 3, 1, 65], [112, 120, 55, 48, 62, 59, 118, 121, 56, 63, 124, 41, 110, 57, 105, 116, 38, 86, 102, 22, 53, 47, 29, 46, 20, 98, 25, 60, 84, 50, 126, 125, 122, 27, 58, 93, 119, 81, 117, 12, 61, 97, 89, 76, 88, 14, 101, 37, 78, 17, 113, 19, 15, 51, 91, 49, 83, 39, 24, 79, 127, 111, 10, 18, 114, 115, 54, 34, 94, 103, 30, 90, 109, 106, 123, 33, 107, 32, 72, 74, 82, 35, 43, 104, 77, 11, 5, 9, 92, 6, 96, 45, 13, 42, 31, 23, 52, 87, 16, 108, 26, 64, 44, 69, 73, 80, 7, 71, 0, 28, 2, 66, 21, 75, 95, 85, 36, 99, 8, 70, 68, 40, 67, 3, 4, 100, 65, 1], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 124, 41, 105, 110, 57, 116, 86, 38, 47, 22, 102, 53, 29, 20, 84, 60, 98, 119, 27, 81, 93, 126, 50, 25, 17, 125, 46, 122, 58, 88, 37, 117, 12, 19, 89, 76, 24, 78, 34, 113, 109, 79, 83, 51, 97, 49, 15, 94, 61, 39, 14, 18, 111, 91, 107, 90, 32, 33, 115, 114, 74, 54, 101, 10, 103, 127, 106, 82, 52, 30, 123, 35, 87, 77, 104, 72, 26, 42, 31, 11, 13, 16, 9, 108, 96, 23, 80, 28, 45, 92, 43, 73, 44, 21, 99, 71, 95, 5, 85, 7, 0, 64, 69, 75, 6, 66, 36, 70, 3, 2, 100, 40, 68, 4, 8, 67, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 63, 118, 41, 124, 110, 105, 57, 116, 86, 102, 47, 38, 53, 22, 119, 29, 84, 98, 58, 60, 20, 125, 93, 24, 27, 50, 126, 37, 122, 109, 89, 88, 117, 17, 49, 25, 91, 81, 113, 114, 76, 61, 19, 111, 12, 33, 51, 39, 46, 54, 15, 78, 34, 123, 83, 115, 90, 127, 107, 79, 52, 94, 32, 103, 35, 18, 92, 101, 26, 82, 74, 97, 14, 30, 96, 28, 72, 10, 77, 87, 44, 23, 13, 45, 106, 31, 80, 108, 104, 42, 85, 16, 21, 71, 43, 9, 99, 95, 100, 36, 11, 7, 69, 40, 5, 73, 70, 75, 0, 64, 6, 66, 67, 2, 68, 4, 65, 3, 8, 1], [112, 120, 55, 48, 59, 62, 56, 121, 63, 118, 124, 41, 105, 57, 110, 116, 86, 102, 38, 47, 22, 60, 53, 125, 58, 98, 84, 93, 119, 50, 29, 126, 27, 46, 25, 20, 61, 122, 113, 109, 12, 91, 49, 37, 117, 81, 17, 88, 89, 111, 115, 19, 76, 35, 24, 127, 34, 94, 101, 15, 54, 52, 97, 51, 79, 14, 114, 78, 33, 30, 107, 123, 72, 32, 18, 39, 26, 83, 90, 10, 82, 103, 74, 13, 104, 31, 23, 87, 96, 80, 77, 73, 92, 7, 16, 71, 106, 0, 42, 69, 45, 21, 95, 43, 108, 44, 70, 9, 99, 11, 85, 5, 28, 64, 40, 75, 100, 36, 66, 2, 4, 67, 6, 68, 3, 65, 1, 8], [112, 120, 55, 48, 62, 59, 56, 118, 121, 63, 41, 124, 110, 105, 57, 116, 86, 38, 102, 47, 98, 125, 84, 22, 29, 119, 53, 25, 20, 93, 60, 27, 12, 17, 58, 24, 126, 81, 50, 37, 117, 76, 113, 94, 46, 34, 49, 91, 78, 61, 19, 15, 88, 89, 14, 39, 122, 109, 111, 101, 97, 79, 35, 52, 83, 30, 18, 51, 72, 33, 54, 115, 10, 127, 123, 103, 82, 74, 90, 107, 13, 32, 26, 31, 23, 114, 73, 87, 106, 45, 92, 96, 9, 80, 71, 16, 77, 43, 69, 44, 64, 70, 104, 108, 11, 85, 5, 7, 0, 36, 99, 28, 95, 66, 40, 42, 21, 75, 6, 67, 68, 3, 2, 8, 100, 65, 4, 1], [112, 120, 48, 55, 62, 59, 56, 63, 118, 121, 41, 124, 110, 105, 57, 116, 86, 102, 38, 47, 22, 53, 29, 58, 98, 84, 20, 27, 125, 17, 117, 46, 76, 119, 126, 25, 60, 37, 81, 93, 113, 49, 50, 61, 24, 12, 34, 122, 89, 15, 91, 88, 19, 14, 94, 101, 78, 107, 79, 123, 51, 109, 52, 97, 115, 111, 83, 127, 35, 10, 54, 74, 39, 33, 18, 30, 82, 13, 90, 31, 32, 106, 72, 114, 103, 9, 26, 43, 77, 11, 73, 80, 44, 87, 45, 104, 92, 96, 42, 69, 71, 23, 95, 7, 108, 16, 70, 40, 85, 64, 5, 21, 75, 28, 99, 36, 0, 8, 2, 66, 67, 6, 100, 3, 68, 4, 1, 65], [112, 120, 48, 55, 62, 59, 56, 121, 118, 63, 41, 124, 105, 110, 57, 116, 86, 38, 102, 22, 53, 47, 98, 126, 29, 60, 58, 119, 20, 84, 125, 27, 46, 88, 50, 34, 49, 93, 37, 91, 89, 17, 81, 24, 25, 117, 122, 19, 113, 76, 109, 127, 12, 51, 33, 90, 83, 61, 32, 111, 94, 52, 79, 15, 97, 54, 78, 82, 39, 107, 92, 101, 18, 14, 114, 123, 30, 26, 74, 115, 43, 103, 35, 87, 96, 31, 10, 106, 13, 104, 77, 44, 23, 80, 16, 85, 28, 95, 42, 21, 45, 108, 40, 99, 36, 72, 8, 73, 5, 71, 9, 64, 70, 0, 11, 7, 75, 69, 100, 66, 6, 3, 2, 68, 4, 67, 1, 65], [112, 120, 48, 55, 62, 59, 56, 118, 121, 63, 41, 124, 105, 110, 57, 116, 38, 86, 102, 47, 22, 53, 98, 20, 29, 93, 113, 60, 46, 119, 84, 58, 125, 37, 89, 27, 91, 126, 25, 122, 17, 88, 50, 12, 49, 24, 76, 19, 117, 81, 109, 94, 15, 14, 79, 90, 127, 123, 83, 78, 61, 32, 34, 33, 30, 82, 74, 114, 107, 111, 39, 97, 115, 101, 103, 52, 54, 35, 51, 26, 10, 92, 18, 43, 13, 104, 87, 77, 31, 96, 23, 80, 0, 42, 73, 71, 16, 45, 28, 85, 106, 8, 21, 36, 7, 9, 5, 69, 108, 44, 95, 6, 64, 99, 75, 72, 11, 40, 70, 66, 2, 100, 67, 3, 4, 68, 65, 1], [112, 120, 48, 55, 62, 59, 63, 118, 56, 121, 41, 124, 110, 105, 57, 116, 38, 86, 47, 102, 53, 98, 22, 46, 125, 126, 29, 50, 93, 117, 27, 20, 25, 58, 84, 122, 89, 17, 127, 113, 37, 119, 114, 115, 91, 60, 81, 49, 19, 61, 12, 24, 94, 34, 33, 14, 109, 76, 52, 88, 123, 79, 83, 97, 15, 30, 101, 107, 51, 32, 90, 39, 54, 92, 104, 35, 82, 78, 74, 103, 8, 18, 31, 111, 43, 10, 87, 26, 45, 13, 96, 108, 77, 106, 44, 73, 23, 36, 6, 40, 16, 0, 80, 9, 71, 64, 69, 7, 42, 21, 28, 5, 11, 75, 95, 99, 85, 100, 3, 2, 70, 68, 66, 72, 4, 67, 65, 1], [112, 120, 48, 55, 62, 59, 63, 56, 118, 121, 41, 124, 110, 105, 57, 116, 86, 102, 38, 47, 53, 98, 22, 29, 125, 46, 84, 58, 25, 126, 27, 117, 89, 37, 20, 93, 119, 91, 17, 61, 24, 12, 50, 34, 60, 76, 49, 113, 109, 83, 14, 127, 111, 88, 81, 51, 115, 122, 33, 94, 114, 79, 32, 19, 123, 15, 78, 10, 54, 8, 97, 74, 107, 90, 104, 103, 82, 101, 39, 43, 30, 52, 92, 31, 77, 35, 106, 18, 13, 96, 26, 87, 80, 108, 44, 45, 28, 23, 6, 21, 73, 16, 9, 11, 95, 71, 69, 99, 75, 42, 5, 7, 64, 85, 0, 40, 36, 100, 66, 2, 70, 68, 72, 4, 1, 67, 3, 65], [112, 120, 48, 55, 62, 56, 59, 118, 63, 121, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 53, 22, 98, 119, 25, 126, 125, 29, 20, 93, 27, 58, 60, 117, 61, 46, 84, 114, 89, 127, 113, 50, 91, 76, 37, 24, 122, 49, 109, 17, 19, 83, 81, 115, 78, 88, 123, 12, 94, 79, 14, 97, 30, 34, 52, 51, 15, 54, 107, 33, 18, 103, 32, 10, 43, 82, 111, 39, 101, 90, 8, 28, 106, 77, 13, 26, 35, 9, 104, 74, 44, 42, 92, 45, 87, 31, 80, 21, 23, 96, 16, 75, 95, 69, 6, 71, 73, 85, 11, 99, 7, 108, 5, 0, 36, 40, 64, 2, 70, 4, 67, 66, 3, 100, 68, 72, 65, 1], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 41, 124, 110, 105, 57, 116, 86, 38, 102, 47, 58, 22, 98, 53, 126, 29, 119, 93, 20, 117, 49, 50, 125, 61, 25, 27, 51, 60, 46, 89, 84, 37, 115, 17, 91, 113, 123, 83, 114, 109, 88, 122, 76, 12, 127, 24, 81, 54, 52, 94, 14, 79, 19, 97, 18, 15, 103, 78, 32, 33, 34, 8, 30, 39, 43, 107, 92, 26, 82, 45, 111, 74, 90, 10, 13, 77, 87, 35, 31, 101, 96, 104, 21, 28, 80, 23, 44, 95, 9, 64, 0, 108, 69, 40, 106, 42, 7, 99, 5, 16, 71, 11, 73, 85, 75, 6, 36, 70, 100, 66, 4, 2, 67, 65, 3, 68, 1, 72], [112, 120, 48, 55, 59, 62, 56, 121, 118, 63, 41, 124, 105, 110, 57, 116, 38, 86, 102, 47, 22, 53, 58, 98, 119, 126, 29, 50, 60, 49, 20, 117, 113, 27, 91, 93, 61, 51, 25, 46, 89, 24, 17, 37, 125, 123, 84, 122, 76, 127, 114, 111, 88, 81, 12, 54, 15, 115, 94, 52, 19, 79, 33, 109, 83, 30, 78, 34, 107, 14, 90, 35, 97, 32, 103, 92, 26, 101, 43, 45, 77, 82, 18, 8, 10, 106, 104, 31, 39, 44, 74, 13, 87, 99, 21, 23, 108, 36, 80, 28, 16, 95, 96, 9, 7, 42, 0, 70, 73, 85, 71, 5, 11, 75, 40, 100, 69, 6, 64, 2, 66, 4, 67, 68, 72, 3, 1, 65], [112, 120, 48, 55, 62, 56, 59, 121, 118, 63, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 22, 58, 29, 126, 98, 53, 119, 25, 20, 84, 27, 89, 125, 93, 113, 50, 24, 61, 91, 117, 60, 46, 15, 115, 81, 37, 88, 17, 94, 109, 123, 76, 122, 12, 127, 49, 51, 111, 97, 32, 34, 114, 78, 19, 14, 83, 33, 79, 30, 10, 107, 54, 39, 82, 18, 35, 74, 104, 92, 13, 101, 103, 90, 96, 8, 73, 52, 31, 77, 43, 87, 21, 23, 26, 106, 44, 45, 9, 16, 71, 28, 7, 42, 69, 75, 95, 80, 40, 70, 11, 5, 108, 85, 100, 0, 99, 2, 6, 64, 68, 36, 72, 67, 66, 4, 3, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 118, 63, 41, 124, 110, 105, 57, 116, 86, 38, 22, 47, 102, 58, 98, 60, 20, 53, 29, 84, 25, 113, 126, 89, 119, 125, 17, 88, 81, 24, 50, 27, 61, 12, 93, 76, 49, 78, 97, 37, 19, 117, 123, 46, 91, 127, 109, 34, 122, 15, 111, 14, 83, 107, 54, 51, 103, 79, 39, 10, 33, 101, 115, 32, 94, 82, 114, 18, 90, 26, 30, 74, 13, 77, 52, 35, 43, 92, 31, 106, 87, 96, 28, 23, 9, 16, 45, 80, 104, 7, 21, 42, 44, 73, 70, 75, 69, 108, 8, 71, 5, 72, 11, 85, 0, 64, 40, 36, 100, 99, 95, 66, 2, 3, 68, 67, 6, 4, 65, 1], [112, 120, 48, 55, 62, 56, 59, 121, 118, 63, 124, 41, 105, 110, 57, 116, 86, 102, 38, 47, 53, 22, 125, 58, 98, 119, 29, 60, 84, 20, 113, 126, 49, 27, 91, 88, 46, 37, 89, 117, 122, 123, 93, 25, 61, 50, 76, 107, 12, 24, 34, 17, 127, 109, 111, 81, 32, 19, 78, 114, 51, 83, 54, 33, 115, 94, 97, 79, 103, 15, 30, 26, 35, 90, 14, 74, 39, 18, 82, 101, 43, 92, 87, 10, 96, 42, 44, 23, 31, 77, 28, 13, 45, 52, 21, 40, 16, 7, 95, 80, 104, 108, 99, 106, 72, 36, 70, 9, 85, 75, 69, 73, 71, 8, 5, 0, 100, 11, 64, 2, 66, 67, 6, 68, 1, 3, 4, 65], [112, 120, 48, 55, 56, 62, 59, 63, 121, 118, 124, 41, 105, 110, 57, 116, 47, 38, 86, 102, 53, 22, 98, 119, 126, 50, 58, 29, 20, 60, 125, 122, 113, 91, 93, 27, 61, 37, 25, 88, 111, 49, 84, 114, 46, 89, 107, 117, 51, 34, 24, 115, 109, 81, 123, 17, 76, 127, 15, 30, 12, 90, 39, 54, 83, 97, 33, 79, 14, 32, 101, 35, 103, 19, 94, 43, 92, 42, 104, 18, 78, 23, 10, 82, 52, 74, 44, 45, 26, 87, 77, 72, 96, 13, 16, 106, 99, 95, 108, 21, 28, 36, 75, 31, 80, 5, 71, 73, 9, 70, 69, 40, 100, 7, 64, 11, 0, 2, 85, 67, 66, 4, 8, 6, 3, 68, 65, 1], [112, 120, 48, 55, 56, 62, 59, 121, 63, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 22, 53, 98, 126, 119, 84, 58, 29, 125, 122, 50, 20, 46, 54, 60, 89, 93, 113, 17, 24, 27, 127, 25, 76, 12, 49, 51, 37, 78, 115, 81, 61, 91, 15, 117, 19, 109, 79, 123, 111, 83, 88, 97, 14, 114, 34, 39, 18, 94, 30, 32, 33, 10, 35, 90, 103, 101, 72, 107, 52, 74, 45, 13, 82, 77, 43, 92, 26, 87, 9, 16, 23, 71, 73, 64, 31, 44, 7, 5, 28, 96, 42, 108, 104, 75, 80, 21, 69, 0, 95, 99, 106, 40, 85, 70, 11, 36, 66, 6, 2, 8, 100, 3, 68, 67, 4, 65, 1], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 98, 22, 58, 53, 60, 125, 119, 20, 122, 46, 113, 25, 126, 29, 27, 84, 51, 50, 89, 49, 76, 83, 93, 81, 127, 114, 78, 37, 12, 17, 91, 111, 97, 61, 79, 19, 24, 117, 15, 115, 88, 34, 109, 94, 54, 33, 101, 123, 45, 35, 14, 82, 43, 30, 92, 10, 90, 18, 52, 72, 107, 32, 44, 103, 104, 31, 77, 106, 9, 74, 23, 39, 13, 108, 87, 0, 69, 64, 21, 75, 5, 96, 71, 26, 73, 95, 6, 36, 16, 28, 7, 80, 85, 66, 42, 11, 99, 40, 67, 4, 100, 68, 2, 3, 1, 70, 8, 65], [112, 120, 55, 48, 62, 59, 56, 121, 118, 63, 124, 41, 110, 105, 57, 116, 86, 102, 38, 47, 22, 58, 98, 53, 20, 119, 46, 60, 29, 122, 125, 84, 49, 113, 76, 93, 25, 24, 123, 91, 27, 51, 37, 12, 81, 17, 126, 54, 89, 19, 88, 15, 14, 50, 83, 78, 115, 127, 79, 111, 34, 97, 114, 117, 101, 82, 109, 94, 61, 90, 72, 10, 33, 39, 43, 32, 30, 35, 18, 107, 87, 26, 103, 52, 92, 74, 44, 45, 106, 13, 96, 9, 23, 77, 28, 16, 6, 108, 5, 21, 104, 85, 95, 73, 71, 42, 80, 31, 7, 36, 75, 40, 69, 11, 64, 99, 2, 0, 100, 66, 3, 70, 67, 4, 8, 68, 1, 65], [112, 120, 55, 48, 59, 62, 121, 56, 118, 63, 124, 41, 110, 105, 57, 116, 38, 102, 86, 47, 98, 22, 58, 53, 60, 122, 29, 46, 20, 113, 119, 125, 126, 84, 89, 49, 24, 12, 93, 76, 51, 50, 81, 91, 127, 115, 27, 25, 123, 17, 88, 37, 97, 15, 61, 78, 14, 111, 54, 83, 101, 19, 30, 34, 72, 79, 103, 117, 94, 39, 33, 35, 109, 107, 92, 114, 32, 10, 45, 18, 82, 90, 52, 26, 74, 9, 108, 106, 6, 23, 71, 13, 44, 5, 16, 69, 95, 73, 87, 7, 96, 80, 21, 99, 31, 77, 64, 43, 85, 104, 28, 11, 75, 40, 0, 42, 2, 3, 100, 66, 36, 8, 4, 68, 70, 67, 1, 65], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 105, 57, 116, 86, 102, 38, 47, 58, 22, 53, 98, 50, 20, 93, 29, 60, 113, 24, 126, 84, 125, 89, 25, 49, 119, 37, 81, 12, 27, 46, 111, 91, 127, 17, 76, 61, 122, 51, 78, 15, 123, 88, 19, 79, 14, 97, 109, 94, 115, 32, 54, 90, 33, 39, 117, 34, 83, 35, 101, 52, 10, 45, 74, 92, 114, 72, 103, 107, 82, 18, 30, 26, 96, 87, 77, 44, 43, 106, 13, 108, 73, 23, 80, 16, 71, 11, 36, 6, 21, 95, 5, 9, 28, 31, 7, 75, 104, 99, 40, 42, 69, 85, 64, 100, 4, 3, 8, 70, 0, 67, 2, 68, 66, 65, 1]], "model.layers.20.self_attn.q_proj": [[114, 126, 101, 110, 46, 98, 19, 86, 29, 92, 25, 120, 24, 67, 124, 82, 12, 89, 9, 87, 28, 37, 119, 7, 49, 90, 16, 60, 23, 51, 13, 74, 115, 79, 31, 122, 58, 65, 127, 62, 77, 112, 121, 100, 116, 54, 4, 0, 97, 57, 88, 43, 118, 53, 125, 39, 61, 111, 108, 96, 34, 55, 50, 15, 14, 63, 32, 113, 47, 48, 40, 94, 30, 5, 93, 6, 56, 59, 64, 20, 109, 123, 42, 44, 117, 66, 1, 35, 72, 107, 68, 95, 45, 52, 2, 17, 27, 36, 83, 41, 26, 3, 70, 102, 38, 80, 104, 33, 105, 84, 106, 91, 81, 103, 18, 21, 99, 69, 8, 78, 73, 10, 85, 75, 71, 22, 76, 11], [46, 126, 114, 110, 101, 120, 98, 49, 124, 92, 55, 115, 58, 122, 19, 60, 119, 28, 37, 40, 54, 127, 86, 121, 116, 52, 53, 125, 118, 112, 57, 50, 51, 47, 61, 48, 62, 56, 108, 111, 90, 31, 42, 63, 113, 59, 24, 123, 43, 89, 109, 117, 105, 91, 45, 103, 44, 29, 106, 88, 15, 104, 82, 102, 39, 107, 41, 25, 95, 36, 16, 94, 35, 100, 23, 99, 85, 7, 38, 22, 33, 97, 13, 93, 96, 87, 34, 14, 79, 9, 32, 26, 21, 30, 27, 83, 73, 12, 17, 20, 84, 80, 8, 77, 81, 72, 11, 18, 78, 75, 66, 74, 10, 68, 6, 5, 4, 71, 76, 3, 1, 69, 64, 70, 2, 65, 67, 0], [126, 101, 110, 114, 46, 98, 28, 92, 120, 19, 86, 115, 124, 14, 24, 60, 58, 121, 82, 123, 23, 29, 89, 119, 116, 125, 37, 42, 39, 122, 36, 49, 31, 55, 118, 112, 103, 54, 109, 48, 25, 105, 127, 57, 62, 99, 47, 53, 51, 16, 111, 50, 59, 61, 56, 117, 26, 100, 91, 63, 33, 104, 95, 38, 32, 45, 30, 113, 83, 93, 52, 35, 94, 40, 41, 44, 85, 97, 87, 90, 96, 12, 107, 27, 108, 43, 84, 7, 102, 74, 34, 106, 21, 13, 81, 22, 15, 17, 88, 79, 80, 5, 8, 20, 18, 76, 78, 75, 71, 77, 3, 72, 9, 11, 73, 10, 66, 70, 69, 6, 1, 68, 67, 4, 2, 65, 0, 64], [114, 126, 46, 110, 101, 92, 98, 120, 119, 12, 124, 28, 19, 82, 60, 5, 51, 25, 86, 37, 127, 49, 115, 0, 62, 58, 16, 122, 50, 57, 74, 112, 117, 121, 7, 123, 61, 53, 38, 55, 113, 54, 68, 125, 118, 14, 31, 116, 47, 44, 48, 56, 109, 111, 63, 89, 87, 104, 43, 108, 2, 100, 36, 91, 45, 107, 33, 105, 94, 29, 52, 23, 59, 8, 41, 39, 6, 85, 77, 40, 70, 79, 3, 102, 42, 15, 35, 66, 18, 95, 32, 88, 69, 83, 99, 106, 96, 73, 75, 30, 9, 1, 71, 93, 64, 17, 27, 13, 103, 97, 24, 4, 20, 65, 78, 11, 67, 76, 84, 81, 21, 80, 22, 26, 72, 90, 10, 34], [43, 98, 93, 103, 107, 116, 46, 117, 78, 58, 21, 24, 81, 19, 127, 12, 63, 8, 51, 95, 121, 62, 92, 90, 120, 29, 37, 70, 26, 109, 41, 59, 101, 104, 119, 105, 100, 52, 48, 50, 36, 57, 87, 74, 86, 16, 27, 44, 7, 110, 73, 2, 72, 42, 54, 33, 67, 13, 32, 39, 56, 126, 115, 75, 53, 79, 114, 68, 83, 84, 3, 10, 125, 23, 64, 118, 9, 22, 1, 112, 61, 113, 80, 40, 25, 17, 47, 108, 85, 66, 106, 45, 30, 14, 97, 99, 38, 76, 123, 94, 35, 122, 96, 11, 82, 18, 60, 28, 49, 111, 31, 6, 15, 0, 55, 91, 71, 20, 77, 69, 89, 34, 102, 88, 124, 5, 4, 65], [43, 121, 107, 98, 93, 46, 63, 62, 52, 58, 118, 109, 38, 24, 116, 39, 127, 103, 54, 119, 37, 104, 19, 53, 51, 55, 26, 29, 44, 49, 56, 105, 21, 57, 110, 115, 48, 117, 41, 95, 50, 90, 61, 33, 47, 59, 32, 125, 112, 92, 120, 81, 123, 42, 106, 114, 108, 60, 23, 78, 111, 25, 126, 124, 122, 96, 86, 45, 113, 87, 34, 75, 12, 40, 100, 36, 101, 70, 9, 91, 83, 97, 73, 102, 31, 35, 84, 28, 22, 30, 80, 88, 99, 18, 77, 10, 82, 68, 94, 11, 27, 20, 13, 79, 17, 89, 4, 15, 8, 16, 67, 1, 85, 7, 64, 6, 71, 74, 76, 3, 65, 69, 2, 5, 14, 66, 0, 72], [43, 113, 121, 116, 93, 98, 107, 59, 52, 24, 103, 63, 53, 125, 49, 29, 127, 105, 58, 51, 110, 95, 26, 48, 117, 54, 119, 112, 46, 124, 37, 92, 120, 47, 44, 106, 61, 21, 123, 118, 109, 114, 56, 62, 45, 111, 104, 60, 90, 115, 91, 41, 122, 126, 39, 108, 57, 100, 81, 42, 97, 50, 40, 32, 33, 87, 38, 23, 101, 55, 102, 36, 22, 19, 35, 96, 88, 34, 83, 86, 84, 99, 94, 31, 18, 30, 80, 27, 77, 28, 25, 82, 89, 75, 85, 20, 16, 15, 78, 12, 17, 9, 76, 13, 79, 73, 11, 74, 70, 14, 10, 6, 72, 7, 5, 4, 66, 68, 8, 0, 69, 65, 71, 67, 2, 3, 1, 64], [43, 98, 93, 107, 103, 46, 116, 121, 24, 81, 78, 51, 21, 12, 109, 62, 8, 63, 19, 120, 26, 7, 117, 52, 29, 69, 74, 127, 68, 58, 38, 70, 92, 119, 73, 65, 90, 3, 2, 105, 37, 16, 66, 125, 44, 95, 50, 86, 48, 10, 39, 75, 106, 54, 0, 33, 64, 47, 59, 87, 56, 4, 96, 5, 97, 49, 82, 53, 71, 41, 101, 45, 84, 67, 18, 113, 114, 123, 110, 115, 104, 83, 13, 77, 79, 118, 6, 126, 11, 111, 36, 60, 17, 55, 124, 61, 1, 108, 76, 57, 23, 28, 25, 42, 35, 20, 112, 27, 80, 30, 72, 22, 40, 99, 85, 100, 122, 31, 9, 91, 15, 32, 14, 89, 102, 94, 34, 88], [39, 109, 46, 97, 93, 110, 78, 17, 10, 71, 19, 45, 5, 85, 76, 86, 16, 65, 26, 3, 22, 116, 83, 9, 89, 66, 2, 121, 35, 90, 87, 88, 107, 81, 24, 64, 79, 21, 91, 80, 119, 103, 75, 11, 28, 12, 74, 29, 15, 82, 54, 8, 92, 72, 127, 7, 18, 111, 59, 13, 23, 77, 14, 94, 1, 84, 4, 123, 48, 69, 68, 67, 20, 32, 73, 6, 70, 0, 27, 118, 30, 25, 115, 34, 99, 33, 114, 96, 102, 113, 31, 100, 38, 43, 95, 101, 58, 44, 41, 51, 125, 120, 122, 57, 98, 53, 50, 55, 126, 40, 60, 36, 104, 52, 47, 106, 62, 37, 42, 61, 105, 63, 117, 124, 112, 49, 56, 108], [109, 39, 45, 93, 97, 85, 107, 123, 110, 16, 99, 116, 54, 26, 76, 59, 53, 119, 57, 90, 23, 121, 56, 60, 118, 58, 115, 24, 55, 125, 51, 112, 40, 126, 63, 111, 43, 114, 29, 48, 102, 19, 122, 113, 127, 52, 86, 44, 124, 9, 105, 47, 62, 49, 117, 92, 46, 108, 120, 37, 61, 50, 41, 30, 106, 21, 42, 83, 6, 31, 104, 36, 94, 98, 100, 17, 64, 28, 5, 103, 35, 101, 34, 88, 96, 38, 4, 32, 25, 95, 0, 18, 87, 12, 27, 80, 10, 89, 22, 78, 1, 33, 3, 65, 68, 66, 84, 70, 73, 20, 91, 71, 82, 67, 81, 79, 77, 11, 72, 15, 75, 13, 2, 69, 8, 7, 14, 74], [109, 39, 45, 97, 93, 107, 123, 85, 110, 46, 26, 16, 86, 76, 23, 19, 121, 24, 83, 90, 29, 119, 59, 116, 51, 102, 111, 127, 57, 118, 99, 54, 17, 22, 10, 125, 53, 78, 48, 122, 58, 114, 55, 112, 60, 113, 63, 21, 124, 56, 126, 115, 27, 30, 104, 120, 50, 49, 62, 52, 12, 92, 44, 105, 61, 108, 47, 96, 35, 103, 42, 41, 89, 40, 117, 43, 15, 36, 88, 101, 37, 5, 79, 100, 32, 38, 71, 64, 33, 106, 94, 80, 95, 34, 9, 11, 6, 98, 18, 68, 82, 28, 87, 3, 77, 31, 81, 91, 25, 84, 75, 13, 20, 66, 70, 1, 0, 65, 67, 14, 4, 8, 74, 72, 73, 69, 7, 2], [39, 109, 110, 46, 97, 45, 93, 102, 86, 26, 85, 118, 117, 96, 19, 24, 123, 17, 111, 92, 57, 35, 30, 15, 60, 125, 49, 59, 116, 53, 58, 29, 121, 119, 113, 54, 99, 127, 78, 44, 63, 48, 27, 89, 122, 52, 107, 55, 51, 124, 47, 11, 112, 36, 56, 98, 126, 95, 41, 37, 62, 61, 83, 115, 40, 108, 42, 114, 106, 101, 50, 43, 105, 100, 10, 120, 32, 87, 31, 34, 28, 104, 88, 22, 76, 75, 84, 90, 16, 9, 91, 23, 25, 94, 18, 5, 79, 38, 20, 0, 21, 4, 82, 6, 13, 80, 33, 77, 70, 12, 1, 2, 8, 71, 103, 81, 65, 14, 64, 69, 72, 67, 68, 73, 3, 74, 7, 66], [51, 101, 120, 119, 97, 127, 25, 115, 82, 37, 113, 110, 118, 46, 104, 22, 28, 62, 89, 31, 93, 78, 53, 109, 21, 94, 116, 42, 122, 121, 98, 75, 30, 27, 123, 117, 126, 23, 87, 20, 59, 52, 96, 54, 47, 69, 44, 114, 48, 60, 9, 124, 81, 61, 63, 57, 56, 83, 58, 111, 55, 16, 50, 80, 125, 86, 91, 49, 95, 77, 112, 88, 24, 29, 41, 10, 90, 92, 14, 105, 45, 107, 108, 73, 3, 19, 84, 99, 71, 85, 39, 34, 43, 103, 106, 102, 100, 35, 70, 18, 38, 64, 32, 26, 5, 40, 15, 17, 8, 36, 12, 68, 67, 65, 66, 7, 74, 72, 33, 76, 13, 2, 1, 4, 11, 79, 0, 6], [127, 120, 51, 119, 101, 109, 97, 56, 113, 107, 82, 118, 102, 54, 106, 62, 59, 117, 25, 93, 124, 123, 116, 52, 126, 122, 57, 37, 38, 121, 114, 104, 55, 105, 48, 50, 125, 49, 98, 115, 112, 58, 47, 91, 44, 60, 89, 63, 35, 45, 61, 111, 110, 40, 17, 30, 46, 92, 28, 53, 23, 85, 90, 94, 42, 108, 18, 34, 36, 43, 31, 39, 96, 84, 27, 103, 41, 100, 99, 24, 88, 95, 79, 29, 26, 15, 12, 19, 32, 81, 73, 33, 83, 87, 21, 11, 78, 20, 22, 16, 77, 75, 13, 86, 1, 66, 76, 7, 0, 80, 14, 72, 69, 3, 70, 6, 9, 4, 68, 74, 10, 5, 2, 8, 67, 64, 65, 71], [120, 51, 119, 101, 25, 89, 97, 37, 47, 118, 127, 115, 62, 117, 113, 123, 84, 122, 54, 121, 52, 60, 126, 53, 55, 116, 125, 46, 56, 16, 57, 49, 50, 59, 124, 61, 63, 109, 114, 58, 48, 110, 93, 111, 28, 44, 30, 22, 42, 108, 112, 45, 12, 29, 107, 40, 92, 41, 43, 80, 103, 39, 38, 104, 105, 86, 106, 87, 73, 20, 27, 31, 95, 102, 18, 35, 98, 33, 69, 7, 36, 3, 66, 100, 34, 14, 99, 88, 77, 72, 94, 82, 76, 17, 32, 10, 1, 96, 13, 91, 70, 85, 15, 8, 74, 75, 4, 19, 83, 24, 78, 26, 11, 0, 5, 68, 21, 9, 23, 71, 79, 81, 90, 6, 67, 64, 2, 65], [119, 101, 51, 127, 120, 84, 97, 113, 115, 54, 59, 123, 62, 118, 82, 92, 30, 50, 37, 46, 27, 124, 126, 89, 25, 28, 43, 47, 110, 125, 61, 52, 45, 121, 40, 109, 53, 100, 122, 42, 102, 18, 107, 116, 58, 60, 49, 104, 48, 35, 15, 57, 114, 44, 63, 117, 95, 56, 105, 91, 96, 36, 22, 111, 108, 77, 106, 112, 73, 55, 90, 34, 41, 32, 99, 20, 38, 33, 39, 103, 98, 12, 79, 19, 26, 94, 86, 17, 93, 31, 81, 88, 29, 24, 87, 11, 13, 16, 21, 23, 74, 83, 76, 80, 10, 85, 14, 75, 72, 9, 71, 69, 6, 78, 70, 7, 68, 8, 0, 5, 67, 2, 4, 1, 3, 66, 65, 64], [110, 50, 37, 46, 26, 38, 88, 114, 41, 58, 98, 30, 36, 82, 53, 94, 90, 85, 97, 28, 34, 84, 117, 109, 100, 122, 20, 96, 123, 86, 57, 87, 104, 39, 51, 21, 25, 111, 81, 126, 120, 102, 99, 62, 22, 61, 113, 52, 45, 76, 112, 48, 116, 115, 27, 95, 29, 43, 78, 44, 79, 42, 103, 59, 47, 101, 108, 17, 105, 107, 83, 18, 121, 127, 106, 23, 31, 56, 60, 119, 54, 40, 125, 49, 124, 63, 10, 33, 91, 35, 32, 89, 55, 24, 92, 16, 4, 93, 118, 64, 75, 6, 66, 13, 72, 65, 15, 80, 77, 7, 74, 11, 73, 14, 19, 5, 12, 3, 67, 70, 68, 71, 8, 1, 9, 69, 2, 0], [110, 50, 46, 37, 58, 26, 97, 88, 41, 84, 96, 62, 114, 90, 106, 109, 30, 78, 126, 20, 92, 85, 86, 81, 34, 28, 87, 53, 123, 94, 99, 83, 104, 91, 23, 122, 38, 117, 57, 31, 93, 39, 10, 124, 63, 98, 112, 11, 21, 42, 51, 56, 48, 127, 61, 125, 17, 108, 111, 107, 116, 29, 16, 105, 115, 3, 24, 52, 118, 49, 121, 60, 7, 55, 6, 113, 70, 43, 54, 103, 47, 45, 36, 68, 102, 59, 80, 4, 32, 33, 120, 25, 95, 119, 100, 64, 44, 76, 40, 89, 77, 27, 19, 14, 82, 15, 18, 73, 35, 0, 75, 5, 79, 66, 74, 71, 72, 101, 22, 8, 9, 12, 65, 67, 13, 1, 69, 2], [110, 50, 37, 58, 46, 26, 41, 98, 97, 30, 109, 87, 122, 90, 84, 88, 94, 31, 85, 62, 60, 81, 28, 125, 78, 29, 63, 35, 20, 91, 36, 43, 126, 48, 17, 86, 121, 124, 61, 83, 51, 32, 102, 100, 54, 15, 118, 117, 114, 123, 52, 93, 92, 47, 45, 127, 56, 11, 112, 111, 113, 40, 107, 120, 116, 4, 103, 22, 27, 108, 106, 55, 104, 53, 24, 42, 101, 76, 49, 99, 89, 21, 115, 34, 96, 16, 38, 57, 33, 44, 39, 66, 6, 18, 59, 75, 119, 7, 95, 19, 79, 25, 105, 23, 8, 80, 72, 77, 10, 73, 14, 64, 74, 82, 70, 12, 5, 3, 65, 13, 67, 69, 71, 9, 2, 68, 1, 0], [110, 50, 37, 46, 58, 26, 97, 84, 30, 41, 109, 88, 28, 78, 90, 96, 117, 94, 126, 17, 47, 20, 91, 87, 85, 53, 106, 108, 81, 114, 29, 122, 105, 93, 86, 107, 11, 31, 111, 48, 34, 23, 40, 95, 45, 57, 43, 124, 16, 92, 119, 123, 49, 59, 83, 51, 15, 21, 118, 62, 63, 112, 104, 55, 98, 36, 24, 56, 116, 4, 61, 10, 127, 113, 44, 65, 54, 60, 121, 18, 103, 5, 120, 125, 39, 77, 52, 38, 100, 72, 115, 35, 89, 102, 76, 6, 99, 33, 42, 32, 74, 79, 75, 7, 70, 3, 27, 80, 73, 66, 14, 22, 25, 82, 12, 13, 101, 71, 0, 19, 8, 68, 9, 64, 67, 2, 1, 69], [125, 48, 112, 37, 53, 124, 46, 126, 61, 117, 100, 113, 121, 54, 51, 116, 57, 56, 59, 58, 114, 34, 120, 127, 118, 52, 60, 122, 123, 50, 49, 63, 62, 55, 110, 115, 97, 111, 119, 47, 109, 45, 108, 42, 44, 102, 43, 105, 103, 107, 41, 99, 104, 26, 39, 40, 106, 32, 35, 84, 38, 101, 86, 90, 96, 94, 95, 82, 88, 36, 28, 33, 30, 98, 31, 29, 87, 22, 93, 73, 14, 76, 20, 23, 16, 92, 18, 24, 78, 25, 80, 27, 7, 89, 5, 12, 91, 3, 65, 69, 9, 85, 1, 71, 83, 0, 67, 17, 21, 64, 2, 4, 15, 66, 19, 70, 10, 11, 13, 75, 77, 68, 6, 8, 74, 81, 72, 79], [48, 100, 46, 112, 25, 85, 125, 17, 34, 87, 14, 15, 83, 27, 86, 80, 96, 12, 10, 40, 29, 84, 97, 98, 103, 13, 94, 95, 37, 93, 16, 9, 22, 75, 24, 99, 8, 78, 42, 124, 73, 28, 32, 110, 69, 6, 3, 36, 77, 118, 20, 117, 11, 81, 68, 26, 21, 5, 92, 71, 74, 76, 18, 30, 82, 90, 126, 88, 50, 121, 89, 23, 79, 47, 38, 19, 0, 91, 39, 105, 1, 31, 70, 7, 115, 35, 2, 41, 116, 67, 45, 54, 114, 65, 101, 120, 53, 4, 102, 63, 104, 61, 109, 43, 64, 72, 58, 113, 33, 57, 127, 59, 56, 119, 62, 123, 52, 49, 66, 108, 106, 111, 122, 51, 55, 107, 44, 60], [48, 112, 27, 100, 46, 85, 83, 34, 25, 97, 17, 13, 15, 125, 75, 8, 28, 87, 86, 32, 37, 30, 29, 4, 12, 10, 18, 72, 124, 68, 66, 6, 96, 16, 2, 24, 77, 40, 47, 91, 33, 20, 76, 93, 26, 70, 71, 7, 31, 84, 95, 14, 90, 22, 88, 9, 36, 0, 23, 82, 39, 121, 89, 126, 81, 21, 11, 79, 38, 92, 94, 74, 118, 73, 1, 99, 54, 98, 43, 103, 19, 78, 69, 110, 114, 117, 35, 80, 115, 5, 65, 102, 67, 53, 113, 3, 45, 59, 57, 101, 56, 64, 116, 50, 105, 44, 42, 123, 111, 41, 58, 61, 52, 49, 63, 62, 104, 120, 107, 60, 119, 51, 109, 108, 127, 55, 122, 106], [46, 48, 112, 125, 121, 37, 124, 54, 126, 57, 56, 117, 62, 53, 113, 59, 120, 58, 116, 61, 114, 127, 122, 49, 123, 51, 52, 63, 118, 115, 60, 110, 55, 50, 111, 119, 45, 47, 109, 43, 44, 100, 108, 102, 104, 106, 103, 107, 41, 39, 42, 105, 40, 38, 96, 34, 101, 98, 99, 97, 33, 32, 36, 95, 86, 35, 90, 88, 28, 22, 94, 84, 82, 26, 31, 30, 92, 29, 93, 25, 89, 65, 91, 87, 20, 78, 18, 7, 16, 24, 80, 27, 76, 14, 23, 0, 3, 12, 85, 67, 73, 2, 69, 71, 5, 83, 9, 1, 21, 17, 70, 4, 81, 19, 10, 64, 13, 15, 66, 8, 75, 79, 68, 74, 72, 77, 11, 6], [116, 122, 103, 57, 126, 55, 119, 118, 40, 127, 54, 51, 59, 100, 120, 114, 125, 124, 121, 61, 115, 112, 111, 50, 123, 85, 63, 53, 62, 113, 108, 56, 47, 117, 60, 30, 48, 58, 52, 46, 104, 49, 109, 42, 33, 110, 44, 102, 45, 106, 41, 43, 107, 29, 105, 38, 36, 24, 37, 27, 34, 99, 101, 35, 25, 32, 96, 98, 19, 31, 94, 95, 97, 39, 93, 23, 78, 28, 91, 87, 83, 21, 14, 22, 92, 89, 17, 86, 90, 26, 10, 1, 72, 0, 88, 8, 5, 3, 4, 16, 81, 80, 74, 75, 68, 13, 71, 64, 67, 66, 6, 18, 84, 69, 12, 2, 70, 7, 65, 77, 82, 9, 76, 79, 15, 11, 73, 20], [122, 116, 103, 55, 54, 120, 50, 40, 114, 118, 33, 53, 124, 24, 126, 100, 115, 59, 127, 19, 113, 117, 31, 125, 63, 121, 61, 57, 119, 112, 111, 60, 123, 56, 62, 58, 46, 110, 52, 51, 30, 102, 47, 42, 49, 48, 108, 27, 85, 74, 106, 109, 104, 22, 94, 80, 25, 44, 35, 107, 36, 45, 93, 43, 91, 41, 105, 32, 29, 38, 37, 84, 101, 28, 34, 26, 20, 98, 89, 82, 86, 23, 99, 97, 87, 96, 90, 21, 95, 83, 15, 72, 92, 88, 39, 5, 12, 0, 14, 18, 16, 78, 77, 3, 17, 73, 71, 6, 79, 76, 81, 68, 13, 10, 70, 9, 4, 2, 8, 66, 69, 64, 67, 75, 1, 65, 11, 7], [116, 122, 103, 55, 118, 54, 126, 50, 127, 61, 114, 57, 120, 112, 119, 121, 125, 115, 124, 62, 117, 40, 123, 33, 52, 53, 59, 60, 56, 113, 58, 104, 63, 111, 51, 100, 34, 91, 47, 49, 26, 36, 48, 93, 22, 110, 19, 46, 42, 44, 45, 25, 109, 102, 108, 24, 27, 107, 21, 106, 41, 31, 43, 80, 101, 94, 37, 35, 99, 38, 105, 29, 85, 28, 98, 86, 32, 39, 97, 96, 95, 89, 30, 92, 23, 78, 90, 74, 16, 14, 83, 82, 17, 88, 84, 87, 3, 18, 77, 12, 15, 10, 11, 1, 9, 68, 4, 81, 66, 0, 8, 20, 65, 13, 6, 79, 67, 70, 64, 71, 75, 69, 5, 2, 72, 73, 76, 7], [103, 116, 122, 93, 100, 84, 82, 55, 81, 33, 88, 15, 22, 75, 91, 87, 77, 52, 24, 11, 118, 79, 17, 71, 50, 26, 31, 119, 102, 126, 27, 40, 120, 13, 57, 32, 54, 89, 29, 23, 20, 34, 25, 90, 12, 21, 28, 72, 83, 125, 115, 121, 14, 18, 86, 59, 68, 94, 114, 61, 112, 85, 117, 124, 113, 127, 53, 96, 92, 123, 63, 8, 30, 111, 80, 58, 76, 62, 16, 110, 46, 47, 51, 7, 56, 104, 78, 69, 9, 10, 19, 60, 73, 97, 67, 95, 5, 37, 44, 49, 4, 66, 3, 108, 48, 99, 6, 74, 43, 109, 42, 70, 45, 98, 38, 36, 0, 107, 35, 106, 1, 64, 2, 105, 41, 101, 65, 39], [102, 113, 56, 97, 23, 49, 79, 29, 77, 81, 20, 75, 10, 72, 6, 3, 22, 16, 18, 32, 67, 38, 28, 15, 34, 5, 9, 88, 48, 17, 112, 65, 115, 92, 25, 70, 26, 74, 53, 78, 0, 39, 69, 86, 84, 63, 87, 64, 11, 31, 12, 105, 83, 19, 89, 66, 58, 117, 57, 80, 30, 85, 21, 13, 91, 93, 73, 8, 55, 82, 90, 52, 100, 41, 27, 1, 44, 7, 95, 96, 103, 36, 76, 94, 59, 99, 24, 119, 121, 14, 51, 50, 116, 35, 118, 33, 47, 60, 106, 101, 110, 46, 122, 2, 98, 123, 45, 62, 40, 109, 4, 42, 71, 37, 126, 104, 54, 124, 68, 61, 43, 114, 120, 125, 107, 108, 127, 111], [102, 56, 113, 97, 29, 49, 23, 20, 79, 77, 75, 81, 10, 3, 6, 72, 22, 69, 0, 30, 18, 65, 88, 25, 53, 84, 74, 13, 70, 17, 87, 86, 59, 26, 93, 71, 1, 89, 64, 66, 15, 76, 52, 80, 92, 11, 119, 73, 24, 4, 95, 96, 8, 31, 34, 90, 91, 14, 33, 19, 32, 28, 38, 62, 58, 82, 9, 41, 36, 112, 94, 110, 12, 83, 78, 16, 5, 121, 27, 57, 21, 103, 7, 48, 115, 50, 63, 117, 85, 35, 99, 2, 101, 43, 98, 39, 100, 67, 125, 60, 68, 45, 123, 122, 105, 51, 126, 44, 54, 127, 47, 61, 37, 124, 114, 106, 55, 108, 111, 40, 120, 46, 42, 107, 109, 104, 118, 116], [113, 102, 56, 121, 48, 49, 97, 39, 29, 22, 44, 126, 16, 115, 52, 50, 60, 59, 124, 118, 51, 117, 123, 38, 57, 119, 63, 54, 47, 112, 28, 114, 127, 58, 46, 122, 90, 62, 120, 110, 53, 125, 43, 55, 40, 109, 104, 61, 116, 108, 24, 41, 37, 111, 105, 103, 107, 32, 101, 106, 42, 45, 91, 23, 20, 26, 36, 100, 86, 25, 34, 98, 81, 99, 88, 80, 96, 93, 31, 94, 35, 85, 14, 30, 27, 95, 92, 83, 76, 33, 18, 78, 82, 9, 89, 19, 21, 12, 73, 77, 75, 69, 84, 10, 71, 79, 7, 4, 2, 68, 87, 17, 66, 5, 72, 64, 65, 0, 3, 1, 6, 15, 11, 13, 67, 8, 70, 74], [56, 102, 113, 48, 97, 49, 29, 23, 41, 22, 117, 52, 28, 115, 16, 46, 36, 20, 114, 103, 81, 86, 121, 44, 31, 50, 53, 63, 105, 58, 25, 77, 39, 119, 51, 106, 126, 91, 54, 90, 76, 78, 27, 60, 89, 55, 59, 123, 62, 92, 79, 118, 10, 38, 107, 104, 42, 120, 122, 109, 108, 124, 72, 69, 112, 14, 12, 127, 47, 116, 57, 110, 33, 43, 19, 111, 93, 32, 83, 24, 40, 101, 45, 125, 30, 61, 80, 75, 37, 35, 99, 82, 84, 26, 98, 88, 18, 34, 85, 96, 100, 9, 94, 21, 3, 64, 95, 66, 68, 71, 5, 7, 74, 1, 0, 17, 73, 6, 4, 65, 2, 87, 13, 70, 15, 67, 11, 8]], "model.layers.20.self_attn.k_proj": [[126, 114, 46, 37, 86, 34, 50, 64, 124, 28, 12, 16, 25, 51, 19, 66, 119, 57, 120, 7, 60, 58, 115, 82, 53, 127, 123, 62, 116, 61, 74, 122, 14, 113, 112, 9, 117, 63, 56, 125, 54, 121, 55, 47, 118, 48, 44, 111, 43, 79, 59, 49, 13, 45, 109, 29, 39, 101, 95, 6, 52, 107, 24, 3, 41, 36, 42, 108, 102, 104, 87, 105, 106, 40, 94, 103, 21, 38, 98, 4, 33, 90, 27, 32, 23, 8, 35, 72, 96, 5, 1, 99, 15, 100, 91, 80, 30, 110, 77, 97, 89, 84, 20, 93, 70, 88, 17, 2, 31, 11, 18, 65, 68, 26, 81, 75, 92, 69, 76, 85, 78, 73, 67, 0, 10, 83, 71, 22], [107, 34, 43, 29, 116, 51, 90, 78, 81, 8, 127, 19, 31, 70, 12, 56, 0, 110, 7, 120, 87, 32, 68, 61, 63, 3, 74, 62, 75, 54, 2, 39, 121, 88, 21, 66, 100, 24, 92, 73, 85, 58, 46, 95, 28, 86, 109, 45, 52, 59, 69, 77, 126, 117, 37, 1, 48, 113, 57, 36, 33, 47, 42, 114, 53, 27, 64, 4, 11, 84, 99, 18, 15, 5, 50, 119, 108, 97, 112, 65, 82, 80, 124, 44, 125, 16, 30, 71, 102, 22, 25, 41, 122, 40, 94, 60, 118, 105, 55, 101, 104, 10, 79, 13, 106, 38, 89, 115, 20, 26, 49, 9, 123, 111, 23, 35, 91, 72, 76, 96, 17, 14, 67, 93, 83, 6, 103, 98], [45, 103, 46, 33, 29, 109, 86, 19, 85, 123, 90, 119, 5, 10, 16, 64, 78, 57, 116, 76, 71, 52, 17, 112, 54, 126, 127, 121, 65, 3, 53, 115, 125, 51, 122, 107, 55, 48, 113, 58, 56, 114, 118, 59, 120, 60, 24, 124, 9, 2, 47, 44, 61, 63, 34, 111, 89, 62, 108, 106, 67, 13, 41, 11, 49, 32, 117, 100, 43, 99, 50, 72, 35, 38, 101, 104, 6, 31, 69, 105, 23, 42, 37, 15, 18, 88, 30, 95, 82, 87, 36, 91, 94, 102, 25, 84, 92, 75, 0, 40, 110, 27, 93, 98, 14, 1, 77, 96, 28, 20, 8, 4, 73, 79, 66, 81, 26, 83, 68, 22, 7, 70, 74, 39, 12, 97, 80, 21], [51, 37, 119, 120, 127, 33, 22, 46, 122, 123, 121, 50, 118, 56, 45, 124, 113, 52, 126, 61, 116, 53, 62, 117, 48, 125, 106, 115, 54, 55, 60, 57, 30, 59, 58, 63, 40, 89, 91, 49, 108, 114, 101, 47, 110, 104, 111, 112, 41, 109, 80, 44, 102, 103, 43, 42, 107, 38, 105, 39, 16, 92, 93, 95, 23, 36, 32, 82, 35, 34, 31, 21, 84, 29, 25, 86, 99, 96, 100, 28, 77, 76, 26, 98, 87, 88, 19, 72, 12, 27, 90, 94, 24, 83, 70, 78, 15, 85, 10, 75, 97, 7, 9, 74, 81, 14, 17, 13, 68, 3, 79, 11, 5, 2, 20, 18, 73, 65, 6, 4, 71, 64, 67, 8, 69, 0, 1, 66], [46, 50, 110, 101, 26, 30, 58, 20, 114, 85, 78, 33, 17, 4, 66, 88, 11, 76, 79, 41, 62, 7, 72, 126, 64, 75, 5, 53, 122, 70, 6, 3, 19, 34, 87, 73, 18, 23, 91, 86, 15, 68, 83, 16, 32, 77, 60, 84, 81, 49, 43, 10, 65, 117, 100, 118, 105, 127, 24, 112, 109, 31, 51, 92, 45, 67, 61, 120, 48, 35, 57, 63, 56, 29, 40, 107, 96, 25, 89, 22, 27, 113, 119, 125, 82, 52, 80, 124, 55, 116, 21, 93, 0, 1, 115, 121, 103, 108, 44, 54, 106, 99, 28, 74, 38, 123, 95, 13, 104, 39, 47, 59, 98, 36, 102, 111, 42, 8, 97, 14, 9, 37, 12, 2, 90, 94, 71, 69], [112, 48, 36, 86, 15, 17, 125, 110, 83, 75, 25, 10, 27, 33, 13, 29, 85, 70, 32, 30, 8, 124, 117, 46, 56, 126, 121, 54, 84, 82, 115, 61, 28, 57, 53, 2, 116, 98, 0, 47, 4, 62, 90, 120, 59, 51, 80, 63, 102, 60, 114, 87, 123, 35, 127, 44, 113, 50, 109, 52, 122, 118, 49, 58, 92, 105, 111, 55, 41, 119, 77, 45, 11, 104, 26, 107, 42, 73, 43, 38, 6, 103, 101, 106, 37, 31, 14, 108, 40, 74, 79, 65, 99, 7, 88, 16, 34, 23, 72, 39, 12, 76, 96, 95, 67, 21, 93, 78, 3, 24, 20, 89, 91, 9, 69, 94, 97, 100, 19, 18, 68, 71, 81, 5, 66, 22, 1, 64], [122, 116, 39, 22, 55, 97, 54, 61, 126, 118, 52, 117, 112, 124, 57, 121, 50, 123, 120, 115, 62, 113, 127, 27, 58, 56, 114, 53, 59, 63, 51, 95, 119, 98, 60, 26, 34, 49, 125, 111, 47, 110, 104, 45, 42, 44, 46, 82, 109, 48, 24, 107, 108, 80, 32, 38, 35, 41, 43, 84, 102, 37, 106, 16, 105, 101, 12, 36, 96, 40, 14, 99, 85, 92, 23, 15, 29, 9, 33, 94, 90, 17, 77, 19, 30, 31, 93, 100, 28, 10, 71, 75, 103, 81, 87, 25, 91, 89, 20, 79, 78, 76, 70, 67, 83, 18, 11, 8, 72, 1, 88, 13, 69, 86, 74, 5, 3, 21, 73, 0, 6, 66, 7, 4, 64, 2, 68, 65], [113, 56, 38, 33, 22, 93, 81, 23, 79, 72, 77, 20, 6, 10, 75, 112, 0, 3, 120, 69, 117, 27, 57, 65, 16, 49, 52, 115, 103, 54, 59, 110, 50, 126, 46, 122, 51, 28, 53, 60, 124, 26, 58, 55, 61, 119, 125, 102, 118, 105, 62, 123, 5, 121, 48, 95, 116, 2, 94, 47, 63, 45, 127, 83, 108, 30, 114, 99, 109, 106, 9, 107, 66, 100, 89, 67, 32, 104, 44, 43, 35, 21, 14, 76, 40, 42, 18, 96, 78, 85, 88, 24, 101, 39, 111, 34, 73, 68, 41, 25, 4, 36, 31, 37, 29, 90, 98, 12, 92, 1, 80, 19, 7, 71, 74, 82, 64, 91, 87, 84, 8, 17, 70, 11, 13, 15, 86, 97]], "model.layers.20.self_attn.qk_proj": [[46, 113, 56, 116, 122, 112, 126, 48, 110, 51, 114, 45, 107, 43, 119, 120, 50, 109, 127, 37, 93, 125, 121, 54, 49, 22, 52, 123, 86, 101, 58, 39, 102, 118, 55, 29, 115, 53, 57, 90, 63, 124, 60, 62, 103, 26, 33, 92, 61, 97, 59, 17, 81, 34, 23, 83, 38, 19, 84, 89, 47, 117, 91, 87, 85, 24, 14, 111, 44, 32, 108, 41, 78, 98, 21, 16, 74, 30, 80, 88, 25, 20, 105, 36, 79, 100, 75, 77, 10, 15, 106, 95, 11, 12, 13, 28, 104, 27, 76, 31, 94, 72, 6, 82, 40, 18, 8, 96, 35, 7, 42, 67, 71, 3, 66, 9, 64, 0, 99, 73, 5, 2, 70, 4, 65, 1, 69, 68], [46, 113, 56, 116, 126, 112, 122, 48, 110, 45, 51, 114, 120, 107, 119, 43, 50, 109, 127, 93, 37, 125, 121, 101, 49, 52, 58, 123, 86, 22, 54, 102, 39, 29, 53, 103, 115, 55, 60, 97, 61, 90, 118, 117, 63, 34, 57, 59, 33, 124, 26, 62, 38, 81, 92, 23, 85, 87, 83, 24, 47, 89, 19, 17, 30, 98, 91, 84, 14, 41, 78, 16, 88, 32, 111, 21, 80, 36, 75, 95, 20, 76, 100, 44, 25, 108, 10, 77, 15, 12, 11, 74, 104, 79, 105, 106, 13, 42, 28, 94, 96, 6, 27, 72, 82, 31, 40, 9, 8, 64, 18, 35, 3, 71, 66, 2, 7, 0, 99, 70, 5, 68, 67, 1, 73, 69, 65, 4], [46, 113, 116, 56, 126, 112, 48, 122, 110, 51, 114, 45, 119, 107, 43, 120, 50, 109, 127, 93, 125, 37, 121, 22, 86, 101, 49, 54, 52, 58, 55, 39, 29, 63, 102, 123, 103, 115, 33, 118, 60, 90, 53, 124, 61, 97, 62, 92, 26, 34, 57, 59, 30, 81, 17, 23, 24, 83, 38, 117, 47, 14, 85, 87, 44, 88, 19, 21, 41, 84, 111, 108, 98, 91, 32, 95, 78, 100, 89, 80, 25, 105, 16, 28, 36, 75, 104, 74, 79, 31, 20, 15, 77, 106, 12, 27, 76, 13, 94, 10, 82, 72, 18, 11, 64, 35, 96, 67, 0, 42, 8, 6, 9, 40, 70, 99, 3, 66, 7, 69, 2, 5, 71, 68, 73, 4, 65, 1], [46, 113, 116, 56, 126, 122, 112, 48, 110, 51, 114, 45, 107, 120, 119, 43, 109, 50, 127, 125, 93, 37, 49, 22, 101, 121, 54, 58, 55, 53, 39, 102, 115, 86, 63, 29, 60, 103, 52, 61, 97, 90, 57, 118, 62, 34, 124, 33, 117, 26, 59, 92, 47, 17, 81, 123, 19, 14, 87, 30, 83, 38, 75, 23, 24, 78, 88, 16, 80, 85, 95, 20, 98, 106, 89, 91, 84, 21, 44, 111, 31, 79, 15, 105, 25, 12, 100, 32, 28, 74, 42, 41, 104, 13, 27, 76, 36, 94, 10, 11, 77, 82, 96, 108, 67, 72, 40, 18, 3, 8, 2, 70, 7, 66, 64, 0, 5, 71, 6, 35, 73, 69, 9, 99, 68, 4, 65, 1], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 114, 45, 120, 107, 119, 43, 109, 50, 127, 125, 93, 49, 37, 54, 121, 22, 86, 39, 115, 58, 101, 63, 123, 55, 29, 61, 103, 53, 102, 57, 52, 60, 59, 62, 26, 124, 97, 81, 90, 118, 34, 33, 47, 117, 19, 17, 98, 38, 23, 87, 78, 83, 84, 24, 92, 30, 85, 20, 80, 21, 75, 16, 111, 89, 79, 91, 25, 95, 14, 88, 105, 74, 44, 15, 12, 104, 40, 36, 10, 108, 31, 28, 41, 77, 100, 106, 76, 32, 27, 13, 11, 82, 42, 72, 70, 73, 18, 94, 71, 8, 7, 35, 0, 3, 96, 66, 64, 99, 67, 9, 69, 2, 6, 5, 68, 65, 4, 1], [46, 113, 116, 56, 122, 126, 112, 48, 110, 51, 45, 114, 119, 120, 107, 43, 50, 109, 127, 93, 125, 49, 37, 22, 86, 121, 58, 54, 101, 39, 60, 115, 29, 123, 102, 52, 97, 90, 63, 124, 53, 117, 26, 57, 81, 103, 118, 34, 19, 59, 55, 61, 84, 16, 87, 80, 85, 62, 33, 17, 23, 47, 79, 78, 98, 38, 92, 24, 89, 14, 20, 111, 30, 12, 83, 44, 75, 31, 88, 95, 104, 91, 15, 25, 21, 10, 74, 76, 32, 28, 11, 77, 100, 36, 105, 82, 13, 108, 106, 41, 27, 72, 96, 8, 70, 40, 42, 94, 18, 73, 9, 3, 67, 7, 0, 64, 71, 35, 99, 66, 6, 5, 2, 69, 4, 65, 1, 68], [46, 113, 56, 116, 48, 126, 112, 110, 122, 114, 45, 51, 120, 119, 109, 107, 43, 50, 127, 125, 93, 37, 22, 86, 49, 39, 102, 101, 121, 58, 29, 123, 115, 57, 54, 60, 103, 90, 34, 118, 26, 117, 55, 52, 63, 53, 33, 61, 124, 87, 19, 97, 17, 23, 16, 84, 24, 47, 81, 62, 85, 59, 38, 92, 83, 91, 88, 30, 98, 80, 20, 79, 78, 14, 44, 21, 89, 25, 15, 76, 104, 32, 36, 75, 100, 108, 40, 10, 12, 28, 31, 95, 27, 41, 74, 77, 13, 111, 11, 42, 105, 106, 82, 18, 94, 35, 9, 96, 8, 72, 70, 99, 7, 73, 64, 71, 67, 3, 0, 6, 66, 2, 69, 5, 4, 68, 1, 65], [46, 113, 56, 116, 126, 48, 112, 110, 122, 51, 45, 114, 120, 119, 107, 43, 109, 50, 127, 125, 93, 22, 37, 86, 101, 121, 123, 102, 49, 58, 39, 29, 53, 57, 117, 54, 55, 60, 90, 62, 115, 118, 103, 124, 52, 33, 26, 87, 34, 63, 91, 92, 59, 24, 81, 23, 61, 19, 97, 83, 85, 38, 30, 17, 47, 98, 89, 21, 111, 80, 108, 41, 88, 32, 36, 84, 25, 15, 14, 20, 79, 104, 27, 16, 10, 44, 31, 78, 28, 40, 100, 95, 77, 94, 75, 12, 42, 13, 76, 18, 74, 11, 105, 106, 82, 99, 35, 70, 8, 96, 7, 72, 64, 3, 9, 0, 71, 66, 73, 69, 2, 6, 67, 5, 68, 1, 4, 65], [46, 113, 56, 126, 116, 48, 112, 122, 110, 51, 45, 114, 120, 107, 119, 50, 43, 109, 127, 93, 125, 49, 37, 121, 101, 22, 123, 58, 102, 29, 86, 55, 117, 60, 39, 26, 53, 63, 54, 61, 57, 62, 103, 90, 52, 118, 115, 34, 33, 124, 92, 17, 81, 38, 97, 47, 85, 87, 83, 23, 78, 59, 24, 30, 21, 91, 36, 88, 32, 84, 19, 89, 14, 16, 20, 104, 41, 111, 15, 98, 25, 44, 80, 79, 31, 100, 95, 27, 28, 11, 10, 105, 77, 94, 76, 12, 40, 75, 108, 13, 106, 74, 18, 96, 42, 8, 35, 7, 82, 99, 72, 3, 70, 0, 2, 73, 6, 67, 64, 9, 71, 69, 66, 68, 1, 4, 5, 65], [46, 113, 116, 56, 126, 48, 112, 122, 110, 51, 45, 114, 120, 107, 43, 119, 50, 109, 127, 93, 37, 125, 121, 49, 58, 22, 54, 52, 123, 101, 117, 86, 102, 53, 55, 124, 29, 57, 115, 39, 63, 118, 61, 62, 103, 90, 34, 97, 60, 38, 26, 17, 33, 92, 81, 47, 78, 87, 85, 59, 23, 19, 83, 88, 24, 89, 25, 30, 32, 98, 10, 16, 80, 21, 41, 84, 14, 111, 11, 95, 100, 15, 36, 44, 91, 77, 79, 104, 12, 75, 28, 40, 31, 20, 27, 8, 74, 76, 106, 96, 13, 105, 42, 108, 94, 7, 6, 71, 0, 72, 3, 18, 35, 2, 64, 9, 82, 70, 73, 69, 67, 66, 99, 68, 5, 4, 1, 65], [46, 113, 56, 116, 122, 126, 48, 110, 112, 51, 114, 45, 120, 107, 43, 119, 50, 109, 127, 125, 93, 37, 121, 49, 115, 22, 58, 101, 54, 86, 118, 123, 39, 63, 52, 124, 53, 62, 102, 97, 29, 103, 55, 26, 57, 61, 60, 90, 81, 17, 19, 117, 33, 111, 87, 84, 92, 34, 38, 83, 85, 78, 16, 15, 47, 25, 79, 24, 14, 80, 95, 30, 11, 59, 23, 21, 44, 75, 20, 88, 10, 91, 98, 12, 104, 77, 32, 28, 106, 6, 76, 8, 74, 36, 100, 82, 72, 89, 105, 27, 31, 41, 108, 13, 18, 40, 94, 42, 7, 64, 96, 2, 73, 71, 0, 3, 9, 35, 67, 69, 66, 99, 5, 70, 4, 1, 65, 68], [46, 113, 56, 116, 48, 126, 110, 122, 112, 51, 45, 114, 120, 107, 119, 43, 50, 109, 125, 127, 93, 37, 22, 86, 49, 101, 58, 102, 121, 39, 29, 115, 53, 54, 123, 63, 117, 34, 55, 52, 90, 62, 26, 103, 61, 17, 19, 124, 60, 97, 33, 118, 81, 57, 83, 38, 85, 87, 59, 84, 92, 23, 78, 47, 15, 98, 16, 80, 24, 14, 10, 30, 88, 91, 20, 100, 79, 12, 21, 44, 111, 36, 25, 32, 11, 95, 31, 89, 104, 75, 28, 41, 77, 76, 27, 74, 106, 82, 8, 108, 94, 96, 40, 18, 105, 6, 13, 72, 71, 73, 9, 42, 67, 35, 66, 3, 0, 64, 7, 99, 69, 2, 70, 5, 1, 65, 4, 68], [46, 113, 56, 116, 126, 48, 112, 110, 122, 51, 45, 120, 114, 107, 50, 109, 43, 119, 93, 127, 125, 37, 49, 22, 102, 86, 58, 123, 39, 121, 54, 101, 29, 55, 117, 115, 124, 62, 63, 52, 33, 53, 97, 60, 103, 34, 118, 90, 61, 47, 38, 57, 26, 92, 30, 19, 85, 59, 87, 81, 83, 89, 88, 98, 17, 32, 91, 23, 84, 36, 25, 44, 24, 16, 111, 80, 78, 21, 100, 20, 108, 15, 95, 11, 31, 105, 77, 28, 10, 79, 27, 12, 14, 41, 74, 82, 76, 94, 40, 13, 104, 75, 106, 96, 35, 71, 8, 42, 18, 6, 72, 0, 66, 64, 73, 7, 9, 3, 99, 2, 67, 70, 5, 69, 1, 4, 68, 65], [46, 113, 56, 122, 126, 116, 48, 112, 110, 51, 45, 114, 120, 107, 119, 50, 43, 109, 127, 125, 93, 37, 22, 49, 58, 54, 101, 121, 86, 39, 60, 29, 102, 123, 57, 52, 53, 62, 103, 115, 55, 63, 117, 90, 61, 124, 118, 34, 26, 59, 17, 81, 47, 33, 92, 97, 83, 111, 38, 87, 19, 32, 85, 78, 98, 23, 30, 80, 21, 91, 24, 89, 84, 11, 36, 20, 14, 95, 15, 44, 10, 28, 104, 25, 105, 108, 16, 79, 41, 106, 31, 88, 76, 96, 74, 12, 77, 100, 82, 75, 27, 72, 94, 42, 8, 18, 64, 0, 13, 71, 73, 6, 35, 9, 70, 7, 99, 66, 67, 40, 5, 1, 2, 3, 69, 4, 65, 68], [46, 113, 116, 56, 48, 126, 112, 122, 110, 51, 114, 45, 107, 120, 43, 119, 50, 109, 127, 93, 125, 37, 22, 86, 121, 49, 102, 101, 58, 52, 123, 39, 55, 29, 118, 54, 117, 103, 63, 61, 57, 33, 62, 90, 26, 17, 53, 81, 19, 60, 124, 59, 34, 115, 87, 97, 24, 92, 78, 85, 83, 74, 11, 91, 79, 16, 21, 10, 89, 104, 23, 80, 32, 38, 98, 20, 84, 36, 111, 95, 47, 12, 27, 30, 14, 28, 88, 76, 15, 25, 100, 75, 41, 108, 106, 77, 96, 94, 44, 18, 31, 73, 105, 40, 8, 82, 70, 35, 13, 71, 72, 0, 64, 67, 7, 2, 3, 9, 6, 66, 42, 69, 5, 68, 1, 65, 99, 4], [46, 113, 56, 116, 126, 48, 122, 110, 112, 51, 114, 45, 120, 107, 43, 119, 109, 50, 125, 127, 93, 37, 49, 22, 101, 121, 86, 54, 58, 102, 57, 123, 52, 63, 60, 117, 115, 29, 53, 55, 62, 118, 103, 39, 81, 59, 26, 124, 97, 61, 90, 83, 87, 33, 17, 34, 38, 19, 74, 47, 23, 78, 85, 32, 89, 11, 91, 24, 84, 95, 92, 98, 20, 15, 79, 104, 111, 36, 16, 30, 77, 14, 25, 10, 88, 75, 27, 12, 21, 80, 28, 40, 44, 41, 106, 105, 13, 70, 31, 100, 108, 76, 94, 8, 72, 18, 42, 96, 82, 71, 73, 67, 7, 35, 9, 3, 0, 66, 2, 64, 99, 6, 69, 5, 65, 1, 4, 68], [46, 113, 56, 116, 126, 122, 48, 112, 110, 51, 45, 120, 114, 107, 119, 43, 109, 50, 127, 125, 93, 37, 121, 49, 22, 86, 101, 102, 58, 29, 54, 103, 123, 60, 52, 39, 115, 118, 124, 117, 57, 55, 26, 34, 53, 62, 97, 90, 63, 33, 47, 92, 59, 83, 61, 81, 19, 17, 23, 24, 111, 38, 87, 30, 20, 91, 89, 85, 84, 32, 98, 16, 21, 88, 36, 78, 80, 25, 100, 40, 104, 44, 79, 74, 108, 95, 28, 14, 27, 11, 31, 15, 12, 77, 94, 41, 76, 10, 96, 82, 13, 72, 18, 70, 75, 35, 105, 106, 42, 64, 71, 9, 0, 7, 67, 2, 8, 5, 73, 99, 66, 3, 6, 68, 69, 65, 1, 4], [46, 113, 56, 116, 126, 48, 110, 112, 122, 45, 51, 114, 107, 120, 43, 119, 109, 50, 127, 93, 125, 37, 121, 22, 86, 49, 101, 117, 58, 55, 29, 102, 53, 39, 118, 54, 103, 63, 124, 33, 115, 90, 123, 92, 60, 26, 34, 57, 62, 52, 97, 17, 61, 19, 81, 24, 111, 14, 83, 91, 85, 98, 23, 30, 16, 38, 87, 36, 89, 44, 11, 21, 59, 20, 47, 88, 74, 84, 28, 94, 95, 32, 15, 76, 79, 41, 80, 108, 10, 40, 78, 27, 31, 25, 13, 12, 100, 104, 75, 77, 42, 70, 72, 18, 96, 82, 106, 0, 105, 9, 35, 64, 67, 71, 99, 5, 3, 2, 8, 69, 7, 66, 73, 6, 65, 4, 68, 1], [46, 113, 116, 56, 126, 112, 48, 122, 110, 51, 45, 114, 120, 107, 43, 119, 109, 50, 127, 93, 125, 37, 49, 121, 117, 58, 63, 54, 22, 102, 39, 52, 101, 86, 62, 61, 29, 55, 53, 118, 103, 123, 115, 97, 57, 124, 33, 60, 90, 34, 26, 59, 23, 38, 83, 111, 24, 92, 17, 81, 14, 36, 30, 47, 21, 95, 87, 88, 85, 19, 20, 89, 78, 40, 16, 80, 98, 91, 44, 104, 84, 32, 28, 41, 11, 106, 76, 100, 25, 10, 74, 75, 15, 27, 31, 94, 105, 77, 42, 96, 79, 72, 108, 12, 82, 13, 70, 0, 66, 71, 35, 8, 67, 18, 6, 64, 7, 9, 4, 3, 5, 2, 99, 73, 1, 68, 69, 65], [46, 113, 116, 56, 126, 48, 110, 122, 112, 51, 45, 114, 107, 120, 43, 109, 50, 119, 127, 125, 93, 37, 121, 49, 22, 58, 101, 86, 117, 102, 52, 54, 97, 118, 63, 123, 29, 115, 39, 62, 55, 60, 124, 61, 103, 53, 34, 26, 38, 90, 57, 33, 87, 19, 95, 59, 81, 85, 92, 47, 24, 23, 83, 32, 98, 80, 11, 111, 84, 17, 16, 30, 89, 20, 78, 88, 91, 36, 14, 25, 21, 74, 104, 106, 15, 79, 100, 108, 40, 76, 28, 94, 12, 31, 10, 13, 72, 96, 44, 41, 27, 75, 77, 42, 8, 35, 6, 18, 71, 9, 0, 7, 82, 66, 73, 70, 105, 67, 64, 99, 3, 2, 4, 68, 69, 5, 1, 65], [46, 113, 116, 56, 126, 48, 110, 122, 112, 51, 114, 45, 107, 43, 120, 119, 109, 50, 127, 125, 93, 49, 37, 121, 22, 58, 86, 117, 54, 53, 123, 115, 63, 101, 124, 52, 118, 29, 39, 102, 55, 60, 26, 57, 103, 34, 90, 62, 97, 33, 81, 92, 78, 61, 87, 16, 38, 59, 83, 19, 111, 47, 17, 95, 32, 30, 91, 98, 11, 20, 84, 21, 23, 89, 88, 14, 36, 24, 25, 15, 44, 74, 28, 85, 108, 79, 76, 106, 12, 31, 94, 105, 13, 80, 104, 42, 75, 27, 41, 100, 82, 40, 77, 6, 10, 72, 8, 67, 18, 7, 35, 9, 96, 73, 99, 5, 0, 71, 3, 70, 66, 69, 2, 1, 68, 64, 4, 65], [46, 113, 116, 56, 48, 126, 112, 110, 122, 51, 114, 45, 120, 107, 43, 119, 50, 109, 127, 125, 93, 49, 37, 121, 54, 58, 117, 22, 53, 118, 39, 63, 86, 124, 29, 115, 55, 52, 101, 102, 123, 97, 103, 62, 61, 34, 57, 90, 26, 60, 33, 59, 92, 38, 17, 95, 81, 91, 83, 19, 87, 24, 47, 84, 16, 21, 88, 85, 111, 78, 108, 23, 89, 14, 30, 11, 32, 25, 44, 104, 36, 100, 28, 98, 74, 20, 80, 42, 79, 94, 41, 15, 106, 31, 76, 40, 75, 27, 82, 96, 77, 8, 64, 10, 13, 6, 105, 12, 71, 0, 18, 7, 3, 72, 9, 73, 67, 35, 2, 70, 99, 66, 5, 69, 68, 65, 1, 4], [46, 113, 116, 56, 126, 112, 122, 48, 110, 51, 114, 45, 120, 107, 119, 43, 109, 50, 125, 127, 93, 49, 121, 37, 123, 54, 22, 86, 117, 101, 58, 115, 52, 39, 102, 124, 118, 53, 29, 57, 60, 63, 26, 103, 61, 55, 34, 97, 62, 38, 90, 33, 92, 83, 47, 59, 17, 111, 88, 87, 81, 44, 19, 21, 91, 89, 24, 23, 32, 80, 36, 85, 78, 84, 98, 95, 30, 108, 14, 20, 74, 16, 94, 25, 27, 41, 31, 28, 106, 100, 79, 104, 12, 40, 15, 96, 10, 76, 11, 75, 77, 105, 18, 82, 13, 6, 35, 8, 73, 42, 7, 99, 64, 0, 72, 3, 71, 67, 66, 2, 9, 5, 69, 70, 4, 68, 65, 1], [46, 113, 116, 122, 56, 126, 48, 112, 110, 51, 114, 45, 107, 120, 119, 43, 109, 50, 127, 125, 93, 37, 49, 54, 22, 58, 86, 121, 101, 115, 117, 123, 29, 118, 39, 57, 52, 53, 124, 102, 97, 63, 26, 55, 90, 103, 60, 111, 61, 34, 59, 33, 92, 87, 47, 81, 62, 17, 19, 83, 85, 84, 23, 16, 98, 32, 36, 30, 91, 80, 78, 89, 38, 24, 95, 75, 14, 20, 88, 21, 31, 25, 44, 76, 74, 28, 79, 11, 12, 106, 15, 27, 13, 41, 10, 100, 104, 72, 96, 77, 105, 94, 108, 8, 82, 18, 42, 6, 40, 71, 9, 35, 73, 67, 7, 3, 70, 0, 64, 66, 5, 2, 69, 99, 1, 4, 68, 65], [46, 113, 116, 56, 122, 112, 48, 126, 110, 114, 51, 45, 120, 107, 43, 119, 109, 50, 125, 127, 93, 37, 22, 49, 86, 121, 115, 58, 54, 124, 63, 29, 102, 118, 53, 117, 39, 52, 123, 57, 101, 90, 55, 61, 26, 81, 34, 83, 19, 103, 97, 87, 60, 33, 59, 62, 23, 111, 24, 92, 80, 47, 84, 38, 17, 85, 89, 78, 21, 20, 88, 44, 91, 30, 16, 75, 12, 15, 25, 36, 95, 14, 74, 98, 10, 32, 31, 28, 76, 100, 104, 27, 13, 79, 41, 11, 94, 8, 77, 42, 40, 106, 96, 108, 18, 7, 9, 99, 72, 105, 35, 0, 82, 6, 71, 73, 70, 67, 5, 2, 66, 64, 3, 69, 1, 4, 68, 65], [46, 113, 116, 56, 112, 126, 48, 122, 110, 51, 45, 114, 120, 107, 119, 43, 109, 50, 127, 125, 93, 37, 124, 121, 22, 49, 58, 86, 115, 54, 118, 53, 101, 117, 102, 123, 29, 60, 39, 63, 55, 52, 61, 90, 34, 97, 62, 57, 103, 26, 33, 92, 38, 24, 83, 36, 59, 47, 111, 98, 19, 21, 32, 30, 88, 91, 17, 85, 95, 84, 87, 20, 81, 89, 23, 74, 14, 16, 104, 25, 100, 78, 75, 27, 80, 108, 44, 28, 41, 94, 79, 15, 31, 12, 96, 40, 105, 10, 11, 77, 35, 76, 8, 82, 13, 42, 72, 73, 106, 18, 70, 99, 64, 67, 71, 9, 66, 6, 7, 3, 69, 0, 5, 2, 1, 4, 65, 68], [46, 113, 122, 56, 116, 126, 112, 48, 110, 114, 45, 51, 107, 120, 119, 109, 43, 50, 127, 125, 93, 121, 123, 115, 49, 37, 101, 54, 58, 102, 118, 22, 124, 117, 86, 55, 53, 39, 61, 57, 103, 29, 60, 63, 52, 97, 90, 34, 62, 26, 92, 47, 38, 111, 33, 19, 81, 83, 17, 23, 44, 98, 59, 36, 87, 30, 95, 84, 32, 21, 85, 24, 91, 14, 108, 100, 78, 88, 89, 25, 75, 28, 94, 41, 79, 20, 80, 104, 10, 31, 16, 76, 74, 105, 15, 77, 13, 96, 40, 12, 27, 18, 106, 70, 82, 11, 42, 99, 8, 35, 2, 3, 9, 64, 72, 71, 69, 7, 73, 67, 5, 0, 66, 6, 65, 4, 68, 1], [46, 113, 116, 56, 126, 122, 112, 48, 110, 51, 114, 45, 107, 50, 120, 43, 119, 109, 127, 93, 125, 37, 49, 121, 58, 22, 55, 54, 118, 101, 115, 86, 124, 102, 39, 123, 29, 63, 53, 117, 57, 97, 90, 61, 52, 103, 62, 33, 59, 26, 60, 92, 19, 81, 17, 87, 34, 83, 47, 23, 85, 14, 75, 106, 16, 20, 38, 78, 88, 21, 111, 25, 84, 30, 24, 80, 95, 32, 91, 89, 44, 74, 10, 15, 8, 100, 31, 76, 12, 79, 11, 70, 41, 13, 98, 77, 94, 28, 104, 40, 96, 36, 27, 105, 108, 7, 42, 82, 72, 35, 9, 67, 0, 18, 64, 71, 69, 66, 73, 6, 99, 2, 3, 68, 5, 1, 4, 65], [46, 113, 116, 56, 48, 126, 122, 112, 110, 51, 114, 45, 119, 107, 43, 120, 109, 50, 127, 37, 93, 125, 49, 121, 54, 22, 123, 86, 118, 102, 57, 39, 58, 52, 55, 101, 62, 115, 53, 29, 124, 103, 34, 117, 63, 60, 26, 97, 33, 90, 61, 47, 59, 81, 78, 19, 17, 87, 95, 38, 85, 83, 84, 100, 75, 111, 21, 92, 89, 91, 16, 23, 14, 98, 32, 74, 44, 80, 104, 24, 30, 10, 41, 12, 79, 20, 88, 15, 36, 105, 31, 76, 40, 108, 25, 27, 28, 77, 64, 8, 11, 70, 3, 72, 96, 82, 35, 106, 18, 94, 67, 13, 2, 9, 0, 7, 71, 42, 5, 66, 73, 69, 6, 4, 1, 68, 99, 65], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 114, 45, 119, 120, 107, 43, 50, 109, 127, 125, 93, 37, 121, 54, 49, 22, 118, 86, 101, 123, 63, 57, 102, 58, 117, 29, 53, 124, 52, 59, 103, 26, 115, 39, 97, 61, 62, 34, 60, 90, 33, 47, 55, 81, 92, 111, 87, 19, 23, 16, 17, 83, 38, 14, 80, 30, 84, 78, 24, 21, 44, 91, 98, 75, 15, 20, 88, 89, 12, 10, 85, 104, 25, 95, 79, 36, 32, 100, 31, 74, 40, 105, 13, 28, 76, 41, 77, 11, 27, 8, 106, 96, 108, 72, 18, 9, 94, 70, 82, 6, 71, 42, 3, 7, 67, 35, 66, 73, 64, 5, 0, 99, 69, 2, 68, 65, 1, 4], [46, 113, 116, 56, 122, 48, 112, 126, 110, 51, 45, 114, 107, 119, 120, 43, 50, 109, 127, 125, 93, 54, 37, 49, 121, 86, 22, 58, 101, 52, 102, 124, 118, 123, 63, 39, 57, 115, 53, 60, 29, 117, 103, 55, 90, 62, 97, 61, 34, 47, 33, 59, 26, 92, 83, 111, 17, 81, 91, 87, 44, 23, 38, 19, 98, 14, 32, 75, 85, 78, 88, 80, 24, 104, 21, 30, 89, 10, 100, 36, 16, 25, 79, 72, 74, 12, 20, 76, 95, 84, 28, 41, 15, 31, 40, 13, 108, 11, 105, 77, 35, 96, 71, 27, 8, 18, 94, 73, 6, 64, 82, 106, 66, 7, 70, 67, 0, 42, 9, 69, 99, 5, 3, 2, 68, 65, 1, 4], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 45, 114, 120, 119, 107, 50, 43, 109, 93, 127, 125, 37, 22, 121, 54, 86, 101, 49, 52, 102, 58, 124, 39, 118, 123, 53, 29, 60, 103, 57, 90, 55, 61, 97, 34, 115, 63, 62, 117, 33, 92, 47, 26, 81, 83, 38, 59, 85, 111, 24, 19, 91, 98, 23, 87, 17, 89, 88, 14, 30, 78, 32, 21, 12, 84, 75, 44, 20, 95, 36, 16, 100, 80, 10, 15, 79, 25, 41, 76, 104, 27, 74, 77, 105, 13, 31, 72, 94, 40, 28, 18, 106, 108, 11, 96, 42, 6, 8, 82, 70, 9, 7, 35, 73, 71, 3, 64, 5, 99, 0, 69, 2, 66, 65, 67, 4, 68, 1]], "model.layers.21.self_attn.q_proj": [[103, 49, 112, 48, 97, 82, 15, 84, 29, 86, 12, 113, 99, 62, 8, 26, 54, 78, 13, 57, 4, 70, 96, 52, 90, 33, 17, 72, 23, 37, 80, 56, 89, 79, 77, 88, 73, 120, 98, 93, 24, 21, 20, 92, 18, 9, 27, 22, 59, 87, 83, 16, 5, 76, 94, 39, 19, 107, 85, 123, 14, 1, 124, 91, 46, 50, 28, 67, 31, 81, 108, 66, 25, 10, 6, 40, 51, 74, 95, 30, 122, 106, 102, 2, 116, 127, 44, 68, 115, 38, 60, 7, 125, 100, 75, 119, 109, 55, 63, 58, 34, 101, 53, 118, 61, 36, 121, 69, 32, 64, 117, 114, 47, 104, 35, 11, 126, 41, 110, 43, 45, 42, 71, 65, 111, 105, 3, 0], [49, 103, 112, 48, 97, 29, 113, 62, 56, 84, 87, 35, 38, 26, 123, 52, 82, 63, 95, 115, 120, 59, 57, 51, 124, 61, 116, 104, 86, 89, 53, 106, 119, 78, 45, 12, 122, 125, 60, 118, 126, 47, 40, 54, 117, 127, 50, 46, 88, 110, 108, 111, 55, 96, 30, 58, 114, 43, 37, 105, 34, 107, 44, 121, 102, 42, 15, 11, 19, 41, 109, 8, 100, 16, 99, 36, 92, 101, 31, 94, 32, 98, 39, 67, 23, 14, 25, 33, 85, 91, 3, 28, 17, 73, 70, 10, 4, 13, 18, 7, 22, 27, 90, 93, 24, 21, 68, 80, 71, 66, 5, 20, 83, 75, 76, 77, 9, 79, 69, 2, 65, 72, 81, 64, 6, 1, 74, 0], [112, 62, 49, 103, 48, 56, 52, 123, 122, 59, 120, 113, 50, 97, 115, 121, 51, 57, 54, 109, 46, 119, 63, 125, 35, 114, 118, 60, 105, 104, 117, 58, 108, 124, 29, 61, 127, 126, 42, 41, 110, 116, 45, 47, 88, 55, 53, 99, 87, 106, 43, 111, 107, 74, 3, 71, 37, 44, 82, 40, 38, 2, 102, 68, 6, 100, 95, 0, 84, 86, 101, 36, 30, 85, 34, 5, 26, 31, 98, 32, 14, 33, 27, 72, 94, 92, 96, 65, 89, 81, 9, 21, 90, 76, 18, 39, 91, 7, 28, 1, 19, 23, 83, 22, 11, 12, 25, 75, 93, 24, 67, 15, 17, 80, 16, 4, 73, 70, 77, 13, 20, 79, 10, 69, 78, 64, 8, 66], [112, 103, 49, 48, 97, 113, 29, 0, 65, 50, 26, 11, 95, 84, 78, 17, 54, 82, 31, 86, 62, 5, 56, 57, 123, 2, 66, 122, 88, 114, 70, 52, 42, 46, 118, 120, 59, 121, 4, 64, 8, 104, 115, 67, 125, 63, 116, 60, 108, 1, 106, 44, 35, 10, 127, 38, 119, 15, 75, 51, 12, 109, 13, 58, 117, 61, 93, 7, 3, 25, 124, 69, 126, 90, 110, 111, 45, 53, 47, 81, 99, 43, 34, 55, 96, 9, 94, 107, 37, 41, 6, 105, 100, 83, 72, 98, 74, 71, 102, 77, 85, 73, 22, 33, 40, 24, 92, 19, 87, 101, 32, 30, 89, 27, 68, 91, 14, 28, 23, 16, 20, 21, 36, 80, 39, 18, 76, 79], [102, 120, 118, 54, 53, 84, 56, 11, 15, 9, 18, 1, 67, 76, 6, 5, 87, 31, 71, 64, 66, 0, 93, 68, 77, 78, 89, 72, 13, 80, 25, 65, 86, 74, 46, 21, 109, 63, 48, 92, 69, 114, 52, 22, 32, 8, 127, 40, 126, 75, 16, 7, 10, 117, 107, 70, 115, 82, 4, 73, 57, 12, 116, 121, 113, 3, 94, 60, 38, 14, 24, 23, 79, 41, 45, 44, 110, 20, 91, 90, 98, 2, 85, 125, 103, 36, 124, 95, 17, 59, 30, 100, 47, 119, 50, 101, 62, 105, 29, 43, 34, 19, 42, 61, 99, 35, 33, 112, 108, 123, 97, 27, 51, 28, 39, 49, 96, 122, 106, 26, 37, 81, 104, 55, 83, 111, 88, 58], [102, 54, 118, 120, 84, 15, 18, 56, 72, 76, 53, 11, 25, 6, 93, 9, 78, 5, 80, 71, 31, 67, 89, 66, 74, 16, 22, 64, 1, 69, 3, 77, 46, 87, 48, 29, 92, 24, 85, 8, 117, 10, 13, 90, 65, 124, 40, 110, 75, 20, 107, 23, 109, 57, 12, 70, 83, 115, 125, 114, 79, 41, 103, 0, 42, 113, 99, 82, 63, 45, 86, 36, 14, 127, 73, 68, 32, 19, 44, 37, 39, 47, 116, 26, 61, 55, 88, 121, 51, 105, 7, 30, 81, 96, 112, 59, 123, 33, 95, 2, 38, 60, 49, 119, 4, 52, 126, 91, 17, 100, 34, 122, 97, 35, 50, 27, 104, 21, 111, 62, 98, 101, 43, 108, 28, 94, 106, 58], [102, 120, 54, 118, 56, 80, 18, 84, 76, 25, 9, 78, 93, 11, 89, 87, 31, 53, 15, 6, 71, 72, 5, 67, 77, 66, 13, 1, 64, 0, 22, 86, 74, 113, 46, 68, 65, 116, 115, 90, 32, 63, 26, 4, 81, 16, 7, 107, 8, 52, 82, 69, 40, 17, 125, 117, 73, 48, 92, 99, 36, 23, 70, 75, 12, 98, 43, 88, 51, 14, 126, 124, 19, 29, 55, 79, 37, 39, 20, 61, 114, 85, 3, 10, 122, 112, 49, 57, 109, 38, 127, 59, 30, 2, 27, 119, 24, 21, 33, 121, 35, 104, 83, 60, 100, 103, 105, 45, 34, 91, 42, 96, 94, 110, 28, 108, 62, 106, 41, 111, 101, 123, 50, 58, 47, 95, 97, 44], [102, 120, 54, 118, 56, 78, 53, 84, 72, 31, 18, 5, 15, 1, 93, 76, 71, 67, 66, 11, 9, 89, 25, 87, 13, 64, 65, 0, 117, 6, 77, 46, 68, 80, 73, 70, 107, 2, 26, 75, 22, 88, 8, 3, 39, 113, 32, 48, 69, 12, 4, 63, 82, 85, 60, 19, 109, 127, 74, 16, 116, 90, 125, 27, 28, 52, 79, 14, 110, 86, 29, 41, 126, 98, 114, 81, 124, 7, 34, 55, 92, 62, 105, 91, 96, 44, 101, 51, 115, 57, 40, 17, 42, 20, 104, 111, 36, 61, 119, 24, 94, 83, 43, 45, 50, 58, 121, 33, 38, 122, 59, 108, 100, 23, 10, 99, 106, 112, 103, 97, 95, 123, 30, 47, 37, 21, 35, 49], [59, 106, 60, 36, 99, 62, 28, 119, 42, 87, 123, 55, 61, 85, 121, 18, 53, 127, 31, 114, 32, 98, 89, 122, 124, 86, 47, 56, 92, 54, 117, 104, 112, 111, 126, 116, 125, 58, 115, 13, 118, 30, 41, 57, 113, 25, 90, 107, 100, 49, 110, 97, 109, 120, 46, 43, 50, 39, 38, 51, 52, 105, 45, 44, 101, 103, 48, 91, 63, 108, 35, 93, 102, 21, 80, 94, 29, 37, 34, 33, 40, 14, 75, 19, 79, 20, 96, 27, 4, 26, 15, 65, 22, 74, 23, 95, 82, 7, 17, 71, 83, 24, 6, 88, 77, 76, 73, 9, 68, 12, 8, 1, 84, 64, 81, 2, 69, 67, 3, 66, 16, 0, 5, 11, 10, 78, 70, 72], [106, 99, 59, 80, 28, 13, 20, 74, 8, 14, 86, 6, 60, 42, 75, 4, 18, 65, 58, 55, 30, 32, 85, 89, 64, 2, 67, 114, 31, 111, 27, 24, 91, 10, 119, 25, 62, 92, 21, 95, 22, 90, 19, 35, 107, 84, 81, 72, 82, 36, 96, 123, 16, 7, 70, 102, 112, 97, 78, 125, 17, 101, 57, 77, 79, 11, 88, 29, 87, 56, 12, 23, 69, 9, 5, 68, 73, 109, 46, 53, 113, 34, 71, 93, 0, 83, 61, 15, 127, 122, 94, 51, 45, 54, 37, 26, 66, 41, 116, 76, 117, 3, 1, 108, 52, 121, 104, 98, 115, 33, 43, 100, 120, 39, 110, 118, 105, 38, 103, 40, 124, 49, 47, 48, 50, 63, 44, 126], [106, 99, 59, 20, 14, 80, 86, 75, 74, 32, 6, 67, 8, 28, 64, 92, 60, 42, 18, 111, 25, 58, 55, 4, 2, 3, 91, 114, 69, 29, 87, 97, 65, 123, 102, 66, 127, 119, 57, 30, 1, 62, 107, 90, 36, 95, 31, 7, 88, 113, 112, 35, 12, 22, 70, 10, 122, 96, 21, 68, 89, 24, 73, 82, 11, 109, 17, 56, 125, 13, 45, 16, 72, 100, 61, 77, 78, 53, 79, 27, 15, 5, 116, 81, 19, 9, 71, 34, 84, 104, 0, 26, 108, 51, 23, 101, 85, 94, 54, 76, 98, 83, 121, 43, 48, 37, 52, 39, 115, 93, 41, 33, 46, 105, 103, 110, 38, 117, 120, 47, 118, 40, 50, 44, 49, 124, 126, 63], [106, 60, 36, 62, 42, 119, 58, 123, 55, 112, 61, 114, 121, 59, 87, 124, 111, 53, 125, 41, 115, 47, 110, 118, 63, 103, 122, 101, 127, 113, 85, 99, 18, 126, 54, 104, 56, 50, 120, 49, 107, 25, 51, 46, 108, 52, 57, 48, 28, 116, 44, 109, 43, 45, 100, 117, 105, 40, 32, 92, 89, 39, 35, 38, 102, 90, 27, 31, 37, 15, 34, 98, 33, 91, 88, 19, 30, 97, 86, 26, 93, 96, 95, 21, 94, 29, 13, 83, 22, 24, 17, 23, 76, 79, 82, 20, 81, 77, 71, 14, 80, 73, 12, 7, 9, 84, 65, 75, 4, 74, 68, 78, 1, 16, 6, 69, 11, 67, 2, 5, 8, 64, 10, 70, 3, 0, 66, 72], [102, 57, 58, 59, 114, 90, 87, 127, 26, 96, 60, 29, 38, 93, 85, 74, 82, 15, 61, 18, 110, 21, 46, 35, 48, 71, 111, 121, 62, 12, 44, 51, 84, 113, 32, 22, 49, 117, 79, 16, 50, 112, 13, 41, 119, 109, 86, 63, 43, 124, 72, 100, 4, 23, 123, 53, 52, 105, 115, 56, 118, 122, 107, 40, 106, 54, 120, 116, 98, 45, 126, 125, 55, 2, 39, 108, 33, 104, 101, 17, 94, 27, 37, 30, 103, 47, 31, 95, 20, 42, 34, 5, 91, 25, 88, 8, 28, 97, 0, 92, 89, 36, 99, 11, 83, 81, 3, 80, 19, 24, 78, 68, 1, 75, 9, 77, 14, 70, 66, 73, 65, 76, 7, 6, 64, 10, 69, 67], [102, 58, 57, 59, 114, 90, 87, 127, 121, 96, 26, 29, 38, 15, 93, 85, 49, 74, 41, 110, 82, 32, 60, 84, 63, 43, 109, 113, 50, 116, 117, 79, 62, 21, 124, 35, 52, 54, 105, 22, 48, 53, 46, 56, 13, 55, 18, 20, 125, 44, 71, 51, 12, 86, 126, 123, 61, 119, 111, 112, 108, 118, 122, 115, 120, 103, 45, 95, 23, 104, 106, 36, 47, 101, 42, 39, 31, 16, 100, 107, 33, 40, 89, 80, 28, 98, 37, 78, 2, 97, 88, 94, 34, 30, 4, 91, 99, 27, 25, 0, 72, 92, 77, 24, 83, 11, 19, 17, 73, 5, 81, 14, 68, 3, 75, 64, 70, 1, 8, 7, 66, 6, 9, 76, 10, 65, 67, 69], [102, 59, 58, 114, 57, 90, 87, 26, 15, 29, 85, 96, 74, 93, 121, 71, 18, 82, 21, 38, 109, 13, 79, 47, 44, 84, 48, 2, 116, 117, 12, 115, 49, 32, 86, 60, 54, 0, 4, 119, 55, 127, 105, 124, 35, 45, 112, 5, 43, 46, 113, 110, 17, 118, 63, 53, 52, 123, 41, 111, 37, 72, 122, 65, 61, 23, 120, 51, 50, 66, 25, 108, 1, 126, 62, 81, 56, 106, 89, 3, 22, 20, 104, 125, 16, 100, 24, 101, 36, 107, 83, 98, 8, 91, 28, 40, 94, 67, 19, 103, 31, 39, 42, 7, 68, 77, 10, 99, 95, 64, 27, 33, 75, 6, 88, 92, 73, 78, 30, 9, 34, 70, 80, 97, 11, 76, 69, 14], [102, 59, 58, 57, 127, 114, 90, 60, 26, 87, 96, 15, 85, 71, 38, 29, 74, 121, 18, 93, 105, 35, 12, 84, 49, 108, 82, 79, 41, 47, 4, 120, 110, 32, 22, 54, 21, 52, 119, 43, 118, 46, 122, 117, 124, 62, 20, 123, 116, 109, 16, 111, 104, 44, 55, 99, 112, 113, 13, 115, 61, 86, 63, 89, 40, 5, 2, 48, 107, 50, 45, 30, 103, 23, 98, 51, 0, 39, 106, 94, 125, 36, 126, 37, 33, 56, 34, 53, 42, 101, 100, 24, 27, 97, 8, 83, 17, 66, 28, 73, 31, 25, 95, 68, 88, 92, 19, 91, 1, 78, 3, 77, 72, 81, 7, 65, 64, 11, 70, 80, 9, 76, 14, 75, 10, 69, 6, 67], [126, 104, 99, 87, 28, 32, 83, 92, 85, 81, 127, 110, 79, 47, 42, 76, 115, 111, 119, 72, 11, 15, 30, 74, 51, 113, 31, 108, 21, 95, 116, 78, 37, 50, 24, 57, 122, 45, 6, 48, 25, 12, 90, 96, 5, 121, 56, 68, 26, 53, 23, 107, 55, 106, 120, 62, 59, 101, 70, 123, 89, 17, 49, 43, 46, 109, 105, 41, 97, 13, 19, 2, 33, 112, 93, 39, 52, 124, 82, 40, 65, 114, 117, 34, 0, 125, 102, 61, 29, 118, 3, 44, 80, 60, 20, 84, 58, 54, 100, 88, 27, 22, 94, 36, 98, 103, 64, 18, 86, 91, 75, 63, 38, 66, 16, 10, 67, 14, 9, 77, 1, 73, 4, 71, 35, 7, 69, 8], [126, 104, 99, 87, 28, 83, 85, 32, 127, 92, 47, 115, 81, 79, 119, 51, 106, 76, 50, 118, 72, 48, 74, 111, 113, 42, 108, 15, 116, 45, 120, 31, 122, 121, 6, 58, 5, 61, 110, 57, 11, 41, 12, 70, 68, 37, 53, 91, 54, 21, 75, 43, 9, 97, 89, 49, 23, 38, 46, 30, 105, 13, 55, 59, 56, 40, 35, 65, 0, 62, 123, 17, 117, 109, 107, 34, 78, 33, 114, 60, 95, 103, 98, 112, 25, 67, 52, 39, 3, 93, 101, 36, 82, 19, 63, 102, 125, 18, 44, 26, 90, 29, 94, 2, 24, 66, 124, 20, 96, 86, 100, 84, 27, 73, 88, 80, 1, 69, 14, 22, 10, 16, 7, 71, 77, 4, 8, 64], [126, 104, 99, 87, 28, 32, 85, 81, 92, 83, 115, 47, 127, 79, 110, 42, 76, 45, 121, 116, 58, 95, 108, 68, 31, 15, 78, 72, 41, 113, 60, 86, 106, 5, 54, 122, 117, 11, 119, 96, 50, 9, 24, 26, 6, 7, 52, 91, 22, 0, 101, 21, 57, 8, 12, 90, 17, 111, 66, 13, 44, 18, 23, 93, 94, 30, 55, 105, 114, 70, 118, 37, 49, 51, 25, 123, 48, 97, 88, 46, 74, 63, 38, 56, 39, 43, 59, 27, 120, 82, 34, 53, 62, 89, 102, 33, 103, 19, 80, 36, 61, 98, 124, 125, 14, 112, 109, 29, 100, 16, 107, 73, 84, 2, 20, 10, 35, 4, 77, 40, 71, 3, 64, 67, 1, 75, 65, 69], [104, 126, 99, 28, 87, 85, 32, 83, 81, 79, 92, 127, 116, 89, 111, 76, 47, 48, 25, 50, 72, 122, 15, 74, 54, 6, 11, 106, 51, 12, 119, 2, 52, 68, 105, 43, 58, 57, 42, 118, 3, 45, 108, 1, 13, 14, 31, 70, 23, 94, 5, 0, 115, 17, 29, 110, 41, 107, 113, 64, 93, 21, 46, 63, 120, 30, 123, 26, 61, 82, 39, 8, 125, 62, 102, 59, 112, 75, 90, 124, 53, 37, 60, 9, 19, 86, 65, 101, 84, 78, 73, 117, 69, 44, 40, 109, 33, 114, 77, 18, 24, 121, 34, 56, 38, 103, 66, 10, 95, 36, 100, 49, 97, 55, 98, 27, 7, 22, 67, 4, 35, 91, 88, 96, 20, 16, 80, 71], [62, 126, 48, 39, 116, 53, 30, 27, 122, 88, 20, 35, 117, 76, 121, 15, 91, 106, 112, 83, 55, 57, 79, 81, 94, 58, 86, 32, 123, 124, 52, 46, 49, 72, 19, 84, 33, 60, 98, 63, 89, 115, 100, 127, 50, 29, 24, 34, 56, 107, 114, 5, 41, 105, 108, 109, 51, 61, 44, 113, 118, 110, 26, 125, 47, 42, 23, 119, 87, 103, 18, 28, 43, 120, 36, 111, 17, 54, 22, 37, 59, 45, 31, 75, 95, 104, 93, 10, 97, 13, 102, 12, 85, 99, 40, 25, 38, 21, 74, 101, 90, 92, 78, 2, 14, 8, 82, 80, 96, 9, 16, 77, 3, 1, 11, 71, 73, 69, 6, 7, 70, 67, 66, 68, 4, 0, 65, 64], [116, 126, 39, 62, 48, 55, 57, 117, 123, 46, 53, 125, 111, 49, 121, 30, 50, 60, 41, 20, 122, 52, 127, 51, 120, 63, 114, 124, 89, 56, 115, 35, 59, 58, 113, 118, 43, 109, 32, 86, 54, 112, 47, 61, 88, 42, 106, 119, 108, 44, 45, 110, 107, 37, 34, 91, 40, 103, 105, 27, 104, 81, 23, 92, 102, 80, 38, 98, 29, 94, 101, 33, 28, 99, 36, 22, 87, 93, 9, 79, 100, 90, 73, 95, 96, 13, 97, 25, 31, 76, 26, 14, 82, 85, 84, 68, 83, 17, 1, 77, 21, 15, 75, 7, 78, 18, 2, 24, 16, 3, 71, 70, 11, 72, 4, 67, 0, 74, 12, 10, 5, 66, 65, 6, 19, 64, 8, 69], [126, 62, 116, 39, 48, 117, 55, 57, 53, 111, 123, 112, 49, 122, 114, 58, 120, 118, 124, 50, 20, 35, 127, 121, 42, 51, 30, 109, 88, 46, 56, 32, 52, 60, 28, 41, 115, 119, 43, 113, 125, 59, 47, 63, 106, 108, 103, 94, 44, 61, 110, 54, 101, 45, 89, 104, 107, 40, 86, 34, 38, 102, 100, 105, 81, 99, 37, 95, 87, 29, 93, 23, 92, 14, 36, 98, 80, 33, 97, 79, 21, 26, 31, 90, 27, 85, 91, 18, 22, 96, 25, 13, 83, 15, 77, 17, 84, 24, 76, 9, 82, 12, 69, 73, 75, 70, 78, 2, 19, 11, 16, 72, 10, 64, 67, 66, 3, 4, 0, 6, 71, 65, 68, 74, 1, 8, 5, 7], [126, 116, 39, 48, 55, 57, 27, 123, 30, 117, 83, 86, 122, 52, 37, 20, 53, 26, 62, 118, 110, 89, 112, 91, 72, 17, 19, 35, 32, 121, 16, 41, 94, 10, 79, 4, 74, 76, 88, 15, 125, 124, 114, 60, 25, 120, 115, 49, 29, 107, 56, 63, 22, 13, 106, 97, 50, 58, 51, 93, 42, 46, 34, 113, 127, 80, 111, 109, 81, 61, 108, 28, 5, 105, 66, 95, 100, 59, 31, 47, 75, 119, 38, 24, 23, 87, 33, 92, 44, 98, 9, 54, 21, 78, 45, 14, 36, 18, 77, 102, 84, 67, 7, 104, 43, 101, 68, 103, 8, 73, 6, 12, 90, 40, 11, 99, 82, 69, 85, 0, 71, 65, 64, 96, 70, 2, 3, 1], [55, 103, 62, 52, 117, 98, 25, 57, 89, 92, 115, 28, 119, 54, 20, 83, 22, 78, 120, 126, 48, 121, 73, 19, 58, 50, 76, 27, 51, 82, 45, 125, 60, 111, 49, 32, 109, 12, 123, 112, 107, 94, 81, 124, 53, 40, 33, 113, 110, 38, 61, 118, 114, 43, 42, 116, 86, 17, 100, 84, 59, 14, 63, 47, 46, 108, 127, 97, 79, 95, 101, 18, 56, 96, 122, 106, 88, 44, 5, 104, 105, 41, 6, 37, 77, 99, 36, 15, 35, 8, 75, 39, 70, 93, 80, 30, 102, 91, 26, 72, 31, 16, 90, 3, 23, 85, 29, 66, 24, 64, 71, 34, 68, 69, 21, 1, 11, 9, 87, 0, 74, 67, 13, 65, 7, 10, 2, 4], [55, 103, 98, 20, 25, 89, 120, 92, 112, 51, 117, 28, 126, 78, 54, 84, 48, 62, 57, 73, 58, 52, 50, 121, 32, 118, 125, 99, 95, 127, 22, 108, 56, 119, 122, 59, 124, 110, 116, 83, 104, 109, 47, 61, 15, 111, 106, 113, 46, 53, 115, 49, 101, 82, 43, 79, 63, 114, 100, 107, 40, 44, 37, 67, 123, 60, 24, 97, 45, 35, 102, 27, 38, 42, 31, 11, 76, 74, 88, 105, 87, 36, 68, 91, 81, 23, 71, 94, 29, 19, 18, 26, 93, 33, 85, 30, 41, 90, 21, 39, 17, 9, 96, 86, 12, 10, 77, 3, 14, 80, 69, 34, 1, 8, 16, 2, 70, 13, 65, 6, 75, 72, 4, 7, 5, 66, 64, 0], [55, 103, 57, 62, 25, 98, 89, 83, 92, 78, 117, 20, 52, 54, 28, 120, 112, 22, 126, 125, 115, 48, 32, 76, 84, 61, 73, 51, 118, 50, 58, 19, 119, 41, 116, 60, 53, 64, 8, 108, 5, 56, 113, 105, 109, 27, 59, 49, 46, 127, 121, 95, 111, 114, 47, 12, 82, 124, 63, 123, 110, 66, 43, 65, 106, 44, 107, 30, 17, 122, 6, 38, 104, 3, 39, 79, 99, 45, 16, 72, 77, 81, 102, 86, 40, 13, 37, 94, 100, 36, 91, 14, 42, 75, 93, 2, 35, 23, 69, 101, 29, 97, 33, 90, 85, 34, 26, 70, 68, 31, 88, 24, 74, 10, 80, 96, 1, 15, 87, 18, 0, 21, 9, 67, 11, 7, 4, 71], [55, 103, 62, 52, 98, 20, 82, 92, 89, 22, 25, 117, 63, 51, 108, 112, 84, 57, 58, 54, 73, 56, 125, 50, 28, 44, 71, 119, 124, 48, 32, 79, 11, 15, 109, 87, 120, 74, 46, 53, 49, 43, 18, 97, 42, 126, 121, 110, 116, 77, 59, 113, 101, 45, 122, 61, 115, 21, 39, 107, 114, 106, 37, 30, 118, 26, 88, 123, 75, 111, 2, 104, 100, 31, 41, 127, 68, 14, 38, 60, 93, 99, 81, 102, 33, 94, 29, 17, 40, 105, 95, 85, 1, 47, 83, 23, 35, 7, 10, 27, 12, 8, 24, 90, 91, 0, 80, 19, 86, 4, 70, 96, 13, 36, 78, 16, 67, 69, 34, 6, 3, 76, 9, 5, 66, 72, 65, 64], [121, 127, 56, 55, 39, 118, 63, 98, 31, 51, 122, 89, 25, 57, 124, 120, 119, 84, 112, 125, 18, 105, 92, 53, 62, 59, 58, 52, 115, 15, 50, 49, 110, 28, 23, 47, 76, 54, 108, 60, 117, 116, 61, 114, 111, 113, 106, 126, 123, 10, 46, 30, 16, 107, 40, 24, 104, 48, 32, 96, 102, 43, 101, 6, 44, 90, 45, 41, 80, 87, 109, 99, 88, 42, 94, 79, 22, 36, 100, 14, 83, 95, 38, 37, 85, 69, 82, 93, 19, 75, 20, 86, 27, 77, 9, 81, 35, 97, 8, 34, 74, 71, 64, 13, 78, 2, 26, 103, 33, 12, 67, 72, 29, 65, 21, 11, 5, 91, 70, 68, 7, 17, 3, 1, 4, 0, 73, 66], [55, 127, 56, 39, 121, 118, 63, 122, 51, 89, 31, 84, 57, 120, 125, 18, 124, 119, 105, 98, 116, 62, 112, 117, 52, 53, 25, 115, 59, 50, 92, 95, 28, 76, 60, 15, 61, 49, 106, 47, 96, 111, 58, 54, 48, 123, 114, 126, 40, 113, 23, 108, 90, 85, 110, 46, 45, 107, 43, 44, 30, 109, 36, 102, 16, 101, 41, 38, 42, 79, 104, 24, 10, 103, 99, 20, 100, 22, 37, 35, 14, 87, 32, 33, 34, 97, 93, 88, 81, 80, 83, 9, 86, 82, 77, 27, 94, 29, 91, 78, 19, 75, 8, 74, 13, 21, 72, 71, 4, 26, 12, 17, 70, 73, 2, 6, 11, 5, 68, 65, 0, 69, 1, 64, 7, 3, 67, 66], [127, 39, 56, 63, 118, 55, 31, 98, 89, 51, 122, 57, 84, 92, 53, 25, 120, 125, 62, 105, 47, 115, 54, 96, 30, 126, 119, 52, 124, 18, 112, 114, 87, 59, 110, 116, 40, 44, 50, 58, 49, 90, 48, 113, 103, 85, 117, 60, 107, 23, 45, 16, 123, 111, 61, 121, 95, 106, 43, 76, 108, 102, 21, 35, 46, 104, 42, 79, 109, 26, 22, 34, 99, 41, 101, 83, 93, 15, 36, 32, 28, 38, 91, 37, 100, 24, 20, 27, 33, 19, 82, 97, 81, 94, 10, 80, 86, 74, 88, 29, 14, 77, 72, 17, 75, 8, 12, 13, 78, 9, 5, 69, 71, 6, 4, 11, 2, 70, 66, 68, 64, 73, 65, 67, 3, 7, 1, 0], [56, 39, 127, 55, 121, 118, 51, 31, 63, 122, 57, 84, 92, 25, 105, 120, 124, 62, 59, 115, 119, 125, 53, 50, 112, 52, 98, 47, 58, 116, 60, 49, 95, 89, 61, 113, 126, 110, 123, 114, 117, 54, 18, 44, 111, 41, 79, 48, 43, 108, 45, 90, 23, 76, 109, 101, 106, 104, 107, 42, 46, 96, 40, 102, 88, 100, 38, 10, 28, 15, 37, 36, 99, 35, 22, 29, 94, 97, 32, 103, 30, 82, 33, 83, 16, 86, 93, 77, 34, 8, 21, 27, 75, 87, 20, 85, 24, 80, 9, 72, 5, 91, 26, 81, 12, 13, 74, 17, 19, 71, 78, 14, 2, 4, 70, 73, 6, 69, 64, 3, 11, 67, 66, 0, 68, 7, 1, 65]], "model.layers.21.self_attn.k_proj": [[112, 49, 39, 86, 93, 57, 84, 15, 33, 26, 54, 123, 35, 50, 82, 122, 12, 8, 121, 52, 59, 64, 119, 60, 113, 116, 127, 118, 13, 125, 109, 58, 51, 73, 126, 117, 70, 111, 114, 53, 120, 55, 108, 29, 63, 48, 115, 78, 66, 10, 110, 46, 32, 47, 56, 43, 61, 44, 45, 40, 107, 88, 124, 62, 41, 68, 87, 42, 101, 106, 105, 16, 102, 38, 36, 65, 17, 95, 104, 100, 34, 94, 98, 89, 67, 27, 5, 92, 24, 4, 91, 37, 9, 30, 6, 7, 83, 96, 19, 80, 85, 28, 81, 69, 31, 21, 99, 22, 3, 90, 75, 71, 25, 2, 97, 11, 79, 74, 23, 18, 1, 77, 20, 14, 76, 72, 0, 103], [120, 54, 64, 118, 9, 84, 78, 76, 15, 5, 18, 53, 71, 11, 72, 65, 56, 38, 6, 67, 25, 2, 3, 0, 66, 13, 87, 29, 80, 4, 95, 69, 1, 89, 93, 22, 85, 102, 92, 31, 10, 70, 7, 116, 68, 43, 86, 73, 40, 24, 88, 17, 90, 19, 8, 50, 110, 32, 39, 114, 26, 112, 55, 63, 57, 115, 46, 27, 83, 21, 124, 49, 48, 99, 36, 52, 45, 113, 51, 16, 111, 33, 74, 109, 91, 47, 28, 103, 125, 96, 81, 30, 59, 41, 105, 23, 98, 60, 108, 97, 121, 119, 100, 117, 77, 75, 34, 62, 94, 12, 127, 58, 61, 82, 35, 126, 122, 123, 44, 42, 104, 14, 37, 107, 20, 101, 106, 79], [42, 59, 35, 86, 55, 64, 8, 119, 58, 47, 80, 6, 14, 20, 92, 74, 114, 75, 25, 2, 62, 60, 18, 53, 127, 106, 13, 116, 112, 123, 57, 61, 56, 122, 4, 50, 125, 118, 95, 54, 87, 65, 67, 124, 96, 117, 51, 111, 126, 109, 48, 99, 110, 121, 49, 113, 103, 52, 69, 45, 120, 108, 30, 115, 85, 46, 36, 38, 90, 44, 76, 63, 40, 28, 32, 26, 91, 5, 100, 43, 83, 107, 94, 0, 98, 37, 3, 41, 101, 33, 105, 34, 15, 71, 97, 17, 102, 93, 79, 1, 104, 39, 27, 68, 73, 9, 88, 19, 31, 23, 29, 66, 21, 7, 24, 89, 72, 81, 12, 84, 16, 22, 77, 11, 78, 82, 10, 70], [38, 57, 59, 58, 93, 32, 26, 85, 87, 84, 15, 18, 74, 71, 0, 13, 123, 45, 52, 12, 111, 121, 22, 126, 63, 116, 110, 1, 55, 117, 42, 127, 16, 107, 125, 108, 62, 51, 114, 54, 61, 109, 56, 53, 60, 124, 99, 122, 25, 106, 3, 112, 46, 115, 48, 118, 41, 119, 49, 5, 43, 50, 89, 44, 105, 102, 4, 120, 2, 24, 104, 75, 47, 113, 68, 35, 101, 95, 14, 103, 39, 17, 40, 66, 19, 69, 100, 83, 78, 36, 70, 27, 81, 9, 34, 98, 88, 37, 33, 92, 30, 28, 97, 72, 82, 31, 94, 91, 11, 73, 8, 80, 7, 20, 29, 86, 6, 77, 65, 23, 21, 67, 76, 90, 64, 79, 96, 10], [40, 126, 83, 81, 92, 35, 115, 85, 87, 15, 74, 111, 96, 89, 72, 76, 46, 42, 79, 52, 122, 12, 68, 58, 0, 28, 70, 106, 127, 114, 48, 6, 66, 5, 14, 54, 44, 57, 116, 11, 119, 31, 2, 56, 41, 1, 3, 43, 123, 110, 65, 125, 32, 118, 60, 75, 107, 9, 109, 29, 50, 82, 112, 120, 78, 49, 101, 93, 7, 13, 25, 121, 84, 59, 77, 88, 90, 86, 24, 34, 64, 8, 67, 108, 94, 30, 53, 20, 61, 47, 27, 39, 55, 117, 113, 69, 100, 26, 36, 22, 33, 105, 97, 103, 99, 102, 91, 51, 124, 95, 73, 98, 38, 23, 37, 16, 62, 45, 4, 63, 80, 10, 18, 71, 21, 19, 17, 104], [126, 103, 116, 86, 99, 55, 96, 52, 94, 62, 48, 26, 53, 112, 57, 109, 42, 91, 88, 63, 28, 58, 113, 56, 123, 20, 29, 110, 121, 102, 117, 50, 114, 47, 92, 124, 83, 35, 13, 115, 60, 81, 119, 127, 120, 59, 105, 125, 51, 37, 97, 43, 98, 49, 122, 61, 54, 111, 107, 95, 89, 46, 30, 33, 44, 79, 45, 118, 106, 10, 70, 40, 101, 108, 15, 36, 31, 104, 34, 7, 76, 27, 41, 85, 65, 38, 100, 72, 73, 87, 23, 66, 21, 14, 93, 0, 18, 5, 3, 90, 32, 17, 80, 25, 82, 6, 39, 71, 68, 16, 4, 8, 74, 24, 22, 84, 75, 77, 78, 12, 19, 9, 2, 11, 67, 69, 1, 64], [55, 39, 34, 22, 28, 89, 62, 125, 78, 117, 20, 53, 58, 50, 51, 54, 83, 109, 96, 112, 61, 59, 44, 113, 49, 114, 48, 121, 118, 119, 111, 60, 116, 63, 124, 126, 46, 127, 8, 47, 52, 64, 5, 56, 123, 82, 73, 43, 120, 91, 42, 122, 115, 12, 106, 1, 108, 110, 45, 30, 40, 18, 104, 79, 105, 107, 95, 57, 37, 93, 100, 81, 98, 23, 76, 41, 70, 77, 36, 3, 38, 66, 102, 27, 26, 90, 101, 35, 33, 31, 2, 32, 4, 11, 85, 99, 92, 10, 75, 24, 88, 97, 0, 80, 6, 65, 15, 17, 29, 21, 71, 94, 69, 103, 16, 25, 74, 87, 84, 13, 19, 7, 67, 68, 14, 9, 72, 86], [103, 127, 56, 34, 22, 55, 121, 95, 28, 118, 63, 122, 89, 116, 51, 125, 115, 62, 119, 57, 59, 58, 60, 53, 43, 52, 38, 124, 49, 48, 47, 117, 114, 61, 84, 92, 120, 123, 46, 106, 111, 44, 112, 113, 54, 126, 107, 50, 108, 71, 45, 23, 105, 29, 80, 109, 42, 75, 110, 40, 104, 35, 16, 41, 78, 101, 99, 37, 27, 98, 96, 102, 100, 18, 36, 10, 12, 64, 25, 33, 81, 86, 97, 30, 94, 87, 68, 79, 91, 39, 93, 88, 13, 90, 32, 66, 85, 83, 82, 17, 24, 26, 69, 21, 31, 1, 9, 20, 6, 8, 73, 14, 77, 19, 4, 5, 7, 76, 11, 65, 15, 74, 67, 2, 3, 72, 0, 70]], "model.layers.21.self_attn.qk_proj": [[126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 53, 106, 121, 39, 28, 48, 103, 84, 20, 79, 119, 114, 25, 38, 15, 89, 29, 113, 86, 82, 122, 92, 40, 60, 22, 52, 23, 87, 99, 90, 35, 115, 18, 32, 12, 14, 111, 46, 78, 76, 6, 85, 72, 50, 117, 93, 51, 64, 75, 61, 21, 96, 125, 19, 10, 43, 26, 63, 123, 30, 108, 31, 34, 0, 83, 110, 124, 9, 45, 17, 44, 47, 8, 16, 11, 74, 80, 104, 73, 105, 5, 7, 69, 66, 95, 109, 67, 81, 107, 2, 97, 65, 13, 1, 71, 98, 68, 27, 77, 37, 33, 3, 41, 70, 36, 24, 88, 101, 94, 91, 4, 100], [126, 55, 120, 54, 59, 112, 49, 127, 57, 58, 56, 116, 42, 102, 118, 62, 53, 106, 121, 39, 28, 103, 48, 79, 20, 114, 84, 25, 29, 38, 89, 113, 119, 92, 22, 23, 60, 15, 99, 90, 52, 122, 87, 82, 12, 115, 86, 32, 35, 18, 78, 14, 51, 111, 40, 6, 76, 72, 117, 123, 93, 85, 75, 50, 125, 46, 0, 43, 44, 63, 108, 19, 26, 17, 21, 61, 64, 47, 30, 69, 31, 66, 124, 8, 74, 73, 11, 9, 96, 10, 109, 105, 104, 34, 110, 45, 2, 5, 83, 7, 95, 71, 13, 67, 27, 80, 16, 3, 88, 81, 33, 98, 1, 65, 107, 36, 97, 77, 41, 37, 68, 70, 4, 101, 91, 94, 24, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 102, 62, 106, 118, 53, 121, 28, 39, 103, 84, 48, 29, 20, 25, 15, 99, 22, 92, 23, 122, 119, 38, 79, 114, 86, 89, 115, 90, 113, 60, 87, 35, 117, 52, 78, 40, 82, 18, 76, 14, 12, 111, 32, 6, 51, 64, 85, 72, 46, 0, 96, 124, 44, 125, 75, 63, 93, 31, 21, 69, 43, 30, 34, 108, 2, 50, 19, 123, 10, 110, 5, 74, 8, 17, 65, 66, 11, 47, 109, 83, 9, 104, 61, 26, 98, 73, 70, 16, 1, 95, 7, 80, 67, 13, 3, 81, 77, 41, 105, 45, 36, 88, 27, 71, 107, 4, 68, 33, 24, 97, 37, 91, 101, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 102, 118, 62, 121, 106, 53, 39, 28, 48, 103, 84, 20, 89, 79, 15, 29, 92, 119, 22, 25, 99, 113, 23, 60, 38, 52, 115, 40, 14, 82, 122, 90, 35, 18, 114, 12, 86, 78, 111, 76, 32, 87, 117, 46, 85, 6, 0, 17, 51, 72, 125, 96, 50, 30, 5, 11, 75, 93, 63, 123, 43, 110, 69, 47, 44, 34, 70, 19, 64, 2, 8, 10, 104, 83, 108, 124, 31, 21, 26, 61, 16, 66, 109, 65, 73, 74, 1, 9, 77, 98, 27, 81, 95, 45, 105, 67, 80, 7, 107, 71, 3, 88, 41, 101, 33, 36, 4, 13, 68, 91, 24, 97, 37, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 118, 102, 62, 106, 121, 53, 39, 103, 113, 28, 20, 84, 48, 60, 79, 38, 15, 23, 89, 119, 114, 29, 115, 82, 25, 92, 22, 86, 52, 90, 76, 122, 99, 40, 18, 78, 12, 32, 117, 35, 111, 14, 51, 50, 75, 46, 87, 85, 72, 96, 63, 125, 8, 70, 64, 0, 21, 93, 26, 43, 109, 44, 17, 6, 31, 69, 10, 30, 47, 2, 19, 5, 110, 61, 11, 7, 74, 16, 83, 34, 73, 123, 104, 66, 81, 98, 9, 108, 95, 65, 1, 105, 124, 27, 80, 77, 4, 67, 45, 71, 33, 88, 3, 107, 97, 13, 37, 36, 68, 101, 41, 91, 94, 24, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 103, 28, 39, 20, 84, 79, 29, 48, 114, 89, 86, 15, 115, 82, 119, 23, 113, 60, 76, 22, 25, 40, 38, 92, 35, 99, 78, 122, 18, 32, 14, 90, 12, 87, 52, 111, 85, 50, 75, 63, 72, 70, 43, 51, 46, 125, 117, 8, 17, 31, 64, 21, 96, 93, 34, 0, 124, 26, 16, 83, 30, 44, 69, 19, 11, 123, 10, 108, 109, 9, 81, 47, 104, 74, 95, 66, 5, 2, 61, 73, 97, 7, 27, 110, 13, 65, 80, 45, 67, 1, 105, 3, 98, 107, 6, 4, 71, 77, 33, 88, 24, 41, 68, 37, 94, 91, 36, 101, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 42, 116, 118, 102, 62, 106, 121, 53, 39, 28, 20, 84, 79, 23, 25, 29, 103, 38, 60, 15, 89, 86, 90, 22, 99, 82, 119, 18, 76, 40, 122, 12, 92, 113, 48, 35, 114, 87, 32, 78, 115, 111, 70, 85, 14, 51, 75, 46, 52, 117, 50, 93, 64, 19, 8, 13, 125, 72, 16, 17, 11, 10, 30, 21, 63, 9, 0, 77, 26, 31, 83, 44, 96, 123, 43, 34, 1, 66, 47, 74, 104, 80, 81, 45, 124, 105, 69, 110, 61, 7, 73, 2, 71, 65, 108, 109, 95, 97, 5, 41, 27, 4, 3, 107, 98, 36, 37, 88, 6, 67, 33, 24, 91, 101, 68, 94, 100], [126, 55, 120, 54, 59, 112, 49, 57, 127, 58, 56, 116, 42, 102, 118, 106, 62, 121, 53, 28, 39, 20, 25, 122, 38, 103, 23, 84, 22, 29, 90, 48, 60, 119, 15, 79, 32, 86, 111, 89, 82, 99, 52, 113, 18, 92, 12, 40, 70, 117, 85, 115, 87, 35, 78, 14, 76, 114, 51, 46, 125, 0, 93, 21, 75, 123, 30, 64, 19, 96, 11, 63, 8, 31, 83, 43, 26, 50, 104, 34, 44, 16, 47, 17, 110, 61, 13, 10, 72, 108, 74, 69, 124, 9, 80, 73, 2, 65, 45, 95, 77, 81, 66, 109, 67, 7, 41, 5, 3, 36, 68, 1, 98, 6, 105, 33, 71, 107, 37, 4, 27, 88, 24, 97, 91, 94, 101, 100], [126, 55, 120, 54, 59, 112, 57, 127, 49, 58, 56, 42, 116, 102, 118, 62, 106, 121, 53, 28, 39, 48, 84, 38, 20, 29, 15, 103, 89, 122, 25, 119, 111, 23, 79, 60, 22, 92, 90, 87, 51, 52, 99, 32, 86, 78, 82, 35, 18, 76, 115, 40, 114, 12, 113, 14, 85, 117, 46, 70, 93, 123, 125, 47, 17, 0, 21, 8, 96, 44, 26, 31, 19, 109, 110, 50, 30, 63, 83, 11, 75, 34, 64, 43, 45, 61, 72, 9, 10, 81, 104, 13, 73, 5, 74, 108, 16, 2, 65, 66, 6, 69, 77, 67, 124, 95, 80, 41, 105, 7, 68, 27, 3, 98, 71, 107, 33, 1, 88, 97, 101, 4, 36, 24, 37, 94, 91, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 121, 53, 106, 39, 103, 28, 48, 84, 79, 38, 20, 119, 114, 92, 89, 15, 29, 52, 25, 60, 86, 51, 23, 99, 90, 115, 122, 22, 113, 35, 40, 78, 76, 87, 111, 46, 50, 32, 12, 18, 14, 125, 82, 123, 8, 64, 85, 43, 117, 108, 110, 0, 63, 96, 21, 6, 74, 109, 75, 31, 70, 83, 11, 17, 124, 5, 30, 10, 72, 44, 93, 66, 47, 34, 45, 26, 2, 19, 73, 104, 9, 105, 81, 61, 69, 95, 98, 3, 67, 16, 7, 1, 27, 41, 71, 65, 80, 33, 77, 107, 13, 97, 36, 37, 88, 4, 91, 24, 68, 101, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 56, 58, 116, 42, 102, 118, 62, 53, 121, 106, 39, 103, 28, 20, 48, 79, 84, 113, 119, 89, 15, 38, 115, 99, 86, 25, 92, 23, 22, 29, 78, 40, 51, 35, 52, 60, 76, 82, 114, 8, 122, 90, 18, 14, 12, 0, 87, 32, 85, 6, 5, 50, 46, 64, 111, 125, 96, 66, 123, 72, 47, 117, 30, 34, 75, 74, 83, 43, 110, 93, 17, 11, 70, 124, 19, 9, 31, 73, 81, 44, 21, 65, 98, 61, 108, 2, 45, 67, 63, 69, 80, 3, 26, 71, 109, 1, 10, 105, 27, 7, 95, 16, 77, 104, 68, 33, 97, 13, 107, 88, 41, 4, 91, 94, 24, 101, 100, 36, 37], [126, 55, 120, 54, 59, 112, 57, 127, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 39, 89, 20, 25, 84, 48, 86, 23, 22, 60, 15, 79, 103, 29, 38, 115, 99, 76, 40, 114, 35, 12, 90, 122, 78, 32, 113, 51, 92, 6, 18, 82, 119, 85, 111, 87, 8, 14, 96, 52, 93, 75, 19, 47, 17, 74, 11, 72, 0, 46, 50, 64, 83, 13, 34, 31, 43, 21, 123, 9, 73, 117, 30, 110, 125, 44, 81, 10, 16, 80, 26, 63, 5, 65, 77, 69, 109, 71, 108, 66, 104, 3, 1, 97, 67, 98, 45, 95, 61, 2, 105, 124, 27, 4, 7, 107, 70, 88, 41, 68, 36, 101, 91, 33, 94, 24, 37, 100], [126, 55, 54, 120, 112, 59, 49, 57, 127, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 103, 39, 28, 25, 48, 20, 84, 60, 29, 23, 38, 15, 122, 90, 86, 89, 115, 52, 79, 92, 119, 32, 22, 111, 113, 51, 87, 35, 114, 18, 40, 82, 12, 99, 6, 78, 85, 76, 123, 50, 96, 43, 14, 46, 93, 125, 21, 47, 117, 108, 31, 72, 34, 64, 83, 124, 19, 63, 110, 11, 61, 104, 74, 73, 75, 17, 8, 26, 109, 10, 105, 77, 41, 30, 45, 44, 95, 0, 16, 5, 81, 2, 97, 9, 13, 69, 67, 80, 36, 66, 27, 98, 71, 3, 33, 1, 65, 7, 88, 107, 68, 37, 91, 101, 94, 4, 70, 24, 100], [126, 55, 120, 54, 112, 59, 57, 49, 127, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 39, 103, 119, 48, 84, 79, 22, 20, 86, 29, 15, 99, 89, 25, 60, 38, 23, 113, 114, 122, 51, 111, 87, 52, 12, 115, 40, 50, 35, 18, 92, 90, 76, 32, 14, 82, 6, 64, 78, 125, 0, 85, 72, 46, 123, 117, 8, 96, 63, 17, 75, 43, 45, 11, 21, 110, 61, 104, 10, 19, 1, 73, 83, 30, 31, 26, 74, 47, 2, 34, 5, 124, 93, 80, 66, 81, 9, 70, 108, 3, 69, 109, 13, 65, 105, 71, 41, 44, 16, 95, 67, 7, 27, 77, 107, 98, 97, 4, 33, 68, 101, 88, 36, 24, 37, 94, 91, 100], [126, 55, 120, 54, 112, 59, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 39, 28, 103, 22, 79, 84, 20, 29, 38, 23, 89, 48, 15, 25, 86, 40, 60, 113, 119, 92, 122, 114, 99, 12, 18, 76, 90, 82, 52, 32, 35, 115, 51, 78, 111, 0, 14, 46, 85, 72, 87, 8, 6, 123, 64, 96, 75, 21, 50, 5, 125, 117, 17, 83, 30, 74, 10, 26, 34, 70, 31, 11, 47, 45, 73, 81, 9, 93, 105, 66, 98, 44, 7, 43, 110, 2, 19, 61, 108, 63, 69, 1, 67, 16, 71, 80, 124, 95, 109, 3, 65, 27, 13, 41, 4, 77, 68, 91, 97, 33, 104, 37, 24, 88, 36, 107, 101, 94, 100], [126, 55, 120, 54, 112, 59, 127, 57, 49, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 103, 39, 22, 84, 79, 48, 89, 119, 29, 20, 115, 23, 99, 60, 25, 122, 15, 32, 86, 114, 12, 38, 113, 92, 40, 76, 18, 78, 90, 35, 52, 46, 111, 82, 85, 72, 87, 14, 51, 125, 8, 93, 0, 123, 75, 43, 50, 11, 96, 70, 30, 6, 10, 5, 21, 61, 64, 110, 17, 63, 34, 26, 2, 19, 83, 74, 73, 69, 105, 31, 47, 95, 7, 44, 109, 117, 108, 81, 9, 65, 98, 66, 13, 16, 124, 1, 80, 68, 45, 104, 71, 27, 4, 41, 97, 67, 77, 3, 107, 24, 91, 88, 33, 36, 37, 101, 100, 94], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 42, 116, 102, 118, 62, 121, 106, 28, 39, 53, 103, 25, 84, 89, 29, 20, 86, 38, 48, 60, 23, 119, 114, 79, 15, 22, 92, 90, 87, 111, 76, 115, 18, 40, 113, 52, 35, 82, 99, 32, 85, 122, 12, 51, 46, 125, 78, 70, 50, 14, 93, 123, 31, 117, 72, 21, 96, 19, 61, 10, 11, 63, 124, 44, 26, 30, 0, 64, 47, 9, 75, 34, 13, 83, 108, 43, 5, 16, 81, 17, 110, 8, 45, 80, 95, 41, 66, 1, 77, 105, 73, 104, 69, 65, 2, 74, 36, 67, 27, 97, 98, 109, 71, 7, 107, 37, 3, 6, 4, 24, 88, 101, 33, 91, 94, 68, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 28, 53, 39, 48, 84, 20, 103, 29, 89, 79, 38, 22, 60, 86, 15, 99, 114, 23, 113, 25, 12, 90, 92, 87, 76, 119, 122, 46, 111, 70, 82, 18, 14, 40, 35, 78, 32, 85, 115, 117, 0, 52, 123, 72, 64, 51, 44, 31, 26, 93, 63, 47, 96, 11, 30, 43, 75, 21, 125, 17, 10, 50, 83, 19, 69, 74, 104, 61, 66, 9, 5, 1, 34, 8, 110, 73, 80, 16, 2, 124, 108, 3, 95, 65, 13, 81, 27, 71, 77, 7, 45, 4, 67, 41, 36, 68, 109, 107, 6, 98, 97, 101, 88, 105, 37, 33, 91, 24, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 118, 102, 62, 121, 53, 106, 39, 48, 28, 103, 38, 20, 84, 119, 79, 89, 46, 122, 15, 29, 60, 111, 114, 92, 23, 113, 52, 86, 25, 99, 12, 40, 90, 22, 76, 82, 32, 35, 18, 123, 70, 87, 115, 14, 125, 78, 117, 51, 72, 93, 44, 63, 47, 11, 85, 64, 31, 96, 0, 75, 30, 110, 43, 10, 34, 61, 17, 26, 2, 5, 21, 83, 50, 109, 9, 8, 104, 45, 108, 74, 19, 81, 73, 105, 124, 98, 16, 1, 7, 97, 66, 107, 95, 6, 69, 80, 41, 71, 27, 67, 13, 3, 68, 33, 88, 37, 101, 36, 65, 77, 24, 91, 4, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 28, 39, 103, 48, 20, 119, 84, 29, 25, 38, 79, 60, 113, 99, 52, 90, 40, 114, 89, 23, 12, 15, 86, 92, 35, 115, 32, 22, 123, 76, 14, 46, 125, 122, 78, 87, 18, 82, 85, 51, 111, 70, 72, 117, 96, 93, 43, 0, 11, 30, 50, 108, 47, 31, 10, 34, 44, 124, 61, 63, 75, 66, 5, 64, 19, 8, 109, 74, 26, 9, 95, 6, 17, 104, 73, 21, 83, 110, 2, 81, 105, 98, 69, 7, 71, 13, 45, 107, 1, 3, 16, 33, 67, 80, 68, 37, 97, 77, 88, 27, 65, 41, 4, 36, 91, 101, 100, 94, 24], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 48, 103, 39, 119, 20, 113, 29, 84, 79, 60, 38, 22, 86, 23, 89, 25, 15, 35, 40, 122, 92, 90, 46, 99, 114, 12, 115, 52, 76, 32, 87, 18, 78, 111, 82, 125, 123, 51, 14, 117, 43, 11, 85, 72, 96, 50, 34, 6, 47, 31, 26, 104, 30, 0, 17, 93, 44, 19, 63, 75, 64, 10, 61, 45, 108, 21, 83, 9, 8, 110, 70, 74, 80, 95, 66, 2, 5, 81, 124, 109, 73, 71, 67, 16, 69, 1, 13, 98, 3, 7, 27, 107, 88, 36, 91, 97, 33, 77, 65, 68, 24, 41, 101, 105, 37, 4, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 118, 102, 121, 62, 106, 53, 28, 39, 20, 48, 84, 29, 60, 38, 103, 15, 25, 22, 113, 79, 122, 92, 119, 99, 23, 89, 111, 46, 86, 114, 52, 35, 90, 12, 125, 82, 14, 40, 78, 76, 115, 32, 64, 6, 87, 123, 0, 18, 51, 43, 85, 117, 47, 93, 72, 50, 8, 63, 21, 61, 2, 30, 31, 10, 17, 11, 5, 75, 66, 19, 44, 110, 124, 96, 26, 34, 108, 104, 80, 1, 83, 95, 71, 74, 69, 9, 73, 45, 65, 81, 109, 70, 107, 67, 16, 77, 7, 41, 97, 13, 27, 98, 36, 3, 105, 4, 33, 68, 88, 91, 101, 37, 24, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 118, 102, 62, 121, 106, 53, 28, 39, 103, 20, 25, 60, 38, 114, 84, 48, 92, 119, 29, 122, 22, 86, 90, 23, 117, 111, 113, 46, 89, 79, 15, 32, 125, 40, 52, 35, 115, 87, 14, 99, 51, 82, 18, 12, 61, 6, 78, 76, 64, 43, 96, 30, 123, 44, 108, 124, 21, 31, 11, 10, 50, 93, 47, 85, 8, 34, 17, 19, 72, 0, 63, 110, 104, 83, 5, 75, 109, 26, 66, 74, 9, 45, 95, 107, 3, 73, 2, 69, 71, 81, 16, 80, 1, 97, 7, 98, 36, 13, 67, 88, 33, 77, 41, 27, 70, 65, 91, 37, 24, 68, 105, 4, 101, 94, 100], [126, 55, 120, 54, 112, 59, 57, 49, 127, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 39, 48, 103, 114, 29, 20, 84, 38, 79, 113, 89, 25, 119, 22, 15, 60, 40, 23, 92, 52, 115, 86, 82, 90, 12, 14, 99, 35, 122, 76, 87, 46, 18, 32, 111, 6, 78, 8, 123, 96, 43, 85, 0, 104, 51, 72, 10, 50, 125, 63, 11, 21, 30, 61, 26, 34, 83, 44, 17, 31, 9, 93, 19, 108, 117, 110, 47, 2, 5, 74, 75, 64, 71, 7, 45, 69, 73, 95, 81, 13, 109, 27, 1, 105, 67, 66, 16, 98, 80, 70, 65, 124, 3, 97, 107, 33, 4, 77, 88, 91, 101, 41, 24, 37, 68, 36, 94, 100], [126, 55, 120, 54, 59, 112, 57, 49, 127, 58, 56, 116, 42, 102, 118, 62, 121, 106, 39, 28, 53, 20, 84, 25, 23, 38, 103, 60, 89, 79, 15, 119, 86, 48, 22, 29, 113, 90, 12, 114, 92, 40, 18, 122, 99, 115, 82, 14, 46, 52, 32, 35, 51, 76, 78, 123, 111, 125, 87, 6, 85, 8, 21, 11, 83, 0, 61, 10, 108, 64, 34, 72, 96, 75, 93, 19, 17, 45, 26, 63, 74, 47, 43, 50, 71, 9, 104, 30, 124, 16, 31, 109, 73, 80, 117, 13, 81, 70, 5, 66, 77, 44, 2, 95, 110, 65, 69, 1, 67, 97, 7, 105, 98, 27, 107, 4, 91, 3, 33, 37, 68, 88, 36, 24, 101, 94, 41, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 121, 28, 53, 106, 39, 48, 103, 25, 89, 23, 38, 20, 119, 22, 60, 29, 84, 86, 79, 122, 113, 90, 40, 123, 15, 32, 35, 12, 92, 82, 46, 52, 99, 61, 114, 14, 18, 51, 111, 115, 85, 76, 87, 78, 93, 43, 21, 8, 125, 50, 30, 108, 0, 96, 10, 31, 11, 117, 34, 44, 26, 70, 47, 64, 19, 83, 75, 6, 110, 9, 17, 72, 80, 63, 104, 124, 77, 66, 95, 109, 69, 73, 45, 74, 5, 98, 81, 16, 13, 97, 41, 71, 7, 105, 2, 65, 3, 27, 107, 1, 67, 33, 36, 68, 94, 37, 88, 101, 4, 24, 91, 100], [126, 55, 120, 54, 59, 112, 49, 127, 57, 58, 56, 42, 116, 102, 118, 121, 53, 62, 106, 28, 39, 103, 119, 48, 25, 20, 23, 84, 38, 89, 60, 29, 15, 79, 22, 115, 46, 52, 86, 114, 99, 35, 122, 51, 113, 123, 90, 82, 111, 12, 92, 32, 76, 40, 87, 14, 117, 61, 18, 125, 70, 78, 8, 11, 85, 50, 44, 63, 96, 34, 110, 45, 64, 83, 108, 0, 30, 93, 21, 104, 109, 47, 31, 72, 19, 73, 17, 75, 43, 5, 124, 69, 3, 10, 13, 105, 74, 80, 26, 98, 9, 65, 2, 81, 16, 1, 41, 66, 97, 7, 77, 6, 68, 95, 107, 67, 27, 36, 71, 33, 4, 24, 101, 94, 88, 91, 37, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 39, 28, 20, 84, 79, 103, 89, 48, 23, 15, 38, 25, 52, 119, 29, 12, 82, 113, 22, 60, 114, 90, 76, 14, 92, 122, 32, 78, 18, 99, 40, 86, 70, 35, 46, 50, 123, 8, 115, 87, 85, 117, 125, 111, 64, 21, 72, 83, 96, 43, 5, 11, 19, 0, 93, 51, 74, 47, 73, 9, 66, 61, 31, 63, 75, 26, 81, 10, 30, 2, 17, 104, 44, 34, 108, 45, 69, 110, 80, 1, 7, 124, 67, 95, 109, 77, 105, 6, 98, 71, 16, 13, 68, 41, 3, 65, 27, 4, 97, 33, 24, 107, 36, 101, 88, 37, 94, 91, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 53, 62, 121, 106, 103, 28, 114, 39, 84, 38, 20, 48, 52, 79, 25, 89, 15, 22, 29, 92, 40, 23, 113, 86, 119, 60, 32, 12, 99, 50, 35, 76, 70, 90, 18, 115, 82, 78, 14, 122, 64, 8, 111, 46, 31, 85, 51, 0, 123, 125, 63, 87, 43, 45, 5, 61, 11, 72, 34, 108, 75, 96, 66, 69, 74, 117, 10, 93, 124, 17, 2, 30, 19, 21, 83, 1, 110, 73, 104, 44, 109, 7, 47, 95, 65, 81, 71, 9, 26, 16, 33, 98, 3, 107, 27, 77, 67, 105, 13, 97, 4, 80, 6, 37, 101, 36, 24, 88, 41, 68, 100, 94, 91], [126, 55, 120, 54, 112, 59, 127, 49, 57, 58, 56, 116, 42, 102, 118, 121, 62, 53, 106, 28, 39, 103, 84, 20, 38, 89, 15, 29, 60, 79, 52, 25, 48, 86, 119, 22, 113, 23, 32, 92, 114, 82, 18, 99, 76, 14, 40, 12, 35, 78, 115, 90, 46, 70, 87, 50, 51, 122, 125, 111, 8, 63, 123, 93, 117, 96, 19, 21, 85, 72, 31, 11, 43, 75, 124, 17, 73, 30, 83, 10, 74, 34, 108, 69, 61, 26, 110, 0, 45, 104, 80, 44, 13, 7, 64, 9, 109, 5, 47, 81, 27, 67, 66, 95, 16, 1, 2, 65, 105, 77, 71, 98, 3, 6, 68, 107, 97, 37, 33, 91, 4, 41, 101, 24, 88, 36, 94, 100], [126, 55, 120, 54, 112, 59, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 103, 39, 114, 84, 20, 25, 113, 48, 15, 115, 119, 29, 23, 52, 79, 22, 89, 90, 122, 60, 40, 86, 38, 12, 32, 99, 92, 14, 51, 76, 78, 18, 46, 111, 35, 125, 87, 82, 50, 63, 85, 0, 72, 123, 124, 70, 43, 74, 8, 96, 64, 31, 117, 6, 19, 75, 110, 93, 104, 34, 9, 73, 7, 30, 5, 11, 2, 47, 10, 83, 69, 21, 17, 66, 44, 13, 108, 45, 65, 61, 16, 1, 26, 81, 80, 98, 95, 109, 97, 67, 68, 77, 71, 3, 41, 27, 105, 4, 107, 37, 36, 101, 33, 24, 88, 91, 100, 94], [126, 55, 120, 54, 59, 112, 49, 57, 127, 58, 56, 42, 116, 102, 118, 62, 121, 106, 103, 39, 53, 28, 48, 20, 84, 25, 23, 15, 29, 79, 38, 89, 114, 86, 119, 60, 22, 90, 92, 115, 18, 99, 40, 35, 12, 32, 14, 52, 122, 76, 113, 111, 51, 82, 125, 123, 46, 63, 87, 50, 85, 78, 75, 72, 43, 61, 93, 8, 74, 104, 96, 6, 11, 117, 108, 19, 110, 21, 31, 17, 30, 124, 73, 70, 44, 9, 34, 10, 83, 45, 64, 109, 47, 5, 26, 80, 105, 13, 81, 7, 0, 66, 41, 98, 77, 27, 95, 69, 71, 65, 16, 2, 107, 33, 67, 3, 88, 36, 68, 97, 1, 37, 4, 94, 24, 91, 101, 100]], "model.layers.22.self_attn.q_proj": [[57, 40, 50, 121, 54, 118, 51, 125, 52, 119, 123, 124, 63, 60, 117, 58, 61, 116, 33, 47, 62, 45, 114, 127, 59, 48, 56, 111, 126, 112, 49, 38, 55, 44, 53, 113, 122, 115, 120, 36, 102, 92, 108, 42, 109, 46, 107, 28, 17, 43, 110, 106, 94, 77, 41, 26, 25, 85, 105, 103, 83, 86, 89, 99, 39, 32, 87, 95, 19, 35, 100, 30, 37, 104, 27, 101, 31, 11, 22, 97, 21, 96, 34, 24, 98, 81, 13, 90, 29, 93, 69, 15, 2, 88, 84, 71, 0, 91, 79, 20, 16, 3, 75, 82, 1, 5, 9, 23, 67, 78, 65, 64, 73, 18, 66, 7, 80, 4, 14, 10, 74, 68, 72, 12, 8, 6, 70, 76], [50, 40, 121, 57, 51, 118, 54, 33, 114, 125, 52, 119, 123, 45, 44, 63, 58, 60, 116, 117, 127, 124, 112, 62, 49, 47, 59, 126, 61, 36, 113, 56, 48, 53, 94, 120, 122, 111, 55, 46, 115, 28, 17, 107, 26, 109, 110, 92, 25, 41, 108, 38, 43, 77, 99, 30, 42, 89, 102, 106, 85, 105, 86, 83, 19, 95, 87, 100, 24, 103, 21, 27, 31, 35, 39, 104, 101, 22, 37, 32, 98, 97, 11, 90, 84, 34, 2, 81, 96, 16, 13, 15, 69, 64, 93, 88, 82, 78, 0, 29, 79, 5, 66, 73, 23, 1, 65, 91, 67, 3, 75, 20, 7, 72, 9, 12, 18, 4, 14, 71, 68, 74, 10, 80, 6, 8, 70, 76], [121, 40, 50, 114, 125, 51, 54, 118, 57, 33, 63, 45, 52, 123, 119, 60, 117, 127, 58, 44, 116, 124, 56, 112, 59, 126, 62, 108, 61, 48, 113, 28, 46, 53, 120, 49, 122, 47, 55, 111, 115, 110, 17, 102, 26, 92, 42, 107, 109, 36, 94, 25, 38, 99, 106, 85, 43, 41, 105, 89, 95, 100, 30, 77, 87, 86, 19, 83, 103, 21, 35, 39, 104, 24, 37, 101, 96, 32, 22, 27, 97, 98, 34, 31, 81, 11, 13, 84, 2, 82, 88, 90, 64, 73, 29, 93, 16, 0, 15, 79, 9, 78, 1, 91, 69, 65, 23, 14, 5, 75, 66, 3, 4, 7, 67, 20, 71, 12, 18, 72, 10, 74, 68, 6, 70, 8, 80, 76], [40, 121, 50, 57, 82, 114, 78, 12, 33, 27, 16, 25, 99, 118, 23, 38, 29, 74, 19, 86, 6, 10, 93, 18, 36, 89, 72, 94, 51, 14, 70, 76, 104, 8, 28, 20, 84, 63, 102, 88, 90, 30, 95, 68, 54, 21, 80, 58, 119, 4, 55, 87, 91, 22, 125, 9, 15, 85, 49, 31, 1, 81, 96, 26, 62, 47, 42, 73, 24, 7, 127, 66, 116, 83, 92, 3, 52, 123, 39, 11, 32, 13, 115, 2, 124, 60, 64, 44, 98, 34, 117, 17, 105, 61, 79, 35, 126, 53, 101, 112, 48, 56, 5, 122, 45, 75, 77, 109, 71, 107, 67, 120, 113, 106, 37, 108, 43, 110, 59, 41, 46, 111, 100, 103, 69, 97, 0, 65], [42, 54, 32, 35, 123, 99, 106, 50, 89, 85, 27, 87, 16, 23, 58, 29, 96, 18, 78, 36, 57, 40, 98, 83, 103, 116, 107, 117, 7, 84, 62, 31, 17, 108, 61, 48, 60, 82, 44, 12, 47, 37, 105, 95, 100, 77, 26, 21, 125, 74, 76, 118, 46, 55, 45, 41, 90, 9, 114, 10, 20, 115, 94, 113, 101, 119, 92, 127, 110, 19, 124, 112, 81, 97, 122, 109, 126, 102, 49, 28, 24, 121, 59, 75, 104, 56, 22, 70, 88, 79, 43, 120, 34, 68, 33, 51, 25, 53, 63, 93, 15, 8, 91, 30, 52, 86, 39, 111, 80, 38, 4, 11, 14, 0, 72, 66, 13, 3, 69, 71, 5, 73, 64, 67, 6, 65, 1, 2], [54, 42, 32, 89, 117, 99, 123, 85, 87, 23, 29, 58, 35, 106, 96, 28, 50, 121, 55, 44, 36, 60, 15, 111, 33, 101, 57, 40, 46, 82, 122, 52, 126, 51, 112, 53, 31, 47, 10, 16, 100, 78, 103, 104, 41, 18, 125, 119, 45, 110, 109, 38, 83, 114, 95, 81, 17, 105, 115, 39, 118, 21, 12, 25, 113, 56, 76, 127, 19, 124, 48, 116, 34, 107, 37, 14, 27, 8, 98, 90, 108, 91, 49, 77, 63, 62, 79, 59, 102, 43, 120, 97, 30, 61, 69, 93, 84, 80, 92, 24, 72, 26, 94, 86, 74, 22, 88, 6, 67, 20, 75, 66, 71, 13, 11, 7, 5, 3, 70, 65, 73, 64, 68, 2, 4, 9, 1, 0], [54, 42, 32, 99, 35, 29, 89, 87, 96, 115, 23, 85, 36, 106, 95, 27, 110, 62, 105, 101, 18, 61, 123, 119, 94, 44, 45, 50, 83, 82, 117, 118, 63, 59, 48, 41, 86, 113, 28, 40, 121, 112, 55, 58, 31, 93, 120, 47, 114, 92, 103, 109, 125, 107, 37, 127, 122, 104, 57, 53, 108, 34, 60, 100, 126, 38, 52, 49, 16, 51, 20, 43, 116, 15, 124, 102, 90, 46, 33, 56, 22, 39, 26, 97, 12, 77, 98, 78, 84, 10, 25, 111, 30, 91, 81, 17, 24, 13, 75, 88, 14, 19, 76, 21, 4, 6, 8, 72, 74, 67, 7, 69, 79, 73, 1, 68, 5, 80, 70, 9, 11, 0, 66, 71, 64, 3, 65, 2], [42, 54, 32, 85, 35, 123, 106, 99, 89, 23, 16, 87, 76, 78, 18, 12, 50, 48, 27, 84, 29, 125, 36, 11, 14, 6, 72, 117, 10, 81, 4, 115, 31, 21, 80, 51, 25, 17, 8, 116, 28, 26, 101, 111, 79, 104, 83, 77, 3, 74, 109, 55, 92, 82, 2, 19, 9, 73, 60, 88, 57, 62, 1, 98, 69, 86, 64, 124, 97, 103, 119, 33, 66, 37, 44, 90, 93, 49, 102, 40, 105, 108, 30, 95, 114, 7, 110, 96, 67, 100, 41, 94, 75, 46, 56, 15, 127, 24, 126, 34, 38, 107, 53, 70, 63, 20, 118, 91, 43, 5, 112, 22, 39, 47, 52, 13, 45, 58, 71, 120, 122, 61, 59, 113, 121, 68, 65, 0], [117, 38, 120, 97, 25, 58, 63, 93, 89, 83, 116, 52, 53, 121, 55, 49, 102, 39, 86, 112, 115, 75, 122, 126, 127, 113, 119, 57, 125, 123, 50, 109, 47, 21, 48, 60, 114, 118, 46, 62, 20, 84, 41, 88, 105, 61, 59, 111, 110, 51, 44, 73, 56, 29, 124, 17, 100, 54, 104, 45, 80, 106, 42, 107, 108, 103, 40, 43, 22, 87, 16, 101, 37, 24, 26, 98, 99, 34, 36, 35, 15, 95, 18, 96, 31, 77, 94, 19, 23, 71, 32, 30, 33, 92, 4, 28, 79, 91, 14, 81, 90, 13, 11, 85, 27, 68, 1, 7, 82, 72, 3, 74, 70, 65, 64, 10, 76, 69, 66, 78, 9, 5, 0, 2, 8, 12, 67, 6], [38, 120, 117, 63, 97, 93, 17, 83, 25, 89, 86, 20, 29, 88, 102, 14, 58, 76, 74, 121, 65, 72, 15, 53, 116, 77, 71, 73, 49, 64, 41, 126, 122, 60, 100, 57, 4, 91, 46, 55, 59, 22, 11, 119, 123, 98, 94, 50, 81, 127, 115, 34, 10, 26, 39, 66, 104, 7, 47, 111, 109, 125, 16, 113, 52, 42, 118, 19, 56, 48, 51, 112, 90, 54, 114, 61, 44, 45, 84, 79, 62, 96, 24, 28, 35, 37, 78, 82, 2, 103, 87, 70, 108, 31, 75, 101, 124, 99, 106, 67, 40, 105, 32, 95, 21, 30, 36, 80, 5, 43, 27, 85, 110, 92, 12, 9, 13, 69, 1, 23, 33, 68, 107, 8, 3, 18, 6, 0], [38, 120, 117, 97, 58, 25, 83, 86, 93, 63, 73, 89, 17, 102, 29, 121, 15, 53, 88, 3, 14, 49, 75, 22, 126, 57, 20, 71, 19, 109, 41, 76, 77, 113, 116, 55, 127, 84, 46, 111, 79, 72, 52, 119, 11, 4, 47, 122, 65, 118, 112, 39, 125, 87, 115, 51, 60, 62, 50, 59, 123, 74, 70, 45, 21, 80, 56, 81, 9, 44, 114, 61, 16, 48, 42, 104, 10, 82, 100, 24, 108, 67, 124, 78, 34, 105, 103, 2, 85, 110, 33, 90, 23, 54, 7, 107, 40, 8, 37, 98, 35, 91, 28, 101, 43, 36, 0, 66, 99, 13, 69, 26, 12, 94, 18, 96, 1, 106, 68, 92, 31, 95, 6, 27, 30, 32, 5, 64], [38, 117, 120, 97, 63, 83, 70, 93, 25, 15, 73, 89, 3, 86, 58, 53, 41, 14, 17, 88, 80, 74, 29, 109, 78, 77, 75, 121, 57, 84, 65, 22, 55, 111, 115, 46, 49, 19, 79, 105, 0, 102, 118, 98, 122, 76, 42, 126, 106, 116, 119, 52, 125, 72, 4, 20, 107, 113, 127, 50, 56, 21, 54, 104, 62, 61, 47, 60, 9, 6, 123, 45, 59, 110, 11, 10, 44, 39, 51, 26, 114, 48, 12, 103, 2, 112, 81, 124, 82, 85, 8, 66, 108, 67, 43, 87, 71, 24, 40, 16, 36, 69, 34, 101, 94, 1, 23, 31, 7, 37, 95, 30, 90, 99, 28, 91, 100, 5, 32, 18, 35, 27, 68, 92, 64, 96, 13, 33], [122, 58, 53, 38, 125, 124, 121, 56, 87, 90, 92, 55, 59, 34, 111, 127, 76, 117, 63, 126, 30, 81, 123, 113, 62, 50, 120, 54, 52, 110, 49, 57, 61, 60, 114, 119, 48, 118, 51, 116, 115, 47, 100, 102, 19, 112, 8, 40, 107, 109, 91, 3, 22, 44, 89, 46, 83, 45, 21, 108, 41, 23, 2, 32, 80, 31, 103, 84, 105, 42, 43, 104, 106, 72, 36, 94, 39, 5, 37, 29, 35, 85, 28, 96, 33, 16, 24, 93, 25, 101, 70, 65, 95, 17, 14, 27, 26, 99, 10, 86, 0, 97, 13, 6, 78, 64, 12, 73, 9, 88, 98, 77, 1, 18, 15, 82, 4, 11, 75, 79, 66, 71, 20, 7, 68, 69, 67, 74], [53, 38, 125, 122, 58, 87, 121, 56, 30, 127, 34, 92, 90, 60, 50, 113, 63, 124, 55, 59, 123, 48, 120, 126, 32, 51, 41, 57, 114, 61, 62, 52, 118, 54, 119, 116, 47, 111, 115, 49, 21, 117, 76, 102, 110, 19, 108, 46, 85, 112, 40, 45, 107, 83, 106, 22, 109, 81, 44, 94, 37, 43, 80, 42, 100, 27, 28, 26, 84, 104, 105, 39, 103, 33, 91, 101, 23, 29, 36, 15, 31, 72, 97, 99, 16, 89, 77, 35, 86, 13, 78, 96, 98, 17, 93, 3, 0, 95, 65, 8, 5, 14, 18, 88, 2, 25, 69, 20, 73, 10, 75, 74, 24, 11, 9, 82, 12, 79, 71, 7, 6, 68, 70, 66, 4, 67, 64, 1], [122, 53, 38, 90, 121, 58, 56, 59, 127, 34, 87, 60, 55, 124, 92, 126, 123, 63, 117, 52, 30, 120, 48, 57, 111, 113, 61, 49, 110, 114, 125, 115, 62, 118, 119, 54, 50, 51, 47, 116, 102, 83, 22, 112, 46, 81, 19, 94, 32, 89, 109, 85, 23, 104, 108, 41, 14, 43, 84, 103, 107, 76, 31, 44, 105, 17, 106, 45, 36, 40, 42, 101, 25, 86, 28, 78, 100, 13, 82, 80, 37, 27, 39, 35, 98, 33, 26, 21, 29, 15, 97, 99, 88, 95, 8, 24, 91, 96, 16, 65, 93, 72, 18, 10, 12, 79, 69, 20, 3, 11, 6, 9, 77, 75, 7, 68, 74, 73, 71, 70, 67, 5, 0, 2, 66, 64, 4, 1], [125, 122, 53, 38, 58, 90, 121, 56, 51, 59, 127, 113, 120, 110, 81, 87, 55, 50, 63, 30, 61, 48, 126, 124, 123, 102, 49, 108, 57, 52, 117, 60, 114, 119, 34, 118, 62, 47, 111, 46, 109, 54, 115, 116, 19, 44, 106, 112, 92, 39, 42, 107, 43, 76, 103, 83, 22, 101, 32, 100, 94, 37, 105, 41, 21, 40, 45, 104, 36, 99, 14, 31, 35, 33, 8, 85, 91, 17, 26, 93, 24, 96, 29, 23, 97, 27, 84, 78, 95, 88, 28, 25, 98, 86, 80, 72, 10, 89, 74, 12, 15, 79, 13, 16, 18, 82, 11, 77, 3, 9, 73, 20, 70, 69, 75, 6, 5, 68, 7, 4, 71, 66, 0, 64, 65, 1, 67, 2], [102, 110, 97, 126, 46, 73, 21, 77, 82, 68, 11, 15, 26, 66, 90, 70, 53, 0, 7, 25, 87, 71, 85, 121, 31, 18, 1, 23, 29, 79, 100, 93, 4, 54, 118, 81, 88, 124, 10, 117, 55, 9, 57, 106, 78, 28, 64, 13, 80, 22, 20, 127, 2, 75, 32, 5, 108, 76, 83, 65, 69, 17, 12, 58, 6, 105, 92, 3, 89, 63, 16, 24, 116, 115, 86, 125, 84, 27, 72, 39, 40, 104, 123, 98, 91, 48, 59, 30, 62, 96, 49, 74, 52, 122, 37, 101, 67, 60, 14, 95, 112, 34, 19, 8, 51, 45, 99, 35, 94, 103, 109, 50, 61, 120, 56, 47, 36, 43, 107, 42, 44, 119, 111, 113, 41, 114, 33, 38], [102, 110, 126, 97, 21, 26, 82, 46, 15, 77, 73, 11, 87, 90, 93, 68, 7, 23, 12, 25, 45, 115, 40, 53, 78, 70, 8, 18, 44, 76, 85, 118, 84, 55, 66, 120, 54, 116, 109, 127, 0, 96, 101, 36, 95, 31, 37, 91, 13, 112, 79, 72, 125, 24, 83, 1, 98, 121, 124, 27, 41, 48, 19, 71, 32, 114, 17, 103, 52, 56, 75, 122, 58, 69, 14, 9, 123, 39, 89, 49, 42, 106, 59, 88, 63, 22, 81, 105, 6, 20, 51, 107, 62, 94, 80, 119, 117, 28, 34, 29, 3, 86, 60, 38, 47, 99, 61, 57, 108, 43, 92, 104, 30, 111, 10, 16, 100, 50, 35, 113, 74, 4, 33, 2, 5, 67, 65, 64], [102, 110, 126, 97, 118, 46, 26, 21, 15, 82, 87, 11, 31, 63, 95, 90, 93, 68, 49, 73, 77, 44, 41, 120, 53, 116, 88, 127, 106, 122, 123, 25, 54, 112, 22, 7, 101, 38, 84, 42, 56, 37, 12, 58, 52, 17, 51, 80, 29, 59, 109, 70, 23, 43, 92, 48, 119, 55, 57, 124, 66, 5, 113, 94, 45, 125, 111, 115, 0, 65, 104, 121, 117, 83, 10, 50, 108, 60, 105, 1, 62, 85, 100, 47, 96, 91, 69, 61, 36, 107, 27, 78, 98, 99, 28, 114, 40, 30, 39, 19, 8, 86, 76, 3, 32, 103, 79, 74, 16, 34, 71, 64, 67, 24, 72, 35, 89, 20, 18, 4, 81, 75, 33, 14, 2, 6, 13, 9], [102, 110, 97, 126, 21, 46, 26, 77, 82, 11, 15, 90, 73, 116, 87, 70, 92, 91, 68, 7, 25, 66, 37, 54, 32, 45, 85, 99, 62, 72, 49, 30, 83, 108, 41, 51, 19, 124, 18, 44, 111, 52, 71, 0, 113, 53, 122, 112, 127, 105, 104, 88, 24, 118, 48, 58, 100, 89, 40, 106, 50, 17, 123, 8, 75, 101, 16, 95, 36, 121, 96, 78, 63, 42, 93, 31, 94, 22, 38, 98, 3, 76, 115, 79, 28, 60, 55, 5, 107, 64, 33, 34, 114, 39, 35, 125, 109, 61, 119, 20, 57, 10, 47, 84, 6, 120, 29, 103, 59, 13, 43, 1, 86, 56, 81, 2, 117, 14, 9, 23, 27, 74, 80, 12, 67, 69, 4, 65], [111, 55, 101, 47, 25, 87, 113, 95, 21, 28, 48, 109, 58, 119, 82, 56, 59, 31, 62, 97, 91, 118, 94, 44, 54, 45, 122, 89, 33, 127, 126, 42, 116, 51, 115, 50, 120, 107, 61, 102, 80, 106, 23, 15, 76, 85, 123, 38, 53, 74, 121, 112, 32, 39, 92, 18, 57, 40, 110, 52, 105, 19, 104, 125, 117, 72, 124, 22, 41, 77, 63, 37, 49, 46, 30, 108, 16, 98, 29, 103, 99, 35, 60, 34, 93, 114, 26, 100, 14, 90, 84, 71, 96, 17, 36, 83, 69, 43, 86, 88, 27, 24, 75, 81, 20, 11, 13, 73, 79, 65, 9, 7, 78, 70, 3, 12, 67, 1, 5, 4, 8, 66, 10, 2, 68, 6, 0, 64], [111, 55, 101, 47, 25, 95, 21, 87, 28, 82, 33, 94, 91, 15, 74, 53, 76, 59, 62, 97, 22, 89, 72, 29, 114, 60, 56, 121, 49, 116, 107, 37, 92, 45, 93, 123, 38, 32, 11, 70, 43, 18, 110, 109, 67, 122, 98, 54, 113, 48, 103, 57, 23, 99, 115, 78, 102, 4, 69, 31, 80, 44, 51, 40, 83, 85, 105, 124, 36, 119, 50, 34, 86, 46, 66, 41, 13, 117, 77, 104, 120, 58, 26, 52, 108, 118, 42, 27, 71, 17, 61, 127, 63, 112, 96, 39, 125, 106, 84, 19, 90, 126, 79, 10, 81, 100, 30, 8, 7, 24, 12, 35, 73, 88, 20, 9, 75, 16, 64, 68, 5, 3, 1, 14, 65, 2, 6, 0], [111, 101, 55, 47, 25, 87, 21, 95, 15, 74, 82, 70, 4, 89, 76, 94, 64, 97, 67, 77, 28, 80, 113, 2, 114, 72, 84, 50, 73, 18, 0, 91, 58, 31, 126, 29, 79, 66, 11, 85, 6, 122, 23, 92, 40, 83, 16, 10, 71, 45, 3, 116, 107, 75, 41, 65, 5, 7, 86, 35, 27, 88, 13, 98, 93, 9, 14, 12, 121, 33, 8, 78, 19, 123, 17, 68, 96, 81, 69, 20, 24, 90, 26, 105, 34, 22, 117, 44, 30, 32, 51, 115, 46, 38, 99, 54, 37, 1, 62, 124, 60, 109, 127, 103, 100, 59, 61, 48, 39, 110, 108, 102, 53, 120, 125, 42, 36, 112, 52, 104, 56, 63, 106, 57, 119, 49, 118, 43], [111, 55, 101, 47, 25, 120, 87, 21, 121, 28, 95, 56, 116, 31, 82, 110, 53, 33, 123, 37, 126, 38, 46, 58, 50, 124, 104, 122, 115, 89, 60, 44, 62, 51, 97, 112, 59, 29, 118, 61, 57, 127, 76, 23, 63, 48, 94, 52, 45, 39, 107, 103, 113, 91, 49, 117, 114, 85, 80, 42, 109, 125, 105, 119, 32, 40, 108, 41, 98, 102, 99, 106, 18, 15, 90, 54, 30, 26, 84, 43, 34, 72, 77, 92, 35, 96, 86, 100, 93, 27, 36, 16, 74, 78, 20, 67, 17, 11, 83, 22, 79, 88, 24, 81, 13, 14, 69, 19, 71, 70, 9, 12, 65, 75, 7, 73, 8, 4, 64, 3, 66, 5, 10, 2, 0, 1, 68, 6], [115, 100, 51, 23, 91, 32, 96, 84, 86, 9, 77, 15, 81, 39, 71, 5, 11, 27, 20, 108, 49, 10, 94, 36, 24, 83, 22, 44, 68, 21, 67, 118, 17, 0, 64, 65, 57, 82, 126, 127, 121, 117, 114, 16, 54, 45, 110, 2, 87, 79, 28, 35, 95, 107, 33, 13, 12, 80, 73, 25, 101, 74, 3, 69, 19, 76, 40, 85, 116, 37, 18, 42, 75, 30, 112, 8, 90, 89, 104, 92, 72, 124, 70, 88, 122, 14, 41, 78, 48, 120, 52, 34, 99, 50, 62, 4, 93, 97, 102, 123, 47, 105, 29, 98, 63, 119, 125, 109, 60, 53, 113, 31, 66, 38, 6, 46, 103, 56, 26, 7, 59, 1, 43, 106, 55, 111, 61, 58], [115, 100, 51, 121, 36, 91, 32, 23, 86, 84, 39, 82, 77, 81, 127, 48, 96, 113, 15, 118, 33, 20, 29, 44, 63, 104, 55, 71, 116, 120, 45, 119, 123, 49, 37, 47, 108, 22, 102, 114, 41, 126, 107, 50, 42, 106, 46, 54, 35, 110, 124, 53, 125, 105, 27, 10, 43, 38, 52, 111, 112, 16, 59, 56, 60, 58, 57, 30, 61, 122, 40, 95, 11, 19, 18, 62, 99, 109, 97, 117, 66, 34, 101, 98, 64, 31, 103, 83, 17, 26, 92, 24, 94, 5, 68, 85, 87, 90, 12, 21, 93, 89, 28, 76, 73, 6, 88, 8, 25, 13, 79, 78, 2, 75, 9, 80, 14, 67, 74, 70, 72, 1, 69, 4, 0, 65, 7, 3], [115, 100, 51, 121, 36, 91, 23, 32, 84, 86, 103, 63, 82, 20, 114, 104, 44, 22, 77, 15, 48, 39, 117, 46, 107, 99, 55, 43, 127, 49, 116, 29, 34, 45, 81, 110, 31, 30, 40, 119, 61, 118, 37, 33, 57, 108, 38, 41, 112, 42, 120, 113, 54, 125, 101, 35, 111, 60, 87, 106, 96, 58, 109, 53, 59, 56, 52, 95, 123, 47, 98, 126, 92, 105, 62, 24, 90, 97, 102, 27, 50, 94, 124, 122, 28, 9, 93, 71, 26, 17, 16, 85, 74, 25, 88, 76, 18, 19, 14, 89, 79, 13, 21, 11, 83, 80, 78, 10, 6, 12, 66, 64, 7, 73, 1, 4, 0, 75, 5, 72, 8, 68, 70, 69, 3, 2, 67, 65], [115, 100, 51, 36, 23, 91, 121, 96, 84, 86, 81, 11, 32, 15, 118, 94, 48, 117, 39, 9, 50, 108, 116, 107, 102, 5, 20, 2, 119, 25, 105, 45, 77, 44, 0, 37, 113, 27, 63, 82, 58, 62, 68, 19, 57, 46, 47, 110, 120, 49, 114, 126, 122, 43, 112, 123, 99, 125, 127, 53, 59, 22, 54, 41, 61, 55, 24, 124, 109, 52, 104, 42, 95, 17, 60, 65, 40, 56, 33, 38, 106, 35, 111, 70, 101, 71, 103, 87, 88, 93, 64, 16, 26, 98, 30, 79, 34, 29, 73, 83, 10, 31, 97, 90, 72, 75, 69, 8, 28, 14, 89, 12, 67, 6, 92, 13, 21, 18, 66, 78, 85, 4, 1, 80, 76, 3, 74, 7], [115, 40, 122, 53, 93, 124, 49, 55, 58, 51, 52, 48, 120, 33, 59, 60, 56, 62, 127, 111, 125, 26, 44, 61, 54, 47, 50, 123, 63, 113, 43, 126, 114, 118, 121, 116, 107, 57, 119, 45, 46, 112, 117, 22, 19, 109, 42, 110, 90, 29, 108, 88, 84, 41, 101, 77, 99, 38, 104, 106, 105, 32, 94, 92, 95, 103, 39, 83, 36, 102, 37, 34, 81, 100, 89, 97, 35, 86, 16, 98, 96, 25, 82, 28, 87, 71, 27, 10, 31, 13, 21, 23, 91, 79, 30, 80, 17, 14, 7, 3, 24, 74, 15, 12, 78, 85, 2, 20, 64, 66, 0, 18, 9, 67, 5, 72, 11, 69, 65, 6, 1, 75, 73, 4, 68, 76, 8, 70], [53, 115, 40, 122, 124, 93, 49, 52, 55, 59, 51, 33, 60, 56, 62, 48, 111, 126, 61, 58, 113, 117, 120, 125, 43, 114, 50, 127, 54, 26, 121, 63, 47, 123, 109, 119, 57, 116, 118, 46, 45, 112, 90, 22, 108, 81, 107, 44, 110, 35, 29, 88, 37, 42, 41, 39, 105, 101, 106, 103, 92, 77, 99, 36, 104, 32, 34, 102, 95, 84, 100, 83, 38, 94, 19, 89, 97, 86, 98, 17, 25, 96, 31, 27, 16, 82, 13, 28, 20, 30, 15, 79, 24, 10, 91, 85, 14, 87, 21, 71, 80, 74, 78, 23, 72, 5, 7, 12, 66, 64, 11, 8, 0, 2, 1, 3, 75, 4, 67, 18, 69, 65, 68, 9, 73, 6, 70, 76], [40, 115, 53, 122, 88, 14, 33, 19, 82, 12, 22, 90, 80, 9, 92, 49, 28, 111, 91, 26, 30, 1, 55, 73, 93, 6, 31, 50, 124, 29, 25, 75, 20, 72, 27, 4, 95, 23, 17, 89, 69, 86, 78, 15, 96, 85, 32, 58, 10, 38, 117, 59, 76, 18, 51, 24, 34, 45, 98, 83, 70, 112, 21, 100, 104, 35, 127, 52, 16, 99, 105, 46, 56, 11, 123, 94, 79, 84, 42, 81, 106, 108, 61, 113, 121, 13, 8, 68, 109, 62, 114, 3, 120, 54, 110, 87, 116, 103, 101, 126, 37, 63, 125, 119, 39, 102, 71, 77, 48, 107, 36, 47, 43, 60, 118, 41, 57, 67, 7, 74, 44, 97, 5, 0, 66, 65, 2, 64], [122, 40, 53, 115, 33, 124, 93, 55, 52, 60, 49, 59, 26, 48, 62, 111, 61, 56, 120, 58, 125, 113, 101, 47, 127, 51, 54, 116, 126, 121, 119, 114, 50, 117, 123, 57, 63, 118, 109, 45, 84, 95, 39, 88, 112, 90, 43, 29, 46, 108, 22, 41, 107, 110, 44, 77, 34, 32, 19, 25, 42, 89, 94, 92, 106, 99, 100, 105, 83, 103, 37, 21, 81, 97, 102, 104, 35, 38, 24, 36, 98, 96, 86, 31, 28, 23, 17, 87, 27, 16, 91, 30, 2, 80, 3, 82, 69, 64, 13, 74, 1, 65, 14, 5, 0, 79, 66, 10, 7, 71, 67, 85, 15, 20, 78, 12, 11, 68, 9, 6, 72, 75, 4, 73, 8, 18, 70, 76]], "model.layers.22.self_attn.k_proj": [[104, 121, 50, 86, 57, 97, 92, 100, 54, 94, 51, 118, 114, 119, 25, 31, 124, 63, 125, 58, 17, 60, 117, 123, 102, 62, 126, 116, 52, 127, 48, 85, 55, 83, 47, 53, 49, 56, 112, 122, 44, 120, 59, 113, 111, 115, 27, 105, 61, 46, 107, 108, 35, 110, 106, 42, 19, 109, 43, 77, 39, 41, 45, 16, 80, 103, 12, 15, 37, 78, 93, 33, 99, 34, 38, 87, 26, 75, 79, 98, 101, 91, 32, 76, 84, 11, 23, 72, 96, 82, 90, 7, 95, 36, 5, 28, 10, 20, 24, 13, 30, 29, 40, 74, 6, 88, 73, 71, 18, 89, 21, 9, 4, 3, 22, 8, 14, 81, 70, 1, 68, 0, 67, 69, 2, 64, 66, 65], [106, 54, 89, 96, 123, 50, 23, 85, 16, 17, 99, 42, 78, 82, 76, 48, 124, 27, 114, 30, 29, 21, 57, 72, 19, 34, 112, 74, 12, 93, 14, 108, 10, 18, 70, 55, 77, 100, 60, 47, 68, 127, 125, 3, 35, 118, 7, 11, 53, 115, 46, 8, 79, 104, 110, 95, 59, 20, 49, 15, 111, 109, 119, 41, 98, 38, 28, 105, 44, 65, 13, 83, 101, 107, 5, 126, 9, 81, 113, 33, 84, 37, 117, 120, 43, 116, 52, 103, 92, 56, 66, 31, 61, 63, 90, 122, 102, 40, 45, 62, 39, 121, 86, 97, 24, 51, 94, 87, 36, 75, 64, 91, 58, 88, 26, 22, 25, 6, 69, 80, 32, 71, 73, 67, 4, 2, 1, 0], [120, 117, 102, 33, 86, 29, 25, 58, 17, 83, 49, 121, 0, 50, 88, 73, 118, 14, 126, 65, 119, 53, 3, 122, 15, 116, 57, 46, 47, 125, 103, 55, 123, 52, 115, 113, 111, 38, 4, 62, 75, 112, 44, 48, 51, 114, 127, 61, 105, 45, 7, 59, 56, 70, 107, 36, 106, 41, 124, 60, 77, 74, 108, 109, 84, 63, 80, 110, 42, 21, 104, 94, 16, 40, 20, 18, 72, 54, 43, 98, 39, 90, 13, 5, 89, 37, 87, 34, 76, 91, 100, 26, 99, 2, 68, 31, 35, 92, 28, 101, 95, 64, 71, 1, 10, 67, 23, 96, 69, 32, 8, 82, 27, 85, 30, 12, 93, 79, 11, 66, 78, 24, 81, 19, 6, 9, 97, 22], [122, 53, 102, 22, 98, 57, 121, 117, 62, 56, 55, 110, 50, 116, 28, 59, 123, 115, 63, 124, 120, 126, 52, 48, 60, 58, 118, 114, 113, 127, 119, 54, 51, 61, 112, 49, 111, 33, 47, 109, 44, 94, 104, 108, 96, 46, 107, 90, 45, 40, 105, 34, 43, 125, 41, 80, 37, 42, 106, 78, 103, 36, 92, 101, 39, 16, 82, 100, 85, 13, 95, 19, 86, 83, 97, 99, 35, 17, 73, 87, 24, 25, 93, 31, 11, 29, 27, 81, 76, 32, 38, 30, 68, 84, 23, 91, 21, 15, 26, 88, 10, 71, 5, 14, 18, 8, 79, 75, 7, 67, 89, 72, 66, 20, 77, 70, 9, 69, 74, 1, 12, 0, 2, 65, 6, 3, 64, 4], [46, 126, 38, 33, 110, 82, 21, 26, 15, 77, 73, 0, 11, 70, 66, 7, 68, 64, 87, 25, 53, 3, 93, 65, 121, 19, 17, 95, 54, 2, 84, 125, 58, 10, 80, 12, 118, 88, 59, 115, 63, 48, 22, 1, 124, 45, 94, 55, 14, 105, 71, 16, 49, 112, 74, 117, 40, 8, 60, 104, 44, 92, 101, 116, 123, 51, 52, 67, 62, 6, 113, 42, 120, 96, 127, 69, 41, 43, 56, 39, 5, 119, 109, 34, 83, 114, 23, 103, 111, 122, 107, 108, 57, 35, 37, 32, 50, 36, 72, 99, 28, 61, 47, 102, 29, 106, 24, 100, 81, 89, 20, 86, 98, 30, 27, 78, 31, 91, 4, 9, 76, 79, 97, 75, 13, 85, 90, 18], [47, 111, 37, 55, 87, 21, 72, 82, 15, 25, 76, 31, 74, 66, 70, 107, 77, 69, 64, 4, 94, 80, 65, 119, 97, 102, 28, 116, 78, 115, 56, 91, 40, 44, 58, 123, 118, 53, 38, 43, 110, 121, 39, 120, 127, 62, 7, 92, 83, 122, 42, 60, 113, 52, 1, 105, 114, 45, 124, 126, 117, 104, 108, 112, 59, 106, 125, 22, 34, 63, 51, 48, 73, 17, 61, 93, 46, 54, 57, 50, 90, 98, 49, 103, 32, 41, 109, 36, 71, 20, 33, 86, 35, 29, 99, 19, 81, 100, 11, 30, 26, 13, 84, 96, 85, 24, 2, 14, 27, 68, 101, 88, 16, 23, 75, 10, 3, 9, 5, 12, 67, 8, 6, 89, 79, 0, 95, 18], [51, 115, 36, 86, 91, 96, 84, 23, 81, 77, 15, 71, 64, 121, 94, 82, 11, 117, 1, 49, 44, 113, 119, 68, 57, 9, 48, 66, 112, 39, 47, 46, 106, 45, 108, 67, 120, 122, 127, 50, 53, 125, 5, 101, 63, 24, 104, 58, 59, 41, 114, 109, 61, 107, 126, 118, 62, 56, 55, 3, 60, 103, 123, 70, 105, 54, 10, 116, 43, 124, 88, 52, 26, 0, 110, 2, 19, 97, 38, 76, 42, 99, 111, 100, 31, 25, 37, 74, 80, 40, 85, 89, 78, 33, 93, 16, 65, 30, 98, 6, 69, 34, 12, 102, 72, 92, 14, 21, 29, 95, 35, 8, 90, 28, 83, 4, 18, 73, 32, 7, 27, 87, 13, 79, 75, 22, 20, 17], [104, 115, 53, 22, 122, 97, 51, 29, 124, 111, 59, 56, 52, 26, 28, 55, 49, 62, 126, 50, 54, 118, 117, 113, 61, 60, 127, 116, 121, 119, 47, 58, 82, 63, 12, 19, 83, 120, 114, 48, 125, 110, 123, 35, 91, 112, 16, 109, 81, 88, 44, 57, 45, 6, 107, 105, 77, 43, 15, 93, 46, 32, 89, 94, 108, 17, 42, 41, 84, 31, 36, 14, 80, 37, 33, 78, 101, 103, 96, 24, 102, 79, 30, 106, 39, 27, 95, 23, 38, 9, 99, 98, 34, 90, 13, 87, 100, 69, 75, 40, 74, 25, 85, 21, 11, 18, 92, 72, 66, 20, 7, 68, 71, 73, 70, 0, 64, 76, 10, 65, 86, 4, 67, 1, 8, 2, 3, 5]], "model.layers.22.self_attn.qk_proj": [[115, 53, 122, 51, 117, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 104, 89, 58, 22, 123, 118, 125, 25, 87, 97, 85, 49, 40, 27, 56, 36, 21, 90, 86, 23, 93, 18, 62, 63, 127, 79, 60, 101, 59, 96, 15, 33, 82, 77, 114, 124, 99, 13, 116, 81, 48, 17, 26, 112, 52, 61, 19, 100, 29, 44, 83, 119, 11, 37, 113, 28, 84, 32, 95, 92, 45, 9, 30, 94, 73, 91, 31, 75, 7, 108, 20, 16, 109, 41, 14, 80, 78, 76, 70, 12, 39, 35, 4, 0, 71, 34, 74, 10, 105, 107, 8, 98, 64, 43, 68, 88, 103, 67, 3, 2, 66, 6, 24, 65, 1, 72, 5, 69], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 123, 58, 89, 22, 104, 125, 87, 97, 118, 25, 36, 62, 85, 56, 49, 23, 93, 27, 40, 86, 21, 90, 114, 101, 79, 59, 116, 82, 99, 113, 52, 18, 13, 63, 100, 96, 127, 77, 15, 33, 29, 60, 19, 81, 124, 17, 112, 73, 61, 26, 119, 44, 48, 92, 28, 91, 11, 109, 32, 94, 107, 45, 20, 105, 70, 37, 83, 41, 75, 9, 84, 76, 108, 0, 16, 14, 12, 78, 95, 4, 7, 30, 39, 31, 35, 98, 80, 74, 43, 103, 10, 71, 64, 34, 2, 68, 24, 88, 8, 1, 67, 3, 66, 72, 65, 6, 5, 69], [115, 53, 122, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 57, 110, 55, 106, 42, 102, 38, 58, 104, 22, 25, 118, 123, 89, 97, 93, 27, 56, 23, 87, 59, 36, 21, 40, 62, 85, 86, 125, 49, 63, 116, 15, 90, 114, 33, 60, 18, 82, 101, 77, 79, 100, 99, 96, 13, 52, 48, 32, 127, 124, 113, 81, 17, 119, 45, 92, 112, 29, 44, 107, 26, 73, 19, 28, 37, 11, 20, 0, 108, 94, 41, 91, 30, 61, 83, 84, 70, 109, 71, 9, 75, 95, 34, 7, 98, 35, 80, 105, 76, 12, 78, 31, 4, 39, 16, 14, 43, 64, 10, 103, 74, 68, 88, 66, 72, 2, 67, 3, 65, 1, 24, 69, 6, 8, 5], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 57, 110, 106, 55, 42, 38, 102, 58, 123, 104, 25, 22, 89, 125, 56, 62, 21, 87, 63, 59, 27, 97, 93, 86, 118, 85, 23, 15, 114, 49, 82, 18, 77, 40, 90, 36, 100, 124, 101, 127, 13, 52, 33, 81, 79, 45, 116, 48, 99, 96, 119, 113, 112, 32, 60, 17, 107, 11, 37, 29, 84, 61, 83, 70, 26, 73, 12, 44, 19, 9, 92, 95, 28, 64, 41, 105, 109, 91, 7, 71, 98, 14, 94, 75, 78, 43, 16, 80, 30, 76, 31, 4, 108, 20, 0, 74, 68, 34, 66, 35, 72, 10, 103, 67, 39, 3, 24, 65, 88, 2, 6, 1, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 58, 123, 22, 89, 97, 104, 118, 49, 63, 25, 21, 86, 87, 85, 62, 27, 125, 23, 93, 40, 59, 82, 36, 114, 56, 15, 79, 60, 18, 124, 48, 77, 33, 81, 45, 90, 116, 100, 99, 13, 119, 52, 127, 96, 44, 101, 19, 26, 112, 61, 17, 92, 11, 29, 73, 91, 9, 113, 75, 37, 71, 20, 32, 31, 95, 83, 28, 64, 16, 94, 14, 78, 7, 80, 41, 12, 84, 76, 0, 109, 98, 108, 39, 10, 43, 70, 34, 105, 107, 72, 74, 4, 68, 35, 30, 6, 2, 67, 24, 66, 103, 1, 3, 88, 5, 69, 65, 8], [115, 53, 122, 117, 51, 120, 47, 111, 121, 126, 50, 54, 46, 57, 110, 106, 55, 42, 102, 38, 123, 22, 58, 86, 27, 89, 85, 125, 97, 21, 25, 87, 104, 62, 93, 40, 23, 56, 49, 63, 82, 15, 77, 33, 59, 36, 60, 118, 18, 81, 13, 119, 90, 79, 114, 124, 99, 96, 100, 127, 116, 11, 48, 17, 37, 92, 84, 113, 32, 29, 19, 83, 26, 52, 41, 101, 16, 112, 31, 73, 28, 75, 94, 91, 9, 44, 20, 61, 95, 14, 12, 98, 45, 80, 76, 7, 109, 108, 78, 74, 0, 64, 10, 105, 71, 68, 107, 30, 72, 39, 6, 35, 43, 4, 34, 24, 2, 103, 65, 70, 3, 67, 88, 1, 66, 5, 69, 8], [115, 53, 122, 117, 51, 120, 47, 111, 126, 54, 121, 50, 46, 57, 110, 106, 55, 42, 102, 38, 22, 123, 87, 89, 86, 25, 85, 104, 23, 27, 21, 58, 40, 97, 93, 60, 63, 36, 15, 62, 90, 49, 82, 96, 79, 125, 33, 18, 118, 77, 114, 13, 99, 116, 32, 17, 59, 48, 81, 113, 119, 52, 44, 100, 19, 101, 56, 84, 127, 29, 112, 94, 11, 124, 26, 92, 91, 45, 83, 28, 107, 37, 76, 73, 109, 20, 12, 61, 95, 16, 9, 78, 80, 75, 14, 41, 31, 43, 39, 105, 34, 72, 108, 7, 35, 71, 30, 10, 6, 64, 74, 98, 0, 24, 88, 103, 68, 4, 1, 66, 3, 2, 67, 70, 69, 65, 5, 8], [115, 53, 122, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 57, 110, 55, 106, 42, 38, 102, 22, 89, 87, 97, 25, 63, 85, 21, 58, 36, 60, 23, 27, 104, 118, 93, 90, 86, 40, 123, 18, 49, 62, 59, 96, 33, 15, 125, 82, 112, 116, 119, 48, 79, 77, 100, 113, 101, 17, 99, 52, 114, 61, 56, 32, 13, 124, 81, 44, 26, 91, 29, 20, 28, 127, 19, 37, 84, 108, 80, 35, 12, 73, 83, 11, 109, 92, 94, 107, 39, 34, 95, 43, 45, 14, 6, 9, 16, 88, 76, 78, 75, 41, 105, 30, 7, 31, 64, 72, 4, 74, 71, 10, 98, 0, 24, 103, 68, 66, 65, 2, 3, 67, 69, 1, 5, 70, 8], [115, 53, 122, 117, 51, 120, 47, 126, 111, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 89, 58, 22, 125, 25, 62, 36, 27, 93, 87, 21, 23, 97, 123, 118, 90, 104, 85, 63, 49, 59, 114, 82, 86, 112, 127, 40, 15, 77, 96, 116, 56, 33, 79, 101, 124, 60, 61, 52, 18, 48, 32, 100, 99, 119, 19, 107, 13, 28, 113, 81, 9, 17, 109, 45, 29, 26, 37, 44, 11, 20, 41, 83, 91, 92, 84, 35, 94, 108, 12, 78, 105, 39, 80, 14, 34, 31, 76, 73, 95, 30, 6, 98, 75, 16, 7, 103, 43, 74, 72, 68, 64, 0, 88, 71, 4, 10, 24, 66, 65, 2, 1, 3, 67, 5, 70, 69, 8], [115, 53, 122, 51, 117, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 42, 102, 38, 104, 58, 123, 89, 22, 125, 62, 25, 87, 118, 27, 63, 97, 23, 119, 85, 36, 93, 86, 49, 21, 90, 114, 52, 56, 59, 40, 61, 96, 99, 124, 101, 79, 116, 127, 18, 60, 77, 82, 13, 48, 15, 100, 33, 45, 107, 11, 32, 91, 109, 41, 9, 94, 17, 28, 81, 112, 19, 37, 113, 92, 44, 26, 29, 20, 83, 108, 84, 73, 35, 75, 105, 31, 14, 78, 7, 12, 39, 43, 16, 6, 34, 30, 64, 95, 0, 76, 71, 98, 80, 103, 74, 68, 4, 10, 70, 2, 72, 3, 66, 1, 8, 67, 24, 88, 65, 69, 5], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 106, 57, 55, 42, 38, 102, 58, 104, 125, 89, 22, 25, 63, 97, 27, 123, 86, 87, 62, 21, 49, 118, 85, 23, 119, 100, 93, 82, 59, 114, 101, 77, 18, 52, 90, 15, 79, 116, 40, 33, 99, 56, 36, 127, 61, 13, 96, 48, 113, 17, 124, 83, 112, 81, 29, 9, 60, 41, 94, 28, 37, 32, 26, 109, 92, 16, 64, 44, 91, 20, 11, 12, 84, 7, 45, 75, 107, 70, 19, 74, 73, 95, 76, 31, 71, 78, 80, 14, 39, 98, 68, 10, 4, 35, 0, 108, 103, 105, 43, 8, 34, 6, 66, 30, 2, 3, 24, 72, 67, 1, 88, 65, 69, 5], [115, 53, 122, 51, 117, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 22, 89, 123, 86, 25, 21, 104, 58, 40, 87, 27, 97, 36, 85, 118, 23, 90, 82, 93, 79, 62, 125, 49, 124, 63, 77, 18, 116, 96, 59, 15, 33, 119, 61, 114, 100, 99, 127, 17, 13, 112, 32, 113, 81, 101, 19, 26, 83, 60, 94, 28, 44, 84, 92, 20, 48, 11, 37, 29, 9, 73, 56, 52, 12, 75, 91, 41, 95, 14, 70, 31, 76, 80, 78, 10, 16, 7, 45, 107, 64, 98, 0, 74, 35, 34, 105, 71, 4, 109, 108, 68, 39, 43, 30, 88, 24, 8, 103, 66, 3, 1, 6, 2, 67, 72, 69, 65, 5], [115, 53, 122, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 102, 42, 38, 89, 22, 123, 87, 25, 118, 36, 23, 93, 97, 21, 86, 27, 58, 40, 85, 104, 90, 82, 125, 96, 116, 63, 49, 33, 59, 100, 99, 56, 114, 112, 124, 18, 61, 119, 60, 15, 113, 29, 101, 13, 62, 32, 79, 77, 26, 81, 28, 94, 19, 44, 17, 91, 92, 20, 83, 127, 48, 37, 107, 41, 109, 84, 9, 75, 105, 95, 39, 11, 31, 52, 14, 16, 73, 35, 108, 45, 12, 70, 30, 78, 80, 34, 64, 76, 10, 43, 71, 98, 7, 8, 68, 24, 88, 0, 103, 4, 74, 65, 66, 2, 3, 1, 67, 69, 5, 72, 6], [115, 122, 53, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 106, 55, 42, 38, 102, 58, 123, 22, 104, 25, 86, 125, 89, 21, 118, 59, 23, 27, 85, 119, 40, 87, 90, 93, 36, 82, 124, 116, 77, 60, 63, 49, 112, 56, 114, 62, 97, 100, 79, 127, 33, 96, 15, 113, 52, 61, 18, 81, 9, 37, 101, 99, 32, 29, 17, 11, 13, 44, 48, 70, 75, 31, 109, 73, 94, 0, 41, 20, 19, 84, 45, 91, 83, 95, 92, 26, 28, 30, 107, 98, 14, 12, 71, 7, 35, 108, 80, 16, 64, 105, 78, 8, 76, 39, 68, 43, 10, 4, 74, 103, 34, 2, 1, 24, 66, 67, 65, 88, 3, 5, 72, 6, 69], [115, 122, 53, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 110, 106, 57, 55, 42, 38, 102, 58, 22, 104, 97, 86, 25, 89, 87, 27, 123, 21, 125, 40, 36, 62, 23, 118, 82, 79, 93, 119, 63, 85, 77, 90, 49, 112, 60, 114, 99, 15, 56, 18, 81, 127, 33, 124, 96, 13, 100, 61, 59, 101, 113, 17, 75, 83, 32, 48, 116, 109, 9, 91, 45, 29, 52, 37, 26, 92, 19, 28, 71, 76, 84, 14, 11, 31, 44, 95, 73, 12, 107, 20, 39, 80, 7, 70, 41, 78, 94, 35, 64, 30, 0, 43, 105, 8, 74, 108, 34, 10, 16, 98, 4, 103, 68, 2, 24, 6, 3, 67, 88, 66, 1, 65, 72, 5, 69], [115, 122, 53, 117, 51, 120, 47, 121, 111, 126, 54, 50, 46, 110, 106, 55, 57, 42, 38, 102, 123, 22, 104, 86, 21, 25, 40, 125, 58, 89, 85, 87, 97, 27, 23, 63, 93, 36, 82, 90, 77, 79, 49, 18, 56, 100, 60, 118, 59, 15, 116, 33, 81, 62, 61, 127, 114, 113, 52, 119, 48, 96, 13, 101, 112, 124, 99, 29, 41, 17, 26, 12, 32, 45, 75, 73, 109, 83, 84, 76, 94, 19, 91, 92, 44, 9, 64, 28, 31, 7, 16, 37, 20, 78, 11, 34, 14, 10, 8, 95, 35, 39, 68, 70, 74, 108, 103, 105, 107, 30, 43, 71, 98, 80, 0, 24, 4, 6, 66, 88, 65, 67, 1, 2, 3, 5, 69, 72], [115, 53, 122, 51, 117, 120, 47, 126, 111, 121, 54, 50, 46, 110, 55, 57, 106, 42, 102, 38, 104, 89, 22, 58, 86, 123, 87, 40, 25, 85, 23, 90, 36, 114, 93, 125, 97, 21, 27, 63, 96, 118, 82, 79, 100, 33, 59, 49, 60, 56, 18, 77, 116, 15, 81, 113, 29, 124, 61, 94, 99, 17, 84, 19, 26, 44, 101, 127, 48, 28, 119, 62, 112, 32, 83, 13, 41, 31, 52, 75, 20, 91, 73, 37, 92, 30, 11, 45, 16, 35, 39, 78, 76, 109, 98, 12, 80, 108, 107, 9, 95, 6, 34, 43, 103, 71, 0, 10, 105, 7, 14, 24, 8, 88, 68, 64, 4, 74, 1, 2, 66, 70, 65, 3, 67, 5, 72, 69], [115, 53, 122, 51, 117, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 42, 38, 102, 25, 58, 22, 86, 104, 87, 89, 40, 123, 93, 27, 114, 36, 90, 21, 97, 125, 118, 85, 79, 63, 23, 59, 96, 60, 18, 99, 82, 77, 62, 112, 15, 49, 33, 116, 100, 26, 124, 81, 48, 56, 101, 29, 13, 61, 17, 127, 52, 32, 75, 19, 37, 113, 44, 107, 109, 45, 73, 92, 20, 91, 119, 9, 28, 84, 94, 6, 83, 78, 16, 12, 95, 41, 30, 31, 108, 11, 39, 0, 105, 7, 35, 34, 80, 71, 76, 14, 98, 10, 64, 68, 4, 103, 43, 74, 88, 66, 67, 8, 2, 65, 24, 3, 1, 72, 69, 70, 5], [115, 53, 122, 117, 51, 120, 47, 126, 111, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 63, 104, 89, 125, 22, 25, 114, 97, 123, 58, 21, 40, 86, 36, 87, 62, 48, 27, 82, 23, 85, 90, 93, 127, 112, 99, 49, 118, 101, 79, 124, 60, 96, 77, 59, 52, 116, 100, 13, 61, 15, 18, 33, 56, 26, 29, 17, 41, 44, 32, 75, 28, 37, 84, 45, 109, 119, 91, 73, 81, 113, 94, 83, 107, 19, 6, 105, 9, 95, 12, 11, 39, 92, 78, 7, 108, 34, 31, 30, 35, 98, 43, 76, 20, 0, 64, 14, 103, 10, 71, 16, 4, 80, 74, 68, 67, 24, 88, 2, 72, 65, 8, 66, 1, 3, 70, 5, 69], [115, 122, 53, 117, 51, 120, 47, 126, 111, 121, 50, 54, 46, 110, 57, 106, 55, 42, 102, 38, 104, 89, 22, 123, 97, 58, 125, 93, 85, 87, 40, 21, 25, 86, 63, 118, 36, 23, 27, 62, 60, 56, 90, 114, 116, 124, 49, 96, 77, 101, 100, 18, 15, 33, 13, 79, 82, 99, 127, 113, 52, 59, 92, 81, 48, 26, 41, 112, 119, 29, 94, 44, 75, 28, 83, 17, 9, 45, 61, 91, 73, 32, 11, 84, 37, 107, 20, 19, 109, 30, 31, 78, 12, 6, 35, 95, 105, 76, 64, 39, 14, 0, 10, 80, 71, 16, 7, 34, 43, 98, 103, 68, 108, 66, 74, 72, 4, 67, 88, 1, 24, 2, 3, 65, 8, 70, 69, 5], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 89, 22, 104, 118, 25, 40, 123, 97, 63, 86, 87, 85, 27, 93, 23, 116, 21, 60, 36, 49, 90, 125, 56, 79, 82, 59, 77, 124, 96, 33, 62, 18, 99, 112, 113, 48, 114, 100, 15, 119, 52, 13, 127, 101, 81, 92, 44, 31, 26, 32, 29, 17, 19, 107, 61, 94, 75, 41, 73, 109, 28, 37, 20, 83, 11, 91, 78, 7, 39, 30, 95, 45, 84, 12, 10, 108, 9, 71, 64, 76, 98, 14, 34, 68, 6, 16, 80, 103, 0, 74, 35, 72, 105, 4, 88, 43, 70, 67, 24, 1, 2, 66, 3, 5, 65, 69, 8], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 38, 102, 118, 89, 58, 123, 125, 104, 22, 25, 87, 63, 85, 62, 97, 56, 124, 93, 23, 86, 90, 114, 27, 52, 48, 21, 40, 59, 36, 49, 96, 79, 82, 119, 116, 60, 18, 127, 15, 112, 113, 13, 29, 33, 101, 77, 99, 100, 26, 44, 91, 61, 32, 109, 81, 17, 92, 19, 45, 107, 31, 20, 41, 75, 37, 28, 64, 83, 73, 39, 11, 0, 7, 71, 94, 9, 105, 10, 95, 78, 98, 68, 80, 103, 72, 12, 84, 16, 35, 70, 108, 76, 30, 14, 43, 34, 4, 1, 88, 6, 74, 67, 66, 65, 24, 3, 2, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 118, 89, 123, 22, 58, 104, 97, 27, 25, 52, 85, 59, 87, 93, 125, 60, 23, 36, 86, 49, 63, 116, 40, 124, 21, 96, 90, 61, 119, 48, 82, 79, 18, 33, 56, 112, 113, 13, 101, 114, 62, 44, 100, 26, 99, 77, 29, 15, 92, 109, 17, 127, 28, 107, 81, 94, 32, 20, 83, 91, 19, 37, 45, 31, 30, 41, 84, 75, 73, 7, 70, 11, 39, 9, 64, 12, 72, 95, 0, 14, 76, 105, 35, 78, 43, 71, 34, 80, 98, 10, 16, 108, 68, 74, 88, 4, 2, 67, 103, 1, 24, 66, 6, 65, 3, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 118, 22, 89, 86, 104, 25, 27, 97, 58, 49, 87, 85, 123, 125, 93, 40, 36, 90, 21, 124, 56, 23, 18, 63, 62, 96, 15, 79, 119, 59, 82, 116, 114, 77, 127, 52, 60, 13, 81, 61, 33, 99, 113, 48, 100, 112, 92, 17, 26, 101, 37, 29, 83, 9, 28, 75, 11, 19, 44, 70, 31, 84, 107, 41, 73, 76, 94, 32, 20, 91, 95, 109, 10, 108, 105, 7, 45, 12, 30, 98, 14, 72, 78, 16, 80, 71, 0, 103, 68, 64, 43, 74, 4, 35, 39, 34, 88, 65, 24, 67, 3, 66, 2, 1, 69, 5, 8, 6], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 22, 89, 104, 25, 123, 87, 86, 118, 85, 58, 23, 93, 21, 40, 97, 27, 36, 125, 90, 52, 49, 62, 79, 59, 18, 56, 63, 48, 77, 119, 33, 124, 82, 96, 127, 15, 114, 116, 61, 99, 13, 112, 100, 17, 81, 101, 32, 60, 19, 92, 29, 84, 26, 44, 11, 83, 9, 113, 20, 28, 75, 91, 37, 73, 109, 94, 41, 70, 7, 16, 31, 76, 12, 45, 95, 10, 35, 107, 105, 64, 71, 78, 39, 14, 108, 30, 34, 80, 98, 0, 43, 74, 68, 72, 24, 88, 4, 103, 66, 1, 67, 65, 3, 2, 69, 5, 6, 8], [115, 53, 122, 117, 51, 120, 47, 121, 126, 111, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 123, 22, 104, 25, 89, 118, 36, 87, 125, 23, 86, 85, 90, 124, 40, 27, 93, 97, 21, 96, 63, 52, 59, 49, 82, 116, 33, 60, 18, 79, 119, 114, 99, 127, 56, 62, 13, 100, 77, 26, 48, 101, 29, 44, 15, 41, 61, 112, 109, 19, 113, 17, 94, 81, 92, 32, 28, 37, 91, 84, 73, 83, 20, 9, 39, 30, 11, 107, 76, 35, 7, 75, 70, 105, 45, 0, 78, 98, 80, 10, 43, 31, 12, 108, 16, 103, 95, 34, 14, 71, 24, 68, 74, 64, 72, 88, 4, 66, 1, 3, 6, 2, 8, 67, 65, 5, 69], [115, 53, 122, 51, 117, 120, 47, 121, 126, 111, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 118, 125, 89, 104, 123, 22, 87, 25, 27, 63, 21, 59, 36, 62, 86, 116, 49, 124, 40, 85, 96, 52, 93, 56, 23, 90, 97, 60, 114, 119, 99, 127, 82, 100, 112, 33, 48, 18, 61, 101, 13, 79, 15, 107, 77, 81, 113, 32, 44, 26, 37, 19, 92, 109, 29, 17, 45, 83, 94, 28, 11, 20, 9, 30, 12, 73, 91, 34, 105, 98, 108, 41, 76, 14, 95, 31, 84, 71, 43, 35, 16, 103, 0, 80, 39, 78, 75, 70, 10, 74, 7, 68, 6, 4, 24, 64, 88, 8, 66, 2, 1, 72, 67, 3, 69, 5, 65], [115, 53, 122, 51, 117, 120, 47, 121, 111, 126, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 25, 58, 123, 118, 104, 89, 62, 125, 22, 87, 27, 97, 85, 21, 90, 93, 86, 36, 23, 56, 59, 18, 124, 13, 79, 40, 63, 48, 82, 49, 60, 114, 96, 52, 116, 33, 15, 113, 77, 99, 101, 100, 29, 112, 119, 44, 127, 17, 11, 81, 107, 28, 32, 26, 92, 19, 73, 61, 45, 84, 75, 9, 83, 41, 94, 37, 6, 71, 78, 16, 76, 64, 91, 105, 109, 95, 20, 30, 31, 14, 98, 43, 0, 12, 108, 7, 10, 39, 80, 74, 35, 68, 4, 34, 66, 103, 8, 65, 3, 88, 67, 70, 2, 24, 1, 72, 5, 69], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 58, 123, 104, 89, 118, 97, 22, 25, 86, 27, 21, 125, 87, 40, 62, 56, 60, 93, 36, 85, 63, 59, 23, 116, 79, 49, 124, 33, 52, 82, 13, 18, 99, 90, 127, 44, 48, 96, 77, 100, 119, 114, 113, 61, 15, 101, 112, 81, 32, 26, 11, 73, 92, 28, 6, 31, 29, 17, 0, 20, 45, 83, 41, 91, 84, 94, 37, 19, 71, 9, 39, 109, 105, 34, 108, 95, 98, 76, 75, 107, 78, 12, 14, 64, 30, 8, 80, 4, 7, 43, 68, 74, 35, 16, 10, 103, 67, 88, 1, 66, 3, 2, 65, 24, 69, 5, 72, 70], [115, 53, 122, 117, 51, 120, 47, 121, 111, 126, 54, 50, 46, 110, 57, 106, 55, 42, 38, 102, 123, 22, 58, 118, 89, 87, 125, 104, 21, 25, 86, 97, 27, 85, 23, 124, 63, 93, 60, 36, 127, 90, 62, 116, 59, 82, 79, 40, 18, 100, 13, 56, 49, 99, 15, 119, 96, 81, 77, 52, 33, 48, 114, 61, 113, 19, 101, 11, 29, 17, 32, 26, 28, 83, 9, 92, 84, 73, 91, 94, 109, 6, 41, 44, 31, 20, 37, 112, 12, 80, 76, 108, 78, 98, 75, 43, 74, 95, 107, 8, 71, 14, 105, 68, 7, 45, 39, 0, 30, 35, 34, 16, 103, 10, 4, 64, 88, 24, 66, 67, 1, 69, 3, 2, 65, 70, 5, 72], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 106, 55, 42, 38, 102, 58, 89, 104, 123, 25, 22, 118, 97, 87, 56, 27, 93, 86, 23, 40, 21, 36, 85, 90, 59, 124, 100, 49, 63, 125, 33, 127, 52, 60, 82, 79, 116, 18, 13, 96, 77, 99, 15, 101, 29, 62, 114, 113, 61, 119, 11, 81, 17, 48, 19, 44, 84, 32, 94, 9, 31, 28, 7, 73, 6, 30, 75, 92, 83, 112, 45, 91, 71, 26, 78, 74, 109, 37, 64, 0, 20, 41, 12, 95, 8, 16, 76, 4, 35, 98, 108, 80, 34, 14, 68, 107, 10, 105, 43, 39, 66, 103, 2, 67, 24, 65, 3, 70, 88, 1, 5, 69, 72], [115, 53, 122, 51, 117, 120, 47, 111, 126, 54, 121, 50, 46, 110, 55, 106, 57, 42, 102, 38, 22, 123, 58, 104, 89, 25, 87, 86, 27, 97, 85, 118, 56, 21, 125, 63, 40, 93, 36, 23, 90, 18, 82, 49, 79, 124, 96, 116, 60, 77, 15, 62, 119, 99, 101, 13, 114, 33, 59, 100, 61, 11, 48, 29, 81, 17, 9, 94, 26, 112, 127, 44, 19, 52, 113, 83, 32, 84, 28, 37, 92, 75, 73, 91, 76, 41, 30, 107, 8, 80, 35, 16, 78, 12, 109, 39, 31, 20, 45, 7, 14, 95, 108, 71, 10, 105, 6, 103, 74, 64, 98, 43, 4, 34, 68, 24, 0, 70, 88, 66, 3, 2, 65, 67, 1, 5, 69, 72]], "model.layers.23.self_attn.q_proj": [[111, 100, 107, 15, 88, 90, 21, 93, 81, 47, 20, 75, 11, 31, 52, 82, 73, 39, 19, 83, 85, 22, 41, 104, 12, 110, 54, 28, 13, 32, 95, 120, 14, 77, 27, 72, 76, 45, 119, 37, 87, 36, 40, 106, 71, 94, 24, 50, 6, 118, 26, 92, 101, 51, 33, 44, 38, 25, 16, 122, 114, 59, 10, 80, 112, 34, 58, 124, 5, 56, 108, 46, 123, 116, 109, 84, 79, 62, 68, 23, 70, 125, 96, 43, 103, 63, 127, 17, 57, 61, 105, 113, 78, 97, 55, 53, 126, 86, 89, 117, 74, 60, 98, 30, 18, 9, 8, 48, 49, 91, 42, 115, 29, 69, 99, 35, 121, 7, 102, 2, 67, 4, 0, 3, 66, 1, 65, 64], [111, 100, 47, 107, 32, 88, 90, 27, 94, 21, 83, 85, 24, 78, 80, 84, 81, 40, 95, 109, 30, 22, 75, 86, 17, 26, 44, 15, 10, 89, 7, 96, 77, 110, 28, 118, 119, 36, 31, 123, 50, 51, 39, 68, 0, 23, 46, 2, 63, 43, 116, 126, 127, 60, 120, 5, 73, 56, 121, 93, 41, 105, 124, 48, 42, 49, 52, 37, 33, 106, 57, 72, 101, 45, 114, 125, 98, 91, 122, 99, 103, 29, 61, 35, 20, 55, 115, 113, 112, 53, 11, 92, 54, 71, 59, 62, 34, 102, 38, 58, 108, 104, 97, 117, 6, 74, 87, 64, 25, 79, 82, 66, 19, 3, 16, 67, 18, 12, 4, 69, 14, 76, 8, 13, 65, 9, 1, 70], [111, 100, 107, 47, 90, 83, 86, 27, 88, 31, 93, 18, 80, 24, 78, 79, 91, 41, 17, 95, 85, 23, 32, 74, 72, 101, 39, 22, 96, 110, 15, 36, 94, 29, 76, 77, 50, 21, 120, 92, 34, 109, 81, 62, 13, 89, 40, 9, 118, 123, 82, 122, 119, 7, 30, 11, 84, 75, 121, 12, 127, 16, 106, 10, 56, 59, 114, 60, 20, 117, 116, 61, 5, 57, 104, 87, 58, 124, 43, 33, 52, 25, 115, 26, 46, 68, 37, 35, 28, 19, 54, 108, 97, 51, 126, 8, 102, 14, 112, 48, 73, 53, 38, 98, 105, 42, 103, 2, 0, 45, 63, 55, 71, 125, 44, 99, 49, 113, 4, 3, 6, 69, 70, 64, 1, 66, 67, 65], [111, 100, 107, 47, 32, 88, 121, 27, 85, 24, 63, 90, 51, 119, 21, 95, 81, 49, 36, 94, 40, 80, 91, 52, 50, 109, 84, 83, 93, 45, 42, 96, 61, 78, 38, 106, 34, 58, 124, 108, 110, 10, 118, 56, 46, 44, 122, 117, 125, 30, 57, 15, 115, 62, 28, 97, 104, 55, 39, 120, 59, 60, 54, 126, 112, 114, 103, 53, 23, 22, 92, 105, 75, 99, 116, 31, 41, 98, 113, 48, 35, 86, 43, 127, 123, 102, 37, 33, 77, 29, 101, 26, 5, 20, 89, 68, 17, 0, 16, 18, 72, 7, 2, 71, 25, 87, 82, 11, 73, 67, 19, 79, 14, 76, 12, 6, 74, 13, 3, 9, 8, 66, 64, 70, 69, 1, 4, 65], [56, 51, 103, 46, 110, 19, 13, 97, 15, 90, 28, 17, 99, 23, 116, 60, 94, 21, 10, 115, 47, 81, 87, 24, 57, 79, 100, 75, 72, 124, 91, 9, 53, 93, 86, 109, 80, 31, 61, 84, 89, 54, 14, 85, 74, 58, 92, 126, 3, 7, 77, 76, 6, 52, 18, 48, 113, 25, 11, 5, 123, 114, 41, 30, 127, 59, 119, 122, 27, 55, 45, 112, 12, 83, 63, 16, 71, 68, 49, 121, 29, 62, 22, 67, 43, 26, 78, 70, 88, 8, 98, 102, 82, 95, 20, 32, 108, 125, 0, 34, 36, 118, 117, 104, 105, 1, 69, 120, 101, 96, 38, 37, 106, 111, 35, 50, 107, 40, 44, 42, 64, 73, 4, 2, 33, 65, 66, 39], [46, 56, 103, 110, 53, 57, 60, 115, 61, 116, 47, 122, 124, 20, 52, 89, 127, 97, 126, 112, 51, 59, 55, 108, 114, 121, 58, 94, 54, 118, 63, 49, 62, 123, 48, 113, 109, 119, 50, 125, 111, 84, 88, 30, 106, 28, 99, 117, 25, 86, 44, 45, 80, 105, 43, 120, 107, 37, 36, 41, 24, 92, 42, 104, 27, 78, 14, 40, 38, 39, 22, 101, 102, 98, 82, 16, 33, 100, 35, 91, 19, 87, 95, 34, 31, 96, 32, 11, 9, 4, 17, 71, 64, 66, 69, 93, 0, 26, 90, 68, 73, 2, 29, 18, 1, 13, 75, 10, 21, 65, 85, 5, 83, 12, 15, 7, 67, 72, 3, 81, 23, 79, 6, 70, 76, 77, 8, 74], [56, 103, 46, 53, 57, 110, 60, 61, 124, 116, 47, 126, 51, 122, 112, 52, 97, 127, 115, 20, 89, 55, 59, 54, 58, 94, 123, 118, 28, 121, 114, 49, 113, 125, 62, 37, 119, 109, 108, 63, 99, 48, 120, 111, 25, 117, 45, 50, 80, 44, 30, 36, 105, 86, 84, 43, 92, 106, 41, 107, 88, 24, 104, 14, 42, 40, 19, 23, 39, 82, 102, 101, 38, 22, 98, 100, 27, 34, 33, 35, 78, 16, 91, 13, 96, 31, 21, 71, 87, 95, 9, 81, 12, 90, 15, 69, 32, 18, 68, 11, 72, 0, 93, 3, 74, 5, 29, 26, 85, 10, 64, 66, 1, 2, 4, 83, 17, 75, 73, 67, 6, 65, 76, 7, 79, 8, 77, 70], [46, 51, 103, 56, 19, 110, 87, 17, 15, 97, 28, 13, 92, 116, 10, 81, 61, 12, 47, 90, 53, 94, 23, 86, 112, 60, 57, 126, 72, 58, 52, 59, 96, 121, 124, 43, 89, 122, 115, 25, 114, 127, 106, 49, 55, 62, 21, 48, 63, 54, 108, 45, 4, 123, 109, 125, 119, 120, 42, 111, 118, 113, 24, 117, 50, 83, 30, 44, 105, 79, 85, 93, 107, 75, 67, 99, 9, 82, 11, 31, 20, 71, 8, 98, 104, 70, 6, 76, 26, 101, 84, 77, 40, 32, 41, 38, 7, 34, 37, 29, 22, 36, 102, 100, 95, 5, 3, 91, 80, 18, 14, 68, 27, 73, 69, 0, 1, 65, 35, 88, 74, 66, 16, 64, 2, 78, 39, 33], [102, 110, 33, 49, 46, 92, 113, 111, 86, 28, 122, 19, 57, 81, 24, 55, 78, 54, 70, 74, 85, 13, 108, 90, 61, 8, 117, 94, 30, 23, 66, 105, 80, 109, 114, 26, 76, 87, 44, 14, 60, 75, 9, 68, 79, 127, 67, 17, 21, 47, 77, 107, 31, 119, 65, 51, 0, 106, 18, 63, 116, 56, 62, 15, 7, 120, 58, 53, 126, 83, 52, 112, 115, 40, 11, 50, 39, 103, 43, 121, 104, 36, 124, 2, 123, 12, 20, 99, 125, 3, 118, 45, 48, 41, 59, 37, 22, 32, 29, 98, 71, 42, 89, 72, 97, 25, 35, 101, 69, 88, 16, 100, 27, 93, 95, 34, 91, 82, 96, 5, 4, 10, 1, 84, 38, 64, 73, 6], [110, 102, 33, 46, 49, 57, 92, 113, 111, 86, 28, 19, 24, 81, 85, 61, 116, 119, 56, 122, 115, 87, 26, 79, 108, 90, 107, 78, 94, 63, 109, 114, 9, 13, 52, 23, 121, 127, 30, 74, 47, 104, 123, 70, 21, 17, 99, 126, 59, 48, 62, 51, 31, 14, 60, 98, 124, 58, 125, 80, 55, 106, 54, 43, 112, 53, 120, 50, 100, 44, 118, 29, 42, 76, 41, 105, 117, 103, 75, 77, 8, 45, 37, 36, 83, 15, 101, 25, 20, 66, 68, 35, 22, 95, 34, 40, 12, 18, 39, 93, 11, 32, 96, 0, 97, 38, 27, 89, 88, 69, 91, 3, 65, 84, 1, 72, 4, 10, 71, 73, 7, 16, 82, 6, 64, 5, 67, 2], [110, 102, 46, 113, 49, 33, 92, 111, 86, 19, 122, 28, 24, 81, 79, 55, 60, 85, 78, 39, 26, 127, 87, 74, 115, 68, 90, 93, 12, 118, 119, 114, 77, 70, 9, 125, 56, 8, 13, 50, 84, 108, 61, 29, 107, 98, 14, 59, 37, 105, 51, 100, 22, 112, 73, 71, 35, 25, 47, 94, 58, 88, 44, 117, 124, 53, 120, 42, 106, 21, 95, 57, 40, 23, 20, 17, 83, 82, 121, 2, 96, 62, 54, 52, 32, 30, 41, 65, 31, 116, 43, 109, 76, 104, 126, 75, 36, 64, 63, 38, 99, 45, 48, 91, 101, 34, 11, 5, 80, 103, 123, 66, 15, 89, 27, 18, 10, 97, 16, 4, 6, 72, 0, 7, 3, 69, 67, 1], [110, 102, 46, 49, 33, 113, 92, 111, 86, 122, 19, 28, 54, 81, 85, 119, 24, 90, 26, 109, 94, 126, 79, 70, 13, 120, 78, 9, 51, 57, 112, 74, 125, 21, 87, 108, 123, 61, 50, 59, 105, 107, 121, 52, 14, 58, 8, 53, 47, 56, 68, 17, 60, 43, 45, 100, 93, 127, 114, 63, 12, 118, 106, 80, 44, 116, 55, 20, 48, 29, 15, 83, 30, 124, 23, 117, 103, 77, 31, 39, 65, 115, 42, 62, 75, 22, 104, 91, 40, 76, 99, 36, 71, 66, 34, 97, 101, 41, 98, 37, 32, 95, 88, 35, 82, 10, 27, 89, 18, 96, 25, 73, 84, 5, 11, 64, 16, 38, 72, 3, 0, 2, 4, 69, 7, 6, 1, 67], [48, 39, 119, 117, 56, 33, 60, 112, 121, 127, 58, 120, 47, 116, 125, 63, 122, 49, 90, 29, 114, 118, 55, 51, 123, 115, 124, 52, 59, 126, 113, 61, 62, 53, 107, 24, 44, 111, 54, 50, 110, 45, 91, 108, 93, 46, 106, 57, 88, 95, 109, 26, 86, 42, 20, 40, 43, 41, 105, 102, 17, 101, 104, 97, 36, 103, 38, 27, 85, 87, 37, 92, 100, 34, 22, 96, 94, 99, 98, 28, 89, 35, 21, 81, 16, 32, 31, 30, 84, 83, 78, 76, 14, 25, 12, 15, 82, 80, 10, 23, 18, 74, 19, 72, 65, 1, 67, 3, 0, 13, 64, 8, 69, 5, 68, 66, 2, 79, 4, 11, 7, 6, 77, 71, 9, 75, 73, 70], [119, 39, 48, 117, 60, 33, 56, 121, 47, 127, 120, 55, 63, 125, 58, 116, 51, 122, 49, 123, 114, 118, 115, 61, 52, 112, 90, 53, 59, 113, 106, 44, 62, 126, 124, 110, 29, 57, 54, 24, 107, 50, 111, 45, 108, 46, 41, 91, 93, 26, 95, 88, 105, 20, 86, 109, 43, 42, 102, 40, 104, 17, 103, 92, 101, 100, 38, 27, 97, 21, 34, 36, 94, 85, 28, 37, 87, 35, 99, 96, 89, 31, 22, 98, 16, 32, 76, 30, 81, 84, 25, 83, 15, 80, 78, 23, 14, 12, 82, 18, 10, 74, 1, 0, 65, 67, 64, 8, 19, 69, 72, 5, 4, 3, 13, 79, 11, 68, 66, 2, 7, 6, 71, 75, 70, 9, 77, 73], [117, 39, 119, 48, 56, 90, 60, 33, 55, 62, 124, 52, 121, 120, 95, 127, 47, 46, 105, 93, 58, 29, 116, 106, 61, 115, 122, 114, 63, 83, 26, 125, 51, 10, 118, 102, 126, 113, 49, 24, 57, 107, 50, 53, 123, 89, 59, 45, 104, 109, 108, 96, 54, 16, 111, 86, 44, 110, 112, 20, 42, 27, 43, 18, 41, 81, 85, 78, 38, 30, 40, 32, 94, 31, 19, 36, 25, 17, 34, 101, 82, 92, 100, 98, 14, 88, 91, 23, 75, 35, 97, 84, 87, 28, 71, 37, 13, 99, 22, 103, 12, 8, 21, 76, 3, 15, 11, 80, 4, 9, 68, 2, 72, 7, 74, 79, 5, 0, 6, 69, 73, 70, 67, 77, 1, 66, 65, 64], [39, 117, 119, 48, 83, 33, 15, 13, 11, 9, 29, 86, 6, 89, 66, 7, 74, 55, 88, 27, 87, 73, 81, 68, 80, 53, 90, 72, 22, 84, 91, 75, 82, 85, 8, 65, 30, 23, 18, 62, 26, 25, 17, 56, 16, 34, 0, 77, 79, 12, 21, 67, 76, 19, 92, 14, 31, 78, 70, 71, 28, 94, 4, 20, 5, 10, 32, 58, 50, 96, 35, 3, 64, 43, 106, 45, 24, 95, 40, 2, 69, 47, 102, 104, 93, 112, 38, 105, 99, 101, 54, 121, 108, 61, 124, 120, 57, 122, 1, 103, 110, 41, 98, 127, 49, 100, 115, 123, 36, 60, 111, 44, 63, 125, 118, 42, 52, 51, 107, 37, 116, 59, 97, 114, 113, 126, 109, 46], [44, 37, 108, 76, 18, 21, 14, 97, 24, 57, 80, 28, 71, 0, 5, 73, 125, 90, 69, 67, 7, 2, 51, 121, 58, 3, 11, 72, 75, 113, 8, 10, 19, 52, 23, 12, 89, 118, 55, 33, 91, 68, 16, 64, 65, 9, 88, 101, 34, 92, 70, 96, 77, 85, 13, 1, 84, 119, 99, 124, 20, 109, 25, 123, 66, 86, 26, 6, 82, 46, 93, 83, 78, 127, 114, 104, 79, 74, 15, 81, 35, 4, 100, 95, 47, 111, 29, 30, 27, 50, 56, 120, 98, 60, 126, 17, 117, 22, 31, 32, 61, 63, 110, 94, 36, 87, 106, 62, 103, 45, 41, 39, 102, 38, 105, 112, 43, 40, 107, 54, 116, 115, 122, 48, 42, 49, 53, 59], [44, 37, 108, 28, 24, 21, 97, 80, 18, 14, 19, 76, 73, 57, 87, 51, 90, 58, 71, 125, 15, 121, 17, 113, 117, 47, 5, 55, 11, 7, 106, 95, 69, 23, 52, 50, 98, 2, 93, 30, 114, 99, 85, 83, 45, 86, 22, 127, 27, 92, 10, 66, 35, 109, 31, 84, 20, 46, 3, 81, 82, 100, 29, 6, 75, 91, 79, 78, 124, 70, 16, 34, 96, 118, 105, 12, 33, 123, 61, 13, 9, 48, 89, 94, 88, 26, 36, 8, 111, 104, 25, 112, 32, 40, 103, 77, 38, 42, 53, 120, 126, 102, 41, 119, 107, 49, 43, 74, 110, 72, 68, 115, 0, 67, 122, 116, 39, 56, 54, 63, 59, 60, 101, 62, 4, 64, 65, 1], [57, 44, 37, 125, 108, 90, 97, 19, 28, 58, 21, 114, 24, 126, 113, 109, 117, 17, 86, 116, 121, 61, 51, 40, 110, 50, 49, 122, 55, 118, 41, 46, 59, 47, 11, 48, 43, 111, 106, 56, 124, 102, 63, 53, 123, 52, 62, 115, 112, 120, 119, 91, 54, 60, 127, 42, 107, 80, 104, 45, 105, 18, 87, 93, 39, 95, 14, 72, 35, 38, 84, 73, 77, 103, 23, 13, 101, 75, 27, 36, 20, 74, 66, 34, 89, 83, 25, 15, 26, 6, 30, 98, 8, 33, 100, 4, 29, 22, 81, 99, 94, 5, 79, 31, 68, 96, 32, 70, 76, 69, 2, 10, 1, 65, 12, 85, 92, 0, 64, 88, 78, 7, 9, 82, 16, 3, 71, 67], [44, 37, 125, 24, 108, 21, 28, 18, 97, 14, 90, 19, 58, 80, 57, 51, 11, 121, 114, 73, 109, 17, 13, 95, 118, 117, 52, 87, 75, 113, 69, 35, 50, 93, 61, 81, 46, 123, 76, 9, 124, 34, 119, 102, 71, 111, 115, 59, 36, 104, 100, 32, 96, 4, 55, 103, 20, 84, 99, 30, 85, 45, 105, 60, 106, 62, 29, 86, 83, 39, 25, 43, 40, 48, 66, 94, 38, 42, 120, 107, 47, 91, 110, 23, 112, 82, 15, 127, 78, 116, 98, 92, 16, 31, 122, 54, 89, 126, 49, 33, 72, 88, 41, 56, 22, 53, 101, 26, 10, 63, 74, 79, 67, 6, 1, 27, 8, 77, 5, 64, 12, 2, 3, 70, 68, 65, 7, 0], [54, 37, 127, 62, 63, 24, 116, 33, 101, 119, 123, 18, 55, 59, 51, 120, 58, 112, 46, 53, 60, 125, 114, 117, 113, 39, 126, 44, 15, 57, 61, 56, 111, 121, 122, 118, 47, 52, 49, 27, 110, 50, 48, 91, 115, 45, 124, 94, 109, 85, 107, 108, 105, 106, 43, 92, 86, 20, 21, 104, 42, 73, 103, 41, 102, 88, 100, 30, 40, 38, 25, 29, 82, 23, 95, 36, 13, 28, 97, 35, 31, 22, 9, 77, 79, 99, 84, 90, 98, 34, 16, 93, 87, 26, 17, 89, 12, 14, 1, 80, 75, 71, 65, 32, 4, 19, 6, 96, 70, 2, 66, 10, 7, 11, 68, 67, 72, 78, 3, 76, 5, 69, 0, 83, 8, 81, 64, 74], [127, 54, 37, 25, 62, 17, 94, 101, 33, 86, 15, 123, 20, 6, 63, 24, 14, 12, 27, 77, 19, 10, 90, 30, 109, 23, 116, 84, 88, 13, 61, 93, 72, 113, 9, 51, 107, 95, 91, 59, 120, 38, 119, 92, 125, 117, 112, 22, 7, 41, 124, 70, 32, 58, 85, 35, 67, 36, 75, 118, 48, 42, 78, 31, 18, 126, 122, 21, 79, 28, 102, 55, 34, 80, 29, 115, 114, 68, 82, 5, 11, 53, 50, 16, 87, 57, 83, 103, 46, 98, 106, 81, 26, 56, 60, 44, 4, 111, 39, 74, 89, 8, 108, 76, 52, 110, 121, 1, 45, 49, 73, 40, 96, 100, 97, 66, 105, 104, 99, 43, 47, 0, 71, 3, 2, 69, 65, 64], [127, 63, 37, 62, 54, 24, 33, 51, 112, 123, 116, 101, 117, 55, 120, 59, 58, 119, 125, 53, 19, 114, 46, 111, 44, 60, 113, 122, 61, 47, 126, 57, 121, 91, 118, 52, 56, 49, 48, 27, 50, 45, 94, 110, 115, 85, 124, 39, 108, 109, 18, 102, 86, 106, 88, 30, 107, 29, 105, 43, 41, 104, 103, 42, 96, 21, 14, 40, 38, 95, 20, 97, 32, 26, 100, 22, 16, 31, 75, 83, 36, 25, 98, 12, 92, 35, 99, 78, 34, 15, 90, 82, 13, 93, 80, 17, 6, 11, 73, 77, 9, 65, 1, 84, 89, 28, 87, 23, 68, 4, 66, 2, 7, 70, 10, 71, 72, 5, 8, 76, 79, 0, 81, 67, 64, 3, 69, 74], [127, 54, 37, 33, 17, 62, 86, 10, 15, 24, 14, 123, 12, 5, 72, 90, 27, 65, 7, 116, 25, 0, 101, 63, 77, 91, 64, 20, 44, 23, 88, 73, 4, 19, 67, 87, 89, 113, 22, 71, 112, 2, 93, 8, 107, 83, 59, 81, 111, 80, 76, 68, 79, 74, 82, 118, 21, 13, 6, 120, 104, 46, 114, 75, 126, 28, 119, 70, 122, 18, 3, 97, 11, 30, 102, 1, 16, 84, 69, 56, 31, 106, 58, 26, 9, 66, 78, 32, 85, 48, 95, 40, 117, 61, 94, 96, 45, 125, 99, 35, 29, 98, 57, 92, 60, 39, 53, 124, 34, 36, 100, 51, 38, 52, 103, 41, 108, 55, 109, 110, 105, 47, 43, 121, 49, 50, 115, 42], [43, 121, 45, 107, 100, 97, 109, 27, 91, 102, 123, 24, 103, 15, 22, 54, 81, 59, 88, 127, 120, 118, 114, 117, 110, 20, 44, 124, 85, 61, 111, 31, 79, 119, 51, 26, 75, 57, 7, 41, 11, 33, 58, 98, 49, 92, 46, 23, 112, 39, 52, 99, 42, 13, 113, 55, 21, 62, 29, 17, 122, 28, 47, 84, 63, 125, 25, 48, 18, 93, 50, 108, 126, 56, 82, 115, 116, 86, 101, 104, 37, 105, 53, 60, 32, 30, 96, 38, 40, 35, 34, 89, 80, 106, 90, 19, 94, 77, 83, 95, 16, 73, 87, 71, 74, 70, 76, 14, 78, 36, 3, 6, 9, 12, 5, 69, 8, 10, 72, 66, 67, 65, 2, 1, 68, 64, 0, 4], [43, 107, 123, 117, 100, 27, 97, 91, 88, 51, 103, 21, 49, 124, 80, 59, 54, 24, 109, 55, 48, 85, 115, 127, 57, 14, 50, 121, 61, 114, 113, 126, 58, 118, 22, 45, 119, 60, 62, 52, 41, 31, 63, 125, 112, 47, 111, 74, 122, 116, 53, 29, 19, 76, 110, 102, 46, 44, 56, 108, 39, 93, 37, 120, 5, 20, 42, 89, 8, 30, 90, 6, 2, 99, 105, 3, 18, 40, 106, 68, 35, 101, 98, 87, 104, 84, 38, 78, 32, 33, 16, 1, 86, 12, 17, 96, 0, 28, 34, 25, 94, 26, 81, 10, 72, 11, 95, 23, 15, 36, 92, 4, 69, 83, 82, 66, 73, 65, 64, 70, 13, 77, 67, 71, 7, 75, 9, 79], [43, 121, 45, 107, 100, 27, 97, 91, 116, 63, 24, 22, 61, 106, 125, 54, 114, 37, 15, 49, 88, 118, 58, 124, 29, 33, 85, 76, 120, 57, 31, 17, 19, 32, 81, 21, 41, 127, 47, 109, 123, 80, 50, 18, 52, 60, 51, 55, 98, 122, 74, 86, 53, 101, 113, 34, 105, 103, 12, 28, 119, 14, 111, 62, 94, 93, 59, 8, 115, 46, 38, 44, 112, 42, 7, 110, 90, 40, 126, 68, 48, 20, 117, 26, 104, 102, 36, 99, 108, 35, 2, 39, 96, 30, 92, 84, 13, 11, 56, 79, 82, 25, 95, 89, 75, 73, 83, 77, 1, 78, 5, 3, 10, 87, 71, 69, 0, 72, 9, 23, 67, 6, 4, 66, 65, 64, 70, 16], [121, 43, 45, 100, 91, 97, 27, 107, 88, 85, 21, 123, 80, 51, 54, 24, 115, 49, 61, 117, 127, 126, 48, 59, 57, 114, 109, 46, 52, 116, 118, 63, 55, 74, 53, 50, 22, 14, 76, 60, 105, 31, 62, 119, 124, 120, 111, 19, 122, 125, 113, 58, 106, 47, 44, 56, 102, 8, 112, 90, 103, 110, 12, 108, 39, 93, 37, 20, 32, 29, 42, 41, 99, 89, 81, 33, 5, 87, 2, 104, 17, 40, 38, 6, 34, 16, 82, 26, 101, 78, 28, 86, 96, 92, 30, 3, 35, 1, 98, 36, 68, 94, 72, 23, 95, 18, 10, 84, 0, 69, 25, 83, 15, 70, 4, 73, 65, 67, 11, 13, 66, 64, 77, 79, 71, 9, 7, 75], [103, 115, 51, 85, 83, 80, 62, 10, 90, 13, 124, 118, 26, 63, 72, 48, 56, 70, 60, 98, 68, 52, 55, 88, 106, 47, 59, 41, 53, 50, 93, 110, 30, 91, 100, 107, 76, 71, 86, 58, 111, 19, 45, 84, 15, 66, 82, 122, 61, 42, 123, 57, 125, 38, 44, 96, 89, 77, 119, 65, 81, 40, 21, 28, 16, 14, 87, 11, 3, 23, 25, 94, 101, 104, 43, 33, 102, 17, 74, 27, 105, 121, 49, 95, 112, 31, 92, 117, 116, 24, 108, 20, 120, 36, 7, 9, 109, 5, 114, 46, 97, 34, 69, 126, 127, 73, 35, 113, 79, 8, 99, 22, 29, 18, 32, 78, 54, 12, 37, 64, 2, 39, 6, 1, 75, 4, 0, 67], [103, 51, 115, 85, 83, 10, 80, 13, 72, 70, 66, 68, 26, 62, 63, 90, 118, 60, 81, 124, 107, 64, 2, 91, 53, 59, 55, 44, 52, 119, 89, 56, 65, 93, 106, 47, 111, 50, 0, 123, 98, 19, 15, 100, 40, 76, 1, 110, 61, 5, 48, 114, 69, 77, 16, 41, 25, 6, 99, 74, 8, 4, 104, 3, 32, 126, 79, 11, 94, 117, 105, 22, 121, 87, 20, 88, 29, 58, 116, 46, 43, 73, 112, 14, 75, 21, 27, 23, 127, 24, 120, 35, 71, 38, 86, 109, 57, 49, 82, 92, 125, 45, 113, 122, 67, 54, 102, 78, 96, 84, 9, 108, 18, 95, 12, 101, 42, 37, 30, 33, 17, 28, 34, 39, 7, 31, 36, 97], [103, 51, 115, 85, 80, 13, 10, 68, 83, 70, 72, 26, 66, 0, 62, 124, 55, 65, 90, 118, 63, 53, 60, 64, 47, 76, 100, 98, 89, 107, 46, 59, 88, 50, 56, 48, 61, 2, 91, 93, 82, 1, 58, 23, 111, 4, 3, 19, 81, 73, 99, 121, 25, 44, 5, 106, 123, 6, 69, 75, 74, 11, 41, 87, 119, 15, 20, 127, 71, 79, 21, 95, 77, 110, 8, 52, 40, 126, 16, 114, 67, 27, 17, 112, 9, 122, 78, 94, 57, 22, 117, 86, 49, 12, 14, 32, 39, 113, 30, 37, 38, 92, 120, 18, 29, 105, 84, 96, 31, 36, 7, 33, 108, 42, 125, 35, 116, 97, 101, 24, 109, 45, 104, 54, 43, 102, 28, 34], [103, 115, 51, 80, 85, 10, 13, 83, 72, 70, 68, 62, 26, 124, 118, 60, 59, 55, 66, 107, 90, 82, 64, 93, 119, 47, 52, 25, 15, 63, 48, 53, 98, 40, 88, 56, 111, 87, 73, 5, 1, 89, 76, 65, 69, 23, 29, 2, 61, 71, 0, 46, 44, 81, 100, 50, 117, 77, 58, 19, 22, 96, 84, 32, 112, 110, 24, 74, 8, 7, 3, 16, 114, 12, 17, 122, 126, 121, 125, 91, 27, 11, 123, 35, 38, 28, 99, 4, 104, 106, 116, 20, 101, 45, 6, 9, 79, 31, 78, 95, 41, 18, 30, 21, 86, 67, 92, 49, 108, 113, 43, 94, 37, 33, 57, 42, 127, 36, 105, 109, 54, 34, 102, 14, 120, 97, 75, 39]], "model.layers.23.self_attn.k_proj": [[47, 111, 36, 107, 88, 96, 30, 90, 80, 21, 0, 78, 81, 2, 27, 104, 68, 84, 83, 7, 15, 75, 74, 93, 95, 5, 103, 106, 10, 77, 11, 86, 124, 110, 22, 6, 120, 91, 59, 71, 73, 85, 43, 122, 46, 127, 13, 51, 67, 94, 119, 114, 48, 32, 50, 105, 55, 92, 101, 23, 66, 118, 42, 52, 33, 97, 56, 63, 60, 102, 125, 35, 39, 123, 72, 29, 31, 99, 116, 64, 19, 14, 45, 41, 53, 61, 112, 28, 12, 117, 49, 58, 113, 16, 40, 34, 57, 17, 25, 38, 108, 62, 3, 79, 98, 126, 37, 54, 109, 121, 18, 4, 44, 115, 24, 20, 26, 82, 89, 87, 65, 8, 76, 9, 100, 1, 70, 69], [39, 56, 46, 51, 86, 33, 110, 116, 61, 53, 30, 20, 60, 47, 57, 124, 55, 59, 120, 80, 89, 58, 114, 119, 52, 122, 127, 112, 49, 91, 126, 123, 63, 54, 118, 48, 115, 121, 44, 113, 62, 34, 125, 45, 106, 109, 117, 92, 50, 111, 101, 96, 100, 93, 14, 94, 108, 105, 107, 88, 98, 43, 42, 35, 40, 24, 37, 78, 104, 38, 41, 36, 11, 102, 97, 95, 27, 16, 90, 82, 29, 65, 72, 13, 31, 19, 84, 71, 99, 64, 5, 73, 26, 87, 85, 3, 32, 25, 10, 28, 12, 17, 21, 15, 69, 9, 18, 7, 23, 70, 6, 0, 76, 1, 79, 77, 103, 75, 83, 4, 67, 22, 68, 66, 81, 8, 2, 74], [46, 113, 38, 110, 97, 28, 19, 86, 79, 26, 94, 24, 81, 21, 87, 77, 122, 14, 70, 74, 47, 17, 78, 60, 68, 9, 59, 52, 126, 65, 57, 112, 111, 119, 62, 44, 8, 55, 114, 20, 105, 115, 56, 127, 108, 117, 106, 85, 58, 76, 104, 0, 125, 50, 11, 42, 64, 93, 118, 45, 41, 109, 13, 2, 43, 103, 80, 124, 92, 48, 123, 63, 51, 116, 39, 16, 121, 120, 36, 61, 66, 100, 53, 99, 75, 54, 40, 18, 89, 101, 49, 37, 107, 90, 15, 73, 95, 5, 29, 72, 32, 71, 23, 3, 35, 25, 31, 34, 96, 84, 98, 12, 91, 27, 30, 22, 69, 82, 10, 33, 88, 7, 6, 83, 4, 67, 1, 102], [117, 103, 119, 86, 48, 97, 112, 13, 93, 11, 83, 15, 56, 26, 60, 120, 124, 47, 121, 6, 89, 113, 53, 122, 116, 63, 58, 127, 55, 125, 115, 24, 9, 114, 123, 7, 52, 49, 118, 51, 61, 62, 91, 59, 54, 126, 18, 107, 81, 111, 45, 42, 50, 46, 44, 57, 109, 16, 110, 20, 43, 108, 74, 41, 73, 98, 104, 29, 92, 30, 23, 105, 40, 25, 106, 37, 88, 35, 38, 21, 68, 17, 64, 100, 101, 78, 102, 31, 14, 27, 36, 99, 2, 95, 28, 8, 32, 96, 94, 34, 69, 85, 10, 82, 4, 19, 87, 84, 66, 90, 80, 12, 76, 79, 75, 65, 33, 70, 71, 5, 77, 72, 67, 0, 39, 1, 3, 22], [108, 101, 44, 57, 21, 80, 28, 14, 24, 11, 71, 33, 18, 76, 73, 90, 58, 125, 19, 67, 50, 5, 64, 17, 1, 52, 55, 15, 118, 117, 121, 51, 115, 122, 53, 119, 4, 47, 48, 69, 75, 124, 123, 120, 95, 45, 35, 31, 111, 2, 59, 13, 87, 126, 12, 104, 61, 60, 56, 72, 110, 42, 127, 41, 114, 62, 43, 112, 8, 84, 7, 63, 109, 107, 68, 116, 54, 23, 40, 98, 106, 88, 91, 39, 6, 46, 49, 27, 105, 10, 70, 89, 113, 26, 30, 86, 36, 29, 34, 103, 32, 100, 92, 38, 81, 102, 94, 22, 78, 93, 74, 3, 66, 16, 0, 97, 77, 99, 25, 79, 9, 96, 82, 20, 85, 83, 37, 65], [127, 54, 101, 97, 86, 24, 27, 123, 118, 18, 94, 77, 113, 63, 116, 120, 48, 17, 10, 64, 58, 85, 62, 59, 117, 57, 15, 38, 114, 12, 111, 67, 42, 121, 122, 14, 108, 29, 55, 61, 7, 125, 53, 126, 60, 49, 103, 1, 51, 119, 19, 56, 16, 52, 112, 124, 43, 109, 50, 5, 105, 45, 110, 46, 115, 47, 31, 72, 75, 71, 37, 90, 6, 25, 66, 107, 40, 44, 20, 39, 89, 87, 95, 104, 106, 83, 36, 100, 41, 34, 102, 93, 98, 68, 96, 11, 99, 0, 2, 35, 3, 21, 26, 92, 9, 79, 23, 30, 28, 80, 32, 78, 91, 81, 84, 76, 8, 88, 73, 69, 74, 13, 82, 33, 70, 22, 4, 65], [107, 33, 121, 36, 22, 91, 43, 124, 88, 45, 61, 80, 21, 51, 54, 60, 57, 49, 74, 55, 119, 123, 58, 115, 117, 29, 116, 62, 63, 122, 127, 50, 26, 125, 53, 52, 111, 42, 118, 59, 112, 113, 109, 120, 126, 46, 110, 56, 48, 38, 47, 76, 95, 108, 101, 19, 114, 39, 14, 65, 17, 106, 66, 44, 69, 4, 78, 20, 104, 12, 41, 0, 105, 98, 8, 35, 37, 15, 30, 67, 103, 70, 64, 83, 102, 6, 23, 40, 99, 31, 32, 34, 94, 92, 5, 81, 10, 87, 73, 96, 28, 25, 100, 82, 9, 97, 18, 11, 93, 90, 89, 79, 71, 27, 72, 86, 85, 77, 84, 13, 1, 7, 3, 75, 68, 2, 24, 16], [115, 39, 13, 72, 51, 80, 64, 83, 10, 70, 85, 68, 66, 90, 62, 2, 124, 0, 65, 76, 43, 111, 26, 60, 91, 112, 118, 75, 53, 55, 61, 56, 67, 5, 81, 88, 63, 4, 119, 89, 46, 42, 40, 34, 59, 52, 50, 108, 73, 29, 123, 1, 93, 82, 114, 15, 121, 105, 23, 35, 36, 71, 126, 30, 31, 22, 96, 7, 122, 125, 84, 69, 41, 110, 58, 3, 102, 79, 25, 109, 87, 86, 24, 120, 94, 57, 127, 38, 14, 98, 28, 100, 104, 99, 6, 106, 117, 44, 33, 107, 54, 45, 8, 48, 101, 95, 78, 97, 49, 116, 17, 12, 47, 18, 92, 27, 32, 20, 9, 113, 37, 11, 74, 21, 19, 77, 16, 103]], "model.layers.23.self_attn.qk_proj": [[46, 115, 111, 127, 54, 56, 47, 110, 108, 119, 117, 107, 51, 44, 121, 48, 43, 113, 22, 24, 26, 21, 55, 83, 85, 57, 62, 19, 92, 27, 124, 101, 80, 88, 118, 90, 63, 39, 37, 16, 59, 86, 45, 60, 123, 103, 61, 33, 13, 116, 58, 125, 49, 122, 81, 77, 74, 97, 14, 10, 120, 78, 50, 72, 79, 53, 52, 17, 36, 28, 93, 15, 91, 94, 29, 70, 106, 112, 11, 126, 75, 68, 109, 8, 4, 64, 82, 114, 0, 89, 71, 18, 100, 2, 25, 9, 76, 66, 102, 38, 12, 30, 105, 23, 73, 7, 42, 5, 96, 32, 20, 84, 104, 40, 95, 87, 69, 6, 41, 98, 1, 31, 34, 67, 99, 35, 65, 3], [46, 115, 127, 111, 54, 56, 47, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 57, 21, 22, 26, 83, 124, 27, 118, 39, 24, 92, 62, 85, 63, 55, 101, 19, 88, 16, 60, 86, 90, 80, 58, 37, 59, 123, 116, 33, 125, 74, 61, 103, 49, 77, 45, 78, 13, 10, 81, 120, 112, 52, 70, 91, 122, 53, 14, 72, 97, 50, 17, 15, 93, 114, 94, 28, 4, 0, 11, 29, 36, 106, 126, 2, 66, 79, 109, 75, 8, 82, 68, 102, 30, 38, 25, 105, 100, 64, 76, 73, 12, 104, 23, 18, 42, 89, 7, 20, 9, 71, 5, 96, 31, 69, 32, 98, 84, 95, 6, 40, 41, 87, 3, 34, 35, 65, 99, 1, 67], [46, 115, 111, 127, 56, 54, 110, 47, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 22, 124, 26, 62, 85, 21, 24, 118, 83, 27, 55, 39, 92, 37, 63, 19, 101, 60, 90, 80, 88, 33, 58, 16, 86, 103, 59, 97, 61, 123, 125, 74, 53, 77, 122, 116, 45, 14, 13, 78, 91, 120, 10, 52, 72, 81, 49, 112, 70, 36, 66, 4, 28, 50, 15, 17, 109, 93, 64, 79, 11, 29, 126, 2, 114, 38, 94, 0, 68, 106, 25, 12, 82, 75, 104, 100, 23, 76, 89, 18, 42, 71, 7, 30, 84, 8, 96, 105, 41, 20, 5, 9, 32, 98, 73, 102, 40, 69, 87, 35, 31, 95, 6, 34, 99, 65, 67, 1, 3], [46, 115, 127, 111, 54, 56, 47, 110, 108, 107, 119, 117, 51, 44, 121, 48, 43, 113, 57, 22, 21, 83, 63, 124, 26, 62, 118, 60, 19, 24, 85, 92, 39, 88, 58, 27, 55, 80, 101, 37, 16, 122, 53, 123, 86, 59, 90, 77, 61, 33, 125, 10, 116, 17, 81, 13, 14, 78, 120, 103, 74, 45, 72, 112, 49, 97, 15, 70, 52, 4, 50, 91, 11, 66, 114, 106, 126, 2, 109, 36, 100, 68, 29, 28, 93, 94, 0, 18, 75, 76, 79, 12, 71, 38, 89, 8, 102, 7, 69, 64, 42, 73, 82, 25, 9, 104, 30, 84, 20, 5, 96, 105, 23, 6, 31, 41, 32, 98, 87, 95, 35, 65, 67, 40, 99, 34, 3, 1], [46, 115, 127, 111, 54, 47, 56, 110, 108, 119, 117, 107, 51, 44, 48, 121, 43, 113, 57, 62, 83, 124, 63, 22, 21, 19, 26, 85, 55, 24, 60, 27, 101, 39, 118, 86, 16, 37, 92, 58, 90, 122, 123, 88, 80, 61, 33, 103, 97, 72, 59, 14, 13, 10, 77, 45, 74, 49, 125, 52, 4, 81, 78, 68, 112, 17, 50, 116, 75, 53, 120, 91, 66, 79, 15, 28, 11, 109, 0, 126, 106, 93, 114, 29, 64, 2, 36, 12, 94, 70, 6, 7, 18, 20, 76, 25, 100, 82, 71, 38, 102, 9, 23, 73, 69, 89, 8, 42, 30, 5, 105, 84, 40, 104, 41, 32, 98, 96, 35, 34, 87, 31, 99, 95, 67, 3, 1, 65], [46, 115, 111, 127, 54, 47, 56, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 22, 57, 21, 26, 19, 27, 55, 24, 83, 62, 85, 101, 63, 39, 16, 80, 124, 88, 86, 58, 92, 60, 118, 37, 122, 72, 90, 123, 10, 33, 77, 45, 74, 13, 61, 81, 14, 97, 59, 125, 103, 17, 112, 49, 15, 78, 116, 75, 120, 53, 52, 0, 50, 91, 11, 68, 126, 28, 79, 64, 94, 66, 29, 82, 106, 114, 20, 36, 100, 6, 7, 4, 12, 76, 93, 25, 18, 23, 109, 38, 71, 2, 69, 102, 89, 30, 8, 105, 9, 70, 84, 73, 96, 42, 87, 32, 34, 95, 41, 5, 104, 65, 98, 40, 1, 35, 67, 31, 99, 3], [46, 115, 111, 127, 54, 56, 110, 47, 108, 119, 107, 51, 121, 44, 48, 117, 43, 113, 26, 22, 57, 21, 27, 85, 24, 83, 19, 88, 124, 92, 62, 80, 16, 101, 90, 86, 63, 39, 58, 60, 55, 77, 37, 122, 123, 33, 14, 103, 125, 74, 118, 10, 13, 97, 72, 81, 120, 45, 61, 59, 17, 78, 112, 116, 28, 49, 91, 106, 93, 50, 15, 79, 36, 53, 6, 126, 64, 114, 75, 82, 11, 52, 94, 109, 29, 68, 12, 89, 20, 38, 18, 25, 100, 4, 8, 66, 76, 23, 84, 102, 7, 0, 9, 71, 2, 73, 87, 42, 96, 105, 30, 32, 98, 31, 69, 70, 34, 5, 1, 41, 95, 104, 40, 35, 65, 99, 67, 3], [46, 115, 111, 127, 56, 54, 110, 47, 108, 119, 107, 51, 117, 44, 121, 48, 43, 113, 27, 21, 22, 24, 83, 57, 85, 26, 60, 92, 62, 19, 124, 55, 16, 80, 88, 90, 39, 86, 101, 123, 63, 33, 37, 61, 118, 58, 125, 59, 49, 77, 122, 116, 13, 97, 74, 103, 14, 45, 10, 81, 78, 112, 79, 17, 120, 28, 114, 36, 6, 91, 50, 52, 126, 72, 53, 94, 93, 11, 82, 106, 75, 4, 100, 109, 68, 29, 15, 18, 8, 0, 38, 23, 64, 89, 2, 25, 12, 105, 102, 66, 32, 73, 76, 30, 71, 20, 42, 96, 84, 9, 104, 87, 7, 70, 31, 98, 95, 5, 69, 65, 40, 41, 35, 34, 99, 1, 67, 3], [46, 115, 111, 127, 56, 54, 110, 47, 108, 107, 117, 119, 51, 121, 44, 48, 43, 113, 57, 83, 85, 24, 26, 21, 124, 22, 92, 27, 19, 88, 60, 55, 16, 62, 90, 123, 39, 101, 63, 86, 118, 58, 80, 33, 13, 116, 61, 37, 125, 97, 77, 74, 49, 45, 120, 103, 112, 81, 59, 17, 28, 122, 53, 10, 14, 52, 91, 78, 93, 79, 8, 36, 6, 29, 126, 11, 106, 75, 50, 100, 72, 114, 15, 12, 82, 109, 68, 94, 102, 23, 38, 4, 18, 76, 42, 73, 0, 2, 66, 105, 32, 104, 7, 89, 30, 41, 96, 31, 9, 64, 20, 25, 87, 98, 84, 5, 71, 95, 70, 34, 35, 40, 99, 69, 65, 3, 1, 67], [46, 115, 111, 127, 56, 54, 47, 110, 108, 117, 107, 51, 119, 44, 121, 48, 43, 113, 57, 62, 21, 26, 123, 85, 19, 124, 27, 83, 24, 22, 92, 118, 55, 39, 60, 16, 88, 86, 37, 58, 90, 63, 101, 33, 10, 80, 116, 61, 77, 45, 125, 8, 13, 103, 122, 59, 112, 74, 49, 97, 120, 17, 78, 52, 53, 14, 114, 2, 0, 81, 91, 28, 68, 4, 94, 36, 15, 93, 75, 29, 79, 50, 64, 109, 72, 6, 12, 106, 66, 126, 38, 71, 11, 76, 42, 89, 70, 82, 100, 18, 7, 73, 96, 25, 5, 9, 20, 105, 102, 23, 87, 104, 32, 30, 31, 69, 98, 41, 40, 84, 99, 1, 67, 34, 65, 95, 35, 3], [46, 115, 111, 127, 54, 47, 56, 110, 108, 117, 51, 119, 107, 44, 121, 43, 48, 113, 57, 124, 62, 22, 21, 85, 39, 19, 26, 80, 83, 123, 86, 92, 24, 16, 118, 60, 63, 88, 27, 58, 77, 37, 101, 55, 125, 61, 10, 13, 116, 90, 45, 103, 8, 120, 74, 97, 33, 59, 28, 52, 4, 14, 81, 75, 78, 112, 49, 17, 53, 50, 66, 68, 15, 91, 122, 11, 94, 79, 114, 29, 70, 64, 36, 71, 82, 2, 109, 106, 126, 0, 6, 100, 7, 12, 76, 42, 93, 20, 38, 25, 32, 72, 23, 9, 84, 73, 5, 18, 105, 89, 102, 69, 30, 104, 87, 96, 31, 95, 34, 98, 40, 99, 67, 3, 41, 1, 65, 35], [46, 115, 111, 127, 54, 56, 47, 110, 108, 107, 119, 51, 117, 44, 121, 48, 43, 113, 22, 21, 57, 85, 26, 83, 19, 24, 92, 124, 80, 88, 27, 86, 101, 39, 62, 63, 16, 90, 60, 58, 55, 33, 13, 77, 10, 8, 125, 118, 14, 37, 123, 74, 61, 97, 116, 45, 81, 52, 28, 49, 103, 91, 78, 59, 53, 120, 17, 79, 15, 70, 122, 75, 4, 112, 68, 11, 50, 36, 64, 94, 29, 93, 109, 12, 23, 126, 100, 106, 18, 114, 2, 38, 82, 25, 0, 9, 7, 76, 66, 71, 30, 84, 73, 20, 89, 102, 105, 32, 72, 6, 5, 96, 34, 98, 95, 69, 104, 87, 31, 40, 41, 42, 99, 1, 67, 35, 65, 3], [46, 115, 111, 127, 56, 110, 54, 47, 108, 107, 119, 117, 51, 44, 121, 48, 43, 113, 21, 57, 27, 24, 22, 85, 26, 60, 92, 19, 88, 80, 90, 55, 101, 83, 61, 16, 39, 86, 62, 124, 33, 58, 59, 63, 123, 49, 125, 122, 116, 91, 13, 103, 37, 74, 8, 77, 118, 97, 10, 52, 120, 81, 14, 45, 29, 53, 100, 17, 94, 28, 112, 70, 50, 36, 78, 15, 126, 93, 106, 79, 114, 64, 109, 38, 25, 82, 4, 18, 75, 23, 2, 102, 89, 12, 68, 105, 30, 0, 20, 11, 76, 71, 9, 72, 87, 66, 42, 96, 31, 7, 104, 98, 73, 41, 99, 65, 34, 84, 5, 32, 69, 95, 6, 35, 40, 1, 3, 67], [46, 115, 111, 127, 54, 110, 56, 47, 108, 117, 119, 51, 107, 44, 121, 48, 43, 113, 57, 22, 60, 21, 85, 19, 26, 55, 63, 80, 62, 27, 24, 123, 124, 92, 88, 101, 58, 86, 83, 39, 118, 61, 16, 59, 13, 10, 122, 37, 90, 45, 8, 74, 120, 81, 33, 49, 125, 52, 53, 77, 70, 112, 17, 116, 103, 14, 91, 28, 97, 78, 126, 64, 114, 93, 2, 68, 15, 94, 79, 11, 100, 50, 36, 4, 29, 0, 109, 106, 66, 75, 89, 12, 38, 76, 82, 18, 9, 42, 20, 30, 72, 23, 104, 7, 71, 105, 32, 102, 96, 69, 95, 41, 73, 25, 5, 84, 87, 6, 31, 98, 34, 1, 99, 67, 40, 35, 3, 65], [46, 115, 111, 127, 54, 56, 47, 110, 108, 51, 117, 107, 119, 44, 48, 121, 43, 113, 22, 57, 21, 19, 85, 26, 83, 24, 124, 62, 123, 55, 80, 27, 63, 92, 39, 88, 37, 16, 90, 60, 101, 61, 13, 58, 97, 86, 74, 8, 125, 103, 33, 118, 45, 116, 10, 77, 120, 122, 81, 52, 49, 78, 70, 94, 14, 91, 15, 59, 93, 36, 50, 68, 112, 17, 28, 66, 79, 0, 64, 53, 75, 106, 4, 2, 29, 11, 72, 102, 100, 109, 114, 71, 12, 126, 30, 7, 6, 76, 20, 82, 18, 105, 73, 9, 32, 25, 96, 42, 89, 38, 104, 69, 23, 41, 87, 5, 40, 35, 84, 98, 99, 34, 31, 95, 65, 67, 1, 3], [46, 115, 127, 111, 54, 56, 110, 47, 108, 107, 117, 51, 119, 44, 121, 48, 43, 113, 22, 57, 21, 19, 26, 124, 85, 62, 80, 123, 39, 83, 55, 88, 24, 16, 92, 60, 61, 86, 27, 13, 37, 90, 33, 77, 63, 101, 125, 118, 103, 10, 58, 74, 45, 49, 8, 59, 14, 78, 97, 122, 116, 81, 91, 17, 50, 120, 75, 28, 53, 93, 112, 52, 79, 11, 29, 68, 15, 94, 36, 70, 72, 100, 109, 4, 102, 64, 114, 126, 18, 76, 106, 6, 2, 73, 82, 71, 7, 12, 66, 9, 20, 32, 23, 89, 30, 0, 105, 25, 84, 38, 87, 69, 96, 41, 104, 42, 95, 40, 5, 98, 31, 3, 1, 34, 99, 35, 67, 65], [46, 115, 127, 111, 54, 56, 110, 47, 108, 107, 119, 51, 117, 44, 48, 121, 43, 113, 22, 21, 85, 57, 88, 26, 27, 19, 80, 24, 83, 92, 62, 86, 90, 124, 39, 55, 101, 61, 60, 33, 123, 16, 125, 103, 63, 116, 13, 97, 91, 58, 122, 37, 118, 10, 45, 77, 17, 59, 49, 74, 81, 14, 78, 94, 36, 112, 28, 93, 120, 52, 114, 29, 6, 72, 79, 53, 15, 102, 0, 8, 106, 100, 68, 18, 25, 126, 64, 82, 38, 50, 11, 12, 2, 109, 30, 75, 4, 23, 89, 66, 105, 71, 76, 41, 7, 32, 87, 96, 20, 84, 70, 73, 9, 98, 69, 5, 40, 95, 42, 65, 31, 104, 1, 99, 34, 35, 67, 3], [46, 115, 127, 111, 56, 54, 110, 47, 108, 107, 119, 51, 117, 44, 48, 121, 43, 113, 57, 22, 85, 83, 21, 19, 26, 124, 92, 86, 27, 62, 24, 60, 80, 63, 39, 123, 88, 101, 61, 103, 125, 13, 55, 90, 16, 37, 58, 10, 118, 77, 74, 120, 14, 91, 59, 33, 122, 97, 116, 53, 72, 94, 6, 112, 17, 45, 81, 36, 52, 49, 28, 114, 78, 15, 29, 79, 126, 4, 109, 75, 100, 50, 66, 2, 0, 11, 93, 82, 64, 106, 68, 12, 38, 102, 89, 18, 76, 8, 25, 84, 71, 30, 7, 87, 73, 104, 105, 32, 20, 9, 42, 96, 23, 41, 40, 98, 69, 31, 5, 65, 70, 34, 99, 95, 35, 1, 3, 67], [46, 115, 127, 111, 56, 54, 47, 110, 108, 119, 107, 51, 117, 44, 48, 121, 43, 113, 57, 22, 62, 85, 83, 26, 21, 123, 124, 19, 24, 63, 80, 92, 101, 39, 90, 58, 103, 60, 86, 88, 55, 45, 120, 118, 61, 125, 37, 74, 33, 27, 97, 16, 10, 53, 122, 116, 13, 77, 50, 72, 59, 91, 49, 52, 28, 78, 14, 114, 126, 112, 2, 6, 81, 93, 94, 79, 29, 11, 4, 17, 15, 0, 68, 106, 75, 36, 64, 102, 12, 18, 32, 71, 109, 66, 38, 82, 89, 105, 76, 8, 41, 73, 100, 7, 9, 25, 42, 31, 104, 20, 84, 98, 23, 69, 40, 30, 96, 5, 99, 87, 95, 70, 35, 65, 34, 3, 67, 1], [46, 115, 127, 111, 56, 54, 47, 110, 108, 119, 107, 117, 44, 51, 48, 121, 43, 113, 57, 62, 21, 22, 19, 39, 124, 24, 26, 101, 63, 83, 85, 90, 92, 123, 88, 27, 58, 16, 55, 86, 61, 80, 33, 103, 60, 10, 37, 13, 125, 120, 118, 116, 45, 97, 59, 77, 49, 112, 72, 74, 78, 122, 53, 52, 91, 29, 126, 81, 6, 75, 28, 94, 68, 15, 106, 114, 14, 93, 79, 0, 4, 36, 17, 109, 11, 64, 66, 102, 82, 2, 100, 50, 12, 32, 84, 30, 7, 76, 38, 89, 71, 9, 41, 18, 105, 25, 23, 8, 73, 42, 96, 5, 69, 87, 20, 104, 40, 70, 34, 98, 95, 31, 35, 67, 65, 99, 1, 3], [46, 115, 111, 127, 47, 54, 56, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 22, 57, 62, 124, 63, 85, 26, 19, 21, 83, 24, 39, 92, 27, 55, 61, 37, 88, 101, 123, 125, 49, 80, 58, 86, 16, 118, 60, 90, 59, 45, 72, 116, 53, 33, 52, 13, 77, 97, 120, 10, 74, 103, 91, 112, 78, 81, 14, 68, 122, 94, 2, 15, 28, 79, 93, 36, 75, 29, 126, 17, 6, 100, 64, 114, 109, 11, 82, 106, 25, 18, 0, 4, 50, 89, 32, 76, 73, 30, 7, 38, 66, 20, 71, 70, 12, 41, 84, 102, 40, 105, 9, 23, 87, 104, 5, 69, 42, 34, 98, 8, 96, 95, 1, 31, 67, 35, 65, 99, 3], [46, 115, 127, 111, 56, 54, 47, 110, 119, 108, 107, 51, 117, 48, 44, 121, 43, 113, 57, 62, 63, 85, 124, 22, 24, 60, 19, 21, 123, 83, 26, 92, 90, 27, 39, 53, 101, 80, 88, 61, 49, 86, 59, 37, 16, 58, 116, 74, 118, 120, 103, 125, 72, 13, 122, 10, 55, 45, 91, 112, 33, 52, 77, 14, 78, 97, 64, 114, 81, 17, 68, 29, 126, 93, 79, 70, 28, 36, 11, 15, 89, 100, 2, 106, 94, 50, 66, 0, 4, 75, 18, 38, 20, 109, 76, 82, 32, 7, 42, 30, 25, 12, 6, 84, 41, 71, 105, 8, 9, 73, 69, 31, 87, 96, 102, 104, 5, 95, 23, 65, 35, 99, 98, 34, 3, 1, 40, 67], [46, 115, 127, 111, 47, 56, 54, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 62, 124, 22, 85, 21, 19, 27, 24, 92, 101, 90, 59, 63, 26, 60, 123, 83, 61, 39, 86, 55, 80, 125, 33, 118, 16, 88, 37, 49, 58, 116, 103, 120, 13, 77, 45, 74, 52, 97, 122, 10, 53, 112, 91, 14, 70, 72, 81, 50, 29, 17, 68, 15, 79, 78, 114, 28, 36, 94, 64, 100, 109, 93, 4, 126, 18, 38, 106, 75, 76, 11, 8, 66, 2, 89, 42, 0, 82, 105, 25, 7, 9, 12, 20, 73, 96, 30, 102, 71, 23, 31, 32, 84, 87, 41, 69, 5, 104, 99, 6, 98, 95, 40, 34, 35, 67, 3, 65, 1], [46, 115, 127, 111, 54, 47, 56, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 22, 57, 21, 19, 24, 85, 62, 124, 27, 101, 83, 26, 92, 39, 88, 60, 63, 16, 86, 55, 123, 90, 80, 13, 37, 33, 58, 74, 45, 61, 77, 103, 118, 125, 97, 59, 120, 14, 49, 10, 81, 70, 72, 52, 91, 122, 114, 17, 116, 53, 36, 29, 78, 94, 15, 11, 8, 93, 75, 50, 28, 112, 109, 79, 106, 126, 68, 12, 0, 76, 82, 38, 4, 30, 32, 18, 66, 100, 25, 89, 42, 7, 71, 64, 20, 23, 73, 105, 9, 2, 102, 84, 41, 40, 96, 95, 87, 5, 98, 104, 6, 69, 31, 34, 1, 3, 99, 67, 65, 35], [46, 115, 111, 127, 54, 56, 47, 110, 108, 119, 107, 51, 117, 44, 48, 121, 43, 113, 22, 85, 26, 21, 124, 24, 19, 83, 57, 62, 88, 55, 63, 27, 39, 90, 60, 16, 92, 101, 103, 86, 80, 123, 59, 74, 37, 58, 33, 125, 45, 10, 77, 118, 97, 13, 49, 91, 120, 14, 61, 116, 53, 52, 17, 112, 81, 70, 78, 122, 93, 79, 94, 114, 15, 11, 8, 72, 28, 64, 126, 36, 0, 75, 18, 29, 106, 109, 82, 68, 50, 2, 12, 4, 25, 7, 66, 89, 20, 71, 100, 38, 76, 32, 84, 30, 105, 9, 23, 5, 42, 73, 102, 96, 6, 69, 98, 87, 104, 41, 40, 31, 95, 34, 1, 65, 35, 3, 99, 67], [46, 115, 127, 111, 54, 56, 47, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 21, 22, 85, 124, 27, 60, 26, 19, 83, 92, 62, 24, 123, 63, 88, 86, 61, 90, 59, 55, 101, 39, 80, 16, 33, 125, 118, 49, 58, 53, 103, 97, 120, 74, 91, 37, 116, 13, 112, 77, 52, 45, 122, 10, 94, 36, 78, 93, 8, 81, 106, 14, 79, 50, 29, 70, 126, 114, 100, 28, 17, 4, 109, 75, 66, 64, 12, 15, 18, 68, 11, 82, 25, 30, 38, 89, 0, 41, 42, 105, 76, 72, 20, 102, 23, 2, 7, 104, 71, 32, 9, 95, 40, 96, 73, 87, 6, 84, 99, 5, 34, 69, 31, 98, 65, 67, 1, 3, 35], [46, 115, 127, 111, 54, 47, 56, 110, 108, 107, 117, 119, 51, 44, 48, 121, 43, 113, 57, 21, 85, 60, 124, 26, 125, 83, 22, 19, 62, 92, 55, 27, 61, 39, 88, 123, 24, 63, 90, 118, 80, 101, 59, 120, 13, 86, 37, 58, 16, 33, 8, 45, 52, 53, 97, 77, 103, 49, 74, 114, 81, 112, 10, 126, 122, 28, 116, 14, 50, 109, 91, 17, 15, 78, 29, 79, 36, 94, 75, 11, 82, 4, 106, 68, 100, 93, 6, 2, 38, 104, 76, 66, 12, 64, 42, 0, 32, 30, 25, 41, 70, 20, 102, 89, 71, 105, 18, 9, 96, 87, 7, 23, 73, 84, 40, 95, 98, 31, 34, 69, 99, 72, 35, 5, 1, 67, 3, 65], [46, 115, 111, 127, 56, 47, 54, 110, 108, 107, 117, 119, 51, 44, 121, 48, 43, 113, 57, 21, 62, 22, 124, 19, 24, 85, 26, 83, 63, 88, 92, 80, 27, 118, 55, 58, 39, 123, 120, 60, 101, 90, 16, 53, 13, 86, 8, 74, 103, 61, 45, 125, 37, 49, 33, 77, 59, 10, 52, 122, 97, 78, 14, 112, 6, 116, 81, 91, 79, 94, 0, 114, 17, 15, 68, 11, 29, 28, 66, 75, 126, 4, 109, 100, 50, 12, 2, 36, 106, 38, 93, 76, 89, 82, 102, 7, 18, 71, 64, 42, 32, 5, 84, 73, 20, 25, 23, 70, 30, 105, 72, 9, 87, 69, 96, 104, 98, 95, 67, 31, 41, 40, 34, 65, 1, 3, 35, 99], [46, 115, 111, 127, 54, 47, 56, 110, 108, 119, 51, 107, 117, 44, 121, 48, 43, 113, 57, 62, 22, 26, 55, 21, 83, 124, 24, 19, 85, 39, 45, 118, 27, 63, 92, 88, 37, 60, 101, 90, 123, 61, 8, 80, 86, 59, 74, 97, 33, 16, 125, 120, 49, 13, 103, 52, 53, 58, 77, 122, 114, 6, 78, 10, 68, 14, 50, 0, 81, 11, 17, 29, 91, 116, 66, 106, 112, 75, 15, 4, 79, 94, 28, 64, 126, 93, 109, 100, 71, 36, 76, 102, 18, 7, 2, 73, 25, 32, 12, 84, 38, 89, 82, 105, 9, 98, 5, 72, 30, 69, 96, 41, 35, 42, 23, 20, 34, 95, 87, 70, 40, 99, 104, 65, 31, 67, 3, 1], [46, 115, 111, 127, 56, 54, 47, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 22, 26, 19, 62, 55, 85, 21, 24, 39, 27, 124, 83, 92, 86, 88, 60, 101, 16, 45, 80, 123, 58, 118, 90, 59, 37, 61, 63, 13, 33, 8, 103, 125, 77, 74, 97, 81, 122, 120, 53, 10, 14, 49, 78, 6, 112, 116, 17, 68, 52, 11, 91, 114, 29, 50, 94, 15, 36, 75, 28, 79, 93, 18, 4, 109, 100, 66, 106, 126, 25, 82, 0, 102, 76, 12, 38, 71, 105, 20, 72, 30, 7, 9, 2, 89, 73, 32, 64, 23, 5, 104, 87, 95, 84, 69, 70, 98, 40, 42, 96, 41, 34, 31, 35, 67, 1, 3, 65, 99], [46, 115, 127, 111, 56, 54, 47, 110, 108, 107, 119, 117, 51, 44, 48, 121, 43, 113, 26, 22, 57, 21, 62, 27, 124, 88, 19, 24, 55, 85, 63, 60, 39, 92, 59, 83, 16, 86, 118, 80, 101, 58, 45, 37, 103, 125, 90, 53, 123, 74, 61, 97, 8, 33, 10, 13, 116, 120, 14, 49, 122, 77, 91, 78, 112, 81, 6, 52, 0, 15, 68, 36, 93, 17, 114, 94, 102, 29, 50, 11, 2, 75, 4, 28, 126, 109, 64, 7, 79, 76, 18, 71, 100, 66, 12, 89, 25, 82, 70, 72, 38, 106, 9, 20, 5, 30, 69, 32, 73, 98, 105, 84, 104, 23, 87, 96, 1, 3, 95, 65, 42, 41, 40, 34, 31, 35, 99, 67], [46, 115, 127, 111, 56, 54, 47, 110, 108, 107, 119, 117, 44, 51, 121, 48, 43, 113, 57, 26, 22, 62, 19, 27, 80, 21, 24, 92, 63, 39, 124, 55, 85, 88, 83, 86, 60, 90, 16, 123, 101, 118, 37, 58, 61, 77, 59, 33, 97, 45, 120, 10, 125, 116, 49, 74, 13, 53, 103, 112, 122, 8, 81, 78, 14, 17, 93, 91, 36, 50, 29, 79, 28, 11, 52, 15, 94, 114, 75, 106, 76, 72, 126, 18, 68, 25, 12, 100, 82, 6, 109, 102, 70, 64, 7, 38, 66, 84, 73, 9, 71, 89, 4, 42, 105, 20, 30, 96, 23, 2, 32, 0, 69, 41, 98, 104, 87, 95, 31, 5, 34, 99, 40, 65, 35, 1, 3, 67]], "model.layers.24.self_attn.q_proj": [[112, 37, 48, 54, 90, 41, 94, 86, 30, 101, 56, 25, 59, 81, 62, 19, 20, 106, 58, 44, 63, 121, 123, 89, 105, 114, 12, 84, 9, 22, 95, 35, 74, 113, 46, 77, 52, 97, 104, 70, 2, 16, 24, 107, 117, 124, 60, 39, 18, 43, 119, 31, 88, 14, 125, 79, 122, 26, 10, 126, 109, 49, 4, 108, 68, 67, 80, 50, 45, 92, 8, 47, 120, 111, 29, 57, 0, 32, 115, 98, 51, 61, 118, 53, 99, 27, 71, 28, 116, 42, 110, 23, 103, 93, 102, 36, 127, 33, 91, 38, 40, 55, 100, 3, 21, 87, 17, 15, 78, 34, 85, 75, 1, 5, 96, 82, 13, 83, 65, 7, 69, 73, 76, 11, 66, 64, 72, 6], [112, 37, 48, 54, 122, 90, 111, 94, 30, 86, 125, 25, 101, 117, 41, 81, 24, 58, 19, 89, 88, 22, 35, 121, 59, 84, 45, 57, 126, 114, 109, 53, 20, 63, 106, 42, 105, 77, 12, 31, 119, 102, 120, 55, 104, 46, 10, 82, 60, 124, 3, 113, 14, 50, 18, 123, 70, 62, 100, 97, 49, 56, 47, 8, 52, 110, 44, 61, 74, 21, 26, 16, 107, 118, 87, 36, 9, 43, 103, 29, 40, 116, 65, 115, 95, 64, 93, 4, 27, 79, 32, 34, 51, 108, 28, 23, 17, 92, 91, 85, 127, 15, 96, 33, 2, 38, 39, 99, 80, 78, 98, 72, 73, 5, 83, 71, 11, 6, 75, 13, 7, 66, 68, 67, 0, 69, 76, 1], [112, 37, 48, 89, 25, 90, 54, 86, 94, 30, 56, 24, 18, 19, 35, 41, 22, 45, 105, 43, 20, 59, 50, 101, 116, 127, 97, 42, 81, 117, 125, 63, 16, 77, 60, 123, 88, 49, 95, 85, 38, 47, 9, 121, 106, 122, 109, 12, 114, 84, 115, 119, 100, 67, 0, 108, 55, 40, 26, 31, 110, 58, 70, 87, 51, 103, 104, 36, 102, 27, 92, 124, 111, 79, 2, 62, 46, 61, 13, 107, 91, 113, 53, 23, 126, 57, 74, 32, 34, 71, 39, 52, 33, 120, 29, 14, 82, 93, 99, 98, 44, 83, 96, 21, 15, 118, 69, 66, 8, 28, 78, 17, 10, 80, 7, 1, 75, 68, 76, 3, 11, 4, 72, 73, 6, 65, 64, 5], [112, 37, 48, 54, 90, 41, 94, 84, 81, 86, 117, 56, 62, 88, 30, 14, 20, 105, 19, 111, 24, 25, 51, 125, 101, 109, 49, 114, 10, 120, 63, 45, 58, 47, 31, 8, 60, 70, 126, 59, 43, 64, 77, 55, 121, 22, 52, 92, 102, 42, 119, 18, 57, 118, 53, 66, 3, 124, 4, 122, 113, 127, 65, 97, 106, 108, 89, 35, 123, 44, 107, 50, 16, 34, 39, 78, 103, 74, 110, 61, 26, 46, 32, 98, 104, 99, 38, 116, 40, 28, 12, 69, 17, 72, 100, 80, 21, 115, 93, 87, 1, 95, 33, 83, 29, 36, 85, 73, 96, 67, 5, 79, 91, 11, 2, 27, 9, 23, 71, 68, 75, 15, 0, 82, 6, 13, 76, 7], [108, 36, 44, 123, 51, 90, 26, 114, 85, 92, 96, 52, 121, 120, 18, 82, 31, 77, 28, 111, 94, 24, 15, 125, 56, 53, 95, 127, 60, 126, 20, 62, 107, 63, 113, 122, 23, 76, 46, 73, 110, 55, 32, 124, 39, 105, 102, 101, 45, 84, 87, 58, 50, 43, 74, 30, 37, 118, 41, 109, 116, 16, 6, 8, 97, 29, 27, 49, 21, 54, 33, 61, 115, 38, 9, 48, 7, 57, 112, 106, 40, 98, 14, 103, 117, 91, 89, 119, 47, 22, 99, 34, 25, 59, 104, 100, 35, 3, 42, 86, 78, 64, 88, 93, 4, 19, 80, 2, 83, 81, 11, 12, 17, 65, 10, 79, 67, 71, 5, 13, 69, 75, 66, 0, 68, 70, 1, 72], [108, 36, 44, 123, 51, 63, 127, 52, 90, 28, 96, 56, 114, 120, 92, 23, 105, 95, 119, 118, 46, 58, 31, 85, 87, 111, 30, 102, 106, 112, 57, 54, 124, 40, 103, 122, 55, 42, 77, 26, 107, 125, 60, 113, 99, 104, 116, 110, 15, 43, 109, 39, 32, 94, 61, 41, 100, 45, 62, 47, 38, 49, 11, 121, 82, 18, 115, 117, 126, 48, 53, 17, 37, 50, 10, 20, 59, 101, 93, 84, 8, 35, 97, 98, 33, 76, 34, 27, 6, 91, 88, 24, 16, 9, 89, 29, 83, 25, 19, 80, 73, 5, 21, 86, 22, 74, 3, 79, 81, 13, 7, 68, 12, 72, 75, 78, 14, 69, 66, 70, 64, 1, 4, 71, 65, 67, 2, 0], [108, 36, 44, 51, 52, 92, 96, 56, 85, 121, 90, 126, 26, 28, 31, 60, 114, 23, 120, 45, 117, 87, 18, 82, 15, 105, 49, 94, 77, 124, 102, 24, 112, 32, 48, 88, 109, 123, 55, 104, 110, 46, 103, 93, 41, 27, 73, 76, 100, 107, 125, 115, 34, 116, 61, 29, 63, 118, 113, 122, 43, 50, 53, 84, 119, 89, 111, 57, 127, 95, 101, 16, 22, 38, 47, 42, 39, 33, 106, 54, 25, 62, 4, 2, 40, 8, 37, 59, 98, 21, 97, 58, 64, 6, 3, 74, 99, 86, 30, 91, 9, 35, 80, 20, 65, 83, 78, 19, 7, 69, 71, 79, 81, 14, 5, 12, 17, 70, 10, 0, 13, 11, 75, 68, 72, 67, 66, 1], [108, 36, 123, 52, 90, 94, 44, 92, 28, 121, 124, 111, 63, 62, 93, 59, 109, 86, 105, 82, 87, 23, 116, 114, 50, 60, 26, 20, 100, 49, 96, 42, 91, 46, 31, 127, 119, 17, 103, 51, 78, 19, 41, 84, 57, 110, 15, 27, 53, 98, 48, 102, 37, 122, 117, 112, 58, 55, 33, 101, 99, 30, 104, 47, 45, 40, 25, 56, 120, 43, 61, 35, 54, 11, 106, 38, 118, 39, 113, 115, 97, 126, 107, 125, 22, 29, 18, 34, 8, 14, 85, 89, 80, 95, 88, 32, 21, 73, 77, 76, 24, 9, 83, 6, 16, 10, 81, 13, 79, 3, 7, 74, 12, 71, 75, 64, 2, 70, 72, 4, 5, 69, 65, 68, 66, 1, 67, 0], [45, 52, 109, 32, 89, 85, 18, 123, 12, 29, 87, 10, 9, 79, 69, 81, 96, 83, 71, 91, 101, 58, 4, 30, 117, 0, 68, 2, 82, 8, 19, 1, 16, 105, 59, 84, 35, 108, 37, 7, 64, 23, 6, 50, 54, 77, 76, 78, 127, 42, 25, 124, 97, 33, 17, 46, 102, 24, 31, 120, 114, 21, 28, 72, 15, 20, 94, 80, 73, 26, 88, 3, 126, 62, 104, 92, 70, 74, 110, 75, 66, 14, 103, 13, 122, 61, 44, 116, 27, 95, 67, 36, 60, 112, 39, 40, 86, 100, 22, 90, 34, 5, 119, 38, 113, 43, 98, 49, 93, 125, 65, 115, 56, 107, 11, 118, 48, 51, 53, 111, 99, 121, 63, 106, 57, 41, 47, 55], [45, 52, 109, 87, 123, 91, 79, 85, 69, 89, 12, 10, 32, 18, 96, 101, 4, 71, 1, 29, 9, 0, 81, 66, 83, 92, 30, 2, 36, 35, 56, 82, 65, 122, 42, 120, 117, 60, 8, 105, 22, 70, 58, 39, 110, 97, 64, 104, 88, 127, 106, 7, 84, 72, 53, 90, 14, 61, 95, 63, 74, 3, 126, 5, 21, 77, 16, 48, 54, 73, 111, 80, 27, 23, 68, 24, 76, 67, 94, 119, 17, 25, 13, 50, 33, 75, 62, 26, 107, 15, 116, 51, 44, 20, 49, 93, 31, 19, 46, 11, 6, 102, 98, 37, 108, 114, 103, 78, 28, 41, 38, 59, 121, 86, 125, 34, 118, 113, 99, 40, 57, 112, 100, 55, 124, 115, 43, 47], [45, 52, 109, 91, 32, 123, 85, 89, 87, 29, 18, 12, 9, 79, 46, 81, 112, 10, 4, 114, 69, 71, 83, 126, 96, 50, 54, 94, 61, 58, 47, 119, 108, 60, 82, 127, 92, 25, 63, 72, 22, 70, 62, 77, 30, 84, 107, 59, 110, 101, 35, 105, 64, 15, 57, 106, 16, 13, 111, 40, 49, 27, 93, 68, 116, 117, 42, 21, 1, 56, 55, 41, 39, 2, 115, 17, 33, 20, 23, 118, 24, 80, 0, 75, 19, 37, 113, 120, 103, 99, 95, 36, 26, 7, 122, 88, 124, 31, 34, 98, 90, 125, 28, 104, 100, 51, 8, 121, 102, 44, 73, 76, 14, 97, 5, 6, 38, 11, 3, 86, 43, 74, 53, 48, 78, 67, 65, 66], [45, 52, 109, 89, 32, 87, 91, 29, 85, 123, 79, 96, 18, 12, 10, 71, 83, 69, 50, 114, 9, 60, 35, 119, 49, 92, 84, 81, 106, 4, 77, 101, 95, 57, 107, 93, 63, 16, 1, 30, 36, 46, 82, 108, 100, 61, 117, 94, 39, 22, 112, 120, 118, 23, 58, 127, 53, 26, 21, 113, 110, 116, 43, 40, 44, 59, 28, 105, 25, 103, 88, 99, 42, 122, 55, 31, 121, 111, 124, 24, 126, 13, 54, 98, 62, 125, 37, 56, 27, 38, 115, 97, 15, 80, 51, 19, 20, 48, 102, 86, 68, 14, 90, 6, 74, 34, 33, 75, 78, 47, 17, 64, 8, 41, 76, 72, 104, 7, 73, 11, 5, 0, 70, 65, 67, 2, 3, 66], [104, 127, 98, 92, 87, 84, 81, 109, 31, 15, 17, 22, 54, 41, 28, 58, 82, 51, 37, 76, 124, 14, 21, 63, 122, 116, 95, 121, 89, 30, 20, 43, 60, 38, 57, 90, 53, 13, 56, 25, 73, 18, 94, 85, 102, 29, 24, 11, 88, 126, 111, 117, 125, 70, 27, 46, 55, 107, 23, 48, 62, 12, 42, 59, 78, 67, 61, 4, 40, 45, 93, 110, 80, 105, 118, 106, 32, 9, 114, 83, 79, 19, 35, 97, 99, 120, 96, 36, 47, 6, 39, 101, 33, 68, 86, 52, 75, 10, 50, 49, 77, 100, 119, 74, 113, 123, 115, 44, 103, 26, 5, 108, 91, 16, 72, 69, 8, 112, 34, 71, 7, 66, 3, 1, 65, 64, 0, 2], [104, 127, 92, 98, 84, 87, 14, 81, 109, 31, 73, 76, 37, 6, 63, 125, 122, 82, 57, 105, 75, 89, 124, 18, 41, 4, 107, 94, 99, 78, 25, 17, 97, 116, 95, 83, 61, 10, 64, 58, 28, 30, 68, 59, 108, 66, 43, 54, 20, 65, 60, 51, 47, 115, 72, 15, 100, 106, 3, 50, 46, 53, 45, 56, 49, 85, 48, 23, 13, 16, 22, 114, 96, 24, 9, 62, 77, 12, 74, 35, 26, 80, 70, 119, 19, 36, 33, 38, 91, 79, 90, 40, 117, 44, 55, 29, 111, 121, 101, 39, 88, 102, 103, 2, 21, 34, 93, 8, 123, 69, 120, 126, 7, 32, 113, 112, 5, 27, 11, 42, 110, 52, 118, 71, 0, 86, 1, 67], [104, 127, 98, 101, 22, 43, 92, 18, 105, 53, 31, 59, 125, 84, 124, 87, 63, 61, 89, 54, 81, 41, 39, 122, 57, 106, 50, 95, 45, 19, 46, 28, 99, 35, 121, 109, 71, 79, 118, 38, 26, 62, 25, 76, 58, 37, 116, 44, 8, 96, 15, 56, 0, 30, 102, 27, 111, 117, 14, 115, 13, 120, 126, 107, 108, 60, 94, 93, 49, 73, 74, 51, 55, 110, 52, 11, 85, 123, 113, 103, 82, 119, 32, 4, 97, 42, 75, 36, 77, 112, 100, 90, 48, 33, 47, 86, 114, 68, 65, 6, 5, 29, 24, 34, 10, 80, 88, 23, 91, 83, 40, 7, 67, 17, 20, 72, 69, 2, 21, 1, 16, 70, 66, 3, 64, 9, 12, 78], [104, 127, 98, 73, 87, 31, 66, 14, 92, 81, 124, 84, 76, 4, 6, 116, 37, 25, 64, 89, 2, 63, 71, 13, 5, 60, 41, 80, 65, 75, 19, 0, 18, 106, 61, 114, 22, 79, 67, 46, 10, 1, 105, 53, 47, 58, 57, 125, 8, 122, 95, 40, 85, 54, 82, 101, 12, 59, 102, 51, 77, 74, 117, 121, 62, 70, 3, 111, 11, 90, 55, 24, 69, 72, 29, 109, 50, 23, 49, 15, 9, 100, 16, 86, 21, 91, 42, 33, 56, 88, 35, 99, 26, 7, 119, 123, 107, 17, 112, 43, 68, 36, 32, 38, 44, 118, 48, 94, 103, 96, 27, 126, 113, 52, 97, 108, 83, 28, 115, 34, 110, 39, 120, 78, 45, 30, 93, 20], [61, 102, 121, 114, 58, 116, 119, 62, 115, 54, 50, 33, 111, 126, 60, 125, 59, 122, 120, 124, 123, 117, 112, 110, 63, 52, 38, 24, 56, 53, 49, 45, 113, 42, 57, 29, 127, 55, 48, 26, 51, 118, 46, 107, 15, 27, 108, 47, 109, 91, 88, 105, 41, 85, 21, 84, 106, 44, 43, 86, 93, 37, 90, 18, 101, 28, 103, 100, 40, 104, 39, 22, 30, 97, 36, 32, 83, 17, 79, 34, 98, 35, 82, 96, 12, 19, 73, 95, 99, 31, 92, 81, 9, 94, 23, 80, 77, 64, 7, 20, 76, 25, 71, 66, 78, 89, 67, 87, 0, 2, 1, 65, 3, 5, 4, 69, 16, 13, 68, 11, 6, 14, 75, 10, 8, 72, 74, 70], [121, 102, 114, 61, 116, 58, 119, 62, 115, 54, 33, 60, 126, 59, 122, 111, 50, 123, 117, 120, 124, 110, 125, 52, 112, 53, 57, 24, 29, 63, 56, 45, 38, 49, 48, 26, 127, 55, 42, 113, 46, 118, 51, 88, 15, 47, 109, 108, 27, 91, 105, 41, 107, 106, 21, 85, 43, 84, 44, 86, 93, 28, 37, 90, 18, 103, 100, 101, 104, 39, 30, 97, 22, 40, 83, 17, 36, 79, 32, 12, 99, 95, 35, 34, 96, 98, 73, 19, 20, 31, 81, 87, 92, 82, 94, 80, 76, 23, 9, 25, 71, 67, 7, 77, 64, 66, 0, 3, 65, 1, 78, 89, 4, 2, 69, 5, 68, 13, 16, 11, 14, 75, 6, 74, 10, 72, 8, 70], [114, 121, 102, 61, 58, 116, 119, 115, 62, 122, 60, 124, 126, 50, 59, 123, 54, 33, 111, 112, 125, 120, 110, 117, 24, 52, 63, 56, 53, 42, 106, 57, 127, 48, 49, 107, 55, 26, 118, 113, 38, 15, 46, 51, 45, 29, 47, 108, 27, 105, 84, 109, 91, 85, 21, 44, 41, 88, 93, 28, 43, 86, 37, 90, 18, 103, 40, 104, 32, 101, 36, 100, 39, 30, 17, 22, 82, 79, 97, 98, 35, 9, 34, 96, 12, 99, 83, 92, 31, 95, 73, 81, 94, 7, 19, 76, 67, 69, 20, 77, 25, 64, 1, 0, 78, 71, 66, 65, 3, 4, 2, 80, 23, 5, 89, 68, 87, 14, 13, 11, 16, 6, 74, 75, 8, 72, 70, 10], [61, 121, 114, 102, 23, 80, 83, 50, 58, 74, 77, 30, 25, 45, 11, 8, 33, 37, 29, 90, 91, 78, 32, 92, 20, 70, 81, 119, 100, 69, 116, 88, 86, 62, 75, 68, 72, 120, 89, 59, 42, 105, 26, 21, 14, 125, 18, 6, 87, 16, 10, 66, 28, 54, 24, 76, 19, 27, 127, 85, 96, 126, 103, 56, 107, 1, 111, 106, 93, 63, 64, 51, 12, 67, 22, 13, 48, 84, 34, 4, 82, 17, 60, 31, 47, 110, 43, 115, 122, 98, 124, 52, 101, 7, 112, 117, 94, 35, 123, 57, 79, 108, 49, 0, 104, 41, 95, 2, 46, 71, 55, 5, 9, 36, 118, 3, 113, 99, 44, 53, 109, 15, 38, 39, 40, 65, 73, 97], [53, 42, 58, 120, 122, 100, 125, 116, 61, 45, 63, 123, 127, 48, 50, 52, 121, 49, 30, 54, 113, 89, 126, 51, 23, 108, 56, 59, 32, 44, 112, 111, 115, 46, 60, 106, 114, 62, 18, 47, 117, 57, 55, 21, 124, 119, 110, 109, 43, 118, 90, 91, 107, 40, 86, 92, 35, 15, 25, 104, 103, 28, 94, 24, 85, 41, 105, 12, 31, 37, 27, 38, 39, 83, 99, 101, 96, 36, 22, 102, 82, 34, 80, 98, 88, 93, 81, 95, 87, 78, 33, 1, 20, 76, 79, 97, 17, 5, 14, 29, 11, 26, 7, 77, 74, 16, 67, 68, 65, 0, 3, 72, 84, 19, 73, 64, 2, 70, 4, 71, 66, 69, 10, 9, 8, 13, 75, 6], [53, 42, 58, 120, 83, 80, 100, 89, 32, 11, 77, 73, 86, 27, 106, 45, 122, 70, 28, 24, 1, 92, 99, 26, 85, 81, 17, 67, 25, 74, 79, 72, 14, 5, 103, 20, 75, 111, 116, 23, 40, 113, 76, 125, 63, 123, 57, 48, 84, 119, 96, 61, 66, 6, 37, 93, 78, 82, 19, 50, 8, 91, 127, 34, 88, 15, 30, 126, 43, 94, 31, 9, 87, 69, 10, 49, 22, 105, 29, 4, 21, 0, 16, 95, 112, 71, 13, 39, 90, 56, 33, 54, 52, 124, 121, 101, 18, 51, 104, 118, 59, 109, 108, 7, 68, 114, 62, 12, 3, 35, 97, 117, 60, 115, 46, 44, 55, 98, 38, 2, 65, 102, 107, 64, 47, 110, 41, 36], [58, 42, 53, 120, 122, 125, 116, 45, 100, 61, 30, 123, 63, 48, 121, 50, 127, 49, 111, 54, 52, 89, 56, 59, 51, 112, 126, 23, 106, 108, 32, 46, 113, 115, 57, 114, 44, 62, 47, 60, 117, 55, 119, 21, 124, 110, 18, 109, 24, 118, 90, 94, 43, 107, 104, 91, 40, 25, 41, 105, 103, 92, 31, 35, 85, 86, 39, 37, 99, 12, 101, 38, 28, 27, 15, 96, 102, 36, 22, 82, 95, 93, 34, 98, 17, 81, 88, 97, 79, 33, 83, 76, 20, 80, 14, 87, 29, 26, 7, 16, 78, 2, 74, 8, 19, 64, 65, 72, 66, 77, 68, 0, 5, 71, 1, 67, 3, 84, 10, 4, 11, 69, 6, 70, 13, 75, 9, 73], [120, 42, 58, 53, 122, 125, 45, 61, 116, 123, 106, 50, 89, 52, 23, 49, 100, 48, 59, 56, 63, 112, 111, 44, 121, 54, 127, 117, 126, 57, 62, 110, 113, 51, 115, 30, 55, 60, 46, 109, 118, 114, 47, 90, 119, 124, 18, 108, 107, 28, 43, 21, 105, 24, 104, 91, 32, 83, 95, 29, 40, 103, 38, 39, 35, 101, 34, 41, 99, 102, 94, 86, 37, 97, 85, 92, 27, 31, 33, 25, 15, 98, 96, 77, 36, 26, 12, 82, 93, 20, 13, 19, 22, 75, 17, 10, 14, 87, 7, 11, 76, 84, 74, 79, 88, 16, 80, 71, 67, 6, 68, 66, 72, 1, 8, 2, 5, 81, 70, 3, 64, 0, 4, 69, 65, 78, 9, 73], [51, 118, 55, 39, 63, 54, 121, 25, 23, 122, 124, 120, 126, 59, 53, 61, 113, 123, 62, 58, 117, 95, 50, 56, 57, 89, 52, 125, 60, 82, 49, 116, 115, 119, 34, 47, 48, 114, 127, 106, 111, 112, 42, 46, 44, 109, 104, 110, 45, 108, 29, 87, 80, 14, 27, 43, 107, 84, 36, 76, 35, 37, 99, 41, 96, 40, 102, 103, 20, 101, 100, 105, 28, 97, 38, 31, 18, 12, 85, 91, 26, 24, 16, 74, 32, 93, 92, 98, 78, 86, 30, 71, 69, 33, 94, 8, 21, 88, 7, 2, 22, 1, 68, 90, 83, 10, 70, 72, 13, 81, 3, 73, 6, 19, 11, 0, 5, 9, 79, 65, 4, 64, 66, 75, 67, 17, 77, 15], [55, 39, 118, 122, 25, 23, 120, 121, 63, 89, 95, 115, 124, 51, 123, 113, 126, 61, 58, 60, 34, 82, 59, 112, 48, 53, 119, 106, 52, 57, 62, 117, 111, 42, 49, 50, 56, 47, 114, 125, 44, 127, 116, 43, 54, 80, 14, 29, 108, 45, 110, 46, 109, 84, 76, 41, 104, 107, 40, 27, 87, 20, 31, 105, 18, 102, 100, 16, 28, 96, 78, 37, 36, 101, 38, 103, 12, 93, 35, 8, 97, 32, 24, 94, 99, 83, 88, 74, 33, 86, 10, 71, 92, 85, 72, 7, 98, 90, 70, 73, 69, 68, 13, 30, 6, 21, 3, 2, 5, 26, 9, 22, 0, 65, 66, 19, 1, 4, 67, 64, 81, 11, 91, 79, 17, 77, 75, 15], [51, 55, 39, 120, 121, 118, 54, 122, 63, 23, 89, 25, 59, 123, 46, 113, 62, 115, 124, 60, 61, 126, 44, 112, 125, 58, 50, 116, 33, 53, 52, 49, 82, 95, 57, 34, 119, 29, 108, 56, 45, 110, 114, 111, 117, 87, 47, 42, 127, 48, 43, 76, 14, 109, 100, 84, 68, 71, 31, 36, 99, 98, 3, 106, 26, 72, 11, 80, 38, 107, 37, 40, 88, 32, 18, 102, 105, 27, 0, 83, 104, 20, 41, 28, 96, 97, 1, 12, 101, 81, 30, 92, 93, 90, 94, 78, 103, 5, 35, 2, 64, 74, 24, 69, 65, 70, 9, 4, 85, 19, 21, 8, 91, 13, 16, 6, 86, 7, 75, 67, 10, 73, 79, 66, 17, 22, 77, 15], [51, 55, 120, 39, 63, 121, 27, 124, 126, 59, 53, 61, 123, 116, 122, 57, 58, 60, 62, 56, 45, 118, 119, 54, 25, 117, 125, 114, 113, 50, 47, 49, 127, 111, 34, 115, 108, 95, 52, 48, 46, 84, 107, 112, 36, 86, 110, 82, 43, 20, 109, 42, 76, 89, 44, 29, 40, 23, 14, 92, 41, 106, 24, 94, 80, 102, 101, 105, 91, 93, 22, 79, 99, 31, 103, 28, 17, 81, 90, 35, 104, 38, 15, 74, 98, 100, 12, 69, 85, 7, 37, 8, 88, 32, 13, 16, 9, 96, 21, 70, 71, 97, 26, 18, 33, 87, 68, 1, 77, 83, 6, 30, 2, 11, 78, 73, 10, 19, 0, 67, 72, 5, 75, 66, 4, 65, 3, 64], [38, 115, 114, 50, 51, 113, 89, 75, 19, 82, 16, 23, 77, 7, 73, 78, 69, 10, 39, 87, 25, 6, 94, 67, 8, 85, 57, 104, 13, 74, 125, 52, 27, 3, 26, 80, 118, 21, 14, 68, 65, 5, 106, 92, 124, 12, 105, 11, 123, 30, 64, 79, 98, 122, 37, 95, 81, 43, 44, 63, 62, 56, 41, 18, 93, 70, 102, 54, 101, 2, 112, 121, 107, 83, 42, 76, 100, 59, 34, 48, 111, 49, 45, 40, 119, 117, 33, 28, 126, 60, 84, 47, 108, 120, 36, 96, 109, 103, 4, 61, 88, 86, 29, 32, 17, 99, 127, 35, 58, 72, 46, 9, 116, 97, 31, 55, 22, 15, 90, 110, 53, 91, 71, 24, 20, 1, 66, 0], [38, 114, 115, 50, 51, 89, 23, 82, 19, 77, 16, 69, 75, 7, 73, 113, 64, 2, 85, 14, 93, 17, 67, 66, 78, 1, 21, 72, 3, 87, 79, 12, 94, 8, 13, 32, 71, 126, 106, 25, 15, 68, 95, 102, 10, 22, 41, 57, 74, 6, 127, 24, 44, 34, 112, 107, 56, 39, 76, 27, 92, 26, 90, 98, 0, 54, 18, 101, 80, 65, 40, 121, 105, 49, 81, 30, 122, 84, 47, 9, 104, 108, 83, 109, 117, 4, 125, 11, 20, 100, 37, 36, 48, 42, 116, 43, 86, 103, 97, 59, 63, 120, 28, 29, 61, 46, 45, 111, 99, 55, 110, 119, 53, 35, 70, 58, 60, 91, 33, 124, 118, 31, 88, 96, 62, 52, 123, 5], [38, 114, 115, 50, 51, 89, 7, 19, 73, 16, 75, 113, 82, 2, 77, 64, 69, 23, 3, 67, 14, 25, 85, 87, 93, 6, 13, 1, 17, 48, 66, 107, 8, 125, 9, 124, 41, 56, 71, 104, 0, 12, 79, 20, 52, 122, 45, 126, 5, 94, 81, 18, 22, 65, 74, 95, 127, 91, 57, 49, 59, 83, 15, 11, 80, 46, 24, 60, 92, 39, 118, 21, 86, 90, 105, 68, 10, 117, 121, 4, 44, 110, 29, 78, 72, 62, 106, 111, 88, 32, 108, 96, 58, 27, 40, 31, 123, 34, 109, 116, 120, 76, 119, 97, 101, 43, 63, 37, 100, 112, 98, 36, 28, 30, 99, 33, 61, 54, 103, 53, 26, 42, 55, 47, 84, 35, 102, 70], [38, 50, 114, 115, 51, 113, 23, 16, 82, 89, 39, 19, 77, 85, 126, 75, 73, 78, 7, 30, 24, 37, 22, 57, 116, 14, 52, 12, 35, 92, 34, 27, 125, 112, 36, 79, 83, 69, 13, 81, 26, 107, 49, 56, 88, 63, 80, 104, 17, 106, 67, 21, 94, 25, 20, 120, 44, 90, 46, 87, 60, 121, 45, 117, 42, 40, 9, 108, 47, 41, 48, 8, 109, 93, 76, 6, 122, 32, 72, 84, 97, 102, 53, 62, 10, 59, 119, 61, 99, 103, 98, 2, 118, 31, 111, 11, 54, 105, 74, 28, 18, 86, 127, 124, 33, 110, 15, 95, 29, 123, 43, 100, 96, 101, 58, 55, 91, 68, 3, 70, 5, 64, 71, 65, 1, 4, 66, 0]], "model.layers.24.self_attn.k_proj": [[48, 112, 101, 30, 86, 90, 19, 88, 89, 25, 81, 84, 12, 70, 14, 107, 77, 63, 8, 54, 47, 120, 60, 17, 52, 125, 41, 0, 49, 117, 115, 119, 93, 114, 2, 124, 74, 10, 33, 79, 64, 113, 43, 46, 102, 1, 57, 62, 80, 59, 42, 18, 20, 100, 118, 110, 35, 9, 109, 56, 29, 123, 44, 55, 58, 51, 68, 106, 116, 45, 28, 38, 103, 122, 108, 61, 50, 111, 4, 3, 40, 104, 87, 53, 127, 126, 39, 121, 67, 98, 99, 16, 97, 96, 15, 23, 32, 36, 31, 27, 85, 91, 5, 105, 92, 83, 34, 71, 94, 95, 73, 24, 82, 75, 21, 78, 7, 26, 11, 13, 72, 69, 76, 65, 22, 6, 66, 37], [44, 108, 52, 121, 82, 77, 15, 85, 100, 26, 32, 87, 60, 123, 6, 8, 76, 73, 55, 28, 3, 51, 74, 127, 120, 24, 48, 92, 61, 0, 9, 64, 65, 29, 124, 4, 109, 2, 89, 16, 5, 111, 94, 20, 110, 63, 18, 114, 105, 116, 78, 56, 35, 118, 7, 83, 47, 126, 59, 22, 88, 91, 50, 95, 62, 33, 122, 46, 84, 119, 115, 25, 31, 80, 54, 23, 27, 113, 86, 97, 98, 45, 81, 99, 103, 39, 102, 69, 40, 12, 70, 101, 107, 37, 57, 43, 117, 34, 112, 49, 11, 93, 53, 41, 38, 36, 42, 106, 96, 125, 104, 58, 19, 90, 30, 21, 14, 17, 71, 10, 13, 75, 79, 68, 72, 66, 1, 67], [109, 52, 45, 0, 123, 85, 32, 12, 69, 10, 87, 79, 116, 89, 18, 9, 4, 91, 81, 68, 2, 29, 83, 60, 71, 84, 8, 77, 78, 3, 1, 70, 16, 119, 120, 7, 93, 126, 58, 117, 94, 41, 30, 64, 66, 115, 124, 72, 118, 112, 114, 113, 61, 46, 54, 47, 44, 37, 92, 49, 111, 127, 65, 50, 53, 125, 95, 105, 106, 51, 110, 63, 35, 40, 43, 107, 48, 56, 102, 55, 39, 34, 19, 108, 104, 22, 121, 57, 97, 6, 59, 42, 62, 75, 103, 36, 100, 122, 82, 38, 67, 25, 28, 101, 99, 98, 13, 33, 27, 14, 20, 11, 88, 31, 21, 23, 90, 96, 86, 26, 73, 15, 80, 24, 5, 76, 17, 74], [127, 40, 34, 84, 87, 76, 95, 81, 73, 14, 17, 89, 6, 5, 28, 66, 125, 124, 64, 4, 116, 60, 122, 110, 0, 111, 13, 79, 58, 74, 19, 75, 92, 117, 51, 18, 80, 50, 24, 57, 105, 43, 63, 22, 45, 41, 101, 107, 26, 106, 54, 16, 121, 70, 10, 86, 71, 53, 20, 39, 78, 59, 8, 65, 52, 56, 30, 37, 62, 49, 94, 3, 42, 25, 123, 114, 61, 27, 83, 90, 7, 85, 96, 15, 109, 119, 103, 67, 21, 35, 44, 118, 33, 47, 99, 113, 55, 77, 108, 120, 126, 32, 1, 102, 69, 112, 115, 93, 68, 31, 98, 38, 36, 88, 46, 29, 48, 97, 82, 91, 12, 72, 100, 23, 9, 11, 2, 104], [38, 61, 114, 121, 86, 97, 119, 93, 18, 54, 30, 27, 50, 116, 15, 59, 41, 62, 56, 120, 126, 58, 115, 125, 124, 122, 55, 63, 90, 60, 127, 57, 49, 117, 48, 85, 123, 47, 112, 113, 118, 108, 52, 110, 101, 24, 51, 45, 36, 109, 46, 96, 53, 43, 94, 29, 106, 111, 95, 17, 104, 44, 25, 42, 107, 39, 99, 91, 105, 79, 33, 98, 12, 20, 28, 81, 80, 103, 92, 13, 40, 73, 77, 88, 100, 83, 7, 14, 16, 34, 10, 21, 31, 37, 84, 102, 23, 35, 74, 19, 26, 32, 78, 11, 89, 87, 8, 82, 70, 22, 75, 6, 68, 3, 9, 1, 76, 72, 2, 69, 66, 65, 5, 71, 0, 4, 67, 64], [106, 36, 120, 86, 53, 58, 96, 116, 94, 28, 49, 61, 89, 63, 27, 121, 123, 42, 122, 50, 45, 125, 48, 56, 62, 111, 54, 124, 59, 52, 18, 117, 55, 80, 127, 57, 119, 115, 112, 35, 46, 114, 51, 83, 60, 108, 34, 126, 107, 118, 47, 113, 109, 21, 98, 23, 12, 41, 43, 73, 15, 81, 110, 9, 105, 11, 104, 77, 29, 44, 40, 102, 14, 16, 39, 38, 7, 37, 103, 101, 17, 90, 0, 95, 87, 97, 33, 92, 24, 91, 20, 31, 67, 26, 100, 13, 99, 32, 71, 93, 72, 5, 88, 30, 68, 84, 19, 66, 78, 70, 74, 82, 85, 25, 2, 65, 64, 76, 69, 10, 75, 6, 79, 8, 22, 3, 4, 1], [103, 51, 55, 86, 120, 98, 123, 63, 93, 126, 124, 61, 31, 114, 53, 45, 59, 58, 49, 57, 121, 89, 122, 52, 125, 62, 112, 56, 47, 117, 116, 50, 60, 46, 119, 127, 115, 48, 110, 113, 108, 111, 107, 38, 109, 40, 42, 105, 44, 54, 43, 36, 80, 41, 106, 22, 18, 33, 118, 34, 79, 82, 16, 104, 99, 88, 90, 81, 84, 26, 96, 97, 102, 92, 91, 11, 100, 13, 37, 27, 21, 101, 32, 35, 83, 12, 23, 25, 29, 94, 72, 78, 85, 20, 10, 6, 30, 28, 17, 39, 24, 14, 66, 87, 19, 95, 9, 5, 73, 69, 4, 15, 8, 77, 75, 67, 76, 65, 71, 68, 0, 74, 7, 3, 1, 70, 2, 64], [50, 115, 102, 77, 89, 16, 19, 82, 75, 73, 114, 7, 23, 64, 69, 113, 2, 3, 67, 66, 1, 85, 10, 93, 17, 8, 70, 14, 65, 78, 21, 57, 30, 38, 112, 49, 94, 42, 43, 126, 34, 76, 71, 39, 45, 108, 92, 25, 90, 48, 6, 32, 44, 117, 123, 122, 13, 121, 87, 56, 106, 103, 125, 22, 79, 28, 74, 20, 27, 120, 63, 124, 116, 68, 33, 127, 61, 54, 15, 60, 104, 4, 41, 52, 119, 105, 100, 95, 12, 91, 72, 9, 88, 26, 53, 36, 31, 101, 29, 98, 62, 35, 118, 47, 59, 84, 46, 109, 51, 40, 96, 99, 0, 110, 55, 107, 111, 24, 37, 83, 58, 80, 97, 11, 5, 81, 86, 18]], "model.layers.24.self_attn.qk_proj": [[127, 50, 48, 115, 112, 114, 109, 51, 45, 108, 55, 61, 121, 44, 53, 120, 58, 52, 38, 25, 23, 124, 123, 63, 87, 18, 42, 89, 92, 60, 22, 113, 106, 40, 125, 82, 126, 116, 29, 122, 83, 19, 59, 17, 77, 32, 101, 54, 20, 26, 13, 94, 84, 73, 16, 9, 30, 81, 12, 56, 102, 85, 80, 90, 21, 57, 117, 49, 37, 76, 46, 86, 79, 111, 75, 98, 118, 110, 36, 7, 62, 15, 78, 14, 41, 11, 47, 119, 71, 27, 91, 103, 64, 43, 95, 34, 39, 5, 69, 74, 10, 105, 0, 96, 24, 31, 100, 104, 6, 88, 93, 28, 3, 68, 107, 67, 2, 66, 33, 70, 8, 4, 97, 99, 72, 35, 1, 65], [127, 50, 115, 114, 48, 109, 112, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 38, 25, 123, 92, 23, 87, 60, 89, 113, 18, 42, 124, 22, 106, 116, 40, 122, 63, 54, 125, 82, 29, 126, 73, 59, 83, 77, 101, 56, 32, 13, 20, 85, 19, 94, 16, 17, 26, 30, 81, 102, 9, 12, 49, 117, 86, 76, 84, 79, 21, 119, 111, 118, 57, 71, 36, 98, 80, 110, 41, 27, 103, 39, 75, 37, 90, 62, 47, 104, 78, 14, 46, 91, 11, 15, 43, 74, 34, 64, 105, 5, 69, 24, 100, 31, 95, 107, 7, 0, 96, 4, 93, 10, 28, 6, 88, 2, 68, 8, 66, 3, 67, 33, 97, 70, 72, 99, 35, 65, 1], [127, 50, 48, 115, 114, 112, 109, 51, 45, 108, 55, 61, 121, 120, 44, 53, 58, 52, 25, 38, 123, 113, 23, 87, 106, 92, 42, 18, 89, 60, 40, 22, 126, 125, 122, 82, 63, 32, 56, 116, 124, 54, 29, 59, 17, 26, 101, 13, 19, 73, 30, 102, 49, 83, 85, 16, 119, 20, 77, 94, 81, 76, 86, 12, 90, 9, 117, 21, 47, 57, 111, 118, 98, 110, 84, 104, 41, 80, 103, 78, 34, 79, 36, 31, 39, 7, 37, 62, 71, 75, 64, 15, 11, 14, 27, 5, 105, 69, 96, 91, 46, 95, 74, 100, 43, 10, 107, 0, 28, 93, 24, 66, 6, 33, 67, 2, 88, 3, 8, 4, 68, 97, 70, 99, 35, 65, 1, 72], [127, 50, 48, 115, 114, 112, 109, 51, 61, 45, 108, 55, 121, 44, 53, 120, 58, 52, 123, 38, 25, 106, 113, 23, 42, 87, 89, 18, 126, 82, 92, 59, 22, 124, 63, 125, 56, 116, 119, 29, 60, 94, 40, 76, 13, 122, 17, 32, 30, 73, 86, 19, 77, 80, 9, 101, 85, 16, 54, 49, 83, 81, 26, 79, 102, 12, 84, 111, 20, 98, 118, 110, 75, 47, 117, 5, 15, 41, 21, 104, 62, 71, 39, 36, 37, 69, 57, 14, 90, 7, 27, 34, 78, 46, 31, 0, 43, 11, 74, 103, 10, 95, 100, 96, 2, 28, 24, 88, 66, 67, 8, 70, 64, 93, 91, 4, 105, 6, 3, 107, 33, 35, 68, 97, 99, 65, 1, 72], [127, 48, 50, 115, 114, 112, 109, 51, 108, 61, 55, 45, 44, 121, 120, 53, 58, 52, 25, 123, 38, 23, 42, 113, 60, 63, 82, 18, 89, 87, 106, 59, 92, 56, 126, 124, 125, 22, 122, 40, 54, 13, 17, 73, 19, 86, 49, 29, 116, 9, 94, 80, 77, 30, 57, 32, 84, 83, 81, 26, 20, 76, 12, 101, 16, 75, 85, 90, 110, 118, 119, 14, 79, 111, 102, 21, 117, 37, 15, 104, 62, 98, 36, 5, 103, 47, 41, 34, 71, 78, 27, 95, 7, 11, 10, 46, 69, 91, 64, 70, 74, 96, 105, 31, 100, 39, 0, 24, 43, 107, 33, 28, 88, 3, 66, 8, 2, 67, 4, 93, 97, 68, 6, 35, 1, 99, 72, 65], [127, 50, 48, 115, 114, 112, 109, 61, 51, 108, 55, 45, 44, 121, 53, 120, 58, 52, 25, 38, 42, 23, 22, 123, 89, 113, 87, 18, 82, 106, 92, 60, 126, 40, 124, 77, 84, 122, 73, 63, 59, 76, 83, 81, 17, 94, 80, 19, 116, 125, 86, 29, 12, 13, 101, 85, 54, 30, 16, 32, 9, 26, 102, 37, 79, 56, 15, 98, 21, 41, 110, 57, 90, 36, 34, 118, 49, 20, 47, 14, 103, 5, 11, 95, 75, 27, 111, 78, 71, 46, 70, 10, 7, 104, 0, 117, 64, 43, 105, 62, 119, 39, 91, 8, 74, 24, 69, 100, 96, 93, 107, 88, 31, 2, 33, 66, 97, 28, 3, 67, 68, 6, 35, 99, 4, 65, 1, 72], [127, 50, 48, 115, 114, 112, 109, 55, 108, 51, 45, 61, 44, 121, 53, 120, 58, 52, 25, 23, 42, 87, 38, 82, 89, 123, 92, 18, 106, 124, 22, 60, 113, 125, 122, 40, 63, 19, 17, 54, 126, 32, 94, 84, 86, 26, 73, 30, 80, 13, 77, 29, 83, 101, 81, 56, 116, 12, 16, 59, 76, 85, 20, 9, 57, 21, 119, 98, 79, 102, 15, 90, 49, 111, 78, 41, 75, 118, 47, 104, 11, 110, 27, 14, 46, 71, 117, 5, 36, 37, 91, 7, 103, 95, 34, 64, 10, 62, 105, 100, 39, 74, 0, 24, 43, 31, 69, 70, 107, 96, 88, 8, 28, 93, 33, 35, 2, 4, 3, 97, 68, 67, 99, 66, 6, 72, 1, 65], [127, 50, 48, 115, 114, 109, 112, 55, 51, 45, 108, 121, 44, 61, 120, 53, 58, 52, 25, 23, 123, 38, 124, 89, 92, 18, 87, 42, 60, 82, 22, 113, 63, 125, 106, 40, 122, 116, 83, 59, 29, 126, 119, 32, 30, 56, 16, 94, 54, 73, 86, 26, 101, 20, 80, 19, 49, 13, 84, 17, 21, 90, 81, 102, 12, 9, 85, 77, 57, 76, 37, 15, 111, 41, 79, 78, 62, 11, 46, 71, 36, 110, 39, 117, 118, 34, 91, 27, 5, 105, 104, 43, 28, 75, 98, 31, 0, 47, 24, 103, 64, 10, 14, 95, 7, 69, 100, 70, 88, 74, 93, 96, 68, 107, 66, 2, 35, 3, 33, 4, 8, 97, 67, 72, 6, 99, 1, 65], [127, 50, 48, 115, 112, 109, 114, 55, 45, 51, 108, 121, 61, 44, 53, 120, 58, 52, 25, 38, 23, 92, 89, 123, 124, 87, 42, 125, 113, 63, 82, 106, 60, 59, 18, 126, 116, 122, 83, 22, 32, 77, 40, 56, 101, 80, 94, 81, 26, 84, 29, 54, 86, 30, 19, 117, 13, 119, 73, 102, 20, 85, 17, 49, 90, 76, 16, 9, 21, 111, 12, 37, 98, 118, 43, 15, 41, 47, 79, 57, 14, 34, 91, 110, 103, 27, 96, 75, 105, 11, 71, 36, 46, 69, 39, 100, 62, 104, 31, 78, 28, 7, 5, 95, 24, 74, 107, 10, 88, 93, 64, 0, 70, 33, 2, 4, 35, 68, 66, 6, 97, 3, 72, 99, 67, 8, 1, 65], [127, 50, 48, 114, 115, 109, 112, 45, 51, 108, 55, 121, 61, 44, 53, 120, 58, 52, 25, 38, 123, 124, 23, 87, 89, 42, 125, 113, 63, 106, 92, 82, 59, 18, 60, 116, 40, 126, 22, 29, 122, 30, 101, 83, 9, 73, 13, 94, 32, 54, 17, 119, 19, 77, 76, 56, 86, 84, 26, 85, 117, 16, 81, 80, 102, 57, 36, 111, 49, 12, 21, 37, 41, 118, 79, 103, 69, 20, 14, 110, 7, 11, 34, 90, 75, 104, 71, 98, 15, 5, 27, 47, 62, 78, 91, 95, 0, 10, 105, 6, 64, 31, 96, 24, 43, 39, 74, 28, 46, 100, 4, 66, 3, 88, 72, 2, 68, 93, 33, 70, 107, 67, 35, 97, 99, 8, 1, 65], [127, 50, 48, 115, 109, 112, 114, 51, 108, 45, 44, 55, 61, 121, 53, 120, 58, 52, 25, 42, 38, 123, 124, 87, 23, 106, 92, 113, 18, 82, 89, 22, 63, 122, 126, 125, 40, 73, 13, 29, 60, 116, 83, 9, 59, 56, 54, 77, 80, 86, 16, 84, 85, 19, 32, 81, 94, 17, 30, 26, 12, 76, 119, 102, 101, 20, 118, 36, 15, 62, 79, 71, 11, 117, 5, 49, 111, 90, 0, 75, 21, 69, 37, 96, 34, 27, 98, 104, 103, 7, 6, 57, 14, 39, 95, 78, 10, 64, 43, 74, 110, 46, 105, 47, 91, 2, 31, 41, 88, 24, 72, 67, 100, 28, 66, 3, 68, 93, 107, 4, 70, 97, 33, 8, 65, 35, 99, 1], [127, 50, 48, 115, 114, 112, 109, 51, 45, 108, 61, 44, 55, 121, 53, 120, 58, 52, 25, 42, 23, 87, 92, 38, 123, 22, 18, 106, 124, 89, 113, 82, 63, 40, 77, 13, 76, 125, 19, 126, 86, 30, 80, 73, 29, 17, 60, 85, 20, 84, 83, 9, 54, 94, 116, 81, 122, 101, 32, 16, 49, 26, 12, 56, 118, 90, 98, 15, 21, 57, 59, 111, 79, 11, 37, 102, 36, 7, 34, 95, 75, 71, 0, 117, 104, 27, 62, 47, 14, 100, 10, 91, 6, 119, 39, 96, 41, 78, 5, 69, 103, 74, 43, 24, 105, 46, 64, 88, 107, 28, 93, 31, 110, 72, 33, 97, 67, 66, 68, 2, 4, 3, 99, 70, 35, 8, 1, 65], [127, 50, 48, 115, 112, 109, 114, 51, 55, 45, 108, 61, 44, 121, 53, 120, 58, 52, 25, 38, 92, 124, 123, 23, 89, 42, 22, 87, 125, 113, 82, 63, 126, 18, 106, 40, 60, 32, 116, 94, 101, 30, 59, 54, 122, 29, 84, 26, 85, 77, 17, 13, 9, 86, 49, 117, 83, 56, 81, 19, 57, 20, 80, 73, 16, 111, 21, 119, 90, 37, 76, 12, 98, 41, 79, 36, 103, 118, 91, 15, 34, 27, 95, 11, 105, 100, 102, 24, 110, 75, 14, 71, 39, 7, 62, 47, 78, 46, 0, 5, 104, 28, 69, 31, 74, 10, 88, 43, 96, 72, 6, 68, 64, 93, 33, 97, 107, 2, 99, 66, 67, 4, 35, 70, 3, 65, 1, 8], [127, 50, 48, 114, 115, 112, 109, 51, 108, 45, 55, 61, 44, 120, 121, 53, 58, 52, 25, 124, 42, 38, 123, 23, 126, 89, 113, 106, 87, 22, 125, 59, 92, 40, 63, 18, 82, 122, 32, 56, 102, 77, 60, 101, 73, 116, 83, 84, 9, 119, 13, 30, 94, 54, 19, 81, 118, 16, 17, 86, 49, 26, 57, 111, 85, 29, 80, 117, 12, 37, 21, 76, 79, 90, 36, 20, 7, 41, 11, 98, 62, 27, 47, 15, 78, 103, 39, 69, 34, 75, 105, 71, 43, 0, 100, 64, 104, 10, 88, 74, 31, 14, 5, 95, 110, 96, 24, 28, 72, 6, 66, 91, 46, 33, 107, 93, 4, 2, 3, 67, 70, 97, 35, 68, 99, 65, 8, 1], [127, 114, 48, 50, 115, 109, 112, 51, 108, 45, 44, 55, 61, 121, 120, 53, 58, 52, 42, 25, 38, 23, 106, 89, 123, 22, 124, 113, 87, 63, 82, 92, 40, 18, 125, 126, 116, 59, 29, 77, 32, 9, 81, 60, 73, 83, 122, 16, 19, 30, 94, 54, 84, 13, 85, 26, 12, 101, 111, 80, 17, 20, 118, 49, 21, 76, 36, 119, 102, 56, 57, 37, 11, 98, 86, 79, 7, 15, 117, 41, 14, 62, 5, 103, 43, 90, 34, 64, 78, 110, 69, 39, 95, 10, 71, 24, 47, 31, 105, 91, 100, 27, 75, 0, 104, 74, 46, 96, 72, 33, 70, 66, 28, 97, 107, 88, 2, 67, 6, 68, 3, 4, 93, 35, 99, 8, 65, 1], [127, 50, 48, 115, 114, 109, 112, 51, 45, 55, 108, 61, 121, 44, 120, 53, 58, 52, 38, 25, 42, 23, 123, 92, 82, 87, 106, 89, 124, 22, 63, 113, 18, 59, 125, 126, 77, 122, 116, 84, 40, 73, 83, 54, 13, 80, 81, 94, 30, 9, 16, 19, 32, 60, 12, 29, 86, 17, 76, 56, 20, 26, 85, 21, 111, 37, 101, 15, 118, 102, 90, 119, 62, 27, 57, 98, 110, 11, 71, 7, 75, 79, 39, 36, 10, 43, 14, 5, 46, 34, 31, 49, 117, 95, 78, 24, 41, 0, 69, 103, 104, 74, 105, 70, 47, 96, 91, 88, 107, 68, 100, 64, 72, 28, 66, 2, 67, 93, 97, 33, 6, 99, 4, 3, 8, 35, 65, 1], [127, 50, 48, 115, 114, 112, 109, 51, 55, 45, 61, 108, 121, 44, 120, 53, 52, 58, 25, 38, 123, 42, 92, 23, 87, 124, 106, 89, 126, 18, 82, 22, 63, 125, 40, 59, 113, 30, 32, 116, 19, 29, 86, 101, 77, 60, 84, 26, 94, 81, 122, 16, 13, 56, 20, 17, 21, 54, 73, 9, 83, 111, 57, 76, 85, 102, 27, 110, 80, 118, 12, 90, 79, 7, 98, 103, 47, 41, 117, 36, 75, 119, 49, 24, 37, 95, 91, 15, 11, 62, 14, 104, 31, 10, 34, 28, 43, 105, 46, 96, 5, 78, 0, 39, 71, 107, 74, 64, 100, 88, 93, 70, 69, 68, 66, 4, 33, 97, 8, 67, 2, 72, 35, 99, 3, 6, 65, 1], [127, 50, 48, 115, 114, 109, 112, 55, 45, 51, 108, 61, 120, 121, 44, 53, 58, 52, 25, 38, 92, 123, 89, 87, 23, 126, 42, 106, 125, 113, 124, 82, 18, 116, 63, 22, 59, 86, 29, 40, 54, 32, 17, 19, 77, 94, 60, 101, 83, 26, 56, 16, 122, 9, 73, 13, 30, 57, 102, 119, 84, 81, 80, 76, 20, 85, 49, 12, 90, 39, 98, 111, 118, 21, 36, 37, 79, 7, 15, 41, 5, 27, 78, 47, 11, 69, 110, 91, 75, 64, 31, 104, 96, 103, 71, 14, 24, 117, 34, 95, 28, 105, 0, 62, 10, 43, 74, 70, 46, 100, 88, 2, 93, 107, 68, 4, 33, 8, 67, 66, 3, 35, 97, 6, 65, 72, 99, 1], [127, 50, 48, 114, 115, 109, 112, 51, 45, 108, 61, 55, 44, 121, 53, 120, 58, 52, 38, 123, 124, 89, 42, 25, 23, 113, 126, 125, 106, 18, 92, 63, 87, 59, 82, 116, 22, 40, 77, 73, 29, 32, 30, 94, 101, 9, 76, 60, 84, 119, 81, 19, 54, 17, 86, 16, 26, 13, 21, 122, 83, 103, 57, 56, 111, 85, 49, 98, 37, 41, 36, 80, 117, 20, 11, 12, 102, 75, 27, 47, 118, 79, 110, 5, 90, 7, 69, 62, 34, 64, 104, 39, 105, 31, 15, 78, 43, 46, 74, 0, 14, 24, 71, 91, 95, 107, 8, 100, 88, 66, 96, 28, 70, 68, 10, 4, 67, 93, 3, 33, 2, 97, 6, 35, 99, 65, 72, 1], [127, 48, 50, 114, 115, 112, 109, 108, 51, 55, 45, 61, 44, 121, 53, 120, 58, 52, 38, 25, 123, 113, 124, 42, 23, 89, 18, 106, 125, 126, 22, 92, 87, 82, 40, 59, 63, 116, 29, 60, 32, 77, 30, 81, 73, 13, 9, 101, 83, 94, 122, 86, 76, 80, 84, 17, 26, 85, 37, 110, 57, 98, 111, 12, 21, 56, 19, 54, 36, 20, 102, 16, 104, 119, 117, 75, 15, 7, 41, 24, 103, 10, 27, 79, 34, 69, 11, 91, 118, 105, 64, 5, 49, 78, 62, 39, 95, 71, 90, 43, 88, 74, 8, 46, 107, 0, 31, 96, 28, 47, 14, 100, 93, 97, 70, 2, 6, 68, 4, 33, 66, 67, 3, 99, 35, 65, 72, 1], [127, 48, 50, 114, 112, 115, 109, 51, 108, 55, 45, 120, 44, 61, 121, 53, 58, 52, 25, 124, 123, 23, 38, 92, 42, 113, 125, 126, 106, 89, 116, 59, 22, 87, 60, 18, 82, 40, 122, 32, 63, 13, 77, 83, 80, 73, 86, 26, 29, 19, 101, 119, 94, 17, 9, 81, 12, 56, 84, 57, 76, 85, 16, 30, 102, 104, 54, 20, 21, 110, 37, 49, 90, 111, 98, 117, 75, 15, 27, 11, 79, 69, 118, 41, 34, 5, 31, 36, 46, 7, 14, 71, 96, 47, 24, 10, 91, 39, 78, 95, 74, 62, 6, 43, 100, 103, 107, 0, 28, 105, 2, 93, 64, 8, 88, 4, 66, 68, 70, 67, 33, 3, 97, 35, 99, 65, 1, 72], [127, 50, 48, 114, 115, 112, 109, 45, 55, 51, 108, 120, 121, 61, 44, 53, 58, 52, 38, 25, 123, 113, 125, 59, 126, 89, 42, 23, 106, 92, 124, 18, 116, 63, 82, 87, 22, 32, 40, 122, 54, 60, 56, 101, 119, 94, 30, 29, 77, 9, 86, 81, 73, 57, 13, 12, 17, 83, 110, 76, 26, 85, 20, 80, 21, 16, 19, 102, 84, 111, 71, 36, 46, 79, 11, 117, 90, 5, 34, 7, 75, 103, 69, 15, 98, 37, 62, 104, 0, 27, 31, 41, 118, 49, 39, 47, 78, 24, 6, 91, 96, 14, 64, 10, 4, 95, 88, 100, 74, 43, 67, 107, 93, 66, 68, 105, 2, 8, 28, 3, 33, 70, 97, 99, 65, 35, 1, 72], [127, 48, 50, 115, 112, 114, 109, 55, 51, 45, 108, 121, 44, 61, 120, 53, 58, 52, 38, 25, 113, 125, 23, 124, 89, 123, 92, 42, 59, 18, 87, 106, 126, 82, 63, 40, 122, 22, 30, 60, 77, 54, 116, 29, 86, 32, 73, 20, 56, 94, 57, 83, 17, 26, 101, 19, 9, 13, 111, 85, 81, 102, 119, 80, 12, 16, 37, 21, 84, 62, 98, 110, 76, 117, 15, 5, 75, 118, 47, 90, 104, 79, 91, 14, 49, 7, 36, 103, 11, 71, 34, 27, 46, 69, 24, 6, 100, 41, 95, 31, 39, 28, 0, 78, 88, 43, 74, 107, 93, 68, 10, 96, 105, 4, 8, 2, 66, 64, 97, 67, 33, 3, 70, 99, 35, 72, 65, 1], [127, 48, 50, 115, 114, 112, 109, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 25, 38, 42, 124, 23, 89, 123, 87, 18, 106, 113, 92, 63, 22, 82, 125, 122, 60, 126, 40, 77, 54, 59, 9, 29, 86, 73, 116, 32, 17, 101, 13, 94, 83, 30, 12, 19, 56, 26, 84, 20, 80, 16, 76, 57, 21, 85, 102, 119, 81, 98, 37, 104, 36, 27, 90, 7, 15, 111, 79, 75, 110, 103, 41, 78, 10, 34, 69, 62, 47, 11, 118, 95, 74, 5, 71, 31, 14, 91, 46, 28, 117, 88, 43, 49, 0, 96, 39, 100, 107, 6, 64, 8, 24, 105, 2, 67, 66, 33, 70, 68, 4, 97, 3, 93, 99, 72, 35, 65, 1], [127, 50, 48, 114, 115, 112, 109, 51, 45, 108, 55, 61, 44, 121, 53, 120, 58, 52, 25, 38, 123, 23, 22, 124, 82, 42, 87, 18, 106, 113, 89, 63, 92, 126, 122, 125, 84, 40, 60, 13, 54, 116, 83, 12, 32, 73, 59, 101, 94, 29, 86, 17, 9, 85, 57, 30, 16, 77, 21, 19, 26, 111, 56, 81, 20, 110, 80, 98, 76, 75, 117, 90, 79, 102, 91, 37, 71, 15, 119, 36, 103, 34, 78, 5, 46, 27, 95, 39, 11, 62, 7, 104, 14, 31, 47, 41, 24, 69, 64, 74, 105, 49, 10, 118, 0, 96, 43, 28, 88, 93, 6, 100, 107, 3, 2, 67, 4, 70, 68, 33, 66, 72, 8, 97, 99, 1, 35, 65], [127, 50, 48, 114, 115, 109, 112, 51, 45, 55, 61, 108, 121, 53, 44, 120, 58, 52, 25, 38, 123, 23, 124, 92, 126, 113, 42, 63, 87, 89, 106, 116, 22, 122, 82, 125, 18, 40, 32, 59, 54, 30, 86, 101, 29, 94, 84, 77, 73, 17, 57, 9, 83, 21, 85, 13, 60, 19, 119, 12, 118, 26, 110, 98, 20, 16, 111, 81, 41, 56, 103, 79, 90, 80, 37, 76, 102, 117, 15, 75, 27, 49, 34, 71, 36, 43, 78, 47, 62, 46, 91, 74, 107, 104, 69, 11, 100, 96, 24, 14, 28, 7, 64, 31, 39, 0, 95, 5, 105, 10, 72, 93, 68, 88, 97, 2, 4, 33, 70, 3, 6, 35, 99, 66, 67, 65, 8, 1], [127, 50, 48, 115, 109, 114, 112, 55, 51, 45, 61, 108, 121, 44, 120, 53, 52, 58, 25, 124, 23, 38, 123, 92, 63, 87, 106, 89, 42, 59, 113, 60, 122, 54, 125, 18, 116, 82, 126, 32, 40, 22, 118, 83, 111, 119, 19, 57, 77, 56, 9, 29, 20, 26, 101, 73, 102, 12, 94, 86, 30, 85, 81, 84, 13, 17, 49, 117, 80, 16, 36, 76, 98, 75, 21, 27, 103, 41, 110, 47, 104, 90, 71, 37, 62, 78, 79, 91, 95, 15, 34, 105, 46, 43, 11, 31, 39, 69, 14, 74, 5, 7, 88, 64, 96, 100, 107, 10, 28, 70, 93, 24, 33, 0, 2, 68, 4, 66, 97, 72, 3, 99, 6, 35, 67, 65, 1, 8], [127, 50, 48, 114, 115, 112, 109, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 123, 38, 25, 124, 23, 89, 87, 116, 113, 125, 106, 18, 60, 82, 63, 92, 42, 126, 22, 40, 122, 32, 9, 73, 59, 101, 77, 13, 54, 12, 83, 30, 84, 94, 81, 110, 85, 19, 29, 57, 17, 119, 86, 26, 80, 111, 16, 76, 36, 56, 21, 49, 20, 117, 41, 15, 102, 79, 75, 71, 98, 46, 37, 34, 103, 69, 74, 27, 39, 91, 5, 0, 7, 11, 90, 78, 104, 118, 14, 96, 62, 24, 10, 47, 95, 31, 70, 105, 28, 64, 100, 72, 2, 43, 4, 66, 67, 107, 88, 93, 97, 3, 33, 68, 35, 6, 99, 65, 1, 8], [127, 50, 114, 115, 48, 112, 109, 51, 108, 45, 61, 44, 55, 121, 53, 120, 58, 52, 124, 38, 23, 25, 63, 123, 113, 89, 42, 92, 18, 22, 122, 87, 82, 60, 116, 106, 40, 125, 59, 126, 73, 77, 32, 54, 12, 13, 83, 94, 9, 101, 85, 81, 110, 29, 30, 17, 16, 19, 111, 86, 20, 76, 80, 119, 75, 26, 37, 56, 36, 117, 62, 49, 98, 102, 41, 84, 90, 57, 71, 118, 27, 79, 104, 103, 34, 69, 21, 15, 70, 11, 46, 64, 14, 7, 5, 10, 47, 105, 95, 74, 78, 72, 43, 39, 2, 91, 88, 96, 31, 28, 100, 0, 66, 107, 24, 4, 33, 3, 93, 67, 97, 99, 68, 35, 6, 65, 1, 8], [127, 50, 48, 115, 114, 112, 109, 51, 55, 108, 61, 45, 44, 53, 121, 120, 58, 52, 25, 124, 38, 123, 23, 42, 89, 113, 87, 92, 22, 63, 82, 18, 106, 60, 122, 126, 116, 59, 40, 125, 77, 17, 32, 13, 73, 56, 94, 9, 86, 83, 81, 29, 12, 19, 80, 101, 54, 16, 30, 84, 20, 85, 21, 102, 26, 76, 111, 57, 75, 117, 98, 36, 62, 79, 47, 118, 41, 110, 15, 71, 90, 46, 7, 103, 78, 34, 14, 49, 27, 74, 37, 119, 11, 105, 39, 69, 91, 104, 70, 31, 5, 64, 100, 10, 95, 96, 88, 43, 72, 24, 33, 107, 68, 0, 2, 93, 28, 4, 66, 6, 67, 3, 97, 99, 35, 8, 65, 1], [127, 50, 48, 115, 114, 51, 109, 112, 45, 108, 61, 55, 121, 44, 53, 120, 58, 52, 25, 123, 23, 124, 38, 42, 87, 106, 89, 116, 60, 113, 126, 18, 125, 92, 122, 82, 40, 63, 22, 77, 94, 29, 30, 73, 59, 9, 32, 101, 17, 86, 13, 12, 83, 81, 80, 20, 56, 54, 85, 26, 76, 111, 118, 84, 57, 21, 16, 98, 19, 41, 102, 62, 110, 117, 36, 71, 7, 37, 34, 49, 75, 47, 79, 15, 69, 27, 119, 90, 78, 10, 104, 64, 0, 24, 91, 11, 39, 14, 95, 96, 74, 5, 93, 46, 31, 103, 107, 43, 6, 33, 105, 28, 88, 67, 72, 100, 2, 66, 3, 68, 70, 4, 35, 8, 97, 99, 1, 65], [127, 50, 48, 115, 114, 109, 112, 51, 55, 108, 45, 61, 121, 44, 53, 120, 52, 58, 25, 38, 123, 23, 124, 42, 87, 89, 92, 113, 22, 82, 60, 106, 18, 63, 40, 116, 125, 13, 126, 29, 59, 122, 32, 9, 54, 83, 57, 12, 94, 73, 81, 77, 26, 101, 20, 16, 86, 19, 30, 17, 85, 111, 56, 75, 84, 80, 21, 102, 15, 76, 36, 110, 117, 41, 98, 90, 78, 14, 62, 118, 79, 37, 10, 11, 49, 7, 91, 119, 47, 71, 27, 24, 103, 31, 43, 46, 74, 96, 34, 95, 100, 69, 5, 105, 104, 39, 6, 64, 28, 88, 93, 107, 0, 67, 72, 3, 4, 68, 33, 2, 70, 97, 66, 8, 35, 99, 1, 65]], "model.layers.25.self_attn.q_proj": [[62, 102, 124, 121, 56, 60, 97, 47, 50, 63, 38, 93, 114, 24, 127, 117, 95, 86, 105, 53, 89, 41, 35, 29, 43, 106, 115, 51, 52, 22, 37, 49, 103, 122, 20, 25, 123, 59, 87, 55, 33, 83, 91, 116, 57, 58, 100, 36, 99, 79, 27, 26, 46, 113, 111, 92, 108, 112, 109, 94, 61, 48, 125, 82, 19, 110, 54, 34, 42, 17, 39, 44, 12, 119, 126, 18, 118, 80, 120, 101, 104, 40, 45, 77, 98, 32, 16, 96, 28, 30, 107, 31, 90, 88, 21, 23, 81, 85, 70, 75, 84, 74, 72, 9, 4, 78, 15, 2, 14, 8, 76, 13, 67, 11, 0, 5, 66, 71, 10, 7, 73, 6, 68, 65, 3, 1, 64, 69], [102, 62, 93, 121, 124, 24, 97, 83, 56, 35, 60, 127, 85, 81, 13, 21, 17, 63, 86, 20, 22, 98, 36, 25, 123, 114, 92, 19, 120, 77, 84, 46, 74, 78, 48, 116, 111, 76, 52, 49, 11, 95, 33, 30, 32, 79, 115, 1, 38, 9, 71, 28, 122, 88, 34, 69, 31, 12, 104, 75, 101, 7, 66, 80, 108, 0, 43, 99, 117, 67, 29, 4, 72, 107, 8, 90, 113, 70, 58, 96, 23, 119, 103, 27, 26, 37, 18, 59, 54, 87, 14, 15, 126, 89, 16, 91, 94, 105, 53, 125, 10, 40, 106, 2, 51, 6, 118, 39, 82, 68, 42, 100, 73, 57, 50, 44, 3, 61, 65, 47, 41, 110, 55, 64, 45, 109, 112, 5], [62, 124, 102, 60, 97, 122, 123, 116, 93, 38, 44, 117, 43, 127, 59, 121, 87, 48, 114, 53, 118, 24, 89, 54, 39, 29, 58, 46, 94, 56, 40, 25, 82, 119, 57, 95, 106, 50, 125, 108, 99, 51, 49, 20, 109, 55, 47, 86, 126, 52, 111, 112, 63, 113, 45, 41, 91, 98, 104, 42, 110, 22, 79, 61, 115, 120, 103, 100, 31, 27, 92, 107, 17, 23, 105, 37, 35, 76, 85, 36, 34, 90, 101, 80, 32, 33, 96, 26, 28, 84, 12, 30, 13, 21, 18, 83, 19, 77, 15, 81, 16, 8, 9, 88, 74, 78, 7, 6, 73, 11, 75, 14, 70, 71, 10, 4, 5, 72, 68, 69, 2, 3, 67, 1, 66, 65, 64, 0], [62, 121, 102, 117, 124, 114, 60, 49, 50, 53, 116, 97, 123, 43, 38, 108, 122, 93, 45, 63, 127, 94, 59, 113, 104, 46, 58, 25, 44, 87, 125, 106, 61, 52, 54, 109, 17, 31, 118, 111, 56, 48, 47, 103, 33, 42, 119, 51, 120, 112, 34, 126, 29, 110, 107, 115, 57, 41, 40, 55, 89, 28, 105, 39, 22, 99, 37, 27, 98, 95, 82, 85, 90, 86, 100, 35, 20, 30, 26, 91, 36, 101, 24, 79, 96, 76, 80, 32, 19, 18, 23, 12, 92, 21, 8, 11, 84, 75, 77, 64, 1, 78, 10, 6, 16, 14, 0, 83, 4, 66, 74, 68, 69, 81, 67, 72, 7, 2, 70, 5, 71, 9, 65, 13, 73, 88, 3, 15], [58, 126, 117, 39, 111, 124, 59, 48, 51, 112, 25, 125, 54, 127, 47, 118, 123, 35, 62, 21, 55, 57, 89, 113, 96, 49, 63, 50, 60, 53, 61, 119, 116, 56, 29, 91, 45, 120, 110, 87, 52, 23, 114, 82, 42, 121, 122, 115, 46, 85, 109, 44, 108, 80, 27, 76, 107, 20, 105, 14, 93, 106, 83, 99, 41, 40, 43, 104, 38, 102, 9, 103, 28, 18, 31, 32, 16, 101, 36, 22, 95, 100, 79, 24, 8, 88, 81, 12, 13, 4, 84, 78, 92, 33, 71, 90, 37, 86, 97, 10, 69, 6, 94, 34, 3, 11, 30, 74, 75, 98, 73, 19, 77, 26, 66, 7, 70, 72, 17, 15, 2, 5, 0, 68, 64, 67, 1, 65], [111, 117, 58, 126, 39, 124, 59, 51, 48, 118, 25, 21, 50, 57, 54, 127, 55, 91, 125, 62, 112, 49, 113, 47, 63, 53, 110, 96, 61, 52, 119, 123, 116, 35, 56, 60, 120, 29, 87, 89, 23, 42, 121, 115, 45, 114, 46, 85, 82, 44, 108, 122, 109, 80, 76, 27, 107, 14, 106, 93, 41, 37, 105, 9, 102, 38, 43, 83, 104, 88, 40, 8, 20, 99, 100, 18, 24, 12, 28, 31, 10, 13, 81, 79, 32, 71, 101, 92, 103, 36, 69, 97, 30, 78, 94, 4, 33, 6, 16, 70, 22, 3, 90, 98, 95, 34, 86, 84, 26, 73, 19, 75, 74, 7, 11, 66, 5, 72, 77, 17, 65, 64, 15, 2, 67, 1, 68, 0], [117, 126, 39, 58, 111, 124, 59, 51, 48, 112, 21, 25, 57, 54, 127, 118, 62, 29, 55, 110, 50, 125, 49, 53, 35, 96, 91, 123, 63, 52, 61, 119, 113, 56, 116, 89, 120, 60, 45, 47, 87, 115, 121, 114, 82, 23, 80, 46, 44, 108, 85, 122, 42, 109, 107, 27, 76, 83, 14, 105, 102, 40, 38, 43, 9, 103, 28, 41, 106, 104, 20, 93, 8, 99, 32, 10, 13, 31, 37, 24, 36, 18, 81, 79, 12, 88, 70, 100, 16, 71, 92, 78, 101, 34, 22, 84, 33, 90, 94, 3, 7, 6, 97, 86, 69, 95, 73, 11, 30, 4, 75, 98, 19, 74, 72, 77, 66, 26, 15, 17, 2, 5, 64, 1, 0, 68, 65, 67], [126, 117, 39, 58, 111, 124, 51, 59, 48, 21, 25, 112, 54, 57, 62, 55, 127, 89, 125, 96, 63, 118, 29, 123, 50, 53, 113, 35, 110, 49, 61, 52, 60, 119, 120, 56, 116, 87, 91, 47, 23, 42, 45, 121, 44, 114, 80, 115, 122, 85, 46, 82, 108, 109, 83, 14, 76, 107, 27, 20, 40, 105, 102, 9, 8, 103, 106, 43, 38, 104, 41, 99, 93, 32, 18, 31, 4, 28, 79, 37, 95, 36, 24, 16, 84, 10, 100, 13, 22, 101, 81, 12, 92, 33, 90, 71, 78, 97, 30, 69, 70, 86, 88, 34, 94, 6, 11, 19, 98, 73, 3, 75, 74, 26, 7, 72, 77, 15, 17, 0, 66, 5, 64, 68, 65, 1, 2, 67], [40, 59, 34, 90, 83, 12, 53, 91, 63, 88, 17, 95, 45, 112, 85, 31, 79, 9, 124, 123, 13, 6, 108, 119, 113, 5, 109, 74, 126, 72, 99, 93, 96, 15, 78, 114, 62, 58, 47, 28, 29, 61, 55, 8, 73, 107, 18, 97, 52, 35, 41, 20, 86, 48, 14, 33, 87, 76, 94, 127, 122, 92, 16, 50, 121, 11, 98, 82, 116, 24, 101, 57, 110, 77, 120, 103, 10, 51, 71, 111, 49, 38, 60, 125, 54, 115, 42, 56, 105, 118, 106, 80, 23, 44, 46, 69, 100, 26, 117, 32, 39, 30, 102, 43, 36, 81, 89, 7, 37, 27, 22, 21, 84, 19, 25, 75, 70, 3, 2, 67, 4, 68, 64, 1, 66, 104, 0, 65], [40, 59, 34, 90, 124, 88, 83, 13, 63, 126, 79, 17, 85, 6, 95, 50, 121, 118, 53, 48, 46, 127, 12, 45, 10, 91, 106, 103, 22, 29, 4, 72, 116, 68, 74, 96, 60, 98, 55, 99, 112, 11, 58, 9, 125, 35, 69, 41, 24, 115, 120, 43, 31, 37, 61, 122, 62, 0, 39, 33, 113, 123, 105, 110, 78, 21, 77, 84, 107, 51, 67, 20, 7, 87, 54, 119, 28, 38, 47, 14, 73, 114, 82, 93, 26, 75, 117, 32, 94, 66, 18, 42, 56, 108, 111, 52, 44, 36, 1, 109, 2, 8, 102, 15, 30, 70, 23, 19, 71, 57, 86, 92, 49, 76, 81, 100, 101, 80, 27, 89, 97, 5, 16, 25, 104, 3, 65, 64], [40, 59, 34, 90, 85, 83, 127, 124, 63, 53, 17, 48, 88, 110, 116, 118, 79, 95, 13, 113, 29, 58, 96, 91, 31, 42, 111, 121, 12, 24, 11, 50, 94, 115, 98, 38, 45, 60, 10, 112, 47, 14, 114, 103, 125, 22, 55, 108, 25, 35, 44, 119, 109, 100, 36, 9, 57, 56, 86, 117, 102, 49, 26, 39, 72, 28, 6, 71, 61, 101, 23, 89, 52, 43, 27, 122, 41, 21, 51, 33, 123, 62, 30, 126, 54, 106, 105, 87, 37, 99, 46, 97, 92, 107, 84, 93, 120, 20, 0, 18, 104, 81, 16, 4, 82, 76, 78, 32, 15, 19, 80, 1, 7, 68, 74, 69, 75, 2, 8, 77, 73, 70, 66, 3, 5, 67, 65, 64], [40, 59, 34, 90, 88, 83, 124, 79, 17, 13, 95, 53, 12, 118, 9, 58, 126, 72, 85, 24, 63, 81, 4, 48, 114, 86, 2, 71, 11, 6, 45, 44, 112, 121, 75, 103, 116, 1, 50, 10, 105, 14, 0, 110, 3, 31, 69, 60, 78, 42, 46, 106, 39, 120, 74, 67, 115, 54, 32, 29, 43, 26, 98, 62, 8, 51, 113, 68, 52, 7, 20, 22, 21, 80, 91, 65, 92, 107, 57, 35, 37, 47, 56, 127, 84, 18, 89, 125, 16, 77, 15, 19, 94, 41, 96, 82, 108, 70, 55, 25, 109, 104, 76, 28, 49, 27, 61, 64, 93, 97, 23, 36, 100, 33, 99, 122, 111, 101, 66, 30, 119, 38, 73, 123, 102, 117, 87, 5], [58, 61, 127, 56, 38, 120, 125, 53, 89, 52, 115, 112, 107, 50, 83, 122, 57, 106, 124, 59, 116, 85, 60, 119, 23, 114, 118, 121, 123, 117, 55, 126, 87, 48, 99, 46, 96, 102, 49, 63, 110, 54, 62, 51, 111, 47, 76, 109, 113, 29, 45, 94, 41, 18, 44, 91, 42, 16, 28, 43, 36, 25, 104, 108, 103, 92, 78, 95, 40, 90, 82, 105, 80, 100, 35, 84, 39, 14, 17, 26, 34, 73, 70, 69, 12, 71, 32, 21, 13, 22, 37, 3, 74, 7, 27, 101, 20, 97, 98, 24, 31, 11, 33, 86, 79, 9, 30, 93, 8, 15, 88, 68, 72, 75, 5, 19, 10, 77, 81, 66, 4, 2, 6, 1, 64, 67, 65, 0], [56, 127, 61, 58, 38, 120, 89, 125, 53, 52, 50, 115, 59, 57, 124, 83, 114, 107, 119, 116, 112, 85, 106, 122, 118, 60, 55, 121, 123, 117, 126, 102, 23, 99, 63, 51, 54, 49, 110, 48, 46, 62, 96, 45, 29, 111, 47, 87, 94, 109, 113, 18, 25, 43, 76, 16, 41, 95, 44, 42, 90, 36, 91, 108, 104, 103, 92, 28, 40, 35, 80, 105, 73, 78, 84, 100, 39, 21, 82, 17, 70, 32, 71, 37, 13, 31, 98, 14, 8, 74, 26, 97, 22, 12, 101, 79, 24, 34, 27, 86, 11, 3, 15, 7, 20, 30, 33, 69, 77, 93, 68, 9, 72, 10, 19, 88, 81, 5, 0, 2, 6, 75, 4, 67, 1, 64, 65, 66], [61, 127, 58, 38, 56, 125, 120, 53, 89, 52, 115, 107, 50, 124, 83, 57, 59, 119, 116, 114, 85, 112, 121, 118, 126, 60, 122, 55, 123, 46, 23, 117, 48, 49, 106, 110, 62, 99, 63, 51, 102, 96, 54, 94, 87, 111, 47, 42, 29, 45, 113, 109, 16, 18, 25, 76, 43, 28, 41, 44, 108, 103, 40, 78, 95, 36, 90, 105, 104, 91, 80, 92, 39, 84, 82, 100, 73, 34, 35, 17, 21, 37, 14, 32, 71, 13, 12, 26, 22, 33, 70, 101, 97, 20, 74, 86, 11, 27, 98, 79, 68, 8, 15, 10, 24, 30, 93, 31, 9, 72, 77, 3, 69, 7, 88, 19, 75, 6, 81, 5, 4, 0, 65, 2, 64, 66, 1, 67], [127, 61, 56, 38, 58, 120, 89, 125, 53, 52, 57, 50, 115, 119, 59, 124, 114, 83, 116, 55, 107, 112, 23, 85, 123, 122, 121, 63, 118, 126, 99, 60, 62, 110, 117, 29, 48, 49, 106, 96, 94, 102, 87, 51, 54, 46, 45, 111, 47, 113, 109, 16, 18, 25, 43, 42, 44, 76, 90, 41, 91, 28, 108, 36, 92, 103, 104, 84, 40, 82, 14, 105, 26, 37, 35, 80, 100, 34, 95, 32, 39, 17, 12, 22, 69, 21, 3, 78, 13, 97, 70, 71, 101, 73, 74, 24, 98, 11, 20, 8, 86, 27, 33, 31, 30, 79, 93, 68, 72, 7, 10, 9, 15, 19, 81, 88, 77, 5, 75, 2, 4, 1, 6, 65, 66, 0, 64, 67], [42, 98, 106, 112, 26, 36, 21, 49, 118, 16, 94, 18, 54, 22, 48, 31, 60, 14, 13, 95, 101, 77, 93, 9, 123, 116, 88, 90, 114, 125, 58, 38, 113, 127, 53, 92, 15, 19, 8, 63, 47, 20, 52, 17, 99, 61, 124, 46, 12, 55, 111, 23, 73, 29, 27, 120, 115, 117, 84, 28, 122, 87, 10, 105, 51, 85, 33, 72, 24, 109, 104, 44, 34, 97, 74, 126, 45, 102, 43, 40, 30, 75, 59, 62, 107, 50, 4, 39, 100, 81, 108, 119, 121, 57, 41, 76, 2, 83, 35, 110, 71, 89, 64, 103, 37, 56, 67, 80, 7, 32, 91, 11, 82, 79, 25, 86, 96, 6, 0, 3, 78, 5, 1, 69, 65, 66, 70, 68], [42, 118, 98, 36, 106, 26, 18, 21, 124, 116, 63, 49, 111, 127, 54, 112, 123, 52, 88, 94, 24, 122, 126, 48, 31, 22, 99, 117, 53, 110, 55, 121, 58, 16, 38, 101, 20, 92, 47, 62, 33, 59, 61, 90, 95, 12, 51, 119, 60, 108, 46, 109, 8, 57, 29, 30, 120, 114, 28, 50, 44, 93, 11, 14, 113, 86, 125, 115, 107, 100, 39, 43, 77, 105, 56, 4, 34, 79, 97, 103, 40, 41, 27, 104, 45, 15, 37, 82, 102, 35, 23, 2, 10, 13, 96, 85, 74, 32, 75, 91, 25, 89, 0, 7, 87, 83, 84, 17, 19, 1, 73, 80, 71, 9, 81, 76, 6, 64, 67, 68, 5, 65, 78, 72, 66, 69, 70, 3], [42, 98, 118, 126, 106, 36, 26, 18, 112, 127, 24, 21, 48, 88, 46, 31, 41, 94, 63, 116, 20, 113, 125, 62, 53, 11, 54, 7, 8, 16, 52, 38, 49, 12, 124, 121, 123, 99, 111, 122, 39, 60, 4, 2, 50, 58, 109, 114, 120, 86, 117, 95, 90, 47, 30, 119, 57, 61, 1, 55, 33, 59, 79, 14, 56, 77, 37, 44, 51, 115, 103, 101, 92, 91, 107, 43, 75, 29, 6, 64, 102, 40, 110, 97, 105, 22, 108, 9, 80, 23, 45, 89, 19, 96, 93, 27, 82, 13, 67, 0, 104, 35, 83, 34, 17, 69, 85, 100, 32, 10, 74, 73, 25, 5, 76, 28, 84, 15, 81, 66, 87, 72, 65, 68, 71, 78, 3, 70], [42, 98, 106, 26, 31, 20, 16, 21, 18, 36, 112, 54, 75, 14, 77, 113, 9, 73, 49, 94, 79, 76, 15, 93, 116, 123, 118, 90, 101, 70, 10, 7, 19, 8, 1, 60, 53, 48, 6, 69, 58, 84, 74, 3, 34, 127, 78, 107, 114, 71, 99, 88, 0, 17, 2, 41, 105, 125, 67, 87, 109, 81, 25, 68, 119, 11, 28, 23, 22, 13, 27, 80, 40, 12, 64, 91, 82, 37, 72, 52, 4, 61, 47, 83, 62, 85, 38, 92, 24, 33, 32, 122, 44, 66, 115, 111, 46, 39, 86, 65, 104, 110, 102, 108, 120, 45, 63, 50, 96, 89, 5, 29, 55, 56, 95, 126, 57, 117, 59, 97, 100, 35, 121, 51, 124, 43, 30, 103], [39, 63, 64, 0, 66, 1, 12, 121, 3, 4, 2, 69, 79, 77, 9, 23, 87, 74, 70, 122, 43, 49, 7, 108, 127, 42, 83, 85, 53, 46, 67, 118, 48, 61, 18, 115, 31, 58, 93, 103, 113, 68, 45, 59, 105, 16, 99, 40, 47, 124, 52, 60, 75, 17, 20, 78, 89, 97, 19, 71, 114, 100, 112, 111, 33, 65, 55, 11, 84, 126, 25, 50, 57, 5, 88, 51, 116, 81, 6, 90, 102, 120, 98, 117, 14, 86, 21, 30, 109, 34, 32, 41, 92, 94, 80, 13, 10, 27, 28, 106, 73, 38, 56, 26, 123, 76, 96, 82, 101, 35, 29, 54, 15, 119, 36, 107, 22, 37, 8, 110, 91, 72, 104, 95, 125, 44, 24, 62], [39, 63, 87, 122, 79, 77, 121, 74, 12, 118, 9, 31, 46, 69, 4, 66, 83, 127, 23, 115, 108, 3, 17, 70, 64, 1, 93, 49, 100, 18, 43, 75, 20, 41, 52, 15, 72, 90, 85, 58, 42, 89, 7, 82, 16, 48, 78, 59, 13, 60, 113, 27, 124, 109, 50, 10, 94, 8, 112, 80, 92, 95, 125, 99, 106, 35, 67, 96, 117, 53, 114, 97, 81, 29, 19, 24, 91, 123, 55, 86, 11, 104, 56, 126, 111, 36, 21, 105, 101, 57, 71, 6, 65, 116, 62, 119, 54, 61, 45, 51, 26, 22, 33, 84, 47, 40, 120, 110, 88, 37, 38, 28, 14, 73, 34, 107, 25, 102, 44, 30, 98, 76, 68, 0, 5, 32, 2, 103], [39, 63, 64, 0, 1, 121, 12, 74, 4, 3, 23, 9, 69, 79, 77, 66, 87, 70, 2, 122, 43, 108, 49, 42, 127, 83, 46, 53, 93, 7, 68, 99, 48, 52, 61, 67, 5, 118, 40, 103, 31, 18, 85, 59, 113, 45, 58, 112, 114, 6, 65, 81, 60, 55, 120, 105, 51, 100, 116, 88, 71, 115, 25, 90, 126, 80, 34, 91, 22, 86, 14, 33, 11, 124, 17, 16, 78, 47, 72, 73, 20, 101, 94, 56, 123, 57, 37, 19, 89, 75, 8, 111, 35, 106, 109, 82, 29, 10, 36, 26, 97, 62, 27, 98, 96, 54, 28, 24, 92, 84, 119, 125, 117, 38, 104, 30, 76, 32, 95, 41, 102, 50, 107, 44, 13, 15, 110, 21], [39, 63, 74, 79, 9, 87, 77, 12, 4, 121, 3, 66, 64, 23, 1, 70, 69, 43, 108, 18, 72, 127, 93, 49, 42, 2, 122, 46, 67, 20, 31, 115, 40, 83, 5, 113, 48, 7, 53, 118, 61, 27, 68, 60, 45, 6, 19, 125, 100, 97, 52, 24, 114, 16, 124, 111, 71, 112, 58, 81, 47, 89, 13, 85, 17, 34, 10, 99, 86, 8, 14, 78, 76, 73, 59, 26, 82, 90, 51, 92, 33, 84, 120, 11, 75, 0, 65, 119, 29, 22, 126, 57, 62, 32, 80, 28, 95, 30, 117, 56, 88, 25, 116, 15, 94, 37, 105, 36, 104, 91, 109, 21, 102, 98, 41, 35, 50, 123, 55, 106, 101, 107, 44, 96, 110, 38, 54, 103], [106, 36, 85, 42, 63, 83, 77, 113, 80, 15, 6, 73, 122, 124, 91, 74, 119, 3, 71, 48, 27, 115, 68, 58, 127, 0, 2, 76, 25, 98, 50, 112, 95, 66, 32, 57, 118, 67, 93, 69, 94, 75, 12, 61, 9, 87, 65, 56, 5, 14, 1, 10, 19, 29, 22, 121, 79, 126, 4, 16, 11, 81, 13, 84, 125, 8, 54, 37, 64, 109, 7, 110, 88, 21, 116, 114, 70, 39, 46, 30, 120, 105, 33, 41, 18, 99, 55, 20, 23, 123, 86, 78, 59, 72, 102, 90, 52, 51, 34, 49, 82, 104, 117, 107, 26, 89, 96, 24, 53, 111, 45, 28, 40, 31, 17, 97, 44, 60, 35, 108, 62, 103, 43, 101, 47, 92, 38, 100], [106, 36, 63, 42, 122, 85, 91, 58, 83, 124, 48, 80, 113, 77, 50, 27, 61, 112, 119, 15, 93, 116, 29, 25, 87, 98, 115, 120, 95, 73, 59, 32, 74, 114, 46, 123, 51, 57, 35, 71, 45, 109, 117, 126, 44, 2, 121, 54, 94, 30, 96, 118, 127, 52, 47, 111, 65, 56, 125, 108, 60, 68, 105, 49, 110, 6, 41, 62, 90, 55, 26, 21, 39, 99, 86, 17, 53, 10, 81, 8, 38, 107, 11, 101, 43, 19, 78, 104, 40, 103, 75, 33, 82, 66, 102, 18, 20, 37, 88, 5, 14, 0, 23, 79, 97, 92, 24, 31, 16, 76, 28, 34, 89, 22, 3, 12, 84, 64, 72, 7, 13, 100, 69, 1, 9, 4, 70, 67], [106, 36, 63, 42, 122, 113, 85, 91, 50, 83, 27, 80, 117, 61, 48, 124, 25, 112, 29, 118, 98, 87, 93, 52, 123, 77, 95, 60, 15, 119, 32, 59, 58, 127, 115, 57, 96, 90, 53, 126, 51, 120, 17, 73, 54, 35, 49, 40, 94, 114, 41, 109, 45, 46, 44, 121, 55, 111, 116, 21, 110, 62, 47, 105, 74, 71, 56, 30, 125, 108, 99, 39, 107, 38, 88, 103, 24, 43, 6, 104, 11, 86, 23, 101, 78, 102, 37, 34, 20, 19, 97, 5, 79, 31, 75, 28, 8, 33, 14, 81, 22, 92, 68, 89, 100, 84, 10, 16, 66, 26, 12, 2, 82, 0, 13, 76, 69, 72, 18, 65, 3, 7, 4, 9, 67, 70, 1, 64], [106, 36, 122, 42, 85, 124, 113, 48, 91, 50, 83, 127, 93, 58, 63, 59, 27, 98, 80, 118, 32, 126, 77, 47, 15, 87, 29, 25, 73, 95, 35, 115, 57, 109, 119, 96, 39, 90, 54, 114, 17, 112, 55, 123, 99, 56, 94, 21, 71, 41, 117, 30, 74, 45, 11, 51, 61, 38, 6, 75, 52, 121, 49, 103, 5, 68, 46, 62, 120, 34, 111, 2, 116, 40, 60, 125, 110, 53, 108, 105, 86, 18, 24, 33, 44, 43, 107, 97, 81, 102, 104, 26, 19, 12, 88, 37, 101, 79, 10, 72, 89, 78, 8, 20, 31, 82, 92, 16, 28, 14, 23, 84, 0, 76, 3, 22, 65, 66, 13, 69, 100, 7, 9, 4, 70, 64, 67, 1], [108, 111, 93, 44, 24, 14, 125, 116, 15, 72, 85, 74, 82, 19, 121, 98, 76, 6, 47, 124, 81, 36, 63, 20, 35, 37, 119, 95, 1, 25, 11, 87, 3, 33, 67, 68, 126, 38, 97, 57, 64, 27, 34, 100, 29, 55, 77, 60, 96, 13, 51, 65, 40, 4, 62, 49, 70, 105, 123, 75, 52, 5, 30, 0, 79, 43, 73, 86, 41, 92, 71, 9, 56, 46, 113, 88, 32, 117, 112, 80, 31, 50, 7, 109, 103, 127, 84, 53, 107, 23, 10, 48, 104, 26, 106, 114, 45, 12, 115, 122, 83, 120, 90, 78, 2, 61, 89, 21, 17, 18, 99, 42, 28, 101, 110, 102, 59, 118, 54, 8, 94, 66, 22, 16, 58, 69, 39, 91], [108, 44, 111, 93, 74, 14, 24, 116, 72, 124, 125, 82, 76, 85, 19, 68, 15, 6, 36, 1, 98, 64, 13, 47, 87, 81, 37, 63, 4, 20, 126, 121, 100, 65, 29, 53, 3, 67, 35, 119, 46, 7, 51, 60, 43, 49, 11, 34, 123, 57, 95, 2, 50, 80, 41, 113, 0, 55, 62, 86, 101, 56, 127, 69, 45, 88, 70, 40, 73, 99, 75, 104, 97, 38, 96, 90, 92, 117, 105, 9, 5, 31, 27, 58, 10, 114, 61, 18, 23, 21, 12, 89, 25, 71, 33, 42, 83, 66, 102, 103, 77, 120, 107, 30, 112, 79, 54, 8, 122, 59, 78, 118, 28, 115, 48, 84, 16, 110, 106, 91, 26, 22, 32, 39, 52, 94, 17, 109], [108, 111, 44, 125, 93, 24, 82, 15, 14, 76, 13, 74, 50, 68, 36, 116, 124, 72, 19, 98, 1, 6, 121, 113, 0, 87, 20, 86, 89, 67, 9, 64, 47, 37, 126, 95, 34, 53, 60, 35, 11, 71, 63, 4, 33, 32, 119, 92, 29, 100, 127, 41, 22, 3, 62, 84, 73, 83, 80, 55, 59, 49, 46, 123, 85, 70, 16, 57, 94, 23, 43, 7, 56, 88, 27, 65, 52, 115, 51, 8, 120, 90, 96, 105, 30, 26, 31, 102, 103, 69, 110, 40, 97, 61, 81, 91, 42, 17, 28, 114, 38, 18, 106, 104, 10, 117, 45, 21, 58, 118, 12, 107, 78, 101, 112, 109, 99, 75, 39, 122, 77, 25, 48, 5, 79, 66, 54, 2], [108, 111, 93, 44, 82, 24, 125, 14, 68, 36, 76, 6, 72, 116, 22, 13, 74, 64, 19, 67, 50, 1, 37, 0, 124, 87, 15, 35, 98, 75, 119, 85, 103, 113, 47, 121, 18, 57, 51, 11, 55, 49, 29, 70, 62, 63, 34, 3, 53, 95, 88, 9, 127, 112, 86, 25, 65, 10, 60, 46, 126, 59, 20, 100, 89, 5, 97, 52, 2, 81, 40, 90, 109, 115, 120, 38, 92, 61, 117, 94, 104, 83, 30, 12, 96, 4, 123, 31, 84, 78, 56, 26, 43, 66, 33, 110, 48, 23, 41, 122, 54, 21, 28, 42, 106, 102, 58, 91, 39, 99, 107, 32, 7, 118, 73, 16, 105, 45, 114, 101, 27, 69, 80, 17, 8, 79, 71, 77]], "model.layers.25.self_attn.k_proj": [[62, 38, 121, 33, 124, 56, 127, 29, 20, 60, 72, 13, 26, 79, 83, 63, 74, 17, 91, 16, 78, 50, 24, 86, 80, 4, 77, 52, 71, 49, 22, 32, 25, 19, 102, 100, 123, 53, 47, 6, 9, 28, 120, 5, 18, 93, 117, 113, 12, 75, 99, 115, 3, 46, 110, 2, 54, 126, 119, 36, 111, 82, 44, 42, 95, 70, 116, 40, 34, 23, 57, 30, 87, 61, 118, 59, 43, 81, 8, 48, 92, 125, 58, 45, 106, 94, 11, 51, 41, 122, 88, 108, 0, 112, 107, 105, 55, 103, 65, 109, 37, 114, 21, 101, 31, 27, 89, 67, 10, 96, 104, 97, 85, 98, 35, 39, 68, 90, 1, 15, 84, 73, 14, 64, 7, 76, 69, 66], [103, 22, 99, 117, 126, 58, 32, 111, 59, 124, 51, 119, 60, 48, 62, 54, 93, 92, 114, 127, 61, 57, 123, 116, 56, 49, 120, 52, 55, 121, 63, 110, 46, 112, 50, 125, 122, 118, 113, 30, 53, 89, 43, 109, 44, 86, 115, 45, 41, 108, 42, 100, 104, 47, 40, 37, 27, 106, 107, 38, 35, 91, 105, 82, 34, 84, 97, 101, 80, 81, 14, 29, 79, 102, 21, 16, 36, 33, 90, 26, 94, 83, 78, 20, 18, 87, 76, 98, 31, 28, 19, 74, 24, 11, 85, 12, 95, 23, 25, 6, 7, 39, 77, 17, 96, 5, 8, 13, 75, 72, 15, 88, 68, 10, 67, 73, 2, 0, 1, 4, 9, 71, 69, 3, 66, 70, 65, 64], [59, 104, 98, 90, 83, 17, 13, 79, 72, 112, 63, 31, 12, 88, 24, 75, 118, 85, 0, 53, 50, 9, 124, 1, 4, 71, 121, 126, 14, 6, 86, 68, 81, 10, 67, 2, 125, 116, 111, 66, 123, 42, 55, 109, 41, 45, 11, 25, 57, 47, 52, 107, 46, 60, 7, 26, 28, 115, 20, 114, 49, 51, 95, 69, 48, 108, 58, 56, 18, 103, 84, 29, 74, 44, 91, 62, 94, 106, 22, 33, 54, 92, 122, 39, 21, 30, 119, 110, 82, 113, 117, 99, 16, 100, 105, 93, 89, 38, 8, 87, 23, 27, 43, 96, 80, 78, 102, 35, 120, 76, 97, 32, 3, 77, 101, 36, 37, 127, 70, 65, 61, 19, 15, 73, 5, 34, 64, 40], [102, 22, 61, 127, 35, 56, 58, 32, 125, 50, 52, 120, 60, 53, 30, 59, 98, 116, 111, 86, 49, 117, 55, 126, 115, 119, 124, 118, 110, 123, 54, 92, 121, 33, 63, 112, 122, 57, 48, 113, 16, 109, 62, 114, 51, 28, 43, 26, 46, 44, 47, 108, 42, 45, 100, 106, 36, 107, 105, 14, 18, 85, 27, 40, 99, 41, 37, 83, 104, 103, 101, 25, 11, 39, 89, 24, 34, 20, 77, 19, 29, 38, 23, 90, 91, 87, 82, 12, 17, 97, 94, 31, 93, 84, 74, 95, 72, 79, 96, 13, 5, 88, 73, 78, 15, 70, 8, 4, 10, 81, 67, 9, 80, 1, 6, 21, 76, 66, 7, 64, 75, 71, 2, 68, 0, 69, 3, 65], [106, 34, 30, 26, 49, 48, 118, 14, 64, 21, 88, 16, 42, 116, 18, 54, 123, 8, 6, 119, 20, 121, 60, 117, 53, 127, 114, 124, 55, 120, 31, 63, 4, 126, 105, 104, 87, 58, 67, 107, 10, 12, 2, 3, 28, 122, 99, 110, 74, 56, 47, 57, 125, 83, 111, 59, 1, 46, 109, 108, 62, 9, 52, 86, 43, 61, 51, 115, 112, 40, 44, 45, 98, 37, 113, 50, 79, 23, 17, 84, 65, 13, 101, 102, 39, 35, 103, 89, 22, 25, 41, 38, 77, 27, 73, 95, 100, 29, 75, 33, 11, 93, 19, 96, 68, 70, 32, 7, 36, 81, 91, 66, 97, 71, 15, 78, 69, 82, 94, 92, 5, 76, 0, 85, 72, 80, 24, 90], [63, 103, 64, 121, 9, 65, 1, 79, 12, 66, 4, 77, 69, 3, 87, 74, 70, 122, 44, 113, 107, 118, 110, 106, 67, 23, 127, 48, 52, 115, 18, 58, 68, 7, 20, 5, 85, 59, 81, 6, 40, 105, 50, 83, 31, 93, 61, 78, 53, 117, 47, 89, 39, 2, 33, 60, 27, 19, 124, 46, 43, 125, 75, 95, 109, 16, 17, 80, 84, 35, 10, 36, 29, 49, 126, 25, 72, 42, 90, 112, 55, 116, 11, 54, 92, 99, 98, 71, 56, 97, 73, 0, 21, 86, 111, 45, 15, 41, 37, 38, 82, 30, 51, 101, 8, 96, 62, 114, 24, 104, 91, 26, 14, 88, 119, 28, 120, 123, 100, 57, 94, 108, 34, 22, 32, 102, 76, 13], [42, 63, 122, 83, 85, 100, 112, 0, 119, 80, 15, 77, 91, 73, 124, 29, 115, 50, 57, 127, 66, 106, 25, 113, 96, 71, 6, 61, 87, 109, 34, 126, 121, 74, 65, 68, 3, 58, 75, 1, 49, 45, 11, 46, 2, 111, 56, 90, 51, 30, 44, 125, 120, 95, 8, 17, 81, 5, 14, 60, 4, 98, 31, 24, 108, 55, 23, 93, 41, 89, 59, 116, 27, 36, 110, 52, 12, 118, 102, 62, 86, 35, 10, 47, 94, 114, 104, 54, 48, 101, 105, 82, 43, 33, 18, 103, 78, 107, 40, 20, 38, 22, 99, 117, 53, 123, 28, 92, 7, 39, 84, 37, 26, 97, 76, 88, 32, 69, 13, 72, 64, 67, 16, 19, 70, 21, 79, 9], [44, 111, 108, 6, 0, 93, 47, 24, 14, 72, 76, 68, 65, 74, 64, 82, 2, 15, 53, 66, 125, 124, 50, 67, 116, 9, 85, 19, 87, 20, 81, 121, 119, 52, 13, 3, 57, 70, 75, 34, 101, 7, 11, 113, 73, 54, 86, 100, 62, 51, 127, 55, 80, 126, 120, 112, 10, 90, 16, 123, 104, 63, 60, 110, 49, 41, 61, 97, 45, 32, 107, 4, 96, 105, 89, 115, 95, 58, 59, 114, 22, 83, 88, 122, 79, 37, 36, 31, 5, 1, 109, 33, 56, 43, 117, 98, 46, 103, 118, 99, 94, 27, 39, 28, 23, 21, 42, 91, 35, 102, 29, 25, 40, 69, 38, 48, 106, 84, 30, 71, 26, 92, 18, 77, 12, 17, 78, 8]], "model.layers.25.self_attn.qk_proj": [[63, 59, 42, 62, 106, 44, 111, 58, 127, 124, 61, 108, 121, 126, 117, 56, 122, 15, 93, 64, 50, 0, 79, 102, 125, 60, 77, 19, 118, 53, 13, 85, 90, 104, 10, 12, 23, 83, 115, 21, 36, 38, 76, 24, 113, 87, 98, 26, 103, 123, 73, 112, 119, 9, 74, 88, 116, 49, 52, 27, 68, 48, 47, 120, 6, 82, 34, 65, 46, 18, 4, 2, 39, 55, 57, 114, 67, 80, 51, 16, 1, 78, 17, 66, 54, 70, 29, 14, 99, 3, 40, 8, 31, 81, 86, 72, 110, 95, 97, 89, 33, 20, 84, 43, 22, 35, 109, 41, 25, 107, 32, 94, 69, 91, 100, 96, 7, 5, 71, 11, 105, 37, 28, 30, 75, 45, 101, 92], [63, 59, 42, 62, 106, 44, 111, 127, 58, 61, 124, 121, 117, 108, 126, 56, 122, 0, 93, 102, 15, 50, 79, 64, 125, 77, 60, 19, 23, 118, 10, 21, 12, 112, 53, 103, 38, 76, 83, 13, 6, 90, 98, 115, 74, 119, 88, 113, 4, 24, 36, 9, 87, 116, 85, 104, 26, 49, 39, 48, 68, 73, 123, 120, 52, 47, 34, 65, 40, 2, 29, 82, 27, 57, 46, 1, 16, 66, 18, 55, 54, 72, 78, 14, 80, 17, 51, 3, 70, 99, 110, 114, 81, 89, 31, 22, 67, 97, 20, 109, 94, 8, 95, 43, 33, 25, 86, 107, 84, 41, 100, 5, 11, 75, 71, 96, 45, 28, 91, 69, 35, 105, 7, 30, 37, 92, 32, 101], [63, 59, 42, 62, 106, 44, 111, 127, 58, 61, 126, 108, 121, 124, 117, 56, 64, 93, 122, 125, 50, 79, 60, 0, 77, 118, 15, 83, 13, 112, 19, 119, 23, 90, 102, 36, 76, 21, 38, 98, 10, 74, 116, 115, 87, 24, 12, 120, 85, 73, 103, 9, 104, 113, 53, 49, 88, 4, 34, 48, 39, 6, 68, 82, 29, 26, 52, 66, 47, 27, 2, 1, 65, 72, 123, 3, 16, 18, 57, 70, 67, 80, 54, 51, 55, 46, 40, 78, 114, 14, 99, 17, 31, 89, 81, 110, 86, 35, 94, 22, 20, 109, 33, 84, 100, 25, 95, 107, 91, 96, 43, 69, 71, 105, 5, 8, 97, 92, 41, 28, 75, 32, 7, 11, 45, 30, 101, 37], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 126, 117, 56, 124, 122, 125, 0, 93, 64, 50, 79, 53, 102, 60, 77, 15, 83, 115, 13, 112, 113, 19, 21, 10, 85, 118, 38, 12, 119, 74, 76, 87, 73, 23, 26, 98, 104, 34, 9, 36, 24, 88, 39, 6, 120, 52, 49, 116, 55, 4, 103, 68, 65, 66, 67, 72, 40, 90, 80, 48, 3, 47, 16, 46, 123, 114, 18, 54, 1, 78, 82, 27, 110, 57, 29, 70, 14, 99, 2, 17, 81, 51, 31, 89, 86, 94, 33, 20, 43, 22, 109, 45, 97, 69, 35, 84, 25, 100, 41, 5, 71, 8, 7, 95, 91, 96, 105, 107, 28, 75, 32, 92, 11, 30, 101, 37], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 126, 61, 108, 56, 117, 124, 125, 122, 50, 93, 79, 60, 15, 0, 102, 64, 13, 77, 118, 119, 23, 83, 53, 113, 98, 38, 74, 85, 116, 104, 112, 76, 73, 19, 115, 12, 36, 90, 10, 21, 103, 123, 34, 4, 9, 49, 88, 55, 24, 52, 87, 26, 72, 46, 68, 120, 70, 48, 27, 39, 65, 18, 2, 57, 82, 1, 47, 16, 14, 99, 66, 114, 51, 81, 80, 6, 17, 54, 29, 40, 67, 78, 31, 3, 22, 110, 86, 25, 33, 94, 35, 95, 20, 107, 89, 84, 43, 109, 41, 97, 75, 69, 105, 45, 7, 101, 71, 5, 92, 96, 28, 30, 100, 91, 8, 32, 11, 37], [63, 59, 106, 62, 42, 44, 58, 111, 127, 121, 61, 124, 126, 108, 56, 117, 122, 93, 125, 79, 50, 64, 0, 15, 13, 102, 77, 83, 118, 115, 119, 36, 23, 85, 90, 21, 98, 87, 52, 12, 88, 19, 112, 38, 60, 113, 34, 10, 49, 104, 76, 73, 53, 103, 26, 4, 9, 116, 74, 24, 27, 72, 68, 70, 39, 17, 1, 14, 51, 16, 46, 81, 66, 57, 55, 47, 48, 6, 82, 2, 3, 40, 123, 18, 120, 80, 65, 99, 54, 114, 67, 22, 78, 86, 110, 97, 29, 84, 35, 31, 89, 94, 25, 41, 20, 32, 33, 109, 69, 107, 43, 100, 5, 75, 95, 7, 8, 105, 71, 96, 28, 11, 30, 45, 92, 91, 37, 101], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 61, 108, 124, 126, 117, 56, 122, 79, 93, 102, 15, 0, 77, 64, 125, 19, 13, 85, 26, 118, 113, 60, 98, 38, 119, 21, 112, 83, 76, 116, 88, 74, 90, 104, 50, 12, 23, 103, 36, 1, 9, 49, 10, 53, 24, 52, 73, 57, 48, 39, 70, 34, 87, 18, 47, 46, 27, 16, 82, 115, 72, 2, 51, 81, 123, 80, 65, 68, 99, 14, 120, 67, 66, 17, 78, 55, 4, 54, 6, 29, 40, 3, 25, 22, 86, 31, 35, 114, 94, 41, 33, 110, 109, 89, 8, 100, 107, 84, 5, 20, 75, 97, 32, 95, 11, 28, 91, 7, 45, 30, 69, 71, 43, 105, 101, 37, 96, 92], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 124, 108, 61, 117, 126, 56, 122, 93, 125, 0, 64, 79, 60, 15, 50, 118, 13, 77, 116, 112, 38, 21, 90, 102, 52, 83, 19, 23, 103, 26, 85, 119, 10, 48, 12, 36, 98, 113, 24, 53, 88, 76, 87, 74, 70, 57, 39, 27, 73, 65, 9, 49, 47, 104, 1, 51, 29, 4, 68, 34, 46, 18, 123, 115, 2, 55, 66, 82, 78, 16, 120, 54, 72, 81, 99, 80, 6, 110, 22, 95, 86, 14, 17, 40, 3, 67, 31, 33, 114, 89, 25, 94, 35, 8, 84, 97, 100, 107, 41, 43, 28, 20, 109, 45, 91, 69, 5, 96, 7, 30, 75, 32, 37, 11, 105, 71, 92, 101], [63, 59, 42, 106, 62, 44, 111, 58, 127, 124, 121, 61, 108, 117, 126, 56, 122, 93, 125, 0, 64, 15, 50, 79, 85, 13, 102, 77, 38, 90, 19, 118, 112, 83, 116, 21, 23, 26, 103, 60, 119, 113, 24, 88, 53, 76, 12, 10, 48, 52, 74, 68, 87, 73, 34, 46, 27, 39, 36, 9, 98, 47, 115, 4, 104, 51, 29, 49, 70, 1, 120, 65, 82, 123, 14, 80, 114, 55, 6, 17, 66, 16, 3, 54, 57, 67, 78, 81, 2, 18, 99, 110, 31, 22, 8, 86, 40, 25, 72, 89, 35, 100, 97, 95, 94, 20, 43, 33, 109, 91, 84, 107, 41, 69, 5, 45, 30, 96, 75, 28, 71, 32, 11, 7, 37, 92, 105, 101], [63, 59, 106, 42, 62, 44, 111, 58, 127, 121, 124, 108, 61, 117, 126, 56, 122, 0, 64, 93, 125, 119, 50, 15, 60, 79, 102, 53, 83, 116, 118, 77, 74, 13, 38, 90, 12, 120, 23, 112, 98, 76, 85, 10, 73, 103, 21, 19, 36, 113, 9, 88, 104, 115, 49, 68, 24, 46, 48, 6, 4, 87, 26, 70, 34, 47, 39, 1, 52, 65, 57, 66, 18, 54, 2, 8, 51, 3, 123, 80, 27, 29, 67, 55, 14, 17, 40, 81, 114, 82, 86, 16, 110, 72, 78, 97, 99, 31, 25, 22, 94, 33, 84, 43, 89, 20, 35, 7, 95, 69, 75, 100, 11, 28, 30, 32, 5, 71, 96, 107, 105, 45, 109, 41, 91, 37, 101, 92], [63, 59, 62, 42, 106, 44, 58, 111, 127, 124, 108, 121, 61, 117, 126, 56, 122, 93, 0, 64, 79, 15, 13, 125, 119, 50, 77, 102, 38, 116, 118, 83, 53, 60, 21, 10, 74, 85, 112, 12, 19, 23, 76, 98, 73, 49, 113, 104, 34, 36, 6, 90, 9, 115, 87, 88, 68, 103, 4, 48, 24, 120, 2, 39, 26, 8, 55, 65, 80, 1, 47, 52, 46, 40, 66, 54, 51, 123, 14, 70, 16, 99, 81, 67, 3, 82, 86, 29, 27, 78, 18, 17, 114, 110, 94, 22, 57, 35, 84, 31, 33, 25, 20, 97, 72, 89, 69, 96, 95, 107, 43, 7, 75, 71, 5, 105, 11, 28, 91, 100, 30, 92, 32, 41, 45, 101, 37, 109], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 108, 126, 124, 61, 56, 117, 122, 0, 93, 15, 79, 64, 77, 13, 85, 19, 83, 50, 102, 118, 21, 116, 9, 23, 90, 53, 38, 98, 104, 26, 10, 76, 125, 112, 60, 88, 74, 36, 119, 24, 12, 103, 48, 113, 73, 68, 6, 87, 4, 49, 40, 82, 65, 115, 47, 39, 46, 27, 34, 55, 52, 8, 123, 120, 14, 80, 17, 57, 16, 54, 2, 99, 1, 110, 29, 67, 114, 81, 18, 66, 22, 78, 3, 86, 31, 70, 97, 35, 94, 69, 51, 84, 25, 107, 33, 72, 89, 30, 28, 20, 75, 41, 5, 100, 45, 43, 95, 96, 91, 7, 11, 37, 71, 105, 32, 109, 101, 92], [63, 59, 42, 106, 62, 44, 111, 58, 127, 124, 121, 61, 108, 126, 117, 56, 122, 93, 64, 116, 118, 102, 50, 0, 15, 79, 83, 112, 90, 125, 77, 13, 19, 48, 113, 60, 23, 119, 85, 24, 26, 103, 104, 38, 36, 53, 21, 12, 87, 10, 9, 76, 88, 98, 74, 73, 54, 39, 6, 52, 34, 8, 68, 49, 4, 115, 1, 65, 27, 123, 47, 18, 46, 82, 51, 120, 66, 14, 2, 55, 29, 57, 99, 16, 22, 17, 80, 40, 3, 81, 70, 89, 110, 25, 86, 114, 107, 94, 78, 67, 31, 33, 97, 20, 35, 84, 91, 28, 43, 95, 41, 32, 30, 105, 11, 109, 69, 100, 75, 5, 72, 7, 96, 45, 92, 71, 37, 101], [63, 59, 42, 106, 62, 44, 111, 127, 58, 61, 124, 108, 121, 117, 126, 56, 64, 122, 93, 50, 0, 60, 125, 77, 79, 102, 90, 15, 118, 116, 13, 119, 48, 83, 85, 112, 120, 36, 19, 53, 38, 52, 9, 76, 113, 10, 74, 87, 21, 27, 103, 23, 73, 6, 104, 12, 26, 24, 8, 34, 88, 98, 1, 51, 49, 39, 65, 115, 68, 47, 66, 4, 80, 54, 81, 67, 123, 16, 82, 18, 57, 110, 2, 29, 17, 70, 55, 99, 14, 31, 114, 86, 46, 3, 78, 22, 89, 40, 11, 43, 95, 20, 97, 100, 33, 41, 35, 72, 32, 94, 5, 69, 84, 45, 25, 37, 107, 7, 75, 109, 91, 28, 105, 30, 92, 71, 96, 101], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 108, 121, 124, 56, 117, 126, 122, 0, 93, 64, 79, 102, 77, 15, 60, 50, 83, 125, 85, 13, 116, 76, 118, 74, 112, 38, 104, 36, 53, 19, 98, 119, 26, 10, 34, 9, 23, 48, 12, 24, 21, 113, 52, 90, 103, 73, 68, 27, 87, 115, 88, 39, 8, 55, 1, 6, 70, 49, 4, 67, 2, 120, 66, 80, 82, 65, 29, 18, 51, 3, 47, 16, 81, 123, 14, 46, 54, 99, 86, 31, 57, 22, 78, 17, 97, 110, 114, 40, 43, 100, 72, 20, 33, 89, 94, 35, 41, 84, 25, 107, 11, 7, 96, 95, 69, 5, 30, 75, 105, 28, 32, 45, 71, 91, 101, 92, 37, 109], [63, 59, 62, 42, 106, 44, 127, 111, 58, 124, 121, 126, 108, 61, 56, 117, 122, 93, 79, 15, 64, 77, 50, 85, 0, 125, 116, 102, 83, 112, 26, 76, 118, 13, 38, 23, 73, 90, 24, 60, 19, 113, 87, 74, 98, 10, 12, 9, 103, 119, 21, 88, 53, 4, 55, 34, 27, 47, 36, 104, 48, 39, 68, 120, 52, 115, 80, 1, 40, 123, 16, 49, 14, 8, 70, 78, 29, 46, 114, 17, 18, 81, 57, 2, 51, 82, 3, 66, 65, 6, 110, 54, 67, 99, 33, 86, 97, 31, 22, 20, 72, 89, 69, 95, 84, 94, 41, 35, 5, 107, 25, 43, 11, 100, 105, 7, 28, 75, 91, 71, 45, 109, 30, 96, 101, 32, 37, 92], [63, 59, 42, 106, 62, 44, 58, 111, 127, 121, 124, 61, 126, 108, 117, 56, 122, 93, 64, 0, 15, 102, 50, 79, 23, 118, 125, 112, 90, 19, 116, 13, 119, 83, 77, 103, 113, 85, 88, 87, 38, 21, 53, 39, 26, 104, 120, 98, 12, 36, 76, 60, 24, 74, 27, 73, 48, 10, 34, 9, 29, 70, 47, 52, 65, 66, 123, 82, 49, 68, 1, 4, 80, 54, 46, 40, 16, 51, 114, 110, 18, 2, 57, 3, 78, 115, 99, 14, 81, 72, 95, 33, 17, 8, 67, 89, 86, 97, 6, 22, 31, 84, 25, 41, 94, 55, 20, 43, 35, 5, 107, 11, 28, 45, 75, 96, 100, 30, 69, 32, 91, 105, 71, 109, 7, 92, 37, 101], [63, 59, 42, 62, 106, 44, 58, 127, 111, 121, 61, 117, 124, 108, 126, 56, 93, 122, 0, 64, 50, 118, 79, 116, 15, 77, 112, 13, 19, 102, 90, 125, 23, 76, 60, 38, 85, 87, 48, 21, 119, 83, 24, 74, 53, 98, 103, 88, 39, 70, 12, 104, 26, 1, 120, 113, 36, 73, 54, 10, 9, 115, 29, 34, 27, 4, 47, 52, 49, 66, 3, 114, 18, 82, 16, 65, 123, 110, 68, 67, 72, 81, 57, 80, 78, 2, 14, 51, 6, 99, 40, 55, 86, 89, 31, 8, 46, 22, 43, 17, 94, 95, 20, 33, 69, 25, 84, 28, 109, 107, 41, 35, 11, 100, 97, 91, 5, 96, 32, 7, 75, 71, 30, 45, 105, 92, 37, 101], [63, 59, 42, 62, 106, 44, 58, 111, 127, 121, 61, 108, 124, 117, 126, 56, 122, 64, 50, 102, 93, 0, 60, 79, 15, 13, 125, 116, 103, 77, 118, 98, 83, 53, 112, 104, 76, 23, 113, 38, 12, 119, 34, 19, 74, 36, 24, 10, 85, 70, 87, 21, 52, 9, 4, 90, 73, 88, 48, 123, 1, 120, 29, 47, 72, 46, 68, 26, 49, 39, 82, 114, 54, 115, 16, 66, 51, 67, 18, 55, 27, 40, 2, 80, 3, 110, 6, 99, 81, 57, 65, 78, 43, 14, 86, 33, 31, 94, 17, 20, 89, 35, 109, 25, 97, 71, 107, 105, 69, 22, 100, 96, 45, 32, 7, 91, 84, 5, 8, 11, 30, 95, 41, 101, 28, 75, 92, 37], [63, 59, 42, 62, 106, 44, 111, 127, 58, 121, 108, 61, 117, 124, 126, 56, 122, 64, 93, 0, 102, 79, 15, 125, 50, 116, 23, 13, 60, 112, 77, 83, 118, 103, 19, 76, 12, 38, 119, 36, 87, 85, 74, 98, 120, 104, 34, 21, 26, 115, 90, 113, 48, 88, 73, 4, 53, 1, 10, 68, 72, 70, 24, 9, 52, 47, 49, 54, 123, 39, 18, 46, 2, 29, 40, 55, 6, 65, 27, 114, 16, 67, 82, 66, 99, 80, 81, 78, 57, 3, 51, 110, 17, 97, 14, 22, 94, 89, 86, 25, 107, 96, 105, 33, 41, 95, 100, 20, 71, 31, 35, 11, 69, 109, 84, 7, 43, 75, 8, 5, 91, 37, 32, 28, 45, 30, 101, 92], [63, 59, 106, 42, 62, 44, 127, 111, 58, 124, 121, 117, 61, 108, 126, 56, 122, 93, 0, 50, 15, 13, 125, 64, 116, 112, 79, 77, 104, 23, 118, 113, 60, 98, 74, 85, 87, 53, 83, 76, 120, 36, 12, 119, 38, 90, 24, 48, 49, 54, 102, 103, 9, 73, 19, 21, 10, 47, 72, 34, 88, 26, 70, 115, 46, 4, 27, 67, 114, 1, 55, 39, 66, 52, 29, 17, 40, 65, 6, 123, 51, 16, 18, 80, 2, 68, 82, 86, 78, 14, 57, 3, 110, 99, 89, 43, 81, 31, 94, 20, 25, 95, 33, 22, 5, 35, 97, 84, 107, 91, 100, 109, 71, 7, 96, 41, 32, 75, 8, 69, 37, 11, 105, 30, 28, 92, 45, 101], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 117, 126, 124, 56, 64, 122, 0, 93, 125, 15, 50, 79, 112, 77, 60, 13, 118, 53, 116, 23, 102, 119, 74, 24, 83, 12, 113, 120, 38, 85, 87, 10, 36, 115, 76, 104, 98, 19, 73, 103, 21, 1, 54, 9, 48, 26, 6, 72, 88, 90, 47, 49, 34, 66, 68, 51, 4, 82, 52, 123, 39, 114, 3, 46, 70, 65, 55, 18, 2, 29, 27, 40, 57, 67, 99, 16, 14, 110, 80, 31, 78, 17, 43, 33, 86, 35, 89, 94, 109, 84, 95, 81, 8, 22, 25, 20, 97, 5, 45, 11, 107, 7, 69, 91, 100, 28, 32, 71, 105, 30, 75, 92, 41, 96, 101, 37], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 124, 117, 126, 56, 125, 93, 122, 0, 79, 23, 102, 118, 77, 50, 64, 116, 15, 60, 123, 76, 103, 13, 53, 90, 120, 83, 104, 115, 9, 24, 38, 21, 113, 74, 36, 98, 85, 87, 26, 12, 112, 48, 6, 19, 51, 73, 4, 10, 88, 34, 49, 68, 119, 52, 29, 1, 39, 66, 46, 55, 82, 47, 72, 114, 27, 18, 2, 54, 3, 16, 57, 80, 40, 95, 67, 99, 14, 33, 65, 81, 94, 31, 70, 78, 22, 110, 25, 35, 89, 86, 17, 20, 8, 97, 43, 91, 84, 105, 96, 71, 28, 45, 5, 11, 109, 32, 100, 69, 7, 30, 41, 101, 75, 107, 37, 92], [63, 59, 42, 62, 106, 44, 58, 111, 127, 61, 124, 121, 108, 126, 117, 56, 125, 122, 93, 15, 0, 64, 77, 23, 102, 79, 118, 83, 50, 13, 112, 116, 19, 104, 21, 38, 113, 53, 98, 49, 115, 60, 103, 12, 85, 74, 6, 76, 119, 9, 10, 26, 48, 90, 34, 120, 36, 87, 88, 24, 73, 4, 68, 123, 54, 52, 46, 72, 16, 51, 39, 65, 40, 47, 27, 82, 18, 55, 3, 66, 2, 80, 29, 114, 22, 110, 1, 81, 78, 94, 99, 14, 17, 67, 70, 33, 57, 45, 84, 8, 97, 31, 20, 25, 86, 89, 5, 32, 41, 35, 75, 71, 95, 96, 100, 43, 28, 11, 107, 69, 109, 7, 91, 105, 92, 37, 30, 101], [63, 59, 42, 106, 62, 44, 111, 58, 127, 121, 61, 108, 124, 126, 117, 56, 122, 93, 64, 15, 0, 102, 79, 125, 13, 50, 118, 83, 77, 90, 53, 98, 12, 85, 23, 21, 112, 19, 103, 119, 60, 113, 10, 24, 104, 76, 87, 36, 49, 9, 74, 116, 48, 26, 120, 73, 88, 38, 6, 115, 47, 123, 18, 39, 34, 82, 27, 52, 16, 65, 1, 2, 54, 40, 70, 14, 68, 55, 4, 80, 29, 66, 72, 86, 46, 57, 81, 110, 17, 67, 8, 51, 99, 31, 89, 114, 3, 25, 78, 20, 33, 22, 94, 97, 95, 84, 43, 107, 5, 32, 11, 71, 75, 7, 105, 100, 109, 35, 91, 96, 41, 28, 92, 30, 45, 69, 101, 37], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 124, 61, 108, 126, 117, 56, 93, 122, 64, 50, 0, 15, 118, 125, 79, 116, 102, 13, 24, 90, 87, 48, 26, 23, 112, 120, 77, 60, 38, 119, 19, 12, 21, 103, 113, 85, 76, 49, 104, 83, 10, 73, 36, 54, 74, 98, 53, 47, 52, 123, 88, 39, 9, 27, 29, 34, 4, 6, 68, 51, 82, 115, 1, 70, 18, 2, 40, 16, 46, 65, 114, 80, 66, 67, 8, 99, 14, 3, 57, 89, 110, 31, 81, 22, 55, 33, 78, 25, 17, 43, 95, 86, 94, 97, 20, 72, 35, 84, 5, 91, 109, 7, 30, 41, 69, 11, 100, 71, 107, 45, 105, 28, 32, 75, 96, 92, 101, 37], [63, 59, 62, 42, 106, 44, 111, 127, 58, 124, 61, 121, 117, 108, 126, 56, 122, 93, 125, 50, 0, 64, 102, 118, 15, 112, 79, 48, 23, 90, 19, 13, 83, 116, 120, 103, 38, 77, 115, 39, 12, 76, 47, 113, 87, 49, 85, 53, 88, 74, 36, 73, 119, 60, 98, 24, 26, 21, 51, 52, 10, 123, 9, 4, 104, 68, 70, 40, 34, 55, 1, 110, 54, 57, 29, 27, 16, 46, 81, 78, 65, 8, 82, 67, 80, 2, 18, 66, 14, 17, 99, 6, 43, 94, 114, 22, 31, 35, 84, 3, 89, 95, 97, 33, 20, 91, 86, 25, 69, 5, 100, 41, 45, 28, 105, 96, 107, 72, 109, 11, 71, 37, 32, 7, 101, 92, 75, 30], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 61, 124, 108, 117, 126, 56, 122, 64, 93, 79, 0, 15, 125, 50, 102, 13, 118, 60, 77, 116, 53, 12, 112, 23, 21, 38, 83, 74, 115, 10, 36, 119, 73, 85, 76, 103, 19, 70, 24, 87, 52, 98, 49, 26, 88, 114, 9, 113, 68, 8, 104, 51, 90, 34, 54, 47, 1, 120, 123, 4, 39, 80, 18, 65, 48, 2, 81, 16, 27, 29, 82, 66, 55, 57, 110, 78, 46, 14, 40, 94, 99, 3, 17, 6, 67, 43, 97, 22, 31, 89, 33, 20, 86, 25, 69, 84, 100, 45, 72, 75, 95, 35, 96, 11, 7, 107, 71, 109, 5, 32, 91, 37, 92, 30, 28, 105, 41, 101], [63, 59, 42, 62, 106, 44, 111, 58, 127, 124, 121, 108, 61, 117, 56, 126, 64, 122, 125, 15, 0, 93, 79, 102, 53, 115, 118, 23, 13, 77, 50, 10, 119, 104, 12, 73, 60, 98, 83, 19, 113, 103, 112, 85, 9, 24, 38, 70, 21, 90, 52, 36, 116, 8, 74, 26, 87, 49, 76, 120, 1, 48, 88, 39, 54, 46, 68, 51, 34, 123, 27, 47, 2, 65, 18, 57, 4, 66, 114, 16, 17, 67, 55, 81, 82, 80, 99, 31, 110, 3, 78, 29, 40, 97, 6, 14, 22, 86, 84, 89, 94, 33, 20, 25, 35, 69, 43, 100, 71, 7, 45, 109, 107, 72, 75, 32, 105, 5, 96, 95, 101, 30, 91, 41, 11, 28, 37, 92], [63, 59, 62, 42, 106, 44, 111, 127, 58, 124, 61, 117, 108, 121, 56, 126, 122, 93, 125, 15, 79, 50, 0, 118, 102, 13, 85, 12, 23, 116, 64, 77, 112, 113, 119, 19, 38, 83, 60, 76, 90, 26, 21, 52, 24, 49, 104, 87, 10, 73, 120, 4, 115, 88, 9, 98, 53, 103, 39, 70, 48, 123, 74, 34, 68, 8, 47, 27, 36, 81, 55, 54, 57, 40, 80, 51, 46, 1, 3, 18, 78, 16, 2, 67, 110, 114, 22, 65, 82, 29, 17, 66, 99, 14, 6, 20, 94, 31, 97, 25, 89, 33, 86, 84, 69, 43, 100, 5, 35, 95, 107, 75, 72, 41, 109, 32, 105, 71, 11, 28, 7, 30, 45, 92, 96, 91, 101, 37], [63, 59, 42, 62, 106, 44, 111, 127, 58, 124, 61, 121, 108, 117, 126, 56, 0, 122, 64, 93, 125, 50, 15, 118, 79, 102, 19, 13, 60, 98, 77, 112, 119, 36, 85, 53, 23, 116, 12, 113, 10, 120, 76, 26, 74, 83, 104, 73, 90, 103, 49, 9, 88, 24, 38, 52, 4, 21, 87, 115, 47, 48, 34, 123, 68, 70, 65, 8, 39, 1, 54, 27, 66, 18, 80, 29, 2, 51, 6, 40, 46, 55, 3, 82, 110, 81, 78, 114, 14, 57, 99, 31, 16, 89, 94, 67, 33, 17, 43, 97, 25, 22, 72, 86, 20, 35, 95, 100, 5, 84, 109, 32, 69, 28, 11, 75, 105, 45, 91, 71, 107, 92, 41, 7, 30, 96, 37, 101], [63, 59, 62, 106, 42, 44, 111, 58, 127, 124, 61, 121, 108, 117, 126, 56, 15, 122, 93, 50, 125, 79, 64, 13, 102, 19, 60, 112, 118, 12, 0, 77, 23, 74, 52, 116, 76, 98, 83, 87, 104, 10, 21, 53, 90, 103, 115, 119, 73, 85, 38, 9, 24, 26, 36, 113, 48, 123, 49, 120, 34, 54, 88, 39, 47, 8, 6, 29, 18, 27, 55, 70, 4, 82, 46, 68, 16, 65, 78, 1, 81, 3, 66, 80, 51, 57, 14, 110, 17, 22, 31, 94, 2, 99, 67, 40, 114, 89, 20, 25, 97, 72, 35, 95, 91, 84, 33, 86, 11, 32, 69, 28, 100, 43, 109, 107, 5, 7, 71, 75, 45, 105, 96, 30, 41, 92, 101, 37]], "model.layers.26.self_attn.q_proj": [[107, 43, 51, 99, 36, 100, 127, 53, 57, 52, 105, 116, 54, 58, 126, 60, 114, 48, 122, 62, 123, 118, 119, 32, 115, 55, 112, 124, 46, 120, 49, 125, 28, 44, 117, 106, 40, 113, 59, 56, 30, 35, 110, 63, 111, 42, 109, 103, 61, 39, 24, 47, 37, 108, 50, 41, 95, 121, 102, 104, 45, 96, 38, 93, 98, 23, 81, 92, 33, 76, 34, 29, 101, 84, 97, 85, 86, 88, 21, 31, 67, 91, 82, 25, 19, 22, 5, 94, 27, 89, 20, 26, 9, 90, 73, 77, 66, 83, 70, 18, 72, 87, 80, 79, 8, 78, 12, 17, 14, 15, 16, 3, 74, 75, 11, 6, 13, 0, 4, 69, 65, 7, 71, 1, 68, 10, 2, 64], [107, 43, 35, 65, 13, 74, 80, 7, 68, 28, 22, 32, 18, 126, 99, 127, 69, 3, 2, 57, 0, 119, 100, 52, 64, 118, 12, 122, 86, 72, 11, 114, 124, 54, 115, 53, 46, 9, 56, 62, 112, 67, 58, 1, 123, 117, 51, 108, 113, 34, 63, 49, 121, 116, 20, 6, 111, 47, 120, 84, 125, 110, 92, 61, 50, 10, 4, 83, 55, 78, 59, 70, 60, 71, 21, 16, 109, 5, 44, 31, 19, 66, 30, 17, 90, 45, 77, 76, 87, 48, 8, 85, 105, 79, 15, 41, 93, 73, 82, 81, 25, 23, 91, 75, 14, 26, 89, 101, 95, 42, 38, 104, 103, 27, 102, 24, 40, 106, 33, 37, 36, 97, 88, 29, 98, 94, 96, 39], [107, 43, 105, 32, 122, 100, 116, 37, 53, 103, 28, 124, 35, 92, 127, 57, 56, 21, 51, 39, 30, 52, 126, 49, 115, 117, 22, 58, 118, 112, 80, 62, 104, 108, 24, 55, 123, 41, 44, 109, 29, 91, 20, 54, 45, 18, 89, 23, 61, 17, 119, 75, 99, 46, 111, 34, 113, 114, 120, 27, 125, 63, 110, 121, 47, 42, 60, 19, 93, 48, 59, 72, 101, 50, 106, 38, 26, 25, 98, 95, 96, 40, 31, 84, 78, 83, 94, 33, 12, 90, 87, 79, 97, 85, 16, 13, 36, 11, 76, 82, 9, 70, 102, 15, 88, 77, 66, 81, 14, 73, 71, 86, 69, 8, 10, 4, 0, 6, 67, 7, 5, 74, 68, 3, 65, 1, 2, 64], [107, 126, 43, 127, 114, 52, 57, 37, 119, 103, 110, 99, 39, 122, 123, 46, 105, 54, 111, 118, 62, 28, 115, 55, 113, 124, 58, 84, 35, 44, 117, 120, 49, 19, 100, 112, 53, 63, 47, 9, 125, 116, 60, 48, 56, 51, 109, 61, 108, 59, 32, 76, 121, 104, 33, 50, 41, 45, 101, 36, 23, 80, 30, 93, 20, 89, 38, 22, 83, 42, 106, 40, 21, 91, 102, 67, 34, 87, 17, 97, 98, 29, 95, 11, 26, 78, 96, 25, 94, 92, 18, 85, 2, 15, 88, 86, 0, 13, 31, 65, 7, 4, 24, 6, 14, 10, 12, 90, 16, 79, 27, 74, 70, 77, 81, 5, 73, 82, 72, 8, 66, 1, 69, 68, 75, 71, 3, 64], [110, 55, 36, 46, 115, 95, 121, 26, 60, 50, 123, 61, 54, 53, 124, 100, 22, 119, 45, 58, 88, 31, 30, 116, 42, 127, 51, 40, 114, 106, 59, 92, 24, 47, 113, 44, 34, 117, 112, 105, 56, 28, 37, 41, 109, 63, 107, 99, 19, 122, 48, 52, 125, 126, 111, 39, 118, 49, 57, 43, 38, 33, 85, 93, 104, 120, 32, 17, 27, 108, 35, 25, 62, 87, 103, 101, 86, 21, 18, 102, 91, 29, 97, 98, 96, 16, 15, 94, 20, 23, 77, 90, 89, 14, 84, 75, 81, 76, 12, 80, 13, 83, 82, 74, 79, 9, 71, 78, 10, 5, 72, 7, 67, 6, 70, 73, 3, 68, 4, 2, 1, 11, 66, 8, 65, 64, 0, 69], [110, 46, 36, 26, 95, 22, 55, 19, 115, 4, 54, 30, 72, 31, 14, 100, 17, 58, 41, 62, 119, 2, 77, 74, 65, 121, 5, 63, 106, 37, 28, 125, 61, 97, 111, 91, 60, 10, 90, 107, 44, 94, 18, 73, 86, 52, 93, 32, 88, 104, 21, 84, 24, 118, 7, 33, 13, 34, 71, 45, 92, 39, 123, 11, 98, 75, 23, 6, 29, 101, 112, 113, 27, 116, 35, 117, 103, 127, 59, 42, 96, 25, 51, 89, 1, 15, 50, 81, 108, 85, 105, 79, 82, 80, 49, 69, 126, 76, 114, 12, 102, 20, 120, 87, 53, 99, 48, 9, 64, 56, 38, 57, 124, 47, 122, 67, 40, 68, 43, 109, 83, 3, 78, 70, 66, 8, 16, 0], [110, 36, 115, 62, 46, 26, 95, 30, 55, 88, 119, 22, 58, 100, 60, 31, 54, 51, 19, 124, 50, 96, 63, 125, 117, 34, 40, 86, 42, 123, 97, 41, 24, 122, 45, 44, 48, 87, 38, 126, 17, 116, 121, 108, 21, 16, 14, 59, 28, 61, 83, 49, 57, 120, 103, 52, 118, 33, 53, 127, 27, 106, 35, 111, 43, 114, 113, 107, 81, 56, 39, 89, 12, 93, 90, 91, 102, 47, 105, 94, 104, 109, 29, 112, 15, 98, 99, 32, 92, 101, 18, 79, 37, 80, 10, 82, 74, 25, 85, 75, 77, 20, 23, 7, 76, 8, 84, 72, 11, 78, 73, 13, 4, 9, 70, 5, 6, 69, 2, 67, 71, 66, 1, 0, 64, 65, 3, 68], [110, 36, 46, 54, 58, 121, 123, 50, 95, 26, 108, 62, 60, 119, 55, 30, 88, 31, 48, 100, 22, 49, 47, 24, 59, 33, 57, 114, 115, 109, 125, 113, 63, 56, 28, 17, 19, 112, 52, 106, 41, 117, 92, 122, 45, 51, 18, 21, 116, 15, 37, 42, 104, 40, 44, 61, 103, 111, 127, 126, 53, 107, 120, 86, 77, 118, 105, 14, 32, 124, 85, 39, 93, 99, 25, 101, 102, 43, 98, 38, 75, 13, 35, 20, 29, 16, 97, 90, 12, 34, 27, 83, 96, 89, 72, 94, 91, 87, 74, 84, 81, 23, 80, 7, 6, 73, 76, 10, 67, 5, 79, 1, 9, 4, 82, 2, 71, 64, 78, 8, 11, 65, 69, 66, 0, 70, 68, 3], [123, 57, 101, 33, 50, 37, 121, 88, 51, 44, 63, 92, 118, 24, 115, 62, 28, 60, 108, 23, 21, 54, 85, 116, 103, 117, 127, 97, 31, 105, 41, 61, 52, 56, 26, 114, 91, 109, 47, 82, 45, 110, 124, 49, 104, 126, 58, 125, 59, 34, 119, 112, 90, 55, 93, 48, 107, 111, 113, 18, 96, 122, 46, 106, 43, 87, 53, 42, 38, 79, 32, 40, 98, 20, 95, 83, 19, 75, 36, 100, 39, 15, 102, 30, 77, 120, 35, 89, 27, 94, 29, 99, 76, 16, 10, 17, 71, 86, 65, 25, 22, 84, 80, 67, 70, 14, 73, 69, 78, 81, 5, 68, 72, 8, 13, 12, 66, 0, 4, 1, 74, 2, 3, 9, 6, 64, 7, 11], [57, 101, 123, 33, 50, 37, 121, 24, 51, 124, 85, 103, 15, 88, 97, 76, 90, 20, 115, 82, 54, 105, 60, 10, 118, 29, 21, 34, 93, 104, 19, 96, 18, 91, 119, 95, 26, 87, 44, 117, 46, 67, 89, 49, 77, 59, 25, 79, 65, 28, 40, 62, 112, 73, 31, 99, 75, 30, 72, 12, 92, 74, 13, 9, 107, 43, 8, 7, 5, 83, 35, 102, 63, 127, 38, 94, 108, 114, 69, 4, 32, 2, 27, 125, 1, 52, 70, 22, 81, 71, 61, 109, 106, 53, 17, 68, 47, 126, 3, 6, 36, 23, 111, 39, 98, 116, 48, 16, 64, 100, 84, 120, 14, 110, 45, 41, 42, 58, 55, 122, 0, 113, 66, 78, 56, 80, 11, 86], [101, 123, 57, 50, 124, 33, 37, 121, 88, 28, 119, 21, 92, 61, 51, 90, 24, 44, 54, 85, 118, 126, 19, 58, 56, 105, 62, 114, 31, 117, 18, 16, 60, 115, 49, 103, 26, 82, 46, 52, 106, 97, 45, 116, 75, 38, 59, 14, 77, 42, 73, 47, 122, 86, 112, 91, 48, 109, 35, 107, 43, 63, 53, 76, 55, 39, 125, 94, 30, 110, 41, 113, 111, 89, 36, 79, 120, 80, 83, 127, 34, 93, 22, 95, 104, 99, 102, 17, 96, 29, 98, 108, 40, 71, 10, 100, 32, 20, 84, 27, 72, 81, 78, 5, 87, 23, 67, 25, 70, 65, 68, 13, 66, 15, 64, 4, 69, 11, 74, 9, 12, 2, 8, 7, 3, 1, 6, 0], [101, 123, 57, 50, 33, 121, 37, 51, 88, 24, 19, 115, 61, 54, 92, 28, 85, 21, 124, 75, 118, 105, 73, 117, 119, 77, 114, 60, 59, 44, 113, 18, 100, 46, 97, 62, 14, 83, 31, 90, 17, 98, 93, 58, 16, 95, 79, 43, 107, 78, 126, 104, 52, 45, 111, 48, 38, 127, 110, 125, 82, 41, 109, 47, 35, 116, 122, 103, 71, 29, 84, 112, 86, 89, 55, 30, 56, 42, 53, 76, 26, 36, 34, 120, 106, 5, 49, 87, 99, 63, 96, 40, 108, 102, 94, 32, 20, 4, 22, 39, 27, 11, 23, 91, 15, 80, 72, 10, 70, 66, 25, 67, 12, 13, 81, 68, 65, 6, 74, 7, 3, 1, 0, 9, 8, 69, 64, 2], [61, 59, 101, 63, 44, 125, 116, 51, 21, 62, 33, 58, 93, 118, 113, 55, 85, 88, 119, 127, 114, 110, 49, 56, 42, 54, 30, 123, 126, 124, 115, 52, 45, 50, 53, 40, 120, 112, 117, 57, 27, 48, 103, 78, 121, 122, 109, 60, 83, 24, 92, 105, 111, 37, 47, 80, 43, 90, 107, 91, 46, 26, 106, 89, 108, 95, 104, 41, 22, 82, 81, 25, 39, 74, 20, 11, 99, 75, 7, 13, 29, 12, 84, 77, 102, 32, 38, 9, 36, 6, 19, 10, 73, 17, 96, 100, 98, 34, 94, 8, 79, 23, 14, 69, 3, 15, 35, 16, 70, 28, 86, 31, 87, 18, 68, 72, 5, 76, 97, 67, 65, 71, 4, 0, 66, 1, 2, 64], [125, 63, 61, 101, 59, 116, 44, 51, 55, 58, 21, 50, 49, 127, 85, 62, 119, 114, 56, 110, 33, 113, 53, 126, 118, 52, 54, 124, 123, 57, 115, 48, 117, 42, 88, 45, 40, 120, 93, 37, 121, 109, 60, 83, 24, 112, 122, 30, 111, 27, 91, 47, 90, 46, 43, 92, 78, 105, 26, 107, 106, 41, 108, 103, 104, 89, 80, 39, 29, 22, 34, 38, 99, 96, 25, 102, 100, 82, 95, 17, 77, 81, 94, 7, 98, 36, 73, 32, 20, 75, 35, 28, 87, 31, 97, 11, 10, 13, 19, 79, 86, 6, 8, 74, 84, 12, 9, 23, 70, 14, 16, 67, 69, 72, 0, 4, 68, 18, 76, 15, 66, 3, 71, 5, 64, 1, 65, 2], [63, 101, 59, 61, 44, 21, 51, 33, 115, 85, 118, 127, 62, 113, 58, 93, 114, 119, 88, 49, 55, 126, 56, 116, 30, 54, 123, 110, 45, 117, 42, 53, 124, 50, 125, 103, 52, 40, 92, 120, 37, 57, 91, 48, 27, 112, 121, 83, 80, 109, 60, 78, 122, 111, 105, 47, 24, 43, 108, 46, 90, 26, 106, 107, 41, 104, 29, 7, 74, 82, 22, 95, 39, 89, 12, 20, 77, 25, 99, 81, 75, 19, 32, 13, 84, 14, 102, 17, 11, 38, 10, 73, 36, 15, 79, 16, 8, 100, 23, 34, 9, 94, 96, 98, 4, 18, 86, 35, 28, 69, 76, 72, 6, 87, 68, 5, 31, 97, 71, 70, 0, 1, 66, 67, 3, 64, 2, 65], [125, 59, 61, 63, 101, 62, 115, 54, 55, 116, 56, 33, 58, 44, 120, 126, 119, 114, 50, 110, 51, 124, 127, 118, 88, 53, 123, 37, 48, 49, 57, 117, 113, 112, 40, 52, 121, 45, 111, 60, 122, 90, 109, 93, 83, 42, 47, 103, 46, 43, 24, 108, 106, 81, 107, 41, 104, 78, 85, 105, 22, 7, 32, 39, 25, 89, 19, 80, 30, 74, 73, 21, 34, 6, 17, 27, 29, 91, 11, 100, 102, 26, 69, 36, 38, 77, 79, 75, 15, 13, 95, 82, 86, 12, 16, 14, 28, 97, 35, 98, 10, 9, 99, 87, 84, 8, 96, 92, 31, 94, 20, 4, 76, 68, 70, 5, 2, 23, 1, 0, 18, 67, 71, 66, 72, 65, 3, 64], [41, 117, 55, 34, 52, 31, 23, 115, 119, 21, 62, 39, 80, 58, 83, 45, 27, 105, 60, 99, 59, 44, 63, 13, 113, 30, 61, 72, 103, 57, 11, 109, 42, 29, 74, 32, 93, 94, 126, 111, 33, 88, 89, 8, 108, 112, 104, 121, 92, 116, 56, 118, 81, 90, 100, 54, 75, 26, 43, 17, 47, 37, 107, 25, 87, 20, 85, 24, 101, 51, 18, 91, 48, 15, 36, 22, 95, 124, 76, 127, 106, 110, 123, 71, 82, 50, 97, 49, 35, 38, 122, 114, 86, 84, 12, 19, 79, 40, 69, 96, 14, 46, 102, 28, 120, 53, 125, 78, 16, 67, 5, 10, 77, 64, 9, 66, 6, 7, 68, 98, 73, 4, 3, 70, 2, 65, 0, 1], [41, 55, 34, 52, 31, 13, 21, 23, 83, 80, 117, 27, 11, 66, 115, 105, 6, 96, 109, 28, 74, 19, 8, 49, 108, 44, 60, 73, 26, 64, 63, 67, 37, 30, 72, 119, 62, 4, 91, 45, 33, 85, 18, 22, 15, 7, 89, 43, 25, 17, 56, 87, 94, 9, 78, 84, 113, 122, 99, 77, 59, 93, 5, 61, 88, 57, 118, 81, 32, 48, 97, 70, 102, 10, 46, 42, 16, 29, 120, 92, 35, 71, 24, 82, 36, 103, 65, 69, 116, 2, 101, 20, 106, 40, 12, 14, 76, 90, 39, 68, 54, 127, 1, 53, 124, 123, 79, 75, 100, 110, 104, 111, 125, 86, 121, 47, 58, 3, 112, 38, 126, 50, 95, 0, 51, 114, 107, 98], [55, 41, 59, 34, 117, 45, 31, 23, 112, 54, 105, 119, 120, 110, 44, 111, 56, 46, 61, 92, 47, 116, 115, 58, 108, 42, 28, 118, 48, 124, 63, 121, 122, 123, 51, 114, 39, 127, 126, 109, 60, 25, 43, 125, 53, 57, 107, 94, 113, 21, 62, 50, 36, 106, 40, 20, 30, 49, 83, 97, 26, 103, 88, 102, 104, 100, 38, 33, 79, 52, 37, 99, 93, 29, 101, 80, 15, 35, 11, 91, 95, 86, 66, 96, 32, 71, 24, 27, 98, 69, 10, 7, 81, 87, 3, 74, 85, 17, 84, 14, 12, 89, 90, 22, 5, 1, 76, 70, 65, 82, 13, 64, 67, 0, 9, 8, 78, 19, 18, 68, 2, 4, 6, 73, 72, 75, 16, 77], [41, 55, 34, 52, 23, 31, 21, 13, 83, 80, 82, 74, 40, 117, 119, 125, 4, 103, 49, 28, 27, 105, 59, 60, 17, 115, 64, 61, 43, 44, 6, 101, 113, 66, 26, 96, 47, 118, 57, 56, 45, 94, 42, 62, 109, 3, 25, 116, 39, 70, 100, 65, 111, 112, 11, 53, 97, 72, 54, 108, 36, 126, 0, 85, 33, 104, 99, 29, 63, 123, 110, 18, 91, 16, 50, 38, 127, 32, 124, 68, 93, 87, 88, 92, 106, 122, 48, 102, 46, 22, 37, 84, 95, 58, 9, 89, 107, 69, 114, 35, 81, 78, 121, 30, 51, 10, 120, 79, 73, 19, 14, 15, 24, 7, 8, 90, 1, 86, 71, 98, 5, 12, 75, 20, 77, 76, 2, 67], [45, 103, 109, 38, 29, 54, 126, 87, 119, 57, 107, 83, 84, 114, 115, 117, 93, 118, 111, 47, 80, 122, 50, 95, 31, 46, 22, 121, 113, 127, 78, 61, 13, 108, 63, 85, 44, 10, 124, 53, 26, 52, 112, 98, 17, 59, 62, 49, 56, 125, 41, 116, 123, 51, 32, 71, 55, 58, 120, 110, 104, 30, 60, 96, 86, 36, 21, 42, 12, 48, 102, 37, 24, 34, 40, 100, 28, 106, 2, 88, 33, 35, 43, 89, 94, 70, 72, 101, 9, 14, 91, 92, 97, 90, 27, 69, 23, 76, 81, 105, 3, 6, 25, 75, 99, 19, 8, 0, 11, 67, 82, 68, 18, 4, 79, 20, 15, 73, 66, 7, 39, 65, 5, 1, 16, 64, 77, 74], [103, 45, 29, 109, 87, 80, 84, 13, 54, 10, 71, 69, 113, 32, 23, 115, 65, 47, 67, 126, 93, 64, 118, 3, 12, 127, 60, 77, 125, 38, 94, 17, 30, 62, 31, 72, 95, 57, 49, 59, 88, 20, 39, 66, 89, 68, 14, 119, 16, 83, 27, 24, 120, 74, 21, 102, 101, 79, 111, 34, 22, 18, 96, 7, 82, 15, 98, 85, 58, 90, 100, 121, 107, 51, 81, 76, 36, 5, 37, 42, 78, 25, 11, 48, 56, 63, 97, 46, 117, 91, 33, 112, 116, 4, 55, 104, 19, 99, 92, 73, 86, 41, 28, 50, 53, 6, 110, 105, 108, 52, 124, 35, 40, 61, 44, 122, 8, 70, 43, 123, 9, 106, 26, 114, 2, 0, 75, 1], [103, 45, 29, 109, 84, 87, 38, 13, 80, 54, 10, 69, 93, 71, 12, 83, 113, 3, 126, 2, 47, 32, 78, 127, 82, 79, 118, 65, 1, 115, 23, 22, 119, 88, 59, 64, 6, 95, 0, 46, 60, 89, 77, 123, 111, 31, 37, 49, 8, 62, 30, 66, 57, 122, 24, 5, 70, 97, 72, 33, 81, 125, 42, 15, 16, 58, 68, 43, 52, 7, 25, 124, 14, 74, 67, 96, 50, 112, 121, 20, 106, 85, 98, 107, 41, 92, 9, 34, 117, 17, 53, 51, 61, 120, 63, 48, 26, 114, 40, 91, 18, 110, 36, 55, 11, 101, 76, 19, 104, 99, 116, 108, 4, 35, 27, 94, 56, 100, 105, 86, 90, 44, 73, 21, 102, 28, 39, 75], [103, 45, 29, 109, 87, 84, 54, 80, 13, 10, 38, 126, 69, 71, 113, 93, 115, 32, 65, 47, 82, 67, 23, 118, 127, 30, 64, 95, 88, 40, 77, 56, 60, 125, 21, 102, 24, 94, 72, 79, 31, 46, 41, 108, 49, 83, 58, 20, 11, 101, 121, 78, 57, 25, 76, 66, 22, 26, 53, 52, 75, 117, 50, 17, 3, 119, 114, 44, 74, 42, 0, 16, 34, 9, 91, 97, 100, 48, 14, 4, 63, 62, 18, 59, 111, 124, 7, 15, 19, 5, 61, 96, 89, 122, 36, 37, 43, 112, 106, 8, 28, 55, 33, 90, 85, 104, 51, 105, 99, 81, 35, 107, 68, 110, 98, 86, 123, 12, 116, 27, 2, 39, 70, 73, 92, 120, 6, 1], [104, 59, 99, 92, 20, 114, 119, 54, 62, 82, 117, 55, 95, 98, 51, 107, 34, 63, 47, 120, 25, 125, 124, 49, 113, 115, 60, 86, 45, 13, 30, 112, 16, 123, 28, 103, 33, 58, 108, 121, 50, 48, 42, 56, 18, 44, 89, 61, 31, 39, 110, 87, 127, 52, 106, 116, 46, 122, 126, 88, 53, 57, 111, 105, 37, 118, 109, 41, 40, 43, 78, 29, 101, 38, 97, 93, 11, 94, 17, 22, 24, 76, 10, 80, 67, 15, 84, 36, 100, 71, 90, 21, 102, 79, 77, 26, 73, 96, 68, 85, 75, 23, 8, 91, 27, 19, 1, 81, 32, 35, 74, 14, 70, 7, 5, 83, 9, 66, 12, 0, 4, 65, 6, 69, 64, 72, 3, 2], [104, 59, 99, 92, 114, 82, 20, 51, 98, 62, 115, 95, 63, 105, 54, 47, 13, 16, 61, 117, 30, 34, 25, 119, 58, 122, 124, 45, 60, 109, 116, 86, 67, 55, 89, 87, 102, 66, 28, 11, 56, 80, 127, 43, 50, 88, 113, 15, 65, 123, 46, 57, 10, 18, 64, 53, 70, 42, 101, 7, 49, 48, 9, 31, 17, 97, 103, 8, 106, 44, 112, 78, 73, 40, 111, 68, 69, 22, 12, 94, 0, 5, 37, 121, 120, 76, 41, 125, 74, 33, 39, 84, 93, 29, 79, 38, 21, 118, 126, 24, 108, 75, 91, 32, 110, 107, 26, 85, 52, 71, 19, 90, 36, 83, 6, 23, 77, 81, 1, 100, 27, 14, 35, 96, 72, 4, 3, 2], [104, 59, 99, 92, 114, 20, 82, 16, 51, 116, 13, 25, 95, 87, 89, 54, 10, 115, 124, 34, 17, 8, 56, 50, 45, 53, 21, 100, 78, 76, 94, 86, 31, 63, 68, 62, 61, 18, 66, 70, 67, 65, 71, 123, 73, 80, 125, 98, 74, 85, 41, 52, 44, 12, 101, 97, 47, 39, 28, 40, 88, 91, 55, 29, 9, 127, 103, 15, 107, 118, 119, 108, 96, 11, 69, 27, 122, 37, 49, 90, 33, 84, 19, 7, 30, 5, 24, 106, 42, 121, 126, 32, 22, 58, 113, 110, 48, 105, 77, 112, 36, 6, 111, 79, 93, 23, 120, 43, 64, 109, 38, 75, 102, 26, 0, 60, 117, 46, 57, 14, 1, 83, 81, 35, 2, 4, 72, 3], [104, 59, 122, 45, 92, 20, 95, 114, 25, 99, 86, 115, 54, 62, 82, 63, 61, 34, 116, 51, 98, 49, 11, 56, 43, 0, 127, 124, 123, 33, 4, 119, 53, 1, 28, 110, 2, 46, 41, 58, 113, 31, 125, 55, 89, 120, 52, 70, 48, 111, 126, 76, 73, 50, 108, 118, 39, 121, 71, 112, 88, 78, 5, 13, 42, 106, 117, 109, 60, 83, 57, 44, 107, 105, 30, 22, 19, 87, 90, 47, 26, 102, 103, 84, 101, 16, 14, 38, 36, 21, 93, 3, 24, 100, 32, 37, 94, 6, 91, 81, 97, 35, 96, 15, 18, 8, 17, 29, 27, 65, 40, 12, 85, 10, 23, 69, 7, 66, 74, 75, 80, 64, 79, 9, 67, 68, 77, 72], [107, 56, 126, 43, 99, 87, 82, 110, 91, 78, 20, 25, 81, 40, 75, 104, 79, 27, 12, 112, 7, 72, 52, 31, 86, 32, 89, 117, 118, 49, 116, 92, 10, 125, 62, 48, 55, 16, 88, 127, 109, 95, 120, 26, 28, 85, 121, 69, 17, 93, 23, 44, 113, 74, 37, 77, 6, 57, 14, 53, 122, 84, 97, 58, 9, 80, 115, 61, 54, 11, 50, 101, 90, 102, 29, 76, 15, 108, 30, 51, 46, 68, 124, 18, 21, 83, 60, 114, 119, 105, 19, 96, 47, 33, 45, 24, 123, 38, 98, 59, 41, 100, 5, 103, 111, 94, 36, 63, 22, 2, 34, 106, 67, 8, 73, 71, 42, 39, 35, 70, 13, 4, 64, 3, 1, 65, 66, 0], [107, 43, 112, 56, 53, 127, 48, 115, 50, 116, 55, 125, 117, 25, 99, 19, 52, 120, 126, 61, 37, 118, 46, 121, 57, 62, 93, 113, 63, 58, 124, 29, 51, 114, 109, 122, 59, 54, 60, 49, 110, 91, 45, 119, 89, 31, 123, 36, 105, 47, 104, 111, 33, 44, 40, 86, 108, 24, 106, 42, 27, 28, 41, 95, 88, 38, 80, 39, 35, 101, 100, 83, 34, 102, 22, 20, 16, 103, 13, 98, 21, 77, 87, 97, 92, 90, 30, 17, 26, 79, 82, 32, 23, 70, 14, 96, 94, 81, 4, 10, 65, 85, 7, 11, 68, 6, 3, 1, 67, 84, 71, 15, 78, 18, 74, 5, 69, 75, 2, 72, 64, 66, 0, 8, 12, 9, 76, 73], [107, 126, 56, 99, 43, 117, 109, 25, 79, 86, 82, 31, 93, 91, 84, 89, 118, 116, 18, 33, 4, 110, 0, 73, 20, 77, 53, 28, 66, 78, 9, 55, 96, 41, 105, 1, 75, 98, 39, 112, 37, 81, 120, 121, 125, 72, 127, 36, 32, 17, 26, 30, 52, 95, 113, 54, 64, 80, 29, 3, 97, 92, 42, 6, 27, 76, 60, 50, 101, 58, 61, 94, 48, 40, 22, 114, 49, 63, 62, 57, 51, 44, 46, 122, 19, 23, 5, 14, 104, 38, 24, 15, 87, 90, 123, 69, 16, 67, 2, 34, 100, 59, 45, 70, 12, 10, 103, 21, 88, 102, 13, 7, 74, 119, 108, 124, 47, 83, 85, 111, 11, 68, 8, 35, 71, 65, 106, 115], [56, 107, 112, 99, 125, 19, 50, 126, 116, 43, 127, 25, 52, 117, 62, 121, 120, 48, 53, 45, 31, 124, 93, 57, 113, 40, 61, 109, 55, 46, 60, 51, 122, 49, 118, 63, 114, 54, 59, 29, 58, 123, 115, 119, 105, 47, 111, 36, 104, 89, 108, 110, 33, 35, 44, 91, 24, 86, 106, 27, 41, 42, 39, 38, 98, 88, 101, 37, 100, 83, 95, 103, 102, 28, 21, 80, 22, 92, 34, 90, 13, 74, 77, 17, 23, 30, 97, 87, 32, 26, 94, 81, 16, 96, 75, 10, 7, 71, 14, 20, 78, 65, 82, 70, 6, 1, 3, 79, 69, 4, 67, 72, 85, 18, 68, 11, 2, 5, 8, 66, 84, 15, 0, 64, 12, 76, 9, 73]], "model.layers.26.self_attn.k_proj": [[43, 22, 107, 99, 92, 126, 127, 96, 64, 47, 46, 56, 48, 80, 119, 124, 68, 7, 122, 13, 116, 57, 49, 109, 54, 113, 18, 2, 117, 112, 63, 118, 44, 62, 115, 55, 52, 58, 123, 50, 125, 53, 120, 51, 59, 121, 60, 110, 108, 114, 61, 101, 74, 98, 111, 11, 106, 42, 45, 76, 41, 102, 3, 37, 78, 36, 6, 40, 105, 103, 34, 104, 1, 97, 69, 39, 38, 10, 70, 8, 21, 89, 72, 15, 5, 94, 31, 90, 84, 33, 95, 87, 30, 9, 17, 100, 23, 24, 82, 28, 29, 26, 88, 91, 32, 81, 12, 25, 93, 27, 73, 85, 75, 67, 19, 20, 79, 71, 35, 83, 14, 16, 66, 77, 4, 65, 0, 86], [46, 110, 100, 31, 26, 22, 19, 86, 17, 14, 94, 12, 74, 28, 58, 121, 60, 24, 77, 15, 72, 62, 104, 123, 54, 29, 118, 43, 10, 106, 122, 52, 112, 32, 37, 61, 108, 68, 83, 49, 67, 7, 16, 40, 63, 125, 5, 44, 119, 127, 116, 97, 98, 33, 38, 113, 50, 27, 59, 89, 57, 41, 42, 111, 115, 109, 102, 126, 93, 91, 81, 56, 34, 101, 53, 69, 55, 70, 124, 103, 105, 84, 99, 48, 18, 117, 107, 71, 39, 51, 1, 95, 35, 23, 114, 47, 76, 120, 96, 87, 36, 65, 45, 0, 21, 64, 30, 88, 92, 25, 20, 85, 9, 13, 82, 8, 79, 75, 90, 6, 66, 80, 4, 3, 11, 73, 78, 2], [123, 57, 97, 37, 24, 95, 92, 85, 82, 90, 80, 108, 115, 124, 73, 75, 19, 50, 114, 104, 76, 40, 78, 77, 83, 101, 53, 59, 121, 39, 35, 70, 119, 10, 0, 88, 105, 14, 55, 111, 71, 5, 48, 45, 126, 61, 4, 103, 30, 93, 109, 52, 21, 43, 1, 113, 116, 3, 112, 49, 42, 72, 54, 62, 96, 44, 117, 107, 41, 51, 58, 106, 60, 81, 118, 66, 46, 56, 127, 47, 110, 15, 63, 122, 120, 17, 20, 125, 36, 86, 89, 102, 100, 91, 32, 99, 34, 33, 84, 98, 27, 22, 94, 38, 29, 87, 2, 13, 25, 23, 79, 18, 16, 67, 31, 8, 7, 74, 26, 28, 64, 11, 6, 12, 69, 65, 68, 9], [125, 37, 63, 61, 22, 97, 59, 114, 126, 119, 51, 55, 124, 117, 112, 58, 57, 127, 53, 60, 121, 110, 123, 108, 122, 56, 52, 49, 111, 107, 44, 118, 113, 45, 31, 50, 62, 47, 29, 115, 120, 48, 109, 104, 54, 46, 36, 106, 43, 101, 90, 105, 79, 42, 34, 27, 17, 102, 116, 41, 40, 83, 38, 35, 39, 98, 85, 88, 92, 86, 30, 81, 89, 103, 87, 91, 99, 20, 24, 25, 93, 100, 13, 15, 78, 95, 82, 28, 18, 32, 96, 73, 21, 75, 94, 84, 26, 19, 33, 12, 69, 23, 11, 6, 72, 4, 8, 80, 10, 77, 76, 7, 14, 16, 67, 2, 74, 65, 71, 64, 9, 70, 1, 68, 66, 5, 0, 3], [105, 55, 98, 52, 95, 23, 21, 80, 83, 64, 13, 61, 11, 116, 74, 6, 28, 117, 51, 60, 27, 46, 69, 8, 49, 109, 2, 90, 71, 59, 123, 65, 42, 57, 12, 111, 4, 115, 127, 67, 113, 72, 103, 53, 126, 50, 79, 54, 58, 93, 82, 124, 43, 118, 26, 100, 62, 119, 108, 56, 110, 25, 122, 44, 41, 112, 63, 78, 47, 34, 97, 45, 114, 99, 1, 104, 106, 29, 125, 37, 121, 107, 35, 30, 101, 32, 38, 39, 3, 120, 96, 18, 40, 102, 81, 33, 24, 48, 94, 15, 36, 20, 88, 17, 86, 91, 76, 9, 14, 77, 22, 89, 92, 75, 66, 5, 84, 31, 73, 10, 7, 87, 70, 19, 0, 85, 16, 68], [109, 39, 80, 84, 10, 93, 13, 87, 45, 71, 54, 69, 64, 3, 111, 118, 57, 12, 119, 29, 126, 117, 65, 121, 49, 78, 68, 106, 88, 31, 1, 9, 122, 127, 55, 123, 22, 51, 115, 113, 17, 104, 102, 96, 53, 82, 46, 76, 58, 125, 41, 14, 61, 63, 83, 2, 52, 59, 62, 112, 72, 15, 44, 60, 124, 101, 100, 32, 86, 36, 26, 50, 94, 95, 43, 30, 48, 105, 6, 47, 114, 108, 85, 107, 110, 42, 34, 27, 56, 98, 120, 116, 66, 21, 37, 75, 0, 40, 25, 91, 35, 28, 90, 79, 4, 67, 24, 89, 33, 99, 92, 38, 70, 18, 5, 19, 8, 23, 97, 73, 103, 7, 11, 81, 74, 77, 20, 16], [59, 40, 31, 20, 28, 86, 114, 98, 13, 113, 89, 88, 54, 115, 82, 56, 116, 111, 50, 99, 124, 21, 66, 0, 67, 35, 127, 41, 87, 62, 16, 53, 49, 17, 121, 27, 15, 68, 80, 126, 18, 11, 78, 44, 48, 55, 1, 109, 8, 70, 123, 47, 92, 10, 105, 58, 57, 112, 25, 60, 46, 118, 33, 122, 106, 63, 23, 51, 76, 29, 43, 125, 83, 30, 34, 36, 7, 61, 117, 119, 45, 52, 120, 73, 93, 65, 103, 39, 108, 26, 107, 42, 69, 9, 100, 96, 74, 97, 5, 101, 110, 102, 38, 85, 94, 79, 24, 81, 71, 64, 37, 90, 14, 32, 19, 6, 12, 72, 91, 4, 2, 77, 22, 75, 95, 3, 84, 104], [43, 35, 86, 56, 107, 29, 25, 19, 118, 126, 117, 95, 127, 116, 121, 57, 79, 109, 119, 55, 101, 92, 112, 125, 58, 63, 113, 41, 97, 52, 59, 51, 53, 54, 122, 114, 115, 61, 60, 62, 49, 124, 123, 45, 50, 111, 105, 82, 120, 48, 47, 64, 42, 44, 110, 20, 27, 34, 94, 46, 108, 80, 103, 38, 77, 37, 106, 104, 102, 75, 70, 100, 9, 65, 98, 39, 4, 40, 2, 36, 85, 84, 24, 7, 32, 67, 99, 90, 93, 69, 28, 16, 71, 26, 73, 96, 33, 11, 30, 87, 78, 91, 31, 76, 17, 81, 14, 21, 12, 18, 68, 23, 74, 8, 88, 10, 72, 13, 5, 15, 0, 83, 89, 1, 6, 3, 22, 66]], "model.layers.26.self_attn.qk_proj": [[59, 43, 57, 123, 109, 110, 107, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 54, 126, 29, 41, 50, 117, 99, 40, 101, 28, 115, 114, 23, 62, 119, 22, 58, 95, 92, 127, 121, 80, 31, 124, 87, 16, 53, 116, 13, 88, 86, 103, 60, 39, 77, 51, 20, 112, 111, 44, 21, 19, 104, 84, 97, 90, 48, 24, 47, 49, 85, 82, 93, 113, 118, 83, 122, 120, 10, 34, 18, 26, 74, 33, 106, 100, 98, 42, 108, 7, 35, 25, 89, 36, 91, 71, 64, 30, 72, 75, 11, 102, 32, 96, 81, 79, 15, 14, 0, 78, 17, 76, 38, 67, 12, 3, 69, 27, 94, 5, 66, 68, 1, 73, 9, 70, 65, 8, 2, 6, 4], [43, 59, 57, 123, 109, 107, 110, 46, 61, 63, 125, 55, 56, 45, 105, 52, 37, 54, 126, 29, 117, 101, 41, 115, 50, 114, 62, 40, 99, 127, 58, 28, 31, 121, 23, 92, 103, 51, 22, 124, 119, 112, 116, 77, 80, 95, 104, 39, 16, 87, 86, 24, 60, 49, 53, 13, 88, 85, 48, 21, 111, 93, 97, 44, 47, 120, 122, 20, 113, 19, 118, 83, 90, 34, 10, 84, 108, 82, 74, 100, 18, 33, 26, 35, 25, 89, 98, 42, 91, 71, 106, 15, 30, 36, 0, 75, 64, 72, 96, 11, 32, 7, 79, 67, 12, 14, 102, 5, 76, 81, 78, 3, 38, 94, 69, 27, 17, 9, 66, 73, 2, 6, 4, 1, 68, 70, 8, 65], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 125, 56, 45, 105, 54, 52, 37, 126, 29, 117, 41, 101, 114, 62, 99, 50, 40, 115, 124, 28, 104, 127, 121, 119, 22, 51, 112, 103, 86, 53, 80, 95, 92, 116, 31, 23, 87, 48, 77, 44, 16, 49, 88, 39, 58, 113, 97, 13, 93, 34, 47, 24, 21, 60, 20, 122, 118, 85, 83, 111, 90, 74, 19, 84, 120, 98, 108, 82, 10, 18, 33, 36, 26, 100, 35, 64, 106, 42, 91, 71, 89, 25, 0, 7, 15, 78, 38, 81, 17, 30, 11, 76, 79, 75, 14, 67, 96, 102, 32, 69, 72, 94, 3, 27, 5, 12, 1, 6, 68, 2, 4, 73, 65, 8, 66, 9, 70], [43, 59, 57, 123, 109, 110, 107, 46, 63, 61, 55, 56, 125, 45, 105, 52, 37, 126, 54, 117, 29, 114, 101, 50, 49, 41, 121, 40, 115, 62, 124, 99, 53, 44, 112, 104, 80, 119, 116, 23, 28, 22, 16, 92, 95, 86, 51, 77, 60, 113, 31, 13, 103, 58, 127, 111, 88, 87, 93, 120, 19, 48, 20, 39, 21, 118, 24, 85, 10, 90, 84, 97, 122, 108, 47, 74, 34, 83, 82, 33, 98, 26, 18, 0, 100, 106, 71, 7, 36, 89, 35, 81, 79, 42, 15, 102, 64, 91, 25, 17, 75, 78, 30, 14, 76, 5, 67, 11, 3, 38, 69, 72, 8, 96, 27, 6, 4, 12, 94, 32, 65, 68, 2, 1, 66, 73, 9, 70], [43, 59, 57, 123, 109, 110, 107, 46, 63, 61, 125, 56, 55, 45, 105, 52, 126, 37, 117, 54, 114, 50, 41, 29, 40, 121, 99, 119, 44, 101, 49, 124, 23, 22, 62, 127, 28, 51, 104, 53, 80, 16, 13, 116, 88, 115, 60, 86, 95, 118, 112, 92, 87, 77, 120, 31, 47, 58, 20, 84, 93, 103, 111, 74, 39, 34, 19, 10, 21, 97, 122, 83, 24, 48, 85, 113, 90, 82, 108, 18, 26, 98, 33, 35, 7, 100, 106, 64, 89, 71, 36, 75, 96, 0, 15, 81, 91, 25, 79, 5, 67, 76, 30, 102, 8, 78, 3, 11, 17, 32, 69, 14, 42, 4, 94, 12, 1, 6, 2, 38, 65, 27, 72, 68, 9, 66, 73, 70], [43, 59, 57, 123, 107, 110, 109, 46, 63, 61, 125, 55, 56, 45, 105, 52, 37, 54, 117, 29, 50, 126, 41, 99, 40, 23, 80, 115, 22, 114, 101, 86, 44, 116, 124, 16, 95, 28, 121, 104, 62, 13, 60, 92, 88, 49, 119, 118, 19, 103, 93, 51, 127, 77, 31, 39, 58, 85, 87, 112, 20, 21, 53, 111, 120, 84, 47, 97, 10, 74, 24, 48, 122, 34, 113, 83, 26, 82, 33, 108, 90, 100, 106, 36, 98, 15, 18, 35, 25, 7, 64, 0, 91, 11, 89, 81, 71, 30, 14, 12, 75, 102, 79, 32, 96, 42, 17, 5, 78, 76, 38, 8, 3, 67, 94, 72, 69, 66, 70, 4, 6, 27, 9, 73, 65, 68, 1, 2], [59, 43, 57, 123, 109, 110, 107, 46, 63, 61, 55, 125, 56, 45, 105, 52, 37, 54, 29, 126, 41, 50, 117, 28, 121, 114, 124, 40, 99, 101, 22, 80, 119, 95, 87, 23, 86, 49, 115, 16, 92, 31, 44, 48, 88, 77, 13, 51, 116, 104, 62, 20, 122, 60, 58, 19, 84, 97, 127, 85, 118, 39, 103, 47, 34, 21, 120, 93, 90, 24, 112, 83, 74, 100, 111, 10, 108, 18, 26, 53, 82, 113, 98, 35, 33, 25, 30, 36, 106, 91, 89, 15, 7, 42, 11, 78, 71, 102, 14, 79, 75, 17, 64, 32, 81, 0, 96, 5, 38, 12, 8, 76, 94, 69, 3, 67, 27, 6, 4, 73, 9, 68, 1, 2, 66, 65, 70, 72], [59, 43, 57, 123, 107, 109, 110, 46, 61, 63, 55, 125, 56, 45, 105, 52, 37, 29, 126, 54, 117, 41, 101, 115, 114, 28, 121, 99, 50, 22, 119, 23, 116, 31, 86, 40, 124, 95, 92, 87, 51, 62, 16, 49, 103, 80, 88, 112, 39, 127, 104, 53, 113, 13, 44, 48, 60, 58, 97, 20, 19, 122, 90, 77, 21, 93, 85, 24, 100, 84, 118, 120, 34, 83, 82, 42, 74, 108, 91, 47, 18, 33, 26, 10, 36, 25, 106, 35, 111, 30, 89, 98, 7, 14, 78, 79, 17, 0, 32, 75, 81, 64, 15, 12, 71, 8, 27, 94, 102, 5, 38, 11, 3, 96, 67, 69, 66, 76, 65, 2, 4, 73, 9, 1, 68, 70, 6, 72], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 37, 52, 29, 54, 41, 117, 126, 101, 50, 115, 58, 99, 124, 114, 28, 40, 121, 44, 23, 119, 92, 22, 62, 116, 95, 31, 16, 127, 49, 13, 104, 97, 87, 122, 60, 86, 53, 112, 77, 103, 88, 80, 113, 85, 51, 48, 93, 21, 118, 34, 20, 24, 39, 84, 111, 90, 47, 19, 82, 100, 120, 74, 36, 108, 83, 33, 18, 91, 10, 26, 25, 106, 35, 98, 89, 30, 42, 38, 15, 71, 14, 32, 11, 78, 79, 96, 7, 27, 8, 17, 75, 94, 64, 102, 12, 0, 76, 81, 67, 5, 3, 69, 70, 9, 4, 1, 66, 68, 73, 2, 65, 72, 6], [59, 43, 57, 123, 109, 46, 110, 107, 63, 61, 55, 125, 56, 45, 105, 37, 54, 52, 126, 29, 117, 115, 50, 41, 40, 101, 62, 28, 114, 112, 99, 121, 119, 58, 124, 53, 116, 44, 23, 95, 103, 51, 86, 92, 22, 127, 104, 60, 31, 13, 16, 48, 80, 87, 49, 77, 122, 88, 21, 39, 118, 108, 113, 120, 111, 93, 20, 97, 24, 84, 19, 85, 90, 18, 34, 74, 47, 10, 100, 83, 33, 36, 82, 7, 71, 35, 98, 0, 91, 42, 25, 75, 11, 26, 8, 64, 79, 30, 96, 89, 14, 15, 106, 17, 5, 67, 76, 3, 78, 81, 38, 70, 102, 69, 32, 12, 94, 1, 9, 65, 2, 4, 73, 27, 66, 68, 72, 6], [59, 43, 57, 123, 109, 110, 46, 107, 61, 63, 56, 55, 125, 45, 105, 52, 37, 126, 54, 29, 115, 41, 121, 117, 101, 99, 50, 114, 119, 80, 28, 53, 40, 127, 16, 51, 60, 22, 13, 62, 23, 124, 58, 44, 87, 116, 86, 95, 112, 77, 104, 49, 31, 88, 103, 20, 111, 92, 113, 39, 74, 19, 84, 21, 97, 118, 122, 10, 90, 24, 85, 120, 83, 93, 26, 33, 108, 100, 7, 18, 47, 34, 48, 98, 82, 91, 35, 42, 71, 36, 25, 0, 11, 64, 81, 79, 15, 67, 75, 14, 12, 106, 3, 38, 89, 76, 78, 96, 8, 17, 30, 69, 102, 5, 32, 72, 73, 4, 1, 70, 2, 94, 27, 65, 68, 66, 6, 9], [43, 59, 57, 123, 109, 107, 46, 110, 63, 61, 55, 125, 56, 45, 105, 52, 37, 126, 29, 54, 41, 114, 101, 117, 22, 121, 40, 115, 99, 28, 86, 80, 23, 50, 119, 13, 104, 95, 31, 16, 124, 127, 51, 92, 87, 44, 58, 116, 77, 88, 97, 39, 60, 20, 62, 93, 53, 103, 84, 118, 74, 112, 113, 21, 85, 111, 48, 120, 90, 19, 49, 10, 33, 82, 83, 122, 34, 24, 18, 25, 100, 108, 26, 35, 91, 47, 89, 71, 98, 79, 7, 15, 30, 96, 36, 106, 11, 12, 78, 0, 102, 14, 64, 75, 17, 81, 42, 32, 69, 67, 27, 73, 38, 76, 3, 8, 65, 94, 5, 2, 4, 1, 72, 6, 9, 70, 68, 66], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 54, 29, 126, 121, 114, 50, 101, 41, 117, 115, 99, 22, 95, 119, 40, 28, 86, 87, 116, 80, 58, 118, 92, 124, 62, 103, 104, 31, 88, 23, 97, 127, 93, 122, 51, 16, 44, 39, 112, 49, 113, 77, 60, 33, 84, 48, 34, 85, 19, 53, 13, 108, 111, 120, 90, 24, 21, 10, 20, 83, 26, 82, 18, 36, 30, 74, 89, 98, 91, 100, 25, 35, 47, 42, 106, 96, 7, 64, 79, 15, 32, 75, 78, 71, 12, 81, 38, 94, 11, 69, 14, 102, 76, 17, 67, 5, 27, 72, 8, 0, 1, 6, 3, 2, 73, 4, 65, 66, 9, 68, 70], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 125, 45, 56, 105, 37, 52, 126, 29, 54, 101, 41, 116, 50, 117, 114, 121, 124, 115, 119, 23, 40, 99, 62, 120, 51, 86, 13, 28, 103, 122, 95, 104, 87, 31, 16, 22, 80, 127, 92, 118, 44, 53, 49, 113, 39, 111, 58, 88, 19, 60, 112, 77, 24, 97, 84, 10, 85, 21, 20, 90, 48, 93, 34, 108, 106, 100, 47, 18, 83, 82, 74, 26, 33, 98, 42, 0, 30, 35, 36, 7, 89, 71, 91, 64, 25, 72, 15, 81, 79, 75, 38, 69, 12, 67, 3, 11, 32, 14, 94, 96, 5, 78, 17, 76, 102, 65, 8, 4, 6, 1, 2, 73, 27, 9, 70, 66, 68], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 126, 54, 29, 41, 114, 101, 117, 40, 50, 99, 115, 23, 124, 31, 60, 44, 86, 121, 58, 22, 119, 95, 62, 16, 13, 28, 104, 116, 87, 80, 39, 88, 112, 103, 53, 113, 127, 77, 92, 51, 49, 48, 118, 93, 85, 120, 111, 84, 74, 97, 20, 19, 24, 21, 10, 90, 122, 82, 100, 47, 34, 83, 18, 33, 106, 108, 91, 26, 71, 0, 35, 42, 98, 7, 64, 75, 25, 89, 79, 36, 30, 38, 81, 15, 5, 78, 72, 3, 11, 102, 67, 76, 96, 17, 14, 32, 94, 73, 69, 6, 4, 68, 1, 66, 8, 65, 12, 2, 27, 9, 70], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 52, 37, 126, 54, 101, 29, 117, 50, 41, 115, 114, 62, 44, 80, 99, 23, 28, 95, 22, 86, 121, 40, 124, 116, 16, 31, 92, 60, 13, 119, 87, 127, 112, 51, 104, 113, 58, 88, 103, 111, 118, 97, 77, 84, 53, 39, 48, 85, 19, 20, 10, 74, 49, 120, 122, 21, 108, 90, 93, 100, 24, 26, 34, 47, 18, 83, 33, 82, 106, 89, 25, 98, 7, 35, 71, 30, 42, 79, 0, 96, 91, 36, 15, 75, 78, 17, 12, 102, 76, 11, 64, 14, 27, 81, 72, 67, 5, 38, 3, 94, 1, 32, 69, 65, 73, 66, 9, 68, 70, 6, 2, 4, 8], [59, 43, 57, 123, 107, 109, 110, 46, 61, 63, 125, 55, 56, 45, 105, 52, 37, 54, 126, 115, 101, 29, 40, 114, 50, 28, 117, 121, 41, 62, 95, 116, 31, 99, 92, 22, 127, 80, 23, 124, 86, 39, 87, 16, 58, 60, 13, 49, 44, 51, 88, 119, 122, 112, 53, 97, 84, 118, 19, 120, 85, 104, 93, 103, 24, 20, 21, 48, 108, 34, 90, 33, 82, 77, 111, 100, 26, 74, 83, 113, 35, 98, 18, 36, 89, 47, 25, 10, 42, 91, 30, 106, 71, 96, 7, 79, 64, 0, 81, 78, 15, 32, 11, 75, 102, 72, 12, 38, 76, 17, 27, 3, 14, 94, 67, 69, 5, 73, 65, 6, 70, 1, 9, 68, 4, 66, 8, 2], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 37, 52, 126, 41, 29, 54, 50, 115, 101, 117, 40, 114, 80, 22, 28, 49, 121, 23, 104, 31, 99, 95, 86, 92, 103, 119, 60, 124, 116, 58, 13, 87, 62, 16, 51, 39, 44, 127, 77, 88, 93, 97, 113, 24, 120, 122, 21, 118, 20, 19, 112, 84, 48, 74, 53, 34, 85, 111, 18, 108, 26, 90, 100, 82, 47, 33, 10, 83, 42, 89, 35, 98, 25, 71, 91, 79, 0, 30, 106, 81, 36, 72, 7, 17, 96, 64, 11, 15, 78, 5, 67, 14, 69, 94, 32, 102, 75, 3, 12, 76, 65, 38, 66, 68, 27, 73, 70, 9, 4, 1, 6, 2, 8], [59, 43, 57, 123, 109, 110, 46, 107, 63, 61, 55, 56, 125, 45, 105, 37, 126, 52, 54, 117, 50, 29, 114, 41, 40, 44, 101, 115, 121, 62, 124, 104, 58, 28, 60, 99, 80, 119, 116, 103, 113, 23, 53, 95, 118, 22, 51, 31, 122, 92, 13, 86, 120, 49, 39, 112, 16, 87, 88, 77, 111, 97, 100, 24, 74, 93, 108, 84, 21, 20, 127, 48, 85, 47, 34, 19, 90, 82, 10, 18, 106, 33, 83, 35, 98, 42, 26, 71, 36, 7, 30, 0, 91, 89, 102, 64, 79, 11, 81, 38, 25, 75, 3, 72, 32, 94, 96, 78, 76, 15, 12, 17, 69, 67, 27, 5, 14, 70, 73, 65, 66, 9, 8, 4, 1, 68, 2, 6], [43, 59, 57, 123, 109, 107, 110, 46, 63, 61, 56, 55, 125, 45, 105, 37, 52, 126, 54, 29, 117, 50, 41, 121, 101, 62, 40, 114, 115, 44, 23, 99, 95, 104, 28, 16, 58, 116, 124, 31, 60, 103, 13, 51, 49, 92, 22, 80, 39, 122, 87, 127, 86, 119, 88, 93, 118, 77, 120, 112, 113, 97, 85, 111, 108, 47, 24, 10, 53, 20, 84, 74, 21, 34, 100, 90, 19, 33, 18, 83, 98, 48, 89, 26, 82, 91, 42, 35, 30, 11, 71, 7, 64, 36, 75, 106, 32, 3, 25, 12, 79, 69, 15, 0, 102, 96, 94, 17, 67, 76, 78, 38, 72, 14, 8, 5, 81, 27, 70, 65, 68, 66, 9, 73, 1, 2, 4, 6], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 126, 54, 37, 52, 29, 41, 121, 40, 101, 50, 117, 115, 99, 116, 114, 95, 127, 28, 44, 119, 62, 104, 49, 86, 122, 22, 124, 39, 80, 23, 53, 118, 16, 31, 113, 120, 13, 103, 87, 60, 92, 112, 51, 93, 88, 58, 111, 77, 84, 48, 34, 97, 20, 74, 85, 33, 21, 19, 47, 98, 90, 10, 24, 108, 83, 18, 26, 100, 35, 91, 25, 89, 71, 82, 36, 7, 42, 15, 11, 79, 30, 75, 17, 106, 96, 76, 81, 102, 78, 0, 72, 14, 64, 32, 69, 67, 3, 94, 38, 12, 8, 5, 70, 1, 68, 73, 4, 65, 66, 2, 27, 9, 6], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 56, 125, 55, 45, 105, 126, 37, 52, 54, 29, 117, 41, 50, 101, 99, 40, 121, 115, 114, 116, 49, 104, 28, 44, 113, 119, 122, 124, 62, 23, 80, 103, 92, 22, 58, 16, 95, 87, 60, 51, 86, 39, 112, 118, 31, 88, 53, 48, 127, 120, 97, 13, 24, 77, 34, 20, 84, 111, 93, 21, 74, 85, 108, 90, 47, 19, 33, 83, 100, 26, 82, 64, 10, 98, 18, 71, 36, 106, 35, 0, 30, 42, 91, 89, 25, 7, 15, 11, 67, 17, 75, 8, 81, 79, 78, 3, 14, 38, 5, 102, 69, 32, 12, 94, 76, 72, 65, 1, 96, 70, 68, 27, 4, 66, 6, 2, 73, 9], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 37, 126, 54, 52, 117, 29, 50, 101, 41, 115, 40, 99, 114, 28, 119, 116, 121, 104, 124, 23, 62, 53, 95, 22, 92, 127, 51, 58, 49, 86, 103, 80, 122, 112, 31, 118, 16, 44, 39, 97, 113, 13, 77, 93, 20, 87, 88, 21, 90, 111, 24, 60, 120, 19, 85, 48, 34, 83, 84, 47, 91, 30, 108, 74, 33, 100, 26, 18, 35, 10, 82, 42, 98, 25, 89, 36, 7, 15, 71, 102, 11, 106, 32, 8, 0, 75, 79, 81, 94, 27, 14, 12, 17, 67, 64, 3, 69, 96, 38, 78, 5, 66, 65, 76, 68, 73, 72, 1, 70, 6, 9, 2, 4], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 125, 56, 55, 45, 105, 37, 52, 29, 54, 126, 41, 114, 121, 116, 40, 50, 117, 28, 62, 101, 115, 95, 104, 99, 44, 23, 119, 80, 22, 103, 16, 60, 92, 118, 13, 124, 58, 39, 86, 88, 77, 111, 31, 87, 112, 127, 20, 93, 51, 53, 49, 21, 84, 122, 85, 113, 83, 10, 97, 74, 90, 19, 24, 33, 120, 82, 25, 98, 26, 18, 47, 34, 108, 48, 7, 100, 35, 36, 89, 91, 71, 79, 8, 75, 11, 106, 30, 12, 76, 32, 42, 15, 0, 78, 102, 14, 69, 38, 17, 96, 81, 3, 73, 67, 64, 5, 94, 6, 9, 65, 27, 70, 66, 4, 72, 68, 2, 1], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 125, 55, 56, 45, 105, 37, 52, 126, 29, 41, 54, 114, 117, 40, 50, 99, 28, 127, 101, 121, 23, 22, 95, 86, 115, 16, 80, 31, 60, 111, 62, 116, 44, 104, 58, 77, 53, 92, 88, 49, 113, 13, 103, 124, 119, 87, 39, 21, 51, 20, 118, 19, 97, 122, 84, 112, 93, 18, 47, 10, 74, 90, 108, 24, 34, 48, 85, 83, 26, 120, 82, 100, 35, 98, 25, 33, 7, 30, 8, 89, 36, 71, 11, 106, 14, 42, 91, 79, 15, 102, 75, 76, 0, 78, 32, 96, 17, 81, 64, 12, 94, 69, 67, 3, 6, 38, 5, 73, 65, 68, 9, 70, 1, 4, 27, 66, 2, 72], [59, 43, 57, 123, 107, 109, 46, 110, 61, 63, 55, 125, 56, 45, 105, 37, 54, 126, 52, 50, 29, 117, 114, 101, 41, 121, 115, 28, 99, 40, 31, 112, 49, 116, 124, 62, 92, 22, 23, 95, 127, 113, 58, 86, 44, 88, 97, 60, 39, 87, 103, 104, 118, 122, 80, 16, 111, 21, 13, 53, 77, 24, 84, 119, 51, 85, 34, 20, 120, 100, 19, 48, 90, 93, 26, 47, 74, 83, 42, 33, 106, 108, 36, 18, 30, 89, 10, 98, 82, 35, 7, 25, 91, 11, 0, 64, 96, 15, 75, 32, 102, 79, 78, 8, 14, 71, 94, 81, 27, 76, 69, 12, 38, 3, 17, 67, 1, 5, 68, 73, 65, 6, 66, 2, 4, 9, 70, 72], [59, 43, 57, 123, 109, 107, 46, 110, 61, 63, 55, 125, 56, 45, 105, 37, 52, 126, 54, 29, 101, 50, 114, 41, 121, 40, 115, 117, 99, 62, 124, 28, 53, 127, 23, 95, 116, 58, 119, 31, 92, 80, 49, 44, 104, 118, 22, 86, 39, 16, 13, 112, 51, 111, 122, 88, 103, 60, 87, 97, 48, 77, 120, 20, 19, 34, 21, 113, 24, 85, 93, 84, 26, 47, 100, 74, 90, 10, 108, 33, 98, 83, 35, 106, 18, 82, 36, 42, 91, 7, 25, 30, 89, 38, 102, 79, 96, 8, 71, 81, 12, 14, 11, 78, 15, 94, 17, 75, 5, 76, 64, 32, 27, 0, 3, 69, 67, 4, 6, 72, 73, 65, 9, 68, 66, 2, 1, 70], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 125, 55, 56, 45, 105, 37, 52, 126, 29, 54, 114, 121, 41, 50, 101, 117, 99, 40, 116, 28, 115, 124, 119, 80, 118, 104, 44, 95, 86, 49, 16, 23, 103, 13, 127, 62, 92, 22, 77, 58, 31, 51, 87, 112, 60, 39, 21, 88, 53, 113, 24, 111, 85, 93, 48, 84, 19, 10, 20, 122, 34, 74, 83, 120, 108, 97, 90, 18, 82, 26, 7, 106, 47, 33, 36, 100, 98, 42, 35, 71, 25, 0, 89, 64, 11, 79, 30, 69, 75, 3, 91, 78, 102, 14, 17, 12, 8, 67, 32, 15, 76, 96, 81, 5, 38, 94, 65, 6, 70, 72, 27, 68, 73, 4, 9, 66, 1, 2], [59, 43, 57, 123, 109, 110, 107, 46, 61, 63, 56, 125, 55, 45, 105, 52, 37, 54, 126, 29, 41, 50, 117, 53, 114, 101, 121, 40, 99, 60, 44, 104, 28, 95, 116, 58, 127, 124, 115, 80, 119, 86, 23, 111, 22, 92, 16, 112, 118, 103, 49, 77, 31, 88, 87, 39, 13, 62, 113, 20, 21, 10, 48, 93, 74, 122, 24, 51, 84, 97, 108, 33, 34, 19, 64, 106, 90, 83, 35, 26, 7, 85, 82, 120, 47, 36, 18, 98, 71, 100, 91, 11, 79, 0, 75, 42, 96, 102, 30, 15, 76, 78, 69, 17, 25, 89, 3, 67, 12, 32, 38, 14, 72, 65, 66, 81, 8, 5, 70, 4, 68, 94, 9, 1, 6, 73, 2, 27], [43, 59, 57, 123, 107, 109, 110, 46, 61, 63, 55, 56, 125, 45, 105, 52, 37, 54, 126, 29, 50, 41, 101, 114, 99, 116, 117, 115, 40, 112, 127, 28, 22, 23, 95, 60, 119, 16, 124, 104, 80, 86, 13, 118, 88, 92, 58, 44, 121, 103, 77, 62, 53, 31, 87, 49, 97, 113, 21, 85, 39, 93, 111, 20, 84, 10, 83, 19, 51, 122, 34, 90, 108, 24, 26, 48, 47, 120, 74, 33, 100, 106, 82, 18, 36, 98, 7, 91, 75, 25, 71, 35, 30, 15, 78, 64, 32, 17, 89, 12, 42, 14, 96, 81, 79, 11, 0, 102, 38, 3, 72, 5, 69, 76, 70, 67, 9, 65, 4, 8, 68, 2, 94, 66, 6, 73, 1, 27], [59, 43, 57, 123, 109, 110, 107, 46, 61, 63, 55, 125, 56, 45, 105, 52, 37, 126, 54, 29, 101, 50, 41, 28, 117, 115, 40, 116, 121, 99, 114, 112, 92, 104, 119, 124, 62, 127, 23, 16, 95, 58, 103, 80, 22, 31, 86, 39, 13, 88, 60, 53, 44, 113, 87, 118, 97, 111, 77, 34, 47, 85, 49, 21, 24, 122, 84, 108, 120, 20, 51, 48, 19, 93, 74, 90, 10, 33, 83, 18, 82, 100, 26, 0, 98, 7, 106, 35, 91, 71, 36, 64, 89, 11, 25, 42, 79, 67, 75, 102, 72, 96, 30, 15, 69, 14, 32, 12, 3, 5, 76, 68, 38, 81, 17, 70, 78, 94, 9, 2, 73, 4, 65, 8, 6, 27, 1, 66], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 52, 37, 126, 54, 29, 114, 41, 101, 50, 40, 115, 116, 28, 99, 117, 112, 95, 121, 127, 22, 23, 80, 86, 119, 92, 62, 103, 16, 87, 31, 39, 53, 13, 124, 60, 118, 44, 58, 88, 104, 113, 77, 51, 49, 21, 97, 19, 48, 93, 84, 111, 24, 10, 34, 33, 122, 85, 47, 20, 120, 74, 90, 82, 108, 83, 18, 26, 35, 36, 91, 100, 7, 75, 30, 25, 98, 14, 72, 106, 71, 79, 42, 89, 11, 76, 102, 96, 32, 17, 12, 64, 78, 15, 94, 5, 81, 27, 38, 73, 69, 0, 67, 9, 3, 70, 2, 68, 8, 66, 4, 1, 65, 6]], "model.layers.27.self_attn.q_proj": [[109, 45, 94, 90, 33, 23, 83, 81, 21, 60, 54, 117, 79, 125, 76, 28, 78, 123, 111, 112, 39, 58, 73, 62, 56, 61, 51, 5, 32, 59, 38, 114, 57, 115, 6, 52, 48, 105, 11, 35, 97, 43, 55, 100, 9, 122, 113, 106, 46, 7, 37, 53, 121, 10, 116, 98, 118, 63, 24, 120, 3, 126, 19, 0, 110, 49, 87, 17, 44, 22, 85, 50, 119, 75, 104, 124, 47, 103, 42, 127, 95, 101, 31, 29, 88, 108, 92, 25, 13, 4, 26, 14, 36, 30, 40, 80, 18, 107, 41, 20, 84, 86, 27, 93, 102, 99, 96, 15, 77, 89, 91, 82, 34, 74, 16, 66, 70, 72, 65, 8, 12, 64, 1, 2, 68, 69, 71, 67], [109, 45, 33, 94, 90, 21, 83, 23, 81, 79, 76, 105, 54, 28, 112, 73, 97, 123, 99, 7, 117, 75, 121, 6, 5, 127, 60, 39, 4, 38, 11, 9, 47, 125, 78, 106, 3, 0, 37, 48, 32, 56, 115, 111, 62, 63, 26, 71, 14, 74, 2, 46, 120, 18, 58, 119, 35, 113, 114, 17, 51, 110, 87, 16, 19, 49, 85, 1, 66, 107, 36, 70, 77, 59, 98, 126, 53, 89, 57, 44, 24, 50, 31, 124, 43, 22, 102, 122, 88, 55, 52, 42, 103, 108, 10, 92, 118, 82, 86, 104, 96, 80, 34, 30, 65, 116, 20, 84, 95, 41, 13, 29, 8, 61, 101, 25, 91, 72, 40, 100, 67, 69, 93, 27, 64, 15, 12, 68], [109, 45, 94, 90, 33, 83, 23, 21, 76, 79, 81, 123, 125, 28, 54, 39, 105, 99, 6, 73, 112, 111, 106, 5, 127, 60, 62, 58, 56, 9, 78, 38, 70, 32, 50, 3, 115, 44, 42, 72, 75, 66, 37, 40, 17, 48, 7, 97, 11, 63, 124, 52, 18, 84, 89, 51, 120, 10, 113, 101, 26, 87, 25, 98, 47, 41, 88, 80, 57, 59, 14, 49, 85, 0, 108, 19, 117, 126, 24, 31, 30, 95, 77, 116, 104, 103, 119, 107, 12, 74, 121, 110, 22, 1, 35, 102, 13, 4, 2, 55, 46, 34, 53, 118, 92, 96, 43, 91, 61, 114, 29, 122, 8, 100, 86, 16, 36, 82, 20, 93, 65, 15, 27, 71, 69, 67, 68, 64], [109, 45, 94, 90, 33, 125, 83, 23, 117, 81, 21, 76, 115, 32, 124, 57, 119, 113, 112, 79, 54, 121, 114, 9, 28, 123, 58, 39, 47, 51, 52, 60, 91, 118, 122, 53, 59, 29, 48, 49, 106, 61, 24, 14, 30, 101, 13, 110, 37, 38, 43, 100, 126, 116, 44, 17, 85, 80, 87, 50, 127, 5, 108, 95, 92, 88, 102, 103, 25, 63, 111, 99, 73, 96, 97, 55, 19, 107, 56, 40, 120, 89, 41, 75, 46, 6, 36, 84, 78, 22, 62, 42, 82, 20, 18, 35, 31, 16, 27, 26, 104, 98, 105, 93, 34, 86, 15, 12, 77, 65, 10, 8, 7, 74, 72, 11, 71, 0, 1, 68, 3, 69, 70, 4, 67, 66, 2, 64], [118, 53, 120, 63, 101, 60, 127, 121, 84, 57, 126, 119, 88, 56, 50, 112, 61, 124, 92, 52, 17, 54, 125, 117, 123, 24, 115, 113, 33, 55, 116, 49, 58, 59, 62, 51, 94, 110, 122, 93, 48, 47, 111, 45, 114, 43, 30, 37, 108, 39, 46, 44, 7, 14, 107, 90, 41, 109, 11, 104, 26, 9, 25, 74, 78, 20, 91, 77, 42, 103, 106, 5, 105, 29, 31, 75, 40, 22, 32, 80, 38, 36, 100, 21, 18, 6, 13, 72, 95, 99, 96, 73, 79, 4, 34, 16, 86, 35, 98, 83, 12, 3, 19, 81, 66, 10, 102, 85, 28, 97, 69, 23, 27, 87, 2, 89, 64, 1, 65, 70, 68, 71, 82, 0, 15, 67, 8, 76], [53, 120, 118, 101, 63, 60, 127, 50, 56, 57, 61, 124, 126, 84, 123, 121, 112, 119, 54, 115, 62, 52, 49, 117, 88, 93, 113, 116, 58, 110, 125, 55, 51, 122, 59, 108, 39, 111, 48, 47, 92, 114, 33, 45, 17, 46, 107, 37, 44, 14, 24, 77, 30, 90, 7, 109, 41, 94, 104, 26, 43, 105, 20, 103, 74, 106, 91, 9, 42, 83, 4, 19, 100, 25, 96, 3, 5, 11, 22, 29, 23, 40, 75, 38, 34, 80, 12, 6, 16, 73, 21, 32, 18, 36, 78, 98, 102, 95, 31, 89, 99, 86, 35, 27, 85, 72, 97, 79, 28, 81, 10, 87, 64, 65, 13, 70, 66, 1, 2, 71, 15, 82, 69, 76, 68, 8, 0, 67], [120, 118, 53, 101, 63, 60, 126, 121, 127, 56, 57, 84, 50, 119, 61, 124, 54, 112, 52, 62, 123, 115, 117, 125, 58, 116, 55, 110, 88, 93, 39, 113, 122, 49, 59, 47, 17, 51, 48, 111, 92, 108, 114, 33, 46, 14, 45, 43, 94, 109, 37, 90, 30, 11, 26, 41, 7, 25, 44, 107, 104, 24, 106, 9, 103, 42, 77, 22, 20, 100, 105, 74, 29, 91, 40, 78, 6, 34, 80, 5, 38, 31, 102, 18, 98, 95, 36, 32, 96, 73, 3, 85, 83, 16, 4, 86, 12, 79, 10, 28, 19, 75, 99, 27, 23, 21, 35, 13, 97, 89, 72, 81, 64, 66, 65, 1, 87, 70, 15, 71, 82, 67, 69, 0, 76, 68, 2, 8], [63, 53, 120, 118, 101, 60, 57, 50, 84, 127, 62, 61, 54, 112, 56, 124, 121, 88, 123, 115, 119, 117, 52, 49, 55, 126, 116, 39, 113, 58, 110, 122, 30, 59, 51, 125, 93, 111, 44, 48, 108, 114, 47, 14, 24, 45, 90, 43, 17, 46, 94, 77, 37, 33, 109, 92, 103, 74, 107, 11, 5, 7, 104, 73, 106, 98, 26, 9, 91, 41, 42, 105, 100, 36, 6, 83, 3, 18, 4, 40, 80, 32, 22, 38, 102, 85, 28, 25, 96, 20, 27, 75, 35, 34, 31, 95, 21, 23, 16, 78, 19, 99, 79, 72, 97, 86, 89, 10, 66, 29, 81, 12, 87, 13, 82, 65, 68, 15, 0, 64, 71, 2, 70, 1, 76, 69, 67, 8], [40, 98, 63, 23, 31, 85, 80, 26, 121, 60, 13, 19, 125, 54, 82, 122, 79, 8, 49, 74, 6, 1, 117, 55, 113, 105, 59, 57, 9, 12, 106, 127, 111, 52, 46, 66, 50, 119, 56, 58, 53, 112, 43, 103, 11, 27, 120, 108, 39, 126, 4, 124, 37, 24, 62, 123, 109, 28, 115, 38, 67, 100, 107, 64, 0, 90, 51, 15, 73, 75, 47, 61, 76, 42, 116, 97, 78, 21, 99, 35, 18, 30, 25, 104, 36, 118, 41, 87, 48, 88, 44, 102, 32, 93, 45, 94, 84, 101, 114, 3, 95, 33, 68, 5, 83, 110, 34, 96, 77, 29, 20, 89, 16, 91, 86, 69, 92, 7, 14, 81, 17, 72, 70, 22, 65, 71, 10, 2], [40, 63, 98, 31, 80, 23, 60, 85, 13, 74, 6, 4, 19, 8, 22, 64, 52, 66, 121, 46, 108, 28, 124, 56, 122, 106, 0, 79, 125, 104, 54, 65, 11, 55, 107, 113, 75, 59, 18, 90, 48, 82, 7, 1, 58, 73, 126, 119, 12, 68, 117, 84, 127, 83, 105, 34, 57, 49, 21, 120, 35, 77, 109, 20, 101, 111, 94, 47, 14, 15, 39, 81, 30, 97, 24, 100, 96, 2, 3, 16, 62, 114, 10, 53, 33, 32, 70, 9, 25, 112, 95, 17, 123, 116, 38, 89, 5, 91, 87, 88, 72, 86, 76, 51, 71, 78, 29, 67, 69, 43, 27, 36, 103, 44, 115, 50, 93, 41, 110, 102, 99, 26, 92, 42, 45, 37, 61, 118], [40, 63, 98, 31, 60, 8, 80, 85, 19, 13, 23, 6, 74, 1, 66, 121, 64, 122, 117, 124, 4, 26, 46, 106, 15, 52, 28, 54, 57, 109, 55, 125, 0, 89, 104, 87, 12, 24, 120, 56, 79, 119, 105, 68, 69, 108, 72, 27, 7, 113, 11, 71, 59, 126, 42, 21, 48, 18, 76, 83, 33, 78, 84, 103, 96, 75, 32, 16, 82, 65, 115, 20, 41, 90, 81, 30, 77, 50, 2, 62, 3, 51, 25, 116, 92, 123, 5, 114, 127, 67, 37, 93, 88, 61, 10, 47, 29, 118, 58, 36, 38, 107, 39, 94, 17, 70, 111, 97, 14, 49, 86, 110, 45, 91, 73, 100, 101, 35, 44, 53, 112, 99, 102, 22, 43, 9, 95, 34], [40, 63, 98, 23, 85, 60, 26, 80, 31, 13, 8, 74, 19, 66, 82, 6, 121, 59, 52, 106, 4, 58, 41, 122, 113, 46, 119, 28, 49, 48, 95, 57, 84, 5, 120, 111, 105, 125, 65, 38, 67, 79, 54, 55, 108, 109, 117, 2, 73, 102, 90, 7, 123, 47, 61, 76, 64, 0, 18, 107, 50, 83, 45, 69, 56, 53, 97, 32, 115, 77, 81, 87, 71, 124, 15, 42, 89, 103, 91, 11, 30, 44, 114, 126, 21, 118, 78, 100, 43, 24, 39, 96, 36, 62, 17, 68, 35, 127, 93, 20, 29, 51, 116, 16, 3, 22, 94, 27, 70, 75, 9, 112, 88, 33, 110, 92, 37, 10, 101, 25, 14, 12, 72, 86, 1, 99, 104, 34], [104, 120, 98, 46, 95, 126, 115, 52, 44, 91, 108, 59, 56, 60, 54, 27, 84, 58, 88, 96, 118, 82, 24, 122, 51, 36, 50, 21, 45, 42, 48, 112, 101, 57, 62, 124, 114, 55, 63, 113, 92, 119, 117, 116, 61, 76, 125, 86, 85, 47, 127, 23, 53, 123, 111, 121, 49, 43, 38, 13, 107, 110, 109, 22, 41, 106, 74, 99, 103, 105, 80, 89, 33, 97, 31, 29, 32, 19, 6, 78, 37, 102, 25, 39, 69, 94, 28, 100, 35, 64, 90, 18, 93, 8, 30, 15, 26, 83, 65, 87, 34, 14, 12, 0, 40, 2, 79, 20, 66, 81, 72, 7, 1, 17, 3, 73, 16, 67, 11, 4, 75, 10, 5, 68, 77, 71, 9, 70], [120, 104, 98, 95, 44, 108, 126, 46, 113, 125, 54, 27, 88, 91, 55, 58, 127, 122, 61, 59, 41, 60, 115, 50, 110, 106, 56, 84, 36, 118, 116, 109, 123, 22, 38, 103, 62, 124, 111, 57, 96, 63, 114, 49, 112, 43, 28, 121, 40, 117, 53, 48, 35, 42, 24, 52, 45, 51, 101, 107, 119, 37, 47, 39, 105, 82, 94, 19, 85, 86, 83, 97, 102, 21, 12, 79, 93, 73, 26, 25, 81, 71, 100, 99, 31, 29, 92, 9, 30, 4, 33, 32, 89, 34, 23, 76, 74, 78, 90, 80, 15, 20, 18, 87, 8, 14, 10, 17, 70, 69, 72, 66, 7, 6, 2, 68, 13, 75, 3, 67, 5, 16, 77, 11, 65, 0, 64, 1], [104, 120, 98, 95, 126, 91, 80, 84, 13, 82, 22, 86, 115, 27, 74, 44, 31, 93, 116, 36, 16, 97, 69, 88, 20, 77, 60, 100, 52, 59, 58, 90, 56, 6, 108, 125, 72, 94, 45, 2, 19, 122, 32, 25, 3, 62, 63, 18, 54, 114, 92, 96, 42, 64, 41, 43, 75, 15, 29, 113, 68, 55, 87, 30, 46, 106, 11, 118, 102, 33, 79, 48, 81, 76, 24, 99, 78, 103, 117, 70, 89, 83, 57, 85, 50, 65, 37, 8, 23, 21, 35, 26, 67, 17, 61, 73, 47, 107, 9, 4, 7, 1, 111, 38, 49, 71, 10, 12, 28, 14, 110, 53, 5, 127, 0, 34, 39, 66, 105, 101, 124, 121, 119, 112, 109, 123, 51, 40], [104, 120, 98, 46, 95, 82, 84, 96, 91, 80, 115, 14, 126, 44, 66, 48, 125, 27, 13, 67, 60, 29, 42, 116, 68, 74, 93, 8, 28, 85, 63, 102, 113, 36, 0, 23, 55, 117, 101, 105, 38, 6, 76, 65, 1, 21, 35, 24, 111, 54, 39, 45, 56, 99, 88, 26, 32, 119, 103, 86, 37, 50, 17, 108, 107, 64, 9, 97, 109, 52, 79, 40, 90, 16, 22, 31, 100, 92, 124, 41, 94, 122, 25, 58, 61, 62, 59, 47, 121, 72, 57, 112, 34, 123, 81, 15, 43, 110, 5, 89, 70, 20, 19, 106, 30, 114, 127, 83, 51, 118, 49, 53, 12, 87, 33, 11, 73, 78, 18, 69, 75, 10, 77, 3, 4, 7, 71, 2], [111, 100, 47, 56, 24, 53, 95, 121, 120, 58, 31, 54, 77, 36, 82, 103, 86, 57, 1, 124, 59, 119, 60, 84, 127, 16, 52, 83, 104, 97, 94, 10, 63, 45, 105, 49, 64, 116, 110, 28, 122, 62, 51, 126, 108, 92, 42, 112, 67, 113, 39, 27, 55, 125, 43, 102, 61, 46, 19, 118, 26, 40, 50, 8, 68, 123, 4, 38, 44, 114, 85, 71, 2, 109, 115, 22, 73, 6, 15, 41, 117, 99, 48, 66, 69, 106, 98, 37, 107, 101, 20, 35, 90, 0, 96, 81, 70, 30, 32, 89, 33, 17, 80, 78, 9, 88, 65, 23, 29, 91, 34, 87, 93, 5, 25, 11, 12, 18, 79, 13, 72, 21, 3, 75, 7, 74, 14, 76], [111, 47, 58, 100, 24, 31, 122, 95, 61, 106, 51, 125, 82, 54, 84, 28, 113, 26, 85, 56, 53, 20, 126, 110, 117, 112, 30, 77, 127, 50, 13, 46, 118, 98, 83, 36, 104, 22, 55, 6, 12, 79, 120, 39, 45, 37, 49, 15, 123, 86, 116, 41, 94, 52, 119, 78, 59, 124, 14, 121, 17, 57, 114, 92, 43, 108, 105, 60, 109, 101, 63, 25, 107, 99, 88, 23, 103, 115, 44, 68, 96, 40, 27, 90, 80, 35, 62, 0, 29, 97, 32, 73, 48, 74, 33, 89, 87, 38, 102, 3, 42, 21, 81, 10, 34, 72, 66, 76, 18, 16, 93, 19, 91, 11, 65, 7, 8, 69, 75, 9, 4, 2, 5, 71, 67, 70, 64, 1], [111, 47, 100, 58, 56, 24, 95, 31, 114, 115, 52, 94, 106, 83, 121, 36, 82, 119, 53, 86, 54, 120, 112, 110, 28, 85, 113, 55, 103, 51, 49, 92, 104, 59, 109, 118, 17, 84, 126, 63, 127, 20, 48, 50, 60, 62, 45, 117, 30, 122, 116, 105, 38, 57, 124, 61, 77, 96, 46, 125, 16, 107, 43, 41, 123, 39, 108, 88, 42, 102, 22, 80, 90, 26, 40, 79, 6, 78, 44, 98, 101, 10, 89, 37, 97, 35, 12, 81, 68, 99, 34, 15, 11, 33, 0, 19, 23, 13, 29, 73, 9, 21, 32, 27, 91, 93, 71, 72, 25, 18, 74, 87, 66, 3, 14, 8, 76, 7, 4, 75, 64, 67, 5, 69, 65, 70, 1, 2], [111, 47, 58, 100, 56, 24, 95, 31, 127, 121, 122, 106, 115, 36, 94, 120, 44, 62, 82, 28, 83, 108, 116, 126, 59, 53, 52, 112, 85, 77, 54, 92, 118, 84, 105, 51, 113, 104, 60, 49, 17, 107, 110, 124, 45, 109, 20, 63, 57, 61, 103, 86, 55, 43, 114, 46, 48, 123, 125, 78, 6, 80, 119, 102, 96, 42, 30, 117, 38, 26, 39, 37, 33, 16, 50, 22, 41, 90, 34, 19, 12, 10, 40, 79, 32, 15, 88, 101, 74, 98, 91, 27, 29, 93, 81, 89, 35, 99, 97, 66, 11, 23, 68, 13, 65, 21, 25, 87, 71, 72, 8, 7, 3, 18, 1, 73, 76, 14, 0, 9, 4, 75, 5, 70, 69, 2, 67, 64], [48, 41, 51, 62, 125, 55, 121, 112, 24, 88, 30, 57, 52, 54, 60, 53, 123, 114, 49, 119, 115, 118, 117, 113, 56, 124, 110, 126, 61, 97, 50, 127, 94, 116, 47, 44, 27, 120, 122, 106, 58, 90, 103, 82, 111, 91, 63, 59, 36, 109, 107, 108, 89, 45, 37, 43, 46, 79, 21, 42, 19, 22, 104, 28, 39, 101, 102, 105, 93, 15, 83, 38, 35, 80, 33, 40, 98, 32, 99, 34, 18, 73, 86, 100, 95, 76, 96, 71, 84, 13, 16, 29, 92, 77, 85, 87, 25, 9, 31, 20, 26, 65, 17, 23, 81, 7, 3, 66, 68, 5, 12, 6, 1, 64, 0, 2, 67, 4, 70, 14, 11, 69, 78, 74, 10, 75, 8, 72], [41, 48, 51, 119, 112, 62, 20, 89, 105, 14, 81, 115, 80, 11, 97, 10, 12, 30, 8, 125, 54, 27, 60, 5, 70, 78, 25, 121, 22, 49, 52, 84, 56, 117, 0, 28, 118, 72, 57, 87, 126, 61, 29, 127, 104, 17, 37, 110, 96, 124, 55, 93, 50, 94, 67, 3, 111, 88, 123, 114, 58, 16, 122, 23, 90, 53, 36, 45, 120, 2, 74, 100, 44, 92, 113, 76, 32, 46, 31, 116, 83, 24, 102, 43, 47, 19, 26, 75, 59, 108, 103, 107, 21, 106, 63, 109, 6, 71, 101, 95, 91, 98, 34, 42, 39, 77, 18, 35, 33, 99, 7, 68, 38, 40, 79, 86, 65, 69, 4, 15, 82, 85, 13, 9, 66, 1, 64, 73], [51, 48, 41, 119, 125, 24, 30, 121, 52, 62, 57, 124, 115, 113, 54, 88, 55, 60, 53, 123, 49, 117, 110, 27, 118, 127, 56, 126, 97, 61, 112, 50, 114, 103, 120, 94, 122, 58, 36, 91, 116, 44, 111, 47, 106, 101, 59, 82, 90, 107, 109, 46, 63, 108, 43, 45, 42, 37, 39, 22, 104, 102, 15, 35, 93, 77, 40, 79, 89, 21, 18, 33, 38, 19, 28, 105, 9, 83, 26, 32, 84, 17, 100, 99, 98, 86, 13, 76, 5, 95, 92, 71, 73, 12, 65, 87, 34, 31, 29, 16, 96, 70, 7, 3, 66, 80, 23, 4, 11, 85, 1, 25, 81, 68, 6, 0, 2, 20, 14, 10, 64, 67, 75, 69, 74, 78, 8, 72], [119, 48, 41, 51, 125, 24, 121, 57, 62, 52, 55, 60, 30, 97, 54, 88, 113, 123, 53, 115, 56, 124, 49, 110, 117, 126, 127, 118, 114, 50, 61, 112, 116, 94, 120, 103, 58, 91, 122, 27, 36, 111, 44, 82, 47, 63, 109, 101, 59, 90, 107, 106, 46, 108, 37, 45, 89, 104, 43, 22, 39, 19, 79, 42, 102, 99, 98, 73, 28, 105, 38, 80, 35, 21, 34, 18, 93, 83, 40, 96, 100, 32, 15, 33, 86, 71, 77, 95, 65, 85, 13, 31, 26, 92, 29, 76, 16, 3, 84, 9, 23, 17, 87, 68, 66, 5, 81, 6, 7, 4, 75, 64, 0, 1, 12, 70, 2, 67, 25, 20, 11, 69, 10, 74, 14, 78, 8, 72], [106, 35, 42, 110, 91, 85, 50, 89, 52, 96, 83, 54, 113, 31, 51, 56, 122, 123, 114, 55, 14, 17, 32, 126, 57, 46, 117, 59, 53, 120, 62, 60, 27, 107, 75, 22, 9, 58, 29, 87, 119, 49, 70, 125, 15, 115, 16, 112, 109, 21, 61, 13, 48, 44, 63, 127, 39, 28, 68, 86, 23, 84, 41, 69, 19, 71, 45, 33, 92, 97, 25, 116, 111, 5, 20, 26, 34, 81, 121, 37, 79, 36, 38, 40, 124, 47, 94, 90, 11, 76, 82, 80, 18, 93, 105, 88, 43, 100, 104, 30, 108, 118, 103, 77, 24, 98, 102, 99, 95, 101, 78, 73, 8, 12, 72, 74, 10, 7, 65, 3, 6, 4, 66, 2, 0, 67, 1, 64], [106, 35, 42, 85, 31, 17, 83, 96, 89, 50, 52, 55, 51, 14, 9, 27, 113, 56, 69, 75, 62, 57, 110, 53, 93, 91, 122, 6, 117, 3, 66, 1, 61, 65, 127, 64, 87, 123, 32, 126, 7, 15, 59, 68, 13, 70, 114, 25, 23, 76, 84, 120, 0, 21, 63, 5, 22, 74, 10, 109, 54, 119, 71, 29, 11, 19, 16, 82, 72, 20, 8, 18, 4, 67, 78, 81, 77, 118, 41, 37, 73, 79, 88, 90, 38, 30, 121, 86, 115, 12, 48, 94, 98, 45, 43, 44, 26, 24, 33, 80, 39, 108, 100, 105, 99, 47, 36, 92, 28, 125, 107, 111, 102, 34, 49, 124, 116, 46, 2, 97, 60, 101, 112, 103, 104, 95, 58, 40], [106, 35, 42, 110, 91, 85, 50, 122, 83, 126, 56, 57, 17, 89, 117, 96, 113, 14, 54, 55, 31, 32, 59, 52, 114, 62, 127, 63, 61, 53, 22, 9, 116, 112, 75, 51, 94, 29, 27, 123, 48, 37, 65, 45, 99, 47, 25, 76, 7, 23, 58, 3, 21, 107, 69, 124, 46, 33, 66, 19, 115, 64, 93, 109, 41, 71, 120, 39, 118, 36, 84, 70, 119, 79, 97, 13, 105, 60, 92, 100, 87, 28, 81, 11, 5, 98, 12, 88, 111, 6, 44, 125, 86, 104, 102, 43, 108, 68, 49, 121, 26, 18, 103, 80, 20, 24, 90, 38, 82, 16, 30, 4, 95, 15, 40, 78, 101, 8, 73, 34, 74, 10, 77, 72, 67, 1, 0, 2], [106, 35, 42, 110, 56, 91, 85, 52, 53, 83, 122, 96, 57, 17, 114, 50, 113, 51, 32, 62, 31, 120, 89, 14, 55, 59, 117, 29, 112, 27, 22, 54, 46, 126, 49, 119, 75, 45, 99, 109, 94, 87, 123, 124, 115, 116, 61, 9, 118, 21, 92, 11, 43, 86, 60, 25, 37, 47, 39, 125, 127, 28, 58, 38, 34, 19, 23, 121, 108, 44, 33, 84, 107, 103, 41, 97, 111, 63, 88, 48, 40, 100, 30, 16, 102, 93, 104, 26, 101, 105, 81, 24, 90, 82, 98, 36, 15, 78, 20, 76, 18, 79, 95, 71, 4, 12, 13, 80, 74, 77, 68, 70, 73, 7, 3, 10, 8, 65, 69, 6, 72, 66, 67, 64, 0, 5, 1, 2], [61, 122, 118, 102, 49, 54, 109, 45, 59, 50, 116, 90, 103, 119, 63, 125, 101, 112, 108, 94, 55, 127, 56, 57, 113, 62, 111, 37, 124, 60, 38, 22, 110, 44, 121, 114, 123, 115, 47, 40, 117, 41, 42, 51, 48, 46, 96, 43, 120, 58, 53, 104, 93, 52, 107, 105, 106, 126, 36, 100, 39, 92, 31, 23, 11, 34, 86, 99, 97, 98, 32, 79, 30, 35, 33, 85, 81, 4, 83, 20, 88, 14, 18, 2, 26, 66, 95, 25, 76, 16, 15, 91, 73, 28, 29, 13, 3, 5, 27, 72, 10, 6, 82, 21, 24, 68, 89, 78, 80, 0, 70, 8, 71, 19, 84, 65, 75, 17, 74, 12, 87, 69, 67, 7, 1, 77, 9, 64], [102, 118, 54, 61, 116, 77, 9, 29, 7, 122, 23, 1, 64, 81, 69, 20, 4, 68, 49, 3, 15, 67, 82, 65, 26, 74, 45, 33, 66, 70, 90, 11, 113, 53, 0, 22, 75, 86, 19, 5, 112, 83, 73, 124, 71, 25, 87, 10, 96, 72, 18, 107, 119, 14, 36, 85, 24, 30, 78, 89, 6, 79, 91, 80, 13, 108, 59, 12, 94, 31, 17, 88, 27, 125, 93, 8, 16, 21, 120, 2, 84, 99, 63, 32, 76, 95, 110, 28, 55, 92, 56, 127, 98, 100, 101, 35, 57, 109, 46, 50, 39, 97, 47, 37, 34, 126, 42, 40, 104, 103, 38, 43, 111, 44, 60, 117, 123, 106, 58, 41, 114, 115, 48, 105, 121, 62, 52, 51], [102, 118, 54, 116, 122, 61, 45, 90, 49, 23, 112, 53, 33, 93, 113, 127, 15, 81, 38, 43, 107, 114, 57, 31, 44, 96, 125, 101, 20, 50, 119, 30, 24, 106, 83, 28, 26, 56, 59, 29, 124, 48, 55, 21, 39, 77, 11, 100, 60, 117, 6, 76, 22, 42, 67, 14, 51, 52, 126, 82, 37, 94, 111, 66, 92, 97, 47, 99, 108, 64, 58, 40, 5, 41, 19, 74, 46, 110, 63, 34, 98, 103, 4, 123, 62, 75, 36, 71, 105, 7, 86, 109, 32, 78, 120, 85, 89, 35, 121, 9, 12, 69, 80, 8, 84, 115, 1, 70, 18, 72, 95, 91, 17, 104, 25, 79, 88, 2, 27, 16, 3, 10, 73, 87, 68, 0, 13, 65], [102, 118, 116, 61, 122, 54, 49, 90, 23, 20, 45, 9, 92, 81, 74, 77, 29, 15, 22, 93, 33, 113, 107, 7, 124, 53, 11, 64, 26, 38, 112, 55, 6, 94, 82, 31, 21, 70, 79, 66, 100, 50, 28, 119, 72, 57, 4, 83, 44, 37, 41, 127, 86, 59, 101, 36, 89, 5, 3, 85, 58, 110, 121, 67, 126, 125, 106, 35, 43, 115, 75, 1, 103, 63, 108, 60, 117, 123, 68, 56, 114, 98, 39, 109, 69, 47, 19, 34, 104, 42, 51, 62, 48, 24, 97, 111, 96, 27, 120, 65, 105, 95, 30, 52, 84, 14, 2, 32, 88, 46, 99, 10, 16, 80, 78, 76, 87, 18, 12, 0, 8, 25, 40, 91, 17, 71, 73, 13]], "model.layers.27.self_attn.k_proj": [[45, 109, 83, 23, 21, 90, 94, 81, 76, 79, 33, 74, 30, 54, 97, 28, 125, 60, 6, 123, 48, 127, 52, 22, 51, 8, 0, 113, 49, 112, 32, 101, 47, 126, 73, 63, 106, 31, 122, 46, 118, 42, 103, 117, 124, 91, 24, 121, 96, 44, 18, 11, 35, 102, 50, 7, 37, 95, 108, 114, 13, 39, 4, 119, 43, 65, 105, 116, 120, 115, 16, 36, 57, 86, 89, 61, 104, 111, 53, 40, 56, 110, 34, 82, 55, 15, 29, 38, 107, 25, 26, 59, 9, 5, 62, 41, 92, 14, 58, 88, 80, 75, 99, 100, 27, 98, 78, 69, 93, 10, 84, 3, 20, 77, 87, 85, 17, 71, 72, 19, 66, 70, 68, 12, 1, 67, 2, 64], [53, 37, 22, 120, 63, 118, 97, 86, 58, 61, 60, 57, 116, 121, 125, 112, 113, 56, 55, 114, 26, 51, 119, 127, 62, 124, 52, 122, 50, 108, 110, 59, 117, 123, 48, 126, 95, 45, 115, 54, 49, 44, 93, 111, 109, 35, 47, 46, 42, 107, 96, 43, 41, 83, 28, 105, 40, 106, 29, 104, 15, 89, 38, 103, 102, 79, 100, 23, 101, 81, 32, 99, 82, 36, 39, 31, 12, 13, 34, 33, 18, 98, 72, 17, 91, 88, 92, 30, 78, 87, 27, 11, 25, 85, 94, 16, 20, 10, 9, 90, 6, 77, 7, 24, 84, 14, 68, 80, 21, 75, 5, 76, 19, 2, 69, 74, 3, 67, 71, 65, 73, 70, 0, 1, 4, 8, 64, 66], [104, 63, 34, 80, 13, 74, 23, 8, 85, 60, 19, 4, 64, 6, 95, 26, 66, 52, 121, 114, 106, 122, 105, 117, 49, 79, 54, 119, 65, 57, 110, 28, 7, 44, 127, 82, 124, 120, 43, 55, 56, 125, 1, 112, 107, 59, 45, 11, 98, 3, 116, 103, 46, 70, 41, 2, 9, 58, 126, 69, 39, 53, 5, 47, 84, 51, 18, 24, 71, 115, 0, 123, 42, 50, 108, 67, 48, 31, 14, 96, 12, 20, 62, 61, 100, 35, 27, 75, 68, 97, 22, 118, 30, 99, 102, 36, 29, 109, 111, 10, 25, 83, 37, 113, 76, 78, 17, 32, 101, 94, 72, 89, 38, 33, 88, 93, 90, 15, 92, 81, 86, 91, 73, 87, 77, 21, 16, 40], [40, 120, 34, 27, 31, 110, 126, 56, 84, 82, 88, 74, 46, 80, 52, 13, 125, 64, 48, 93, 67, 108, 6, 16, 58, 85, 49, 76, 65, 72, 114, 23, 44, 30, 106, 77, 55, 100, 45, 115, 66, 117, 68, 111, 60, 86, 107, 5, 20, 22, 69, 79, 8, 78, 75, 94, 54, 59, 53, 21, 91, 61, 24, 57, 63, 109, 102, 124, 96, 73, 62, 112, 42, 87, 90, 15, 81, 83, 28, 2, 116, 101, 122, 123, 105, 103, 12, 121, 113, 127, 70, 38, 7, 41, 92, 118, 3, 119, 29, 33, 43, 39, 97, 51, 50, 36, 26, 89, 32, 11, 25, 99, 17, 37, 18, 47, 35, 0, 14, 98, 95, 4, 19, 71, 10, 9, 1, 104], [47, 111, 58, 36, 31, 86, 24, 92, 56, 82, 0, 26, 17, 15, 6, 16, 121, 3, 83, 12, 54, 11, 105, 91, 113, 84, 116, 20, 66, 85, 45, 55, 19, 77, 63, 46, 51, 126, 57, 78, 43, 124, 25, 61, 119, 68, 49, 42, 38, 117, 48, 44, 104, 59, 118, 60, 89, 50, 114, 52, 112, 53, 103, 127, 123, 7, 115, 97, 122, 120, 65, 110, 62, 99, 9, 41, 1, 8, 5, 39, 40, 34, 29, 109, 107, 37, 10, 108, 94, 93, 125, 35, 106, 71, 32, 74, 101, 21, 102, 72, 76, 4, 33, 67, 90, 22, 87, 64, 79, 98, 81, 100, 96, 30, 23, 69, 27, 18, 80, 2, 13, 75, 28, 73, 88, 70, 14, 95], [105, 48, 22, 119, 33, 51, 62, 112, 54, 99, 49, 61, 94, 118, 125, 60, 55, 122, 50, 121, 127, 35, 45, 56, 124, 106, 117, 123, 126, 108, 40, 52, 120, 114, 53, 82, 59, 100, 111, 47, 63, 98, 58, 39, 24, 110, 57, 44, 28, 32, 113, 92, 46, 115, 43, 79, 109, 91, 107, 41, 38, 34, 101, 42, 26, 116, 19, 18, 29, 102, 103, 36, 9, 15, 104, 73, 85, 88, 96, 37, 97, 21, 80, 77, 13, 90, 23, 31, 89, 30, 93, 95, 87, 4, 81, 27, 25, 83, 20, 14, 11, 71, 76, 1, 86, 12, 16, 74, 66, 17, 7, 8, 78, 10, 5, 75, 68, 6, 72, 84, 70, 64, 67, 3, 69, 2, 65, 0], [42, 85, 83, 89, 17, 114, 126, 120, 55, 35, 46, 14, 9, 56, 61, 127, 32, 117, 99, 123, 109, 57, 49, 62, 3, 75, 106, 69, 95, 7, 45, 91, 122, 48, 115, 15, 119, 53, 50, 51, 31, 93, 110, 59, 52, 0, 1, 87, 116, 13, 64, 58, 6, 54, 29, 44, 60, 23, 84, 111, 43, 66, 121, 41, 40, 38, 22, 113, 112, 74, 97, 107, 5, 10, 125, 68, 30, 108, 8, 118, 124, 20, 104, 80, 71, 98, 101, 76, 100, 102, 47, 33, 63, 11, 25, 39, 88, 26, 92, 103, 82, 12, 36, 105, 34, 94, 90, 37, 96, 16, 28, 86, 27, 72, 24, 18, 77, 79, 70, 21, 65, 4, 78, 81, 19, 73, 2, 67], [118, 38, 61, 54, 116, 113, 64, 122, 65, 0, 109, 69, 7, 23, 77, 74, 90, 93, 3, 15, 94, 108, 81, 97, 9, 59, 2, 48, 6, 68, 52, 124, 53, 117, 112, 43, 82, 75, 57, 119, 103, 20, 30, 62, 50, 19, 111, 123, 21, 83, 106, 115, 110, 127, 56, 67, 107, 120, 60, 125, 11, 63, 47, 46, 105, 96, 102, 66, 49, 51, 121, 55, 104, 22, 114, 126, 100, 44, 41, 40, 58, 4, 84, 39, 36, 45, 35, 98, 76, 33, 89, 34, 1, 8, 42, 14, 37, 92, 101, 95, 27, 88, 5, 80, 85, 16, 99, 72, 28, 70, 31, 79, 12, 24, 25, 91, 32, 86, 10, 18, 71, 78, 87, 13, 73, 29, 17, 26]], "model.layers.27.self_attn.qk_proj": [[120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 106, 119, 51, 54, 58, 104, 60, 122, 56, 126, 40, 116, 31, 85, 94, 62, 52, 35, 55, 26, 83, 21, 91, 19, 125, 87, 121, 102, 110, 127, 95, 57, 117, 90, 23, 105, 46, 113, 49, 112, 108, 33, 80, 41, 77, 6, 16, 81, 34, 13, 98, 84, 88, 27, 50, 17, 10, 115, 86, 44, 38, 79, 59, 114, 24, 15, 20, 22, 37, 124, 36, 74, 123, 30, 64, 97, 100, 28, 0, 25, 18, 107, 93, 12, 76, 82, 101, 72, 43, 39, 11, 96, 92, 29, 89, 32, 9, 103, 69, 8, 99, 4, 65, 70, 73, 67, 14, 68, 3, 75, 66, 78, 2, 5, 71, 1, 7], [120, 63, 118, 45, 109, 47, 111, 42, 53, 48, 51, 119, 106, 61, 54, 122, 58, 104, 60, 126, 125, 56, 31, 116, 62, 40, 121, 35, 55, 52, 85, 102, 19, 94, 95, 21, 46, 26, 23, 127, 105, 110, 57, 113, 112, 90, 33, 6, 117, 98, 49, 91, 87, 77, 108, 41, 13, 124, 50, 83, 80, 44, 59, 17, 114, 86, 115, 22, 123, 24, 38, 27, 16, 36, 88, 84, 15, 10, 74, 81, 79, 37, 34, 64, 93, 28, 20, 100, 43, 30, 0, 82, 101, 97, 107, 8, 96, 12, 11, 92, 68, 76, 25, 18, 9, 32, 39, 103, 1, 72, 69, 65, 99, 29, 3, 73, 66, 2, 75, 78, 4, 89, 5, 67, 7, 71, 14, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 60, 56, 122, 104, 40, 116, 126, 62, 121, 125, 85, 31, 19, 105, 127, 94, 21, 46, 26, 83, 49, 110, 112, 52, 55, 35, 6, 117, 113, 57, 91, 102, 87, 95, 98, 23, 90, 80, 33, 17, 16, 108, 41, 123, 13, 27, 77, 43, 88, 24, 107, 115, 22, 38, 79, 37, 36, 34, 10, 59, 50, 44, 81, 84, 86, 64, 114, 93, 97, 30, 15, 124, 100, 0, 20, 8, 18, 74, 28, 92, 39, 101, 12, 82, 96, 11, 9, 68, 32, 4, 29, 76, 78, 99, 25, 65, 103, 1, 67, 73, 69, 14, 75, 89, 3, 66, 5, 72, 7, 2, 71, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 106, 119, 54, 58, 60, 104, 122, 62, 116, 56, 40, 126, 125, 127, 85, 121, 31, 55, 46, 19, 26, 110, 113, 52, 112, 35, 87, 105, 102, 83, 91, 21, 49, 6, 94, 57, 16, 95, 124, 41, 117, 90, 77, 59, 98, 123, 43, 86, 17, 23, 115, 108, 79, 33, 27, 13, 10, 37, 88, 22, 44, 80, 107, 24, 81, 84, 8, 34, 0, 114, 30, 93, 64, 74, 15, 97, 50, 36, 38, 101, 100, 18, 28, 69, 68, 82, 76, 92, 20, 12, 73, 65, 11, 67, 29, 9, 1, 103, 4, 3, 99, 2, 96, 39, 71, 32, 70, 5, 25, 78, 66, 14, 89, 75, 72, 7], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 104, 122, 60, 40, 126, 56, 116, 85, 125, 52, 112, 21, 35, 62, 46, 87, 83, 121, 127, 31, 105, 113, 90, 26, 110, 91, 49, 55, 57, 117, 19, 108, 98, 27, 16, 102, 13, 94, 23, 41, 95, 123, 6, 59, 17, 77, 86, 34, 33, 50, 124, 80, 74, 8, 79, 88, 44, 64, 22, 115, 84, 81, 15, 37, 93, 114, 107, 70, 20, 36, 24, 0, 10, 97, 18, 28, 82, 38, 43, 30, 73, 92, 12, 69, 11, 29, 25, 101, 100, 76, 68, 103, 96, 2, 39, 99, 9, 65, 67, 3, 1, 78, 5, 14, 72, 89, 4, 71, 7, 75, 32, 66], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 51, 119, 54, 61, 58, 104, 60, 122, 116, 56, 126, 40, 127, 21, 87, 125, 85, 62, 52, 83, 110, 113, 35, 90, 112, 94, 46, 49, 105, 33, 13, 102, 41, 31, 26, 91, 77, 117, 57, 98, 23, 108, 17, 19, 16, 124, 15, 121, 86, 59, 80, 34, 95, 79, 22, 55, 8, 44, 27, 70, 81, 115, 20, 88, 10, 84, 74, 64, 43, 37, 24, 0, 38, 123, 18, 114, 50, 30, 107, 6, 39, 28, 12, 97, 100, 36, 93, 82, 92, 96, 73, 3, 75, 4, 25, 9, 101, 5, 99, 76, 65, 29, 11, 66, 69, 103, 78, 68, 2, 1, 67, 71, 7, 14, 32, 72, 89], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 51, 61, 119, 54, 58, 104, 122, 116, 56, 40, 94, 126, 60, 90, 87, 46, 83, 35, 52, 21, 125, 127, 85, 113, 110, 23, 112, 105, 62, 31, 57, 49, 102, 19, 55, 91, 121, 98, 117, 41, 27, 80, 77, 22, 33, 16, 95, 26, 124, 13, 70, 88, 59, 15, 24, 34, 84, 115, 108, 17, 86, 81, 8, 10, 79, 20, 18, 114, 44, 50, 38, 74, 30, 36, 123, 107, 37, 82, 28, 97, 64, 25, 93, 0, 43, 100, 39, 12, 96, 11, 76, 101, 99, 92, 66, 73, 14, 103, 69, 9, 32, 4, 5, 75, 71, 29, 89, 78, 3, 1, 68, 72, 65, 7, 6, 2, 67], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 51, 106, 54, 119, 58, 122, 56, 60, 104, 126, 94, 40, 116, 35, 125, 52, 21, 105, 121, 85, 87, 31, 112, 110, 55, 83, 19, 113, 46, 102, 90, 62, 33, 95, 23, 98, 117, 57, 127, 49, 91, 27, 26, 70, 108, 80, 41, 22, 88, 115, 77, 13, 38, 16, 84, 36, 81, 79, 10, 24, 86, 59, 34, 114, 124, 28, 15, 17, 107, 74, 97, 44, 82, 20, 123, 30, 25, 0, 100, 50, 64, 43, 37, 93, 101, 8, 32, 12, 18, 99, 92, 76, 103, 96, 29, 72, 39, 66, 11, 75, 89, 9, 5, 4, 68, 14, 78, 73, 65, 3, 1, 67, 69, 7, 71, 2, 6], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 61, 51, 106, 119, 54, 58, 122, 104, 60, 56, 126, 116, 21, 94, 52, 121, 125, 40, 31, 117, 19, 85, 105, 110, 95, 35, 57, 62, 90, 87, 26, 113, 127, 112, 33, 23, 102, 46, 91, 83, 49, 16, 27, 59, 13, 55, 98, 80, 44, 114, 41, 77, 70, 108, 115, 22, 88, 50, 124, 17, 38, 24, 123, 34, 37, 10, 79, 84, 92, 86, 81, 43, 15, 36, 18, 97, 20, 74, 107, 30, 100, 28, 82, 64, 103, 93, 96, 0, 32, 101, 9, 76, 25, 89, 39, 12, 72, 99, 8, 75, 4, 73, 14, 29, 11, 68, 78, 5, 3, 2, 67, 7, 6, 69, 1, 65, 71, 66], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 51, 54, 106, 119, 58, 60, 122, 104, 125, 116, 56, 55, 40, 52, 46, 31, 121, 126, 85, 62, 35, 94, 90, 112, 83, 113, 127, 21, 110, 33, 95, 105, 117, 87, 26, 102, 49, 59, 124, 70, 19, 115, 23, 98, 41, 50, 91, 77, 80, 57, 108, 27, 0, 10, 74, 16, 114, 86, 123, 13, 43, 15, 34, 22, 17, 38, 30, 81, 44, 24, 64, 84, 100, 28, 36, 79, 20, 37, 107, 88, 72, 82, 92, 93, 18, 9, 97, 6, 32, 8, 65, 25, 39, 68, 76, 73, 5, 1, 67, 75, 96, 101, 69, 4, 2, 29, 103, 3, 12, 7, 66, 99, 11, 14, 78, 89, 71], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 61, 51, 119, 106, 54, 58, 104, 60, 122, 126, 40, 56, 116, 85, 125, 31, 87, 113, 117, 21, 94, 62, 55, 46, 105, 57, 127, 83, 26, 91, 35, 52, 90, 23, 112, 33, 80, 27, 95, 124, 19, 115, 110, 102, 13, 81, 121, 59, 77, 16, 74, 86, 49, 34, 6, 72, 79, 15, 98, 108, 114, 0, 22, 17, 41, 38, 24, 50, 10, 20, 43, 70, 44, 64, 88, 84, 30, 37, 18, 82, 36, 93, 28, 107, 123, 75, 100, 73, 12, 97, 68, 76, 92, 9, 3, 1, 101, 5, 29, 14, 99, 4, 67, 78, 8, 66, 11, 96, 32, 69, 2, 39, 89, 7, 103, 65, 71, 25], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 61, 119, 51, 54, 104, 58, 56, 60, 122, 116, 126, 40, 31, 87, 21, 105, 85, 83, 113, 49, 19, 127, 35, 46, 23, 94, 55, 102, 117, 90, 52, 121, 125, 110, 6, 33, 91, 115, 112, 95, 77, 41, 26, 62, 80, 57, 34, 86, 124, 13, 98, 16, 79, 27, 72, 17, 59, 88, 50, 44, 108, 15, 114, 81, 22, 84, 74, 24, 123, 38, 10, 20, 0, 28, 43, 97, 93, 64, 30, 18, 75, 37, 76, 107, 99, 25, 73, 82, 92, 14, 100, 36, 12, 67, 68, 96, 9, 29, 101, 39, 65, 11, 78, 103, 2, 32, 3, 7, 70, 8, 89, 69, 66, 4, 5, 71, 1], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 61, 106, 51, 119, 54, 58, 56, 104, 60, 122, 116, 40, 105, 126, 121, 35, 52, 49, 125, 94, 112, 102, 110, 87, 23, 21, 57, 85, 90, 83, 113, 46, 31, 33, 19, 62, 41, 117, 26, 91, 95, 98, 115, 80, 55, 108, 27, 86, 127, 6, 124, 24, 15, 77, 22, 34, 79, 59, 38, 13, 50, 74, 88, 72, 123, 44, 114, 84, 16, 30, 17, 10, 100, 37, 81, 18, 20, 36, 28, 93, 43, 92, 64, 82, 97, 25, 0, 39, 75, 101, 107, 89, 12, 32, 96, 9, 14, 29, 76, 103, 99, 73, 4, 5, 1, 2, 68, 69, 66, 3, 78, 11, 8, 7, 67, 71, 65, 70], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 119, 106, 61, 54, 51, 58, 56, 60, 104, 122, 40, 126, 116, 62, 94, 125, 52, 46, 49, 31, 121, 117, 127, 21, 105, 114, 57, 85, 110, 87, 83, 113, 90, 6, 91, 98, 35, 102, 112, 23, 13, 19, 27, 26, 55, 95, 124, 33, 41, 80, 86, 59, 44, 16, 72, 24, 115, 74, 50, 84, 34, 123, 17, 108, 10, 77, 38, 36, 79, 0, 22, 43, 15, 88, 64, 81, 107, 37, 93, 82, 20, 30, 100, 18, 28, 4, 75, 92, 97, 76, 101, 32, 9, 25, 96, 2, 12, 29, 78, 5, 3, 39, 1, 68, 73, 14, 69, 67, 99, 65, 89, 103, 7, 11, 71, 66, 8, 70], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 106, 61, 51, 54, 119, 58, 104, 60, 122, 116, 56, 126, 40, 94, 87, 52, 125, 21, 46, 31, 35, 62, 85, 83, 19, 105, 112, 121, 90, 113, 102, 127, 117, 26, 95, 57, 91, 55, 110, 27, 33, 124, 80, 49, 59, 114, 13, 81, 6, 41, 86, 15, 10, 108, 23, 50, 72, 34, 17, 64, 88, 16, 84, 98, 115, 44, 43, 36, 74, 77, 79, 123, 20, 100, 30, 37, 0, 38, 24, 73, 97, 75, 93, 12, 18, 22, 82, 32, 107, 28, 76, 39, 101, 92, 89, 69, 65, 68, 3, 5, 70, 4, 25, 9, 67, 1, 96, 2, 103, 29, 66, 78, 11, 14, 71, 99, 8, 7], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 106, 61, 119, 54, 58, 104, 56, 60, 122, 116, 126, 85, 87, 83, 62, 105, 40, 35, 21, 52, 102, 31, 125, 46, 94, 95, 113, 91, 19, 33, 55, 110, 57, 13, 23, 26, 121, 59, 112, 90, 127, 41, 80, 117, 108, 115, 34, 81, 124, 49, 16, 77, 27, 38, 79, 17, 15, 44, 123, 10, 86, 37, 22, 114, 50, 6, 43, 88, 84, 74, 98, 24, 72, 0, 36, 93, 20, 12, 75, 76, 92, 101, 82, 100, 28, 70, 30, 73, 18, 64, 9, 4, 97, 99, 107, 8, 1, 5, 96, 14, 68, 11, 39, 25, 3, 2, 32, 89, 69, 29, 78, 71, 66, 7, 103, 67, 65], [120, 118, 63, 45, 109, 47, 53, 42, 111, 48, 106, 51, 61, 54, 119, 58, 104, 122, 56, 60, 52, 116, 121, 46, 94, 126, 35, 21, 40, 95, 83, 102, 31, 49, 85, 23, 90, 110, 125, 62, 113, 41, 33, 55, 112, 57, 87, 19, 105, 26, 91, 98, 13, 80, 127, 86, 108, 115, 59, 124, 34, 27, 15, 16, 88, 117, 123, 24, 92, 44, 28, 10, 22, 17, 84, 74, 114, 70, 50, 36, 77, 81, 79, 30, 97, 20, 38, 0, 100, 93, 37, 82, 18, 64, 43, 101, 107, 72, 8, 12, 103, 9, 25, 99, 96, 32, 39, 29, 76, 69, 1, 4, 89, 66, 73, 6, 68, 78, 75, 14, 67, 11, 3, 7, 65, 2, 71, 5], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 61, 119, 106, 54, 58, 122, 104, 60, 56, 116, 94, 121, 40, 52, 62, 126, 85, 31, 35, 90, 21, 105, 19, 125, 49, 87, 110, 95, 102, 46, 83, 55, 23, 41, 112, 91, 59, 117, 26, 108, 113, 98, 57, 33, 27, 70, 24, 13, 127, 81, 114, 17, 44, 74, 77, 22, 16, 86, 15, 36, 79, 80, 43, 34, 124, 88, 50, 37, 38, 115, 100, 10, 84, 92, 123, 97, 28, 30, 0, 64, 20, 8, 12, 32, 101, 18, 107, 82, 96, 72, 9, 75, 1, 103, 25, 69, 76, 29, 93, 4, 78, 89, 99, 68, 67, 39, 11, 66, 7, 73, 14, 2, 5, 3, 6, 71, 65], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 119, 51, 61, 54, 58, 104, 60, 122, 116, 40, 121, 52, 56, 125, 62, 126, 46, 35, 85, 19, 49, 110, 21, 105, 102, 90, 94, 31, 108, 95, 70, 87, 55, 117, 127, 124, 112, 59, 26, 113, 91, 33, 23, 57, 83, 41, 37, 123, 114, 98, 13, 115, 79, 50, 44, 38, 80, 16, 8, 81, 27, 10, 36, 17, 15, 34, 24, 88, 43, 84, 64, 0, 77, 22, 30, 86, 107, 74, 1, 20, 97, 100, 82, 92, 101, 75, 93, 28, 9, 99, 18, 32, 69, 12, 103, 96, 25, 2, 73, 4, 78, 76, 5, 39, 72, 89, 29, 67, 68, 11, 3, 65, 66, 7, 71, 6, 14], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 106, 61, 119, 54, 58, 104, 60, 122, 116, 40, 52, 62, 35, 56, 113, 126, 85, 55, 46, 125, 94, 110, 102, 90, 21, 26, 121, 105, 31, 23, 70, 57, 83, 127, 41, 117, 33, 19, 95, 87, 59, 108, 112, 49, 124, 13, 91, 86, 16, 27, 123, 50, 8, 98, 15, 37, 77, 114, 80, 74, 44, 10, 17, 38, 24, 34, 115, 81, 88, 43, 36, 79, 64, 22, 30, 107, 84, 100, 97, 82, 0, 93, 73, 28, 20, 92, 32, 101, 12, 1, 68, 18, 66, 96, 4, 75, 76, 69, 103, 39, 3, 5, 9, 25, 67, 99, 14, 78, 29, 65, 71, 2, 72, 89, 7, 11, 6], [120, 63, 118, 45, 109, 47, 111, 42, 53, 48, 106, 61, 51, 54, 58, 119, 104, 60, 116, 122, 40, 56, 126, 85, 55, 125, 105, 52, 113, 94, 62, 46, 121, 21, 35, 83, 110, 90, 87, 127, 117, 31, 19, 57, 102, 95, 41, 49, 115, 26, 124, 16, 91, 23, 112, 77, 33, 108, 98, 13, 34, 27, 59, 81, 86, 15, 70, 123, 38, 74, 44, 8, 22, 80, 50, 43, 88, 10, 79, 17, 36, 84, 30, 114, 20, 92, 24, 100, 28, 37, 0, 75, 107, 99, 82, 25, 97, 39, 18, 101, 93, 12, 6, 64, 73, 32, 9, 76, 14, 68, 4, 5, 67, 1, 78, 66, 103, 3, 69, 96, 72, 11, 65, 71, 7, 89, 29, 2], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 106, 54, 119, 58, 122, 60, 104, 116, 121, 40, 126, 125, 56, 62, 110, 52, 46, 85, 113, 55, 21, 94, 127, 35, 105, 102, 112, 19, 117, 31, 124, 59, 87, 83, 26, 49, 90, 57, 95, 41, 43, 108, 44, 114, 23, 98, 91, 13, 38, 33, 123, 0, 27, 88, 77, 16, 80, 6, 34, 8, 10, 64, 86, 37, 15, 115, 36, 24, 74, 81, 100, 79, 50, 17, 107, 22, 28, 30, 93, 97, 1, 84, 32, 82, 20, 92, 101, 5, 70, 39, 18, 73, 68, 4, 66, 12, 9, 69, 76, 67, 2, 65, 103, 75, 11, 14, 96, 29, 99, 25, 71, 3, 89, 78, 7, 72], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 106, 54, 51, 119, 58, 122, 56, 104, 60, 116, 40, 52, 117, 85, 126, 125, 105, 46, 35, 62, 121, 31, 94, 95, 21, 83, 87, 55, 102, 59, 49, 127, 57, 90, 113, 26, 110, 124, 112, 19, 6, 41, 114, 91, 27, 23, 33, 108, 98, 34, 44, 123, 16, 80, 77, 115, 88, 13, 15, 38, 10, 22, 36, 24, 28, 81, 84, 79, 8, 74, 50, 100, 37, 97, 86, 107, 30, 43, 0, 17, 93, 18, 92, 39, 82, 20, 101, 25, 64, 9, 4, 68, 103, 96, 76, 75, 1, 89, 99, 32, 12, 11, 69, 73, 78, 2, 3, 5, 67, 29, 71, 72, 66, 7, 65, 14, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 106, 61, 119, 54, 58, 104, 122, 60, 56, 116, 40, 126, 52, 21, 105, 94, 31, 85, 125, 113, 121, 35, 46, 87, 90, 83, 95, 110, 49, 57, 6, 55, 59, 62, 26, 102, 127, 41, 112, 33, 117, 124, 19, 23, 44, 108, 91, 114, 27, 86, 80, 77, 98, 22, 34, 16, 13, 88, 10, 81, 107, 15, 123, 17, 115, 74, 43, 8, 79, 37, 38, 84, 50, 30, 36, 20, 97, 18, 92, 100, 0, 93, 24, 82, 28, 64, 4, 76, 101, 12, 9, 73, 75, 96, 25, 5, 39, 11, 78, 68, 3, 14, 2, 103, 69, 66, 29, 32, 67, 72, 1, 65, 99, 7, 89, 71, 70], [120, 118, 63, 45, 109, 111, 47, 42, 53, 48, 51, 106, 54, 61, 119, 122, 104, 58, 60, 116, 40, 126, 56, 125, 52, 21, 83, 87, 90, 85, 94, 62, 105, 121, 35, 46, 124, 55, 6, 117, 27, 33, 113, 31, 49, 26, 102, 127, 95, 91, 110, 115, 19, 77, 112, 23, 80, 13, 86, 108, 41, 17, 59, 123, 57, 98, 16, 22, 88, 34, 79, 10, 15, 38, 84, 24, 44, 43, 74, 30, 81, 114, 50, 20, 8, 36, 64, 107, 18, 37, 28, 82, 93, 97, 0, 100, 25, 9, 92, 72, 101, 12, 99, 73, 75, 103, 78, 96, 32, 1, 76, 5, 66, 68, 69, 4, 11, 29, 3, 7, 14, 89, 65, 2, 39, 67, 71, 70], [120, 118, 63, 45, 109, 47, 111, 53, 42, 48, 51, 61, 106, 54, 119, 60, 122, 58, 104, 56, 116, 125, 21, 35, 40, 121, 52, 126, 87, 31, 62, 83, 94, 95, 46, 105, 55, 57, 85, 33, 113, 102, 117, 127, 26, 115, 49, 91, 23, 59, 110, 19, 90, 108, 123, 41, 124, 77, 114, 27, 13, 98, 22, 50, 44, 36, 112, 80, 38, 34, 88, 86, 24, 6, 16, 10, 15, 81, 84, 43, 30, 79, 37, 28, 17, 93, 0, 92, 74, 97, 100, 107, 18, 72, 32, 82, 101, 64, 20, 103, 76, 96, 25, 1, 29, 73, 8, 12, 9, 39, 68, 78, 75, 89, 11, 99, 14, 3, 69, 66, 4, 5, 67, 7, 70, 65, 71, 2], [120, 118, 63, 45, 47, 109, 42, 111, 53, 48, 54, 106, 61, 119, 51, 104, 122, 56, 58, 60, 126, 116, 31, 40, 94, 125, 35, 62, 117, 52, 21, 87, 121, 105, 127, 55, 83, 46, 85, 95, 90, 91, 115, 33, 102, 19, 113, 23, 41, 112, 57, 27, 26, 114, 110, 108, 13, 44, 77, 49, 124, 22, 59, 98, 80, 34, 16, 36, 81, 88, 123, 17, 72, 24, 38, 74, 37, 15, 86, 107, 100, 50, 10, 18, 79, 84, 43, 97, 92, 20, 25, 82, 30, 28, 101, 93, 64, 70, 6, 103, 96, 39, 12, 68, 0, 73, 29, 32, 4, 11, 99, 76, 69, 9, 1, 75, 3, 14, 67, 66, 78, 8, 89, 65, 7, 2, 71, 5], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 104, 60, 122, 116, 126, 56, 40, 52, 62, 21, 94, 125, 35, 46, 112, 127, 31, 85, 90, 87, 121, 102, 19, 105, 95, 108, 110, 55, 41, 83, 26, 23, 49, 117, 113, 33, 59, 70, 91, 123, 27, 13, 57, 72, 16, 98, 77, 124, 115, 44, 43, 79, 17, 34, 88, 80, 37, 50, 15, 86, 36, 22, 114, 74, 81, 10, 64, 84, 100, 0, 38, 30, 24, 20, 18, 39, 107, 92, 82, 28, 4, 93, 5, 66, 101, 97, 73, 76, 9, 1, 11, 32, 12, 25, 6, 69, 89, 65, 68, 75, 29, 67, 3, 7, 2, 78, 96, 14, 103, 99, 8, 71], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 106, 61, 51, 54, 119, 58, 122, 104, 60, 56, 116, 125, 40, 126, 46, 85, 55, 52, 62, 121, 35, 70, 49, 90, 31, 87, 117, 19, 105, 26, 83, 21, 57, 94, 112, 113, 110, 102, 95, 124, 91, 27, 108, 41, 17, 115, 50, 77, 33, 13, 64, 72, 34, 127, 23, 59, 16, 81, 44, 80, 86, 10, 43, 123, 38, 36, 84, 98, 0, 79, 15, 100, 74, 88, 22, 1, 37, 20, 28, 24, 107, 92, 9, 30, 93, 114, 82, 76, 101, 97, 11, 2, 5, 73, 25, 12, 18, 29, 96, 3, 67, 66, 39, 4, 103, 75, 68, 99, 69, 78, 32, 89, 14, 7, 65, 71, 8, 6], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 106, 61, 54, 119, 58, 60, 122, 104, 56, 125, 116, 126, 52, 94, 40, 85, 62, 90, 87, 21, 55, 46, 35, 117, 83, 31, 105, 49, 102, 70, 110, 113, 23, 57, 26, 123, 19, 121, 127, 91, 59, 77, 95, 33, 114, 112, 13, 41, 108, 27, 16, 34, 80, 124, 22, 81, 86, 44, 15, 10, 72, 98, 17, 84, 43, 74, 115, 38, 37, 36, 79, 88, 24, 50, 20, 100, 18, 107, 64, 97, 92, 82, 28, 101, 93, 76, 96, 0, 73, 30, 11, 25, 9, 4, 68, 12, 103, 39, 32, 5, 69, 99, 14, 89, 65, 3, 1, 78, 75, 67, 66, 29, 7, 71, 8, 2, 6], [120, 63, 118, 45, 109, 47, 42, 53, 111, 48, 51, 61, 106, 54, 119, 58, 60, 104, 56, 122, 125, 116, 126, 40, 94, 52, 62, 35, 121, 57, 85, 87, 105, 83, 46, 21, 110, 19, 26, 90, 31, 55, 113, 98, 49, 70, 102, 117, 91, 127, 95, 41, 112, 23, 33, 59, 124, 115, 123, 77, 86, 16, 72, 34, 27, 88, 79, 15, 44, 10, 81, 80, 74, 13, 108, 64, 114, 84, 17, 43, 22, 24, 0, 28, 38, 50, 36, 30, 20, 97, 92, 100, 82, 37, 93, 68, 107, 101, 4, 25, 18, 5, 76, 9, 39, 69, 73, 66, 67, 11, 78, 1, 32, 65, 96, 3, 14, 12, 71, 2, 8, 103, 75, 29, 99, 7, 89, 6], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 61, 51, 54, 119, 58, 122, 60, 104, 56, 126, 40, 125, 116, 52, 85, 31, 35, 94, 121, 87, 83, 19, 90, 21, 117, 55, 62, 105, 57, 108, 26, 127, 95, 46, 110, 77, 102, 33, 112, 86, 113, 49, 23, 41, 91, 98, 27, 124, 13, 80, 16, 79, 38, 22, 44, 114, 115, 34, 15, 17, 74, 50, 10, 88, 59, 70, 81, 72, 36, 100, 24, 30, 25, 84, 20, 82, 123, 76, 97, 37, 92, 43, 93, 28, 18, 64, 11, 0, 12, 73, 107, 99, 96, 6, 101, 39, 32, 78, 4, 14, 9, 67, 69, 103, 66, 29, 89, 68, 8, 5, 65, 1, 75, 3, 71, 7, 2]], "model.layers.28.self_attn.q_proj": [[40, 120, 48, 34, 27, 51, 23, 85, 116, 126, 89, 53, 54, 16, 17, 112, 121, 3, 62, 31, 125, 124, 61, 93, 123, 50, 52, 109, 58, 114, 30, 49, 122, 19, 45, 110, 42, 7, 78, 29, 57, 82, 119, 60, 8, 115, 113, 59, 47, 55, 111, 63, 107, 117, 5, 118, 12, 56, 108, 46, 43, 79, 127, 74, 14, 44, 105, 88, 71, 100, 21, 20, 41, 65, 37, 64, 106, 11, 39, 1, 101, 38, 36, 87, 35, 91, 102, 0, 103, 22, 97, 66, 95, 6, 96, 10, 99, 98, 104, 84, 26, 32, 94, 2, 4, 90, 68, 73, 24, 77, 15, 13, 33, 9, 25, 28, 69, 18, 67, 83, 92, 80, 86, 72, 81, 75, 70, 76], [40, 120, 34, 48, 23, 27, 85, 17, 16, 31, 51, 11, 25, 13, 97, 61, 29, 62, 100, 55, 4, 78, 126, 19, 87, 56, 58, 73, 70, 59, 93, 28, 36, 35, 21, 113, 79, 46, 68, 22, 53, 18, 32, 96, 66, 109, 92, 10, 123, 95, 84, 8, 60, 63, 127, 0, 106, 49, 104, 37, 12, 112, 114, 116, 94, 26, 119, 99, 124, 7, 101, 30, 117, 83, 76, 103, 2, 118, 20, 44, 89, 52, 54, 125, 108, 81, 38, 33, 77, 110, 107, 15, 39, 102, 121, 111, 43, 122, 80, 86, 91, 47, 14, 57, 24, 90, 41, 82, 105, 50, 98, 75, 45, 74, 115, 5, 6, 88, 42, 67, 9, 72, 69, 71, 1, 65, 3, 64], [120, 40, 34, 48, 62, 23, 51, 27, 126, 85, 31, 93, 117, 109, 112, 58, 56, 113, 107, 16, 61, 50, 55, 44, 89, 123, 17, 19, 49, 53, 47, 8, 0, 30, 116, 106, 21, 95, 42, 52, 46, 25, 79, 43, 29, 108, 100, 121, 111, 119, 68, 110, 11, 57, 73, 38, 98, 54, 122, 59, 96, 36, 124, 118, 99, 63, 5, 103, 12, 105, 41, 78, 127, 125, 115, 94, 114, 97, 45, 35, 87, 37, 33, 60, 32, 39, 102, 22, 66, 101, 14, 24, 2, 70, 13, 10, 26, 86, 28, 91, 88, 92, 65, 18, 82, 67, 90, 6, 83, 84, 77, 4, 20, 64, 71, 75, 81, 104, 7, 1, 80, 69, 3, 15, 76, 9, 72, 74], [40, 120, 48, 34, 23, 63, 51, 31, 27, 62, 85, 49, 13, 123, 61, 108, 56, 79, 1, 110, 53, 55, 67, 93, 10, 64, 107, 16, 74, 113, 30, 19, 2, 17, 117, 70, 124, 18, 100, 59, 115, 127, 45, 68, 38, 26, 103, 102, 105, 111, 58, 46, 122, 119, 109, 52, 37, 116, 96, 36, 54, 24, 78, 106, 121, 112, 60, 50, 6, 84, 11, 125, 29, 114, 21, 118, 126, 0, 101, 57, 94, 43, 12, 73, 89, 88, 97, 47, 77, 39, 22, 35, 98, 95, 41, 5, 99, 33, 65, 44, 28, 32, 42, 86, 87, 92, 90, 8, 14, 72, 83, 25, 9, 82, 20, 69, 91, 3, 76, 7, 75, 80, 81, 15, 4, 71, 104, 66], [40, 125, 87, 17, 34, 21, 82, 70, 12, 74, 14, 4, 44, 0, 8, 124, 113, 111, 66, 54, 79, 58, 127, 68, 52, 109, 73, 110, 53, 19, 28, 75, 16, 33, 35, 123, 22, 59, 104, 27, 1, 95, 115, 119, 69, 86, 25, 114, 42, 50, 89, 18, 93, 108, 11, 26, 71, 96, 13, 90, 77, 56, 116, 67, 84, 63, 55, 5, 64, 62, 81, 78, 6, 92, 85, 106, 51, 7, 45, 39, 101, 23, 57, 103, 122, 98, 49, 94, 91, 88, 99, 32, 107, 48, 9, 2, 100, 29, 83, 105, 112, 65, 60, 24, 15, 76, 97, 3, 61, 47, 72, 38, 80, 36, 37, 121, 126, 46, 30, 20, 10, 31, 102, 118, 120, 41, 117, 43], [125, 40, 3, 87, 34, 12, 17, 14, 70, 0, 74, 1, 67, 66, 82, 21, 58, 64, 65, 8, 4, 69, 124, 54, 113, 25, 44, 111, 106, 52, 71, 49, 127, 2, 28, 75, 104, 109, 110, 50, 11, 42, 114, 22, 5, 79, 43, 26, 23, 121, 37, 68, 123, 101, 53, 86, 96, 31, 98, 112, 90, 27, 95, 77, 78, 20, 59, 13, 35, 122, 102, 81, 16, 29, 119, 6, 19, 91, 45, 39, 80, 47, 24, 32, 73, 117, 9, 76, 118, 103, 7, 88, 93, 84, 46, 72, 18, 33, 116, 55, 41, 30, 126, 105, 36, 115, 89, 62, 15, 120, 60, 63, 10, 57, 85, 48, 100, 38, 92, 107, 108, 94, 56, 83, 51, 97, 61, 99], [125, 40, 87, 34, 82, 21, 124, 12, 17, 14, 74, 109, 103, 50, 19, 4, 44, 8, 71, 95, 89, 92, 54, 70, 123, 127, 90, 9, 111, 68, 93, 108, 5, 39, 28, 115, 43, 41, 25, 77, 29, 110, 42, 105, 27, 61, 59, 100, 119, 58, 116, 64, 56, 66, 22, 79, 114, 88, 113, 47, 73, 121, 45, 91, 20, 60, 62, 106, 18, 75, 37, 49, 15, 10, 57, 52, 122, 31, 63, 51, 23, 48, 38, 33, 26, 96, 94, 6, 53, 32, 117, 86, 120, 112, 102, 36, 80, 97, 101, 72, 118, 83, 35, 69, 126, 99, 107, 46, 30, 85, 24, 55, 81, 13, 84, 78, 2, 98, 16, 7, 76, 11, 0, 1, 3, 67, 65, 104], [40, 125, 87, 0, 17, 34, 21, 12, 82, 1, 14, 66, 124, 4, 70, 50, 74, 44, 75, 58, 26, 127, 54, 69, 110, 113, 28, 111, 8, 95, 52, 13, 88, 33, 45, 79, 93, 42, 114, 64, 22, 103, 32, 98, 109, 67, 20, 123, 73, 119, 35, 11, 92, 19, 49, 104, 15, 39, 30, 27, 18, 90, 25, 59, 68, 96, 71, 102, 107, 53, 72, 101, 112, 62, 2, 65, 91, 60, 16, 77, 61, 80, 106, 99, 121, 115, 43, 7, 23, 105, 81, 84, 78, 55, 97, 3, 118, 76, 51, 122, 94, 83, 29, 89, 48, 117, 41, 86, 63, 9, 56, 5, 46, 108, 120, 31, 126, 24, 6, 116, 57, 85, 47, 100, 38, 10, 36, 37], [42, 36, 30, 43, 56, 106, 20, 118, 87, 77, 35, 82, 91, 123, 94, 46, 52, 80, 117, 49, 53, 89, 104, 120, 27, 99, 112, 17, 39, 47, 23, 122, 51, 78, 85, 54, 15, 11, 60, 19, 114, 103, 124, 71, 105, 125, 34, 58, 57, 95, 107, 108, 126, 110, 61, 62, 22, 74, 50, 32, 84, 109, 9, 121, 44, 88, 29, 13, 116, 101, 18, 55, 31, 59, 92, 115, 86, 45, 113, 98, 90, 111, 102, 41, 63, 48, 26, 12, 21, 25, 97, 119, 8, 24, 38, 127, 16, 37, 81, 83, 75, 14, 100, 33, 7, 96, 40, 28, 93, 72, 70, 69, 5, 76, 73, 79, 3, 67, 66, 2, 10, 68, 6, 1, 4, 65, 0, 64], [42, 36, 114, 118, 30, 15, 74, 35, 20, 17, 68, 87, 106, 77, 82, 70, 4, 91, 56, 76, 71, 0, 66, 123, 95, 64, 50, 31, 94, 43, 52, 21, 27, 23, 62, 7, 53, 11, 10, 122, 98, 9, 12, 29, 5, 67, 102, 84, 19, 18, 28, 111, 13, 65, 88, 1, 37, 85, 6, 8, 120, 2, 107, 32, 49, 112, 57, 103, 81, 83, 93, 14, 79, 69, 59, 96, 124, 46, 24, 44, 97, 26, 54, 126, 47, 109, 25, 16, 45, 73, 3, 60, 38, 72, 104, 86, 80, 78, 34, 108, 115, 41, 101, 58, 39, 22, 75, 61, 105, 92, 89, 48, 90, 51, 117, 33, 121, 119, 125, 110, 116, 63, 100, 113, 55, 127, 40, 99], [42, 36, 114, 118, 30, 87, 15, 123, 17, 35, 89, 74, 106, 20, 82, 94, 70, 56, 77, 50, 19, 66, 104, 52, 27, 68, 91, 8, 43, 6, 54, 72, 29, 44, 120, 58, 95, 75, 11, 47, 1, 38, 23, 51, 59, 122, 112, 113, 57, 49, 124, 61, 32, 86, 53, 62, 125, 99, 93, 111, 3, 116, 92, 102, 98, 12, 24, 103, 34, 71, 48, 28, 78, 64, 63, 76, 40, 127, 107, 108, 97, 46, 83, 119, 126, 96, 60, 110, 84, 10, 115, 81, 45, 25, 117, 37, 101, 0, 105, 21, 121, 2, 33, 31, 14, 109, 80, 55, 18, 26, 41, 9, 39, 85, 13, 16, 79, 5, 22, 90, 67, 88, 69, 100, 73, 7, 4, 65], [42, 114, 36, 35, 30, 106, 91, 103, 20, 87, 50, 27, 15, 17, 43, 74, 82, 94, 89, 68, 71, 123, 56, 104, 77, 95, 70, 120, 113, 85, 105, 65, 29, 62, 46, 3, 57, 127, 118, 61, 80, 119, 2, 125, 76, 122, 48, 117, 90, 1, 126, 47, 45, 111, 52, 44, 11, 51, 55, 109, 12, 49, 54, 121, 38, 75, 124, 63, 8, 102, 112, 67, 78, 107, 58, 66, 31, 59, 110, 115, 108, 116, 7, 72, 69, 81, 101, 41, 24, 86, 53, 33, 0, 73, 5, 26, 39, 28, 37, 19, 60, 13, 23, 21, 98, 84, 34, 93, 99, 14, 64, 92, 16, 9, 83, 96, 40, 25, 97, 88, 32, 22, 10, 4, 18, 79, 6, 100], [126, 41, 98, 56, 115, 89, 84, 13, 111, 80, 18, 112, 28, 45, 86, 75, 74, 31, 50, 21, 96, 82, 54, 58, 79, 5, 73, 8, 29, 81, 55, 11, 14, 70, 63, 120, 68, 4, 3, 25, 106, 119, 127, 117, 77, 40, 60, 71, 122, 92, 48, 10, 95, 88, 66, 46, 57, 59, 6, 76, 78, 83, 108, 34, 2, 93, 43, 51, 72, 16, 22, 0, 19, 20, 35, 124, 23, 44, 102, 121, 125, 62, 87, 107, 1, 113, 100, 61, 52, 24, 27, 85, 12, 90, 26, 17, 123, 49, 47, 105, 9, 114, 118, 32, 116, 103, 36, 7, 53, 99, 91, 110, 67, 109, 104, 33, 38, 94, 30, 97, 37, 39, 101, 65, 42, 64, 15, 69], [56, 41, 58, 98, 87, 63, 31, 32, 126, 28, 54, 122, 115, 44, 57, 17, 53, 60, 121, 46, 112, 43, 103, 55, 124, 116, 108, 127, 50, 120, 51, 26, 47, 117, 111, 86, 48, 113, 62, 52, 110, 61, 118, 119, 49, 45, 92, 114, 123, 109, 59, 125, 89, 101, 105, 40, 95, 107, 42, 23, 88, 22, 39, 91, 24, 100, 106, 84, 25, 94, 34, 30, 104, 37, 82, 15, 102, 38, 97, 99, 36, 35, 90, 21, 93, 80, 96, 33, 83, 29, 18, 20, 81, 79, 12, 27, 78, 19, 10, 85, 77, 1, 76, 14, 13, 7, 72, 66, 0, 5, 67, 3, 65, 16, 2, 64, 69, 11, 75, 71, 4, 8, 73, 70, 9, 74, 6, 68], [56, 41, 126, 98, 89, 79, 84, 18, 115, 80, 15, 31, 21, 28, 72, 13, 58, 10, 11, 63, 86, 82, 46, 4, 57, 40, 14, 73, 112, 70, 92, 77, 111, 100, 25, 23, 32, 122, 3, 121, 113, 60, 43, 127, 119, 75, 81, 16, 45, 87, 125, 95, 53, 66, 29, 62, 55, 109, 96, 51, 19, 124, 69, 5, 50, 52, 120, 9, 49, 71, 123, 1, 0, 48, 59, 22, 20, 47, 33, 76, 116, 8, 85, 6, 44, 27, 117, 103, 93, 17, 30, 61, 78, 37, 110, 64, 114, 88, 118, 24, 35, 2, 12, 91, 94, 83, 39, 90, 104, 54, 105, 99, 26, 102, 108, 68, 101, 7, 65, 97, 106, 36, 107, 42, 38, 74, 34, 67], [41, 126, 56, 63, 105, 60, 31, 127, 122, 87, 51, 52, 115, 98, 47, 49, 62, 121, 57, 123, 112, 124, 58, 46, 28, 117, 103, 114, 59, 116, 118, 45, 109, 86, 119, 54, 61, 113, 48, 55, 111, 120, 92, 50, 32, 53, 125, 44, 110, 108, 89, 84, 40, 43, 42, 107, 18, 39, 106, 104, 26, 100, 101, 37, 38, 102, 23, 7, 36, 35, 94, 96, 34, 91, 95, 99, 97, 17, 33, 80, 30, 25, 21, 85, 82, 88, 20, 22, 93, 24, 29, 27, 90, 11, 19, 83, 78, 77, 5, 73, 14, 81, 12, 3, 70, 0, 8, 71, 76, 79, 1, 16, 65, 10, 13, 67, 66, 64, 2, 74, 69, 9, 4, 72, 68, 75, 6, 15], [123, 103, 55, 47, 34, 125, 27, 95, 88, 32, 110, 122, 21, 15, 76, 19, 127, 24, 53, 97, 91, 94, 18, 36, 71, 51, 59, 43, 104, 38, 40, 101, 44, 118, 74, 106, 42, 26, 58, 102, 57, 52, 90, 117, 9, 92, 115, 111, 62, 2, 124, 96, 10, 87, 29, 48, 116, 33, 16, 126, 22, 78, 120, 112, 105, 85, 60, 63, 46, 107, 100, 114, 41, 113, 25, 1, 45, 54, 121, 37, 99, 23, 14, 31, 98, 49, 108, 56, 93, 39, 89, 50, 61, 109, 5, 79, 70, 119, 68, 13, 35, 28, 84, 82, 83, 86, 81, 30, 20, 80, 66, 64, 17, 11, 77, 12, 8, 73, 75, 3, 69, 4, 72, 7, 6, 65, 0, 67], [123, 103, 55, 34, 116, 27, 76, 88, 70, 0, 3, 18, 15, 1, 19, 68, 122, 66, 21, 95, 99, 71, 94, 91, 22, 59, 51, 9, 62, 125, 74, 56, 30, 73, 77, 67, 33, 126, 10, 47, 31, 37, 96, 109, 36, 57, 6, 79, 8, 107, 48, 80, 110, 29, 49, 35, 45, 118, 115, 58, 50, 63, 44, 121, 65, 86, 24, 26, 60, 17, 82, 85, 92, 23, 119, 40, 20, 52, 104, 13, 98, 112, 5, 120, 28, 113, 117, 114, 69, 72, 11, 102, 124, 81, 78, 14, 38, 53, 46, 43, 93, 61, 39, 87, 16, 108, 105, 75, 111, 42, 54, 106, 41, 25, 84, 32, 90, 97, 83, 101, 127, 4, 89, 100, 7, 12, 2, 64], [123, 103, 58, 122, 27, 88, 21, 91, 47, 18, 15, 59, 38, 34, 62, 9, 5, 95, 48, 120, 55, 49, 125, 60, 110, 44, 53, 121, 50, 56, 117, 119, 30, 105, 57, 52, 63, 24, 124, 67, 40, 113, 126, 94, 109, 112, 96, 51, 54, 118, 97, 46, 45, 116, 107, 111, 76, 108, 114, 85, 42, 115, 100, 43, 106, 104, 61, 36, 127, 101, 99, 41, 75, 29, 0, 92, 71, 86, 33, 102, 2, 64, 4, 37, 14, 39, 35, 17, 81, 3, 98, 73, 93, 82, 69, 77, 7, 11, 32, 87, 79, 68, 26, 84, 72, 89, 31, 8, 25, 65, 20, 83, 80, 90, 23, 22, 13, 28, 6, 78, 16, 19, 74, 1, 10, 12, 70, 66], [123, 103, 47, 34, 71, 76, 62, 122, 27, 55, 21, 22, 59, 19, 88, 125, 68, 15, 18, 101, 91, 109, 97, 45, 58, 24, 2, 56, 48, 64, 95, 117, 63, 36, 32, 38, 85, 111, 94, 66, 92, 77, 107, 10, 84, 9, 110, 30, 44, 119, 127, 5, 115, 50, 87, 79, 82, 73, 20, 99, 40, 124, 83, 49, 51, 42, 75, 61, 116, 57, 41, 46, 89, 60, 126, 120, 52, 54, 70, 118, 104, 113, 29, 112, 114, 53, 35, 121, 26, 105, 1, 93, 0, 100, 37, 108, 106, 80, 33, 65, 98, 12, 17, 43, 16, 72, 86, 74, 90, 23, 25, 96, 14, 31, 11, 28, 81, 4, 102, 78, 7, 13, 8, 6, 3, 69, 67, 39], [103, 61, 95, 60, 23, 104, 54, 91, 18, 116, 114, 27, 127, 80, 20, 108, 51, 40, 98, 62, 7, 46, 56, 78, 123, 107, 121, 122, 55, 113, 57, 12, 97, 53, 117, 109, 59, 58, 42, 75, 125, 124, 112, 86, 74, 126, 47, 41, 45, 63, 22, 115, 25, 65, 120, 49, 52, 69, 119, 50, 101, 28, 31, 44, 43, 118, 48, 99, 110, 105, 81, 39, 87, 38, 32, 24, 111, 102, 106, 93, 37, 85, 36, 79, 88, 100, 19, 34, 35, 21, 33, 16, 29, 84, 9, 83, 30, 90, 13, 82, 5, 96, 92, 8, 66, 94, 15, 89, 6, 70, 2, 72, 67, 26, 77, 76, 3, 17, 73, 1, 68, 71, 14, 4, 11, 0, 10, 64], [103, 61, 95, 113, 23, 104, 27, 91, 49, 42, 108, 60, 78, 114, 80, 18, 12, 74, 109, 7, 54, 127, 57, 21, 20, 34, 52, 58, 53, 45, 105, 112, 101, 2, 6, 56, 47, 86, 59, 69, 121, 117, 1, 70, 51, 39, 107, 126, 46, 43, 3, 64, 31, 119, 85, 116, 19, 62, 125, 32, 88, 72, 122, 67, 97, 13, 40, 92, 102, 65, 123, 8, 41, 120, 111, 48, 106, 124, 55, 118, 63, 44, 4, 115, 99, 25, 96, 66, 0, 22, 28, 9, 11, 38, 83, 50, 17, 110, 30, 100, 84, 35, 89, 68, 73, 90, 36, 81, 98, 33, 77, 15, 71, 37, 93, 24, 94, 29, 26, 79, 87, 75, 5, 14, 16, 10, 82, 76], [103, 61, 95, 74, 18, 86, 69, 27, 12, 78, 23, 48, 60, 64, 7, 3, 20, 54, 66, 113, 65, 22, 80, 123, 0, 127, 6, 21, 98, 33, 47, 72, 114, 38, 82, 68, 121, 73, 85, 35, 49, 75, 29, 81, 43, 94, 62, 16, 31, 88, 84, 19, 104, 53, 106, 34, 26, 32, 55, 56, 90, 119, 87, 44, 124, 14, 118, 101, 83, 24, 15, 46, 13, 2, 97, 17, 10, 93, 28, 25, 77, 92, 108, 51, 96, 100, 58, 111, 5, 99, 50, 89, 30, 11, 36, 116, 110, 40, 115, 57, 102, 67, 79, 42, 105, 45, 52, 107, 76, 126, 70, 122, 63, 120, 1, 59, 117, 8, 125, 71, 112, 41, 37, 109, 91, 9, 4, 39], [103, 61, 60, 55, 23, 95, 54, 91, 27, 18, 56, 78, 20, 113, 7, 53, 51, 114, 98, 123, 121, 12, 116, 107, 104, 69, 41, 58, 46, 63, 108, 59, 40, 109, 126, 62, 80, 49, 74, 65, 97, 124, 22, 117, 52, 45, 112, 86, 57, 75, 127, 115, 47, 105, 42, 122, 38, 25, 125, 44, 101, 37, 110, 119, 34, 118, 43, 50, 106, 6, 66, 32, 102, 48, 39, 87, 111, 31, 71, 93, 83, 120, 33, 35, 28, 29, 4, 79, 36, 21, 17, 96, 100, 68, 72, 99, 30, 16, 67, 84, 19, 15, 76, 92, 88, 89, 3, 13, 24, 14, 90, 5, 85, 82, 73, 77, 81, 9, 0, 1, 94, 8, 26, 70, 10, 11, 2, 64], [121, 114, 39, 56, 120, 33, 93, 116, 19, 101, 29, 57, 113, 126, 63, 77, 49, 103, 109, 51, 110, 127, 54, 122, 111, 53, 119, 58, 50, 66, 105, 23, 59, 125, 117, 115, 43, 55, 124, 52, 46, 48, 61, 118, 41, 112, 123, 22, 108, 62, 45, 60, 107, 8, 65, 47, 25, 89, 42, 106, 24, 74, 44, 1, 104, 98, 37, 15, 69, 102, 40, 38, 32, 72, 26, 99, 3, 35, 90, 97, 67, 28, 96, 100, 91, 36, 88, 34, 79, 94, 7, 17, 2, 16, 83, 92, 18, 86, 95, 81, 10, 30, 31, 6, 84, 13, 85, 71, 87, 27, 20, 5, 21, 76, 9, 11, 68, 78, 64, 70, 80, 82, 0, 75, 4, 73, 14, 12], [56, 39, 114, 33, 88, 85, 16, 78, 18, 26, 120, 76, 71, 29, 91, 50, 11, 31, 20, 121, 116, 100, 115, 94, 119, 58, 84, 80, 97, 6, 9, 54, 24, 57, 37, 19, 51, 22, 15, 124, 110, 113, 75, 81, 79, 126, 99, 95, 83, 87, 111, 108, 13, 90, 92, 25, 27, 61, 34, 44, 7, 122, 63, 125, 52, 28, 96, 89, 127, 53, 30, 72, 60, 77, 82, 93, 41, 74, 21, 48, 86, 123, 55, 117, 59, 49, 10, 40, 98, 14, 17, 43, 105, 101, 23, 62, 68, 36, 32, 12, 109, 102, 35, 47, 107, 118, 45, 42, 106, 112, 104, 38, 46, 69, 1, 73, 70, 8, 65, 3, 66, 67, 103, 5, 4, 2, 64, 0], [39, 56, 114, 0, 64, 9, 121, 120, 68, 6, 11, 67, 78, 16, 33, 65, 88, 66, 71, 4, 19, 29, 3, 85, 1, 76, 77, 2, 18, 22, 73, 12, 70, 86, 75, 20, 5, 87, 83, 8, 81, 7, 10, 72, 90, 99, 110, 31, 27, 116, 13, 14, 91, 69, 108, 17, 21, 41, 23, 80, 98, 58, 26, 25, 113, 79, 96, 74, 61, 35, 84, 52, 54, 95, 119, 82, 94, 107, 89, 103, 125, 15, 28, 102, 30, 57, 92, 24, 118, 34, 59, 32, 117, 49, 100, 44, 53, 115, 93, 63, 36, 42, 127, 97, 105, 123, 101, 104, 62, 40, 38, 50, 124, 37, 51, 43, 47, 106, 60, 48, 46, 109, 55, 111, 122, 45, 126, 112], [114, 39, 56, 120, 116, 29, 57, 117, 126, 93, 54, 119, 19, 113, 51, 111, 122, 49, 110, 58, 63, 50, 127, 101, 53, 23, 125, 109, 33, 55, 60, 59, 62, 118, 123, 61, 115, 52, 121, 46, 48, 47, 124, 43, 112, 45, 103, 107, 108, 22, 44, 42, 105, 41, 106, 37, 104, 27, 97, 38, 40, 86, 98, 83, 91, 26, 102, 89, 31, 88, 99, 36, 35, 25, 77, 15, 100, 34, 20, 87, 95, 84, 24, 32, 96, 90, 17, 30, 85, 18, 81, 94, 79, 28, 13, 78, 92, 16, 71, 6, 69, 11, 21, 10, 80, 74, 7, 82, 5, 65, 72, 9, 14, 1, 76, 8, 12, 75, 68, 67, 66, 70, 73, 2, 3, 4, 64, 0], [46, 102, 49, 110, 26, 84, 16, 53, 29, 123, 60, 126, 98, 10, 119, 90, 25, 91, 44, 56, 124, 113, 62, 41, 112, 86, 63, 127, 42, 92, 59, 57, 33, 47, 118, 107, 31, 54, 125, 6, 87, 115, 80, 114, 66, 104, 95, 105, 20, 99, 111, 120, 55, 61, 43, 52, 24, 58, 116, 36, 108, 117, 45, 18, 7, 122, 93, 39, 51, 48, 109, 35, 97, 121, 37, 50, 30, 32, 106, 100, 64, 103, 27, 2, 13, 74, 40, 1, 22, 101, 81, 88, 72, 28, 94, 21, 78, 70, 96, 34, 69, 0, 83, 38, 85, 15, 17, 89, 11, 12, 3, 23, 68, 73, 19, 75, 67, 76, 5, 14, 71, 79, 65, 4, 9, 8, 82, 77], [46, 102, 110, 29, 26, 60, 84, 16, 86, 51, 69, 18, 100, 123, 109, 67, 121, 36, 112, 49, 113, 1, 63, 97, 90, 11, 13, 50, 31, 53, 33, 108, 7, 61, 115, 30, 47, 107, 28, 12, 105, 119, 37, 59, 106, 114, 42, 120, 99, 45, 52, 98, 87, 124, 104, 89, 125, 92, 41, 116, 55, 111, 39, 103, 10, 57, 58, 127, 117, 62, 56, 35, 44, 54, 122, 40, 126, 64, 6, 38, 101, 85, 76, 93, 27, 81, 14, 32, 24, 25, 4, 43, 95, 48, 34, 73, 118, 96, 70, 3, 66, 91, 9, 17, 19, 83, 72, 88, 94, 78, 22, 79, 20, 80, 2, 74, 68, 21, 71, 15, 23, 5, 75, 82, 8, 65, 77, 0], [46, 102, 110, 26, 84, 18, 29, 13, 72, 16, 67, 116, 10, 109, 68, 107, 121, 113, 59, 0, 47, 6, 90, 112, 60, 98, 115, 48, 24, 87, 22, 123, 120, 8, 12, 11, 49, 127, 30, 73, 23, 106, 126, 41, 119, 34, 61, 40, 37, 35, 66, 97, 124, 108, 100, 114, 42, 93, 25, 85, 103, 53, 33, 81, 80, 89, 63, 55, 31, 104, 105, 74, 50, 44, 117, 95, 45, 3, 111, 21, 57, 36, 43, 62, 92, 58, 125, 88, 86, 122, 20, 54, 83, 96, 70, 39, 7, 75, 91, 28, 52, 118, 14, 27, 101, 99, 51, 56, 32, 19, 9, 2, 69, 17, 94, 82, 64, 79, 5, 77, 65, 1, 76, 4, 71, 15, 78, 38], [46, 102, 18, 110, 13, 29, 84, 26, 16, 10, 72, 6, 68, 93, 25, 87, 1, 73, 90, 60, 0, 2, 22, 91, 78, 7, 14, 49, 9, 88, 15, 82, 79, 36, 59, 33, 112, 20, 17, 30, 123, 109, 27, 31, 19, 116, 67, 65, 8, 71, 92, 96, 85, 3, 80, 61, 76, 81, 24, 11, 77, 98, 113, 83, 50, 5, 107, 28, 106, 23, 118, 54, 101, 86, 37, 75, 34, 12, 66, 89, 63, 74, 41, 56, 48, 4, 108, 32, 35, 115, 99, 94, 21, 100, 119, 62, 57, 114, 70, 95, 52, 58, 124, 69, 39, 117, 120, 44, 121, 127, 105, 43, 55, 64, 126, 104, 40, 53, 103, 47, 42, 122, 125, 97, 51, 45, 111, 38]], "model.layers.28.self_attn.k_proj": [[120, 104, 98, 23, 85, 0, 29, 11, 17, 16, 27, 61, 58, 30, 19, 95, 110, 73, 78, 125, 66, 114, 91, 108, 42, 50, 115, 51, 62, 52, 68, 121, 127, 57, 47, 124, 60, 45, 59, 53, 122, 55, 49, 70, 43, 67, 112, 119, 109, 65, 117, 54, 123, 116, 48, 102, 113, 56, 44, 8, 118, 111, 63, 38, 4, 46, 79, 105, 107, 89, 106, 5, 12, 126, 41, 25, 77, 2, 39, 36, 86, 88, 13, 7, 82, 31, 92, 35, 103, 101, 100, 37, 10, 90, 33, 28, 71, 34, 84, 26, 24, 1, 32, 22, 94, 20, 99, 96, 9, 72, 6, 15, 97, 83, 18, 81, 3, 93, 80, 14, 21, 74, 75, 69, 76, 64, 87, 40], [125, 104, 64, 87, 98, 14, 70, 74, 17, 12, 82, 0, 21, 65, 8, 68, 66, 124, 108, 58, 69, 3, 1, 90, 4, 114, 47, 79, 28, 49, 2, 54, 19, 127, 73, 9, 75, 46, 50, 123, 61, 93, 52, 25, 16, 77, 103, 111, 31, 67, 41, 113, 53, 106, 92, 20, 109, 89, 45, 122, 105, 27, 97, 107, 71, 94, 60, 33, 13, 55, 59, 121, 120, 86, 26, 11, 63, 99, 29, 95, 91, 51, 101, 119, 5, 62, 118, 88, 83, 112, 7, 102, 24, 80, 22, 115, 56, 38, 42, 44, 48, 37, 32, 116, 110, 57, 15, 36, 84, 100, 23, 117, 43, 39, 76, 30, 35, 126, 85, 96, 6, 72, 34, 18, 10, 78, 81, 40], [106, 114, 100, 94, 118, 87, 68, 0, 99, 71, 77, 74, 20, 82, 15, 17, 50, 70, 52, 119, 42, 27, 47, 44, 115, 62, 75, 89, 56, 66, 123, 1, 122, 109, 102, 120, 76, 2, 110, 53, 57, 45, 48, 9, 65, 41, 43, 107, 121, 39, 35, 127, 95, 73, 29, 54, 59, 91, 31, 58, 124, 61, 72, 69, 51, 22, 113, 49, 28, 12, 111, 93, 85, 63, 97, 60, 3, 125, 46, 40, 116, 96, 30, 98, 126, 108, 117, 33, 55, 34, 32, 37, 5, 112, 101, 11, 92, 64, 19, 80, 104, 78, 103, 90, 14, 7, 83, 21, 86, 24, 88, 8, 16, 67, 105, 38, 26, 4, 81, 25, 79, 18, 6, 10, 84, 36, 23, 13], [105, 126, 56, 34, 95, 86, 92, 89, 115, 84, 82, 79, 57, 119, 51, 62, 110, 63, 87, 54, 59, 80, 120, 112, 58, 121, 113, 127, 117, 55, 124, 116, 118, 109, 48, 60, 123, 61, 122, 46, 49, 114, 52, 53, 47, 125, 111, 45, 32, 104, 44, 107, 99, 50, 108, 21, 17, 91, 39, 101, 43, 26, 106, 42, 103, 96, 11, 36, 77, 27, 28, 102, 30, 13, 38, 40, 97, 33, 37, 73, 29, 8, 22, 41, 76, 35, 10, 19, 93, 75, 100, 98, 14, 94, 70, 85, 18, 88, 24, 12, 20, 31, 16, 23, 90, 25, 83, 4, 5, 3, 78, 81, 0, 72, 9, 65, 74, 7, 71, 68, 1, 6, 66, 15, 69, 67, 64, 2], [123, 39, 88, 21, 27, 64, 76, 15, 18, 30, 1, 68, 98, 71, 122, 95, 70, 125, 9, 47, 63, 121, 66, 46, 19, 57, 60, 86, 116, 110, 2, 49, 119, 120, 118, 33, 113, 3, 45, 52, 117, 44, 112, 48, 51, 124, 40, 10, 50, 54, 59, 114, 20, 69, 111, 31, 102, 37, 126, 8, 108, 61, 56, 106, 109, 115, 55, 34, 43, 87, 53, 62, 13, 42, 14, 41, 127, 58, 104, 29, 107, 105, 32, 101, 100, 11, 36, 75, 93, 92, 23, 90, 28, 35, 99, 22, 26, 74, 89, 97, 25, 96, 84, 65, 77, 17, 80, 81, 0, 16, 94, 38, 82, 78, 79, 103, 72, 4, 5, 6, 7, 83, 67, 73, 91, 24, 12, 85], [61, 39, 31, 23, 91, 18, 64, 74, 80, 78, 86, 62, 69, 49, 56, 12, 54, 3, 51, 112, 66, 7, 55, 113, 110, 124, 121, 109, 50, 20, 52, 117, 114, 58, 0, 34, 122, 59, 60, 123, 115, 42, 111, 126, 118, 107, 63, 53, 127, 116, 44, 41, 108, 45, 79, 47, 57, 35, 25, 46, 98, 65, 106, 6, 48, 88, 119, 75, 1, 43, 40, 72, 120, 33, 125, 21, 32, 105, 67, 68, 19, 37, 100, 36, 38, 92, 70, 73, 17, 2, 102, 96, 97, 81, 99, 27, 90, 94, 101, 103, 4, 11, 84, 104, 77, 93, 29, 9, 28, 15, 24, 13, 22, 89, 26, 83, 30, 85, 16, 5, 8, 10, 76, 14, 71, 95, 82, 87], [56, 103, 64, 114, 93, 121, 97, 1, 50, 19, 9, 68, 6, 22, 120, 67, 116, 11, 23, 113, 78, 66, 77, 16, 46, 88, 57, 58, 110, 54, 3, 69, 76, 71, 51, 122, 105, 118, 126, 43, 95, 8, 62, 61, 117, 24, 55, 59, 47, 49, 53, 123, 125, 42, 119, 63, 60, 44, 48, 115, 85, 124, 12, 26, 37, 52, 109, 18, 112, 2, 45, 74, 127, 111, 108, 41, 89, 106, 21, 107, 40, 81, 38, 65, 101, 91, 14, 15, 35, 36, 84, 104, 0, 99, 102, 92, 4, 34, 100, 94, 30, 28, 96, 25, 32, 17, 20, 31, 98, 72, 80, 10, 75, 90, 70, 82, 27, 79, 13, 5, 87, 73, 7, 86, 29, 39, 33, 83], [110, 38, 46, 29, 84, 26, 13, 16, 18, 6, 2, 0, 112, 10, 25, 121, 123, 68, 11, 113, 73, 86, 1, 47, 7, 97, 72, 63, 60, 109, 115, 34, 119, 53, 114, 30, 59, 55, 61, 124, 32, 41, 103, 107, 101, 88, 49, 50, 116, 42, 87, 51, 104, 62, 37, 57, 111, 108, 58, 127, 120, 45, 56, 122, 118, 40, 44, 105, 39, 125, 117, 54, 52, 64, 92, 43, 106, 95, 15, 76, 48, 14, 69, 36, 96, 91, 4, 35, 99, 31, 100, 126, 98, 33, 75, 22, 67, 23, 5, 94, 81, 19, 24, 85, 17, 78, 12, 27, 83, 8, 28, 3, 89, 21, 65, 9, 79, 71, 93, 82, 77, 74, 80, 66, 70, 90, 20, 102]], "model.layers.28.self_attn.qk_proj": [[123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 118, 103, 82, 18, 91, 64, 0, 29, 95, 58, 39, 40, 50, 85, 41, 27, 10, 70, 20, 76, 74, 105, 84, 30, 12, 55, 62, 98, 93, 21, 81, 51, 68, 115, 80, 79, 13, 77, 122, 124, 44, 90, 6, 60, 15, 4, 48, 112, 59, 113, 54, 31, 116, 78, 49, 57, 47, 53, 109, 66, 65, 17, 127, 16, 2, 22, 14, 63, 28, 102, 24, 11, 34, 1, 83, 9, 86, 43, 73, 52, 19, 38, 25, 7, 88, 108, 36, 8, 119, 94, 89, 26, 33, 107, 3, 71, 117, 75, 5, 45, 67, 111, 99, 97, 35, 101, 100, 72, 69, 32, 37, 92, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 103, 118, 91, 0, 82, 64, 18, 40, 70, 95, 50, 58, 29, 30, 47, 10, 62, 39, 27, 20, 85, 105, 4, 41, 76, 12, 93, 51, 98, 81, 55, 74, 21, 60, 84, 13, 49, 77, 44, 115, 63, 53, 80, 65, 124, 66, 116, 34, 112, 113, 22, 54, 79, 102, 17, 16, 2, 6, 68, 11, 14, 78, 90, 15, 122, 28, 1, 9, 109, 59, 127, 48, 8, 108, 52, 31, 38, 57, 43, 71, 24, 86, 88, 83, 73, 33, 117, 7, 107, 111, 119, 19, 94, 45, 5, 36, 100, 3, 89, 75, 69, 26, 67, 35, 25, 37, 97, 99, 72, 101, 92, 32, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 0, 118, 64, 18, 103, 82, 91, 70, 39, 95, 27, 58, 41, 40, 98, 105, 29, 12, 21, 84, 76, 10, 85, 112, 30, 50, 74, 93, 55, 20, 115, 44, 53, 4, 62, 63, 80, 34, 81, 77, 124, 60, 15, 68, 79, 113, 22, 13, 66, 2, 90, 57, 51, 49, 78, 47, 17, 54, 102, 65, 122, 16, 14, 1, 116, 8, 48, 59, 28, 31, 119, 127, 86, 109, 19, 6, 43, 71, 36, 88, 9, 107, 38, 108, 52, 7, 25, 11, 117, 111, 24, 94, 67, 89, 26, 75, 3, 73, 83, 33, 5, 69, 99, 101, 100, 97, 37, 45, 35, 92, 96, 32, 72], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 103, 87, 118, 64, 0, 18, 91, 82, 70, 40, 39, 41, 74, 76, 58, 95, 53, 105, 12, 27, 50, 29, 10, 98, 21, 84, 112, 122, 62, 4, 63, 81, 30, 124, 55, 2, 77, 14, 78, 85, 34, 93, 80, 1, 20, 44, 68, 66, 113, 16, 17, 115, 60, 22, 13, 15, 54, 6, 79, 8, 90, 49, 57, 51, 59, 65, 47, 25, 116, 31, 7, 86, 28, 11, 48, 36, 33, 19, 67, 9, 83, 119, 71, 102, 52, 38, 88, 94, 73, 107, 109, 3, 89, 69, 101, 127, 117, 108, 24, 75, 43, 45, 5, 26, 97, 111, 35, 99, 100, 37, 92, 72, 32, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 103, 118, 18, 0, 82, 64, 40, 58, 41, 70, 91, 50, 53, 39, 29, 105, 55, 12, 76, 95, 98, 112, 21, 27, 80, 74, 10, 84, 20, 85, 30, 6, 78, 77, 113, 4, 81, 62, 115, 15, 122, 60, 124, 68, 66, 54, 48, 14, 2, 57, 44, 59, 90, 86, 116, 16, 47, 79, 65, 17, 8, 28, 63, 93, 49, 13, 51, 127, 1, 25, 31, 11, 34, 19, 22, 7, 107, 117, 9, 109, 24, 52, 108, 102, 83, 36, 73, 119, 67, 88, 3, 38, 71, 94, 111, 75, 5, 26, 69, 89, 33, 101, 43, 45, 97, 100, 35, 99, 92, 72, 37, 96, 32], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 103, 87, 18, 82, 118, 64, 0, 50, 29, 58, 40, 91, 95, 21, 41, 74, 27, 44, 112, 62, 20, 30, 105, 98, 85, 81, 10, 55, 53, 68, 113, 84, 12, 76, 6, 115, 70, 34, 54, 39, 49, 48, 4, 15, 77, 31, 80, 124, 2, 65, 66, 93, 47, 14, 13, 78, 60, 122, 79, 1, 8, 22, 63, 17, 16, 116, 19, 86, 28, 57, 127, 109, 24, 90, 119, 25, 108, 5, 7, 59, 51, 71, 11, 94, 83, 43, 117, 52, 75, 89, 33, 9, 102, 3, 88, 97, 73, 38, 100, 35, 111, 67, 69, 99, 45, 101, 107, 26, 37, 36, 96, 92, 32, 72], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 103, 41, 118, 82, 40, 50, 64, 58, 18, 91, 0, 29, 98, 105, 95, 27, 6, 39, 76, 44, 30, 21, 113, 85, 53, 112, 55, 20, 12, 81, 90, 10, 115, 15, 84, 4, 93, 13, 80, 74, 16, 109, 34, 47, 77, 49, 14, 68, 17, 124, 60, 54, 79, 48, 127, 51, 63, 66, 62, 59, 70, 78, 1, 57, 22, 122, 86, 25, 31, 116, 2, 24, 19, 36, 65, 8, 28, 7, 89, 111, 119, 9, 43, 83, 108, 73, 94, 88, 52, 117, 71, 11, 38, 67, 45, 97, 102, 26, 69, 3, 101, 99, 100, 75, 107, 37, 35, 72, 33, 5, 32, 92, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 103, 118, 18, 82, 91, 0, 29, 64, 39, 41, 58, 40, 105, 6, 95, 44, 27, 98, 21, 50, 112, 55, 30, 76, 34, 54, 12, 85, 47, 20, 74, 122, 4, 53, 81, 84, 77, 79, 115, 51, 10, 113, 48, 93, 90, 31, 80, 16, 49, 14, 15, 102, 17, 116, 68, 65, 124, 59, 2, 22, 60, 13, 1, 57, 66, 43, 127, 63, 109, 62, 119, 89, 78, 38, 25, 70, 28, 108, 24, 88, 9, 86, 73, 83, 45, 52, 107, 36, 11, 19, 94, 71, 67, 111, 7, 117, 33, 26, 100, 37, 5, 72, 97, 99, 69, 75, 3, 8, 101, 92, 32, 35, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 87, 23, 104, 121, 103, 18, 82, 29, 118, 64, 91, 27, 0, 44, 58, 95, 6, 40, 41, 39, 21, 50, 76, 20, 112, 30, 74, 113, 55, 47, 10, 105, 84, 13, 63, 85, 98, 68, 124, 77, 122, 80, 12, 93, 51, 16, 34, 78, 109, 54, 48, 90, 115, 4, 116, 79, 62, 81, 17, 53, 15, 66, 49, 14, 102, 65, 43, 119, 22, 31, 28, 60, 59, 57, 127, 83, 88, 2, 1, 24, 86, 89, 72, 70, 52, 73, 19, 7, 108, 25, 107, 9, 117, 38, 94, 67, 71, 33, 111, 36, 45, 75, 99, 26, 5, 100, 11, 3, 101, 35, 97, 8, 32, 69, 92, 37, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 87, 23, 64, 58, 118, 103, 0, 82, 18, 91, 6, 29, 41, 40, 30, 27, 76, 48, 4, 39, 50, 98, 85, 63, 55, 12, 74, 113, 10, 122, 44, 20, 95, 112, 93, 21, 51, 124, 105, 47, 68, 2, 62, 34, 22, 53, 84, 54, 70, 66, 16, 115, 81, 13, 108, 49, 60, 57, 43, 79, 77, 90, 14, 80, 17, 116, 65, 15, 102, 78, 1, 59, 86, 109, 127, 119, 72, 24, 28, 31, 11, 73, 7, 9, 38, 25, 71, 111, 19, 33, 83, 94, 3, 107, 36, 88, 75, 52, 89, 26, 5, 67, 8, 45, 117, 69, 101, 100, 97, 35, 99, 32, 37, 92, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 118, 82, 18, 103, 0, 64, 91, 10, 40, 58, 70, 27, 74, 29, 95, 50, 76, 98, 6, 85, 39, 41, 30, 21, 4, 20, 55, 77, 84, 66, 53, 13, 16, 12, 90, 79, 81, 54, 105, 68, 78, 72, 93, 15, 1, 80, 124, 122, 2, 62, 48, 112, 57, 17, 47, 60, 51, 115, 34, 113, 71, 59, 94, 63, 49, 7, 31, 65, 28, 14, 24, 19, 86, 22, 25, 73, 88, 109, 83, 11, 44, 89, 116, 3, 102, 52, 9, 127, 108, 119, 38, 5, 67, 33, 43, 107, 75, 36, 26, 117, 45, 111, 69, 35, 97, 100, 101, 99, 92, 8, 37, 32, 96], [123, 56, 125, 61, 120, 114, 110, 46, 126, 106, 104, 42, 87, 23, 121, 82, 18, 103, 118, 64, 29, 40, 91, 0, 98, 95, 70, 76, 27, 50, 84, 21, 39, 10, 20, 74, 44, 58, 30, 4, 79, 41, 12, 85, 55, 105, 34, 13, 16, 93, 78, 77, 80, 122, 68, 66, 90, 59, 113, 54, 17, 51, 47, 15, 62, 115, 112, 6, 31, 81, 2, 49, 14, 22, 60, 53, 65, 72, 57, 1, 25, 102, 24, 124, 86, 116, 28, 48, 75, 83, 63, 73, 109, 127, 7, 71, 36, 9, 3, 19, 119, 94, 89, 67, 52, 43, 11, 107, 88, 33, 45, 38, 69, 108, 111, 35, 26, 5, 101, 99, 117, 100, 32, 37, 97, 8, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 104, 87, 42, 103, 121, 23, 118, 58, 18, 91, 40, 82, 64, 0, 50, 29, 39, 95, 44, 70, 51, 47, 27, 41, 20, 98, 12, 105, 54, 21, 113, 30, 115, 55, 85, 93, 60, 116, 63, 112, 84, 102, 76, 74, 34, 57, 68, 59, 62, 53, 28, 17, 4, 31, 48, 81, 90, 72, 49, 109, 124, 122, 65, 86, 80, 16, 14, 77, 78, 22, 13, 10, 79, 108, 2, 66, 24, 15, 127, 6, 43, 1, 9, 88, 119, 75, 73, 45, 52, 19, 89, 38, 7, 111, 11, 36, 71, 94, 107, 83, 25, 26, 117, 3, 97, 35, 5, 67, 99, 37, 69, 100, 101, 33, 92, 32, 96, 8], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 87, 23, 64, 103, 0, 118, 91, 18, 82, 70, 58, 39, 54, 95, 76, 29, 40, 21, 41, 74, 50, 12, 27, 93, 30, 122, 68, 10, 20, 124, 85, 63, 57, 55, 105, 115, 84, 4, 98, 2, 44, 22, 34, 13, 81, 77, 116, 113, 1, 60, 112, 16, 78, 90, 79, 109, 49, 51, 65, 66, 47, 15, 14, 62, 17, 6, 48, 59, 80, 53, 43, 102, 119, 31, 127, 72, 24, 108, 111, 28, 88, 86, 71, 11, 38, 19, 9, 107, 52, 7, 36, 75, 73, 83, 25, 33, 94, 97, 99, 117, 5, 89, 3, 35, 67, 45, 69, 100, 101, 26, 37, 8, 92, 32, 96], [123, 61, 56, 125, 120, 110, 114, 46, 126, 106, 42, 104, 23, 121, 87, 18, 64, 0, 91, 103, 118, 82, 40, 70, 41, 74, 58, 76, 29, 98, 85, 95, 12, 50, 10, 105, 84, 21, 39, 30, 4, 2, 81, 20, 68, 27, 63, 90, 16, 55, 66, 124, 6, 115, 13, 15, 54, 77, 112, 53, 47, 44, 79, 22, 113, 93, 14, 122, 34, 78, 80, 17, 62, 116, 109, 65, 51, 57, 48, 59, 60, 1, 31, 9, 7, 86, 119, 75, 72, 127, 49, 28, 33, 108, 11, 88, 52, 24, 94, 67, 38, 89, 3, 25, 19, 5, 111, 73, 71, 36, 83, 107, 69, 43, 97, 35, 117, 45, 100, 26, 99, 102, 8, 101, 37, 96, 92, 32], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 103, 18, 118, 91, 82, 64, 29, 50, 58, 0, 95, 10, 70, 40, 27, 74, 105, 55, 93, 6, 30, 76, 84, 39, 85, 13, 12, 20, 98, 68, 21, 41, 81, 90, 122, 115, 77, 47, 16, 112, 80, 78, 15, 4, 44, 34, 66, 54, 1, 62, 113, 109, 14, 2, 49, 60, 22, 79, 17, 59, 65, 119, 31, 63, 19, 28, 53, 48, 124, 86, 7, 75, 71, 9, 57, 24, 116, 73, 127, 102, 25, 3, 88, 33, 89, 43, 108, 83, 52, 72, 11, 51, 67, 8, 94, 111, 107, 38, 100, 45, 69, 5, 97, 26, 37, 35, 117, 36, 32, 99, 101, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 103, 64, 18, 118, 0, 82, 58, 95, 29, 40, 91, 39, 30, 41, 27, 50, 98, 20, 93, 21, 74, 85, 112, 12, 105, 6, 84, 115, 90, 55, 10, 16, 76, 44, 81, 54, 68, 108, 122, 4, 14, 60, 1, 53, 34, 80, 62, 22, 13, 113, 59, 77, 17, 66, 109, 31, 86, 28, 47, 57, 48, 116, 49, 78, 15, 2, 70, 63, 51, 127, 102, 79, 124, 9, 88, 38, 43, 119, 19, 24, 71, 45, 52, 65, 73, 75, 25, 89, 107, 7, 26, 83, 36, 8, 94, 33, 35, 11, 99, 111, 100, 67, 3, 97, 72, 117, 5, 101, 69, 92, 37, 32, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 87, 23, 121, 64, 103, 82, 18, 118, 0, 91, 6, 39, 29, 40, 27, 95, 74, 76, 30, 50, 85, 58, 10, 112, 1, 12, 34, 20, 41, 54, 44, 55, 21, 84, 13, 93, 105, 81, 98, 2, 68, 14, 47, 113, 80, 122, 63, 16, 53, 4, 77, 90, 51, 115, 119, 60, 78, 62, 49, 102, 15, 17, 79, 22, 124, 66, 48, 70, 31, 43, 116, 86, 59, 65, 9, 109, 71, 57, 75, 19, 108, 8, 33, 94, 88, 28, 25, 26, 127, 24, 38, 36, 89, 3, 52, 67, 83, 111, 107, 7, 69, 99, 117, 73, 101, 97, 11, 5, 100, 35, 45, 37, 32, 72, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 103, 87, 64, 0, 118, 91, 40, 82, 6, 18, 58, 41, 29, 27, 112, 76, 39, 50, 74, 98, 49, 10, 95, 93, 85, 12, 2, 122, 113, 20, 60, 84, 4, 30, 105, 124, 44, 63, 21, 34, 81, 62, 48, 115, 14, 68, 13, 77, 55, 80, 109, 116, 119, 53, 47, 54, 17, 51, 1, 8, 15, 59, 127, 43, 16, 66, 78, 70, 22, 102, 108, 31, 90, 79, 75, 57, 33, 65, 86, 36, 19, 38, 28, 45, 88, 52, 24, 9, 89, 71, 94, 111, 73, 3, 83, 11, 25, 99, 7, 107, 69, 67, 101, 97, 35, 100, 26, 5, 117, 37, 96, 92, 32, 72], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 104, 103, 121, 23, 87, 118, 0, 91, 82, 40, 6, 64, 18, 58, 29, 95, 10, 27, 93, 30, 41, 74, 39, 4, 55, 50, 98, 122, 81, 124, 12, 76, 66, 105, 85, 20, 112, 21, 62, 34, 54, 8, 68, 84, 48, 44, 16, 53, 115, 116, 90, 77, 113, 17, 14, 70, 63, 13, 78, 2, 1, 60, 80, 79, 15, 47, 28, 59, 109, 49, 102, 57, 75, 108, 119, 22, 127, 83, 71, 51, 31, 86, 73, 65, 33, 52, 9, 45, 7, 38, 35, 3, 19, 25, 88, 43, 111, 94, 69, 36, 97, 89, 26, 107, 24, 117, 11, 100, 99, 67, 5, 101, 92, 72, 37, 96, 32], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 87, 118, 103, 18, 82, 40, 29, 64, 0, 91, 27, 44, 39, 58, 95, 21, 105, 50, 55, 30, 41, 6, 112, 74, 12, 10, 76, 113, 85, 122, 115, 20, 51, 98, 116, 47, 93, 54, 62, 34, 63, 70, 84, 1, 68, 109, 60, 17, 13, 8, 16, 80, 79, 78, 4, 90, 119, 124, 81, 2, 49, 108, 127, 102, 48, 15, 77, 53, 31, 86, 22, 43, 66, 59, 107, 52, 57, 14, 65, 28, 25, 75, 88, 7, 24, 19, 9, 94, 3, 71, 38, 73, 33, 67, 36, 83, 100, 5, 89, 97, 26, 11, 117, 111, 101, 69, 35, 37, 32, 99, 96, 45, 92, 72], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 87, 64, 23, 0, 118, 103, 91, 18, 40, 82, 70, 58, 41, 27, 29, 39, 50, 12, 76, 112, 105, 95, 68, 20, 21, 10, 34, 30, 124, 54, 44, 85, 122, 74, 62, 78, 93, 55, 115, 2, 6, 81, 98, 113, 63, 84, 116, 13, 1, 51, 53, 60, 15, 119, 65, 66, 43, 17, 49, 57, 16, 4, 86, 79, 102, 77, 80, 47, 22, 8, 48, 59, 14, 52, 109, 31, 127, 90, 38, 75, 7, 108, 28, 71, 19, 88, 67, 25, 107, 33, 94, 9, 24, 5, 36, 89, 73, 97, 11, 26, 3, 45, 83, 92, 69, 100, 117, 37, 111, 101, 99, 35, 72, 32, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 103, 64, 118, 18, 0, 91, 82, 39, 58, 70, 40, 95, 98, 29, 27, 50, 62, 30, 105, 93, 112, 116, 12, 44, 41, 55, 21, 10, 115, 122, 81, 74, 20, 63, 85, 68, 113, 76, 57, 78, 84, 54, 34, 2, 16, 80, 102, 13, 4, 6, 31, 66, 15, 79, 124, 109, 86, 1, 51, 90, 17, 60, 119, 59, 53, 48, 49, 47, 14, 22, 28, 65, 88, 108, 77, 127, 38, 7, 36, 73, 9, 75, 107, 25, 3, 8, 52, 24, 43, 71, 89, 26, 33, 67, 11, 83, 111, 94, 97, 45, 69, 19, 117, 5, 100, 72, 99, 35, 37, 92, 101, 32, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 18, 103, 91, 118, 82, 58, 0, 64, 95, 70, 29, 74, 40, 122, 105, 21, 55, 27, 85, 39, 10, 30, 12, 50, 115, 34, 98, 20, 93, 41, 76, 13, 4, 54, 81, 80, 68, 15, 84, 62, 78, 113, 90, 63, 44, 51, 59, 16, 112, 57, 14, 124, 28, 48, 116, 22, 86, 79, 1, 77, 17, 47, 6, 66, 108, 60, 102, 119, 49, 2, 71, 25, 109, 31, 94, 53, 127, 24, 9, 19, 75, 111, 11, 43, 52, 33, 7, 83, 8, 36, 65, 88, 38, 107, 72, 97, 73, 3, 5, 89, 100, 69, 35, 26, 67, 99, 45, 37, 117, 32, 101, 96, 92], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 118, 103, 18, 82, 91, 64, 70, 58, 0, 29, 40, 122, 98, 39, 41, 50, 10, 30, 21, 55, 12, 27, 105, 20, 54, 124, 74, 113, 81, 95, 57, 85, 76, 116, 112, 109, 84, 47, 62, 93, 44, 15, 79, 127, 22, 13, 115, 78, 4, 17, 16, 34, 68, 49, 65, 80, 90, 77, 2, 48, 86, 119, 59, 63, 6, 51, 66, 60, 14, 53, 24, 31, 28, 1, 108, 52, 43, 89, 9, 94, 102, 19, 71, 73, 111, 88, 25, 75, 83, 72, 38, 7, 107, 67, 11, 36, 101, 117, 26, 33, 45, 3, 35, 5, 100, 97, 69, 37, 8, 99, 96, 32, 92], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 87, 104, 23, 121, 118, 91, 103, 0, 18, 64, 82, 58, 29, 39, 95, 30, 44, 51, 50, 27, 41, 40, 70, 116, 54, 122, 10, 47, 98, 34, 20, 12, 113, 84, 119, 112, 105, 21, 124, 93, 76, 115, 81, 109, 102, 60, 74, 85, 68, 53, 49, 63, 62, 57, 16, 4, 59, 31, 14, 66, 48, 77, 55, 86, 90, 13, 43, 79, 6, 1, 65, 127, 15, 78, 80, 17, 108, 33, 22, 28, 36, 2, 19, 24, 38, 9, 52, 72, 107, 73, 88, 71, 26, 45, 11, 89, 3, 97, 25, 94, 111, 67, 7, 100, 75, 117, 69, 83, 35, 92, 99, 101, 37, 32, 5, 8, 96], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 87, 103, 118, 64, 0, 18, 40, 82, 41, 50, 29, 58, 91, 47, 95, 55, 39, 30, 27, 98, 60, 122, 124, 62, 57, 21, 20, 105, 93, 10, 76, 34, 49, 54, 12, 51, 115, 74, 85, 116, 81, 90, 6, 112, 53, 63, 84, 113, 1, 70, 4, 48, 13, 17, 59, 43, 16, 14, 77, 80, 15, 31, 2, 68, 78, 127, 102, 44, 28, 109, 72, 86, 79, 22, 119, 65, 25, 52, 108, 24, 36, 107, 7, 66, 88, 19, 38, 89, 99, 33, 11, 83, 9, 73, 71, 94, 75, 111, 101, 67, 35, 97, 3, 117, 100, 26, 37, 45, 69, 5, 96, 8, 32, 92], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 23, 104, 121, 87, 118, 64, 82, 0, 103, 18, 91, 58, 40, 6, 41, 10, 29, 74, 122, 12, 55, 62, 50, 95, 27, 85, 124, 17, 76, 39, 13, 105, 93, 30, 21, 20, 34, 68, 81, 98, 4, 57, 115, 54, 60, 78, 59, 90, 70, 80, 63, 48, 84, 77, 2, 79, 66, 112, 15, 116, 16, 72, 1, 19, 44, 49, 47, 119, 127, 113, 22, 11, 14, 51, 86, 28, 31, 102, 65, 7, 52, 53, 36, 73, 109, 33, 71, 43, 67, 38, 89, 24, 108, 75, 9, 111, 83, 25, 88, 101, 3, 69, 94, 26, 107, 117, 35, 5, 99, 45, 97, 100, 8, 37, 92, 32, 96], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 0, 103, 64, 118, 18, 91, 82, 6, 40, 58, 115, 41, 98, 62, 116, 29, 12, 50, 76, 39, 30, 95, 10, 55, 93, 85, 74, 105, 21, 4, 122, 47, 68, 84, 124, 78, 60, 49, 27, 51, 13, 66, 20, 44, 34, 48, 59, 77, 2, 112, 72, 63, 1, 17, 53, 70, 113, 57, 79, 65, 81, 14, 54, 86, 80, 16, 22, 90, 15, 31, 7, 52, 119, 11, 127, 28, 109, 111, 108, 25, 38, 19, 33, 83, 3, 9, 89, 94, 73, 102, 67, 36, 71, 100, 43, 24, 117, 97, 88, 75, 107, 45, 69, 5, 35, 26, 101, 37, 99, 96, 32, 92, 8], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 103, 18, 82, 58, 118, 6, 91, 29, 64, 40, 50, 0, 95, 44, 85, 39, 10, 12, 30, 62, 74, 21, 47, 27, 122, 51, 20, 41, 84, 124, 115, 105, 63, 76, 116, 98, 54, 60, 55, 112, 81, 68, 78, 17, 77, 80, 93, 4, 14, 13, 34, 16, 48, 113, 65, 15, 53, 119, 59, 66, 57, 31, 109, 90, 2, 49, 70, 79, 83, 72, 86, 1, 11, 7, 28, 22, 52, 127, 71, 24, 9, 111, 19, 73, 108, 25, 38, 89, 88, 75, 3, 67, 45, 43, 94, 33, 102, 97, 117, 69, 107, 26, 36, 37, 100, 5, 101, 99, 8, 35, 32, 92, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 104, 42, 23, 121, 87, 0, 103, 64, 118, 18, 91, 6, 29, 82, 58, 40, 95, 10, 105, 50, 39, 41, 27, 74, 68, 12, 62, 55, 76, 81, 63, 30, 122, 85, 112, 84, 20, 124, 21, 115, 34, 4, 70, 93, 98, 54, 60, 66, 59, 80, 65, 2, 90, 78, 31, 51, 15, 44, 49, 14, 16, 116, 77, 13, 22, 57, 17, 113, 86, 79, 28, 53, 47, 48, 72, 71, 109, 7, 127, 119, 1, 108, 24, 25, 11, 111, 9, 36, 73, 88, 83, 67, 75, 117, 52, 102, 43, 19, 99, 69, 94, 38, 33, 89, 45, 5, 3, 97, 100, 35, 26, 107, 8, 37, 101, 92, 96, 32], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 87, 23, 121, 103, 118, 18, 82, 29, 50, 91, 53, 95, 12, 58, 64, 30, 85, 41, 40, 51, 6, 27, 39, 21, 10, 124, 0, 34, 76, 62, 105, 54, 112, 93, 63, 98, 55, 48, 74, 60, 47, 15, 115, 90, 20, 84, 17, 81, 16, 44, 122, 113, 57, 14, 68, 22, 4, 80, 59, 79, 13, 70, 77, 116, 31, 127, 78, 28, 108, 86, 102, 49, 66, 109, 83, 9, 1, 2, 71, 11, 43, 65, 119, 24, 7, 25, 33, 73, 88, 52, 111, 89, 72, 107, 19, 94, 38, 45, 8, 35, 100, 37, 36, 99, 26, 75, 67, 3, 117, 5, 97, 69, 101, 92, 32, 96]], "model.layers.29.self_attn.q_proj": [[45, 35, 109, 93, 22, 25, 80, 18, 83, 11, 14, 91, 67, 31, 1, 115, 124, 10, 88, 127, 43, 55, 13, 119, 8, 64, 81, 90, 6, 21, 7, 102, 48, 95, 5, 9, 69, 27, 112, 70, 40, 113, 78, 114, 126, 4, 120, 53, 50, 111, 61, 121, 2, 101, 24, 12, 103, 54, 37, 63, 57, 108, 38, 122, 92, 16, 44, 59, 97, 33, 52, 123, 28, 3, 96, 26, 116, 79, 104, 47, 49, 86, 100, 15, 17, 32, 60, 117, 62, 106, 36, 34, 118, 41, 58, 84, 51, 39, 89, 94, 46, 110, 19, 107, 29, 42, 77, 72, 56, 105, 125, 76, 30, 85, 0, 87, 23, 98, 20, 99, 82, 66, 73, 75, 65, 71, 68, 74], [45, 109, 35, 112, 93, 25, 22, 115, 18, 83, 14, 61, 13, 10, 80, 119, 31, 91, 55, 11, 121, 124, 54, 126, 7, 114, 50, 103, 60, 90, 102, 101, 21, 113, 62, 116, 123, 1, 8, 84, 15, 127, 110, 33, 88, 9, 20, 79, 48, 108, 27, 47, 122, 92, 125, 23, 95, 49, 28, 17, 69, 32, 118, 100, 75, 117, 85, 120, 52, 24, 51, 97, 107, 111, 37, 98, 63, 41, 46, 34, 59, 81, 4, 56, 43, 39, 104, 40, 106, 26, 53, 87, 57, 58, 42, 86, 94, 82, 38, 6, 36, 19, 105, 44, 76, 89, 30, 5, 29, 12, 64, 73, 67, 96, 16, 77, 71, 68, 74, 78, 72, 99, 70, 2, 66, 65, 3, 0], [45, 109, 55, 35, 118, 93, 22, 112, 111, 121, 113, 115, 43, 48, 25, 31, 91, 53, 62, 126, 40, 122, 52, 80, 119, 123, 50, 61, 49, 117, 51, 56, 27, 92, 58, 103, 104, 102, 18, 83, 60, 39, 124, 46, 108, 59, 44, 94, 54, 41, 127, 106, 90, 114, 76, 47, 57, 107, 99, 42, 20, 21, 116, 101, 120, 110, 32, 125, 88, 84, 63, 14, 16, 23, 38, 33, 28, 74, 64, 105, 95, 37, 97, 36, 7, 15, 98, 96, 1, 100, 2, 30, 79, 19, 89, 11, 86, 34, 85, 69, 10, 0, 87, 66, 29, 8, 72, 67, 12, 68, 65, 26, 24, 17, 70, 75, 78, 4, 81, 5, 6, 82, 3, 71, 13, 9, 77, 73], [45, 35, 109, 93, 22, 18, 25, 55, 83, 13, 124, 31, 115, 14, 88, 80, 61, 112, 91, 81, 10, 118, 21, 50, 11, 15, 121, 28, 90, 63, 77, 9, 73, 48, 64, 56, 33, 7, 103, 85, 23, 44, 126, 120, 127, 27, 41, 24, 43, 92, 119, 111, 114, 113, 29, 26, 12, 62, 99, 30, 58, 67, 1, 8, 122, 52, 117, 76, 97, 95, 57, 123, 60, 69, 116, 108, 84, 89, 102, 71, 20, 51, 39, 70, 74, 40, 53, 2, 17, 37, 86, 59, 107, 6, 104, 106, 79, 5, 38, 19, 105, 54, 100, 110, 82, 49, 42, 36, 101, 34, 47, 32, 98, 46, 96, 125, 72, 87, 16, 94, 3, 66, 78, 75, 0, 68, 4, 65], [41, 98, 57, 117, 25, 51, 86, 127, 95, 114, 125, 120, 121, 18, 93, 16, 78, 50, 115, 61, 110, 105, 59, 53, 48, 55, 119, 46, 15, 49, 31, 58, 38, 87, 40, 12, 47, 39, 28, 19, 88, 118, 62, 108, 63, 45, 74, 85, 11, 116, 72, 75, 54, 67, 123, 68, 9, 76, 56, 106, 124, 35, 122, 10, 126, 80, 101, 90, 7, 69, 107, 13, 65, 111, 91, 66, 60, 43, 42, 109, 64, 92, 1, 113, 71, 112, 100, 33, 70, 23, 103, 32, 44, 82, 21, 89, 52, 20, 27, 22, 36, 104, 17, 29, 102, 99, 81, 30, 37, 24, 26, 79, 77, 14, 2, 96, 84, 97, 94, 4, 83, 8, 73, 0, 3, 34, 5, 6], [41, 98, 117, 57, 127, 95, 25, 125, 86, 114, 51, 93, 53, 54, 50, 18, 48, 120, 60, 61, 58, 105, 119, 55, 111, 38, 78, 115, 31, 16, 87, 59, 62, 121, 40, 108, 88, 21, 106, 43, 126, 56, 35, 63, 19, 15, 116, 85, 110, 12, 28, 10, 45, 52, 122, 47, 92, 7, 90, 101, 124, 123, 104, 109, 36, 113, 49, 72, 29, 80, 100, 33, 81, 76, 112, 103, 44, 30, 74, 107, 32, 46, 91, 118, 9, 77, 42, 23, 24, 82, 97, 22, 1, 75, 2, 39, 89, 67, 37, 20, 94, 27, 83, 13, 5, 102, 11, 26, 99, 70, 96, 79, 17, 3, 4, 84, 14, 68, 71, 69, 73, 64, 34, 6, 8, 0, 66, 65], [41, 98, 57, 117, 25, 86, 95, 51, 61, 127, 114, 18, 78, 59, 125, 119, 121, 48, 16, 93, 53, 110, 50, 105, 120, 116, 62, 58, 55, 38, 49, 31, 7, 76, 67, 65, 39, 10, 15, 87, 115, 12, 28, 108, 9, 74, 88, 45, 71, 56, 75, 21, 118, 109, 44, 72, 47, 19, 46, 60, 113, 124, 111, 68, 99, 40, 122, 106, 54, 103, 92, 35, 64, 81, 85, 70, 100, 89, 90, 107, 29, 43, 23, 63, 42, 69, 66, 112, 82, 1, 52, 91, 20, 32, 126, 123, 104, 11, 36, 80, 27, 13, 102, 14, 2, 33, 37, 26, 84, 22, 4, 8, 5, 83, 30, 101, 77, 17, 79, 97, 73, 96, 24, 94, 0, 3, 34, 6], [41, 98, 57, 117, 95, 120, 86, 25, 114, 51, 127, 125, 18, 110, 93, 121, 78, 16, 105, 59, 48, 108, 53, 58, 62, 119, 50, 45, 61, 38, 47, 49, 15, 124, 115, 39, 87, 44, 72, 85, 31, 46, 55, 111, 118, 12, 76, 21, 101, 54, 109, 7, 65, 116, 77, 67, 92, 35, 107, 43, 40, 69, 32, 90, 9, 11, 10, 112, 19, 88, 52, 75, 80, 97, 106, 28, 42, 82, 68, 113, 123, 102, 83, 81, 66, 20, 74, 70, 84, 24, 56, 64, 60, 22, 89, 99, 29, 63, 94, 126, 104, 36, 14, 13, 122, 91, 26, 103, 27, 73, 33, 5, 30, 100, 4, 96, 17, 37, 79, 0, 23, 8, 71, 1, 2, 34, 6, 3], [39, 63, 52, 97, 18, 113, 13, 87, 22, 28, 100, 120, 10, 4, 8, 45, 92, 11, 66, 0, 89, 126, 6, 81, 94, 27, 29, 68, 69, 42, 72, 65, 15, 12, 1, 14, 30, 59, 127, 108, 25, 70, 86, 74, 82, 110, 80, 119, 20, 79, 77, 44, 111, 90, 21, 7, 78, 62, 50, 85, 43, 16, 17, 121, 5, 19, 102, 23, 101, 40, 35, 106, 116, 9, 64, 24, 67, 96, 122, 75, 26, 84, 55, 83, 47, 38, 118, 125, 95, 54, 105, 91, 99, 76, 58, 88, 37, 104, 93, 3, 98, 32, 61, 34, 57, 117, 49, 73, 123, 115, 56, 36, 109, 103, 46, 41, 51, 31, 60, 48, 53, 124, 112, 71, 107, 114, 2, 33], [63, 52, 39, 113, 120, 37, 122, 125, 126, 45, 106, 119, 58, 61, 127, 111, 55, 110, 57, 47, 116, 51, 62, 103, 123, 56, 48, 115, 117, 59, 49, 54, 118, 108, 60, 97, 109, 53, 50, 121, 44, 114, 46, 124, 112, 100, 105, 107, 43, 36, 22, 42, 28, 41, 40, 38, 104, 21, 87, 20, 101, 102, 27, 35, 96, 2, 91, 94, 98, 11, 32, 34, 19, 99, 18, 5, 25, 92, 24, 33, 90, 29, 26, 31, 85, 8, 89, 93, 30, 95, 88, 84, 78, 81, 14, 86, 15, 13, 73, 6, 83, 23, 16, 64, 75, 79, 17, 69, 66, 9, 10, 80, 82, 12, 76, 77, 3, 67, 72, 0, 65, 70, 71, 4, 74, 7, 1, 68], [63, 39, 52, 97, 24, 87, 50, 66, 54, 110, 64, 83, 18, 69, 11, 7, 92, 58, 120, 79, 3, 36, 73, 13, 20, 114, 28, 0, 57, 2, 67, 6, 9, 22, 119, 106, 80, 44, 45, 10, 113, 8, 26, 1, 68, 108, 48, 37, 111, 4, 30, 65, 122, 85, 29, 62, 47, 72, 71, 107, 126, 25, 88, 125, 49, 61, 89, 76, 103, 70, 51, 96, 34, 12, 123, 40, 16, 104, 35, 81, 5, 38, 78, 116, 124, 117, 115, 31, 90, 84, 127, 32, 46, 56, 59, 55, 101, 98, 42, 60, 100, 118, 14, 75, 121, 94, 91, 105, 53, 112, 43, 27, 15, 17, 23, 109, 19, 95, 99, 93, 77, 102, 86, 41, 21, 82, 74, 33], [63, 39, 113, 52, 97, 120, 55, 28, 122, 87, 126, 119, 62, 92, 50, 114, 59, 22, 111, 58, 109, 36, 118, 106, 51, 47, 18, 110, 121, 61, 125, 54, 56, 116, 105, 115, 48, 57, 42, 45, 60, 49, 15, 102, 107, 21, 101, 127, 123, 124, 46, 100, 41, 37, 103, 117, 40, 44, 112, 13, 19, 35, 24, 53, 108, 43, 89, 81, 38, 84, 93, 99, 20, 69, 91, 104, 33, 8, 96, 74, 85, 34, 94, 30, 11, 98, 27, 31, 32, 86, 25, 23, 29, 26, 95, 90, 10, 17, 9, 88, 79, 83, 82, 76, 12, 4, 14, 5, 78, 6, 80, 64, 16, 71, 72, 73, 66, 3, 1, 77, 70, 2, 65, 68, 7, 75, 0, 67], [111, 47, 95, 28, 25, 84, 97, 18, 86, 33, 39, 122, 16, 123, 74, 29, 69, 15, 53, 117, 78, 120, 107, 49, 58, 7, 4, 106, 124, 35, 75, 125, 64, 17, 2, 110, 62, 55, 61, 113, 63, 91, 13, 32, 46, 42, 126, 99, 23, 12, 54, 121, 3, 108, 112, 72, 115, 1, 37, 57, 38, 96, 73, 59, 85, 44, 36, 43, 41, 60, 45, 50, 119, 77, 19, 114, 27, 48, 109, 101, 67, 116, 118, 104, 79, 52, 87, 127, 56, 102, 22, 76, 26, 103, 70, 90, 40, 24, 20, 83, 31, 89, 51, 6, 82, 92, 105, 30, 88, 94, 0, 65, 93, 100, 98, 9, 34, 21, 11, 80, 71, 81, 5, 14, 8, 10, 66, 68], [111, 47, 28, 95, 25, 84, 39, 97, 86, 33, 18, 16, 60, 59, 115, 112, 29, 54, 15, 74, 7, 106, 78, 52, 110, 118, 123, 1, 114, 69, 64, 17, 46, 62, 120, 57, 109, 2, 13, 4, 75, 12, 122, 101, 116, 107, 126, 49, 99, 27, 119, 9, 113, 121, 50, 45, 48, 41, 124, 42, 70, 63, 72, 58, 81, 36, 55, 85, 90, 56, 23, 73, 87, 117, 125, 76, 94, 61, 26, 67, 20, 35, 3, 96, 30, 32, 38, 43, 100, 53, 44, 103, 77, 24, 51, 40, 127, 89, 91, 37, 108, 31, 88, 22, 79, 19, 82, 83, 102, 6, 34, 105, 65, 98, 104, 92, 93, 80, 21, 66, 5, 0, 14, 68, 71, 11, 10, 8], [111, 47, 28, 95, 25, 84, 126, 33, 18, 97, 57, 86, 39, 120, 16, 60, 110, 124, 61, 29, 63, 116, 74, 54, 78, 56, 17, 64, 7, 46, 15, 125, 99, 50, 4, 12, 107, 62, 127, 75, 48, 53, 27, 35, 96, 106, 112, 109, 92, 59, 38, 113, 67, 2, 49, 55, 101, 72, 45, 42, 36, 69, 1, 41, 119, 20, 31, 94, 108, 52, 58, 123, 13, 89, 85, 122, 121, 87, 23, 22, 40, 114, 117, 73, 37, 19, 44, 32, 115, 70, 76, 105, 100, 90, 118, 9, 24, 51, 103, 43, 104, 91, 102, 34, 26, 88, 82, 81, 30, 98, 6, 3, 83, 0, 68, 21, 14, 80, 77, 93, 71, 10, 79, 8, 5, 65, 11, 66], [111, 47, 28, 25, 95, 84, 57, 86, 39, 110, 62, 33, 18, 16, 74, 116, 78, 29, 97, 2, 49, 41, 17, 118, 96, 15, 115, 1, 7, 64, 69, 4, 126, 13, 59, 120, 3, 127, 113, 63, 51, 65, 58, 56, 125, 36, 44, 50, 85, 123, 35, 60, 121, 75, 99, 43, 54, 37, 42, 76, 117, 119, 98, 114, 106, 46, 6, 55, 67, 70, 108, 45, 79, 124, 40, 89, 109, 52, 48, 38, 32, 53, 0, 26, 24, 105, 101, 72, 73, 27, 9, 20, 82, 112, 103, 22, 5, 122, 30, 81, 94, 61, 12, 102, 104, 107, 83, 90, 34, 77, 23, 11, 80, 92, 87, 88, 93, 21, 10, 91, 31, 14, 19, 100, 71, 66, 68, 8], [49, 117, 102, 113, 123, 52, 116, 53, 38, 23, 41, 91, 33, 25, 121, 85, 118, 119, 54, 122, 115, 109, 59, 93, 104, 120, 30, 110, 62, 61, 51, 108, 60, 124, 82, 112, 45, 111, 98, 114, 57, 48, 50, 96, 42, 63, 36, 94, 56, 47, 126, 46, 106, 37, 125, 21, 35, 43, 107, 127, 58, 44, 77, 79, 27, 55, 100, 103, 99, 95, 89, 40, 105, 39, 97, 24, 84, 18, 34, 101, 32, 20, 16, 17, 22, 31, 19, 29, 76, 87, 11, 78, 90, 83, 28, 88, 86, 26, 72, 92, 15, 5, 12, 4, 14, 70, 71, 13, 74, 80, 73, 9, 75, 3, 7, 81, 66, 1, 10, 8, 6, 0, 65, 68, 69, 67, 2, 64], [49, 102, 113, 117, 33, 91, 25, 38, 116, 123, 54, 30, 82, 23, 110, 55, 85, 35, 53, 121, 109, 39, 115, 95, 58, 62, 122, 100, 43, 93, 98, 51, 89, 52, 59, 60, 27, 21, 41, 125, 78, 48, 119, 57, 120, 42, 77, 40, 37, 92, 50, 118, 108, 80, 90, 96, 63, 28, 114, 112, 24, 111, 61, 103, 106, 124, 104, 71, 56, 34, 47, 101, 5, 14, 11, 127, 99, 22, 20, 36, 46, 94, 29, 12, 126, 32, 9, 65, 19, 88, 68, 86, 31, 97, 44, 15, 107, 105, 26, 45, 87, 2, 18, 17, 83, 67, 70, 72, 6, 64, 84, 8, 10, 76, 1, 3, 0, 16, 13, 75, 81, 74, 79, 7, 73, 69, 66, 4], [49, 102, 113, 117, 53, 33, 30, 25, 38, 91, 85, 120, 23, 125, 122, 93, 60, 82, 119, 54, 61, 41, 48, 118, 96, 107, 59, 27, 94, 109, 50, 108, 121, 110, 114, 106, 52, 35, 98, 42, 103, 101, 20, 40, 95, 43, 45, 123, 58, 57, 105, 99, 14, 116, 62, 89, 46, 36, 44, 56, 100, 124, 39, 47, 80, 77, 26, 127, 51, 111, 55, 97, 78, 72, 126, 112, 92, 3, 63, 115, 31, 34, 104, 22, 11, 24, 90, 29, 84, 32, 28, 75, 37, 10, 88, 73, 71, 64, 83, 4, 87, 18, 21, 15, 17, 12, 19, 2, 81, 65, 74, 9, 5, 86, 70, 16, 79, 76, 7, 13, 6, 68, 1, 0, 69, 8, 66, 67], [49, 102, 113, 117, 123, 33, 52, 25, 53, 91, 122, 61, 23, 38, 63, 20, 41, 48, 82, 85, 100, 118, 121, 93, 112, 27, 89, 116, 15, 96, 114, 42, 108, 11, 35, 30, 97, 50, 39, 101, 62, 9, 120, 99, 51, 87, 46, 45, 6, 115, 47, 60, 57, 59, 44, 124, 68, 126, 111, 127, 105, 109, 58, 43, 94, 54, 90, 34, 110, 80, 32, 103, 40, 55, 56, 28, 29, 26, 107, 119, 77, 125, 106, 104, 14, 98, 4, 1, 37, 95, 84, 22, 73, 36, 81, 21, 18, 16, 92, 83, 88, 31, 8, 86, 66, 79, 70, 19, 24, 10, 12, 0, 75, 2, 17, 76, 69, 5, 71, 78, 13, 7, 67, 3, 74, 64, 65, 72], [117, 120, 60, 38, 58, 127, 122, 46, 52, 62, 102, 126, 55, 125, 118, 43, 110, 116, 123, 121, 124, 115, 113, 63, 61, 119, 51, 57, 53, 114, 112, 56, 25, 54, 48, 50, 45, 59, 47, 108, 111, 39, 107, 44, 97, 104, 35, 37, 49, 26, 109, 94, 106, 41, 105, 86, 42, 98, 83, 103, 17, 24, 101, 23, 28, 36, 40, 85, 88, 92, 100, 91, 78, 19, 22, 21, 75, 32, 30, 34, 99, 11, 96, 16, 95, 27, 77, 79, 29, 82, 84, 31, 33, 14, 90, 93, 20, 81, 89, 80, 72, 18, 9, 3, 12, 87, 7, 67, 76, 15, 73, 71, 69, 70, 8, 2, 6, 1, 5, 0, 13, 65, 64, 74, 66, 68, 10, 4], [117, 60, 58, 127, 38, 110, 122, 46, 52, 116, 62, 115, 118, 126, 120, 125, 123, 43, 102, 51, 61, 124, 63, 55, 113, 109, 53, 94, 56, 121, 119, 57, 112, 59, 97, 54, 50, 114, 48, 45, 107, 47, 108, 111, 39, 104, 28, 44, 106, 25, 103, 23, 49, 86, 26, 105, 42, 35, 83, 41, 37, 36, 98, 101, 17, 91, 40, 74, 30, 100, 88, 84, 33, 24, 32, 99, 19, 22, 95, 90, 85, 81, 34, 96, 92, 75, 93, 31, 11, 69, 79, 27, 77, 78, 29, 66, 15, 80, 72, 16, 0, 14, 3, 64, 5, 89, 12, 87, 71, 20, 67, 18, 2, 82, 21, 65, 1, 8, 68, 6, 13, 10, 70, 76, 7, 9, 73, 4], [117, 60, 46, 120, 38, 110, 127, 122, 58, 52, 115, 62, 57, 125, 116, 118, 43, 126, 119, 48, 63, 123, 61, 56, 35, 49, 25, 109, 94, 121, 55, 124, 47, 51, 111, 50, 112, 54, 114, 83, 53, 113, 24, 26, 108, 45, 59, 105, 40, 107, 104, 102, 44, 86, 41, 97, 106, 28, 103, 37, 42, 88, 30, 23, 36, 39, 101, 100, 22, 75, 34, 85, 99, 11, 84, 17, 31, 19, 32, 16, 98, 79, 27, 81, 91, 69, 92, 33, 78, 93, 96, 95, 3, 14, 72, 82, 90, 29, 8, 89, 77, 20, 5, 13, 18, 66, 80, 74, 15, 87, 2, 64, 0, 9, 21, 67, 70, 65, 71, 6, 1, 76, 7, 12, 10, 73, 68, 4], [46, 117, 38, 120, 110, 85, 94, 25, 97, 16, 9, 90, 12, 126, 122, 81, 23, 116, 58, 100, 15, 62, 118, 98, 52, 68, 14, 76, 28, 89, 7, 125, 87, 82, 61, 71, 121, 37, 18, 123, 48, 86, 1, 42, 60, 30, 21, 99, 57, 96, 31, 91, 44, 107, 32, 119, 112, 36, 53, 47, 56, 127, 51, 101, 45, 55, 115, 10, 113, 43, 70, 114, 124, 88, 104, 29, 26, 73, 92, 54, 79, 109, 111, 75, 49, 50, 63, 24, 19, 13, 20, 93, 59, 83, 4, 108, 95, 74, 80, 34, 84, 35, 17, 106, 72, 27, 6, 11, 103, 2, 77, 22, 105, 78, 41, 8, 40, 39, 5, 65, 33, 69, 64, 3, 66, 0, 67, 102], [41, 99, 67, 74, 80, 0, 86, 44, 4, 13, 71, 73, 19, 115, 118, 69, 124, 105, 52, 119, 113, 114, 51, 65, 122, 54, 12, 116, 112, 111, 35, 117, 110, 126, 57, 2, 62, 63, 64, 60, 48, 107, 61, 22, 6, 89, 121, 76, 84, 123, 109, 1, 29, 90, 3, 45, 30, 88, 14, 106, 83, 79, 10, 33, 92, 9, 108, 120, 20, 55, 23, 68, 39, 15, 40, 101, 102, 103, 98, 104, 66, 75, 100, 56, 27, 94, 11, 38, 53, 17, 58, 24, 31, 32, 8, 77, 70, 7, 5, 26, 43, 16, 85, 95, 81, 46, 34, 37, 49, 125, 96, 21, 28, 36, 72, 50, 127, 91, 93, 59, 97, 42, 82, 78, 87, 18, 47, 25], [41, 99, 80, 74, 13, 4, 86, 71, 19, 67, 73, 113, 52, 69, 61, 51, 12, 44, 118, 115, 112, 122, 64, 60, 114, 105, 111, 66, 2, 120, 57, 116, 103, 63, 6, 54, 119, 62, 124, 35, 75, 117, 43, 1, 55, 20, 15, 88, 48, 108, 126, 22, 101, 27, 125, 107, 21, 82, 29, 85, 31, 102, 76, 17, 50, 78, 34, 3, 90, 106, 37, 79, 26, 24, 8, 109, 83, 127, 59, 39, 23, 91, 45, 46, 87, 49, 98, 123, 100, 81, 96, 92, 110, 32, 30, 68, 93, 89, 38, 56, 97, 42, 18, 53, 84, 36, 33, 5, 58, 94, 72, 25, 28, 104, 14, 95, 121, 10, 65, 47, 70, 40, 11, 7, 16, 77, 9, 0], [41, 74, 0, 99, 65, 86, 2, 44, 80, 13, 67, 69, 4, 118, 119, 116, 115, 113, 71, 112, 54, 19, 111, 110, 114, 105, 73, 124, 12, 117, 121, 52, 122, 51, 126, 57, 35, 22, 107, 62, 48, 63, 29, 61, 15, 94, 3, 59, 68, 89, 83, 60, 6, 123, 10, 27, 90, 33, 45, 20, 108, 5, 24, 40, 88, 16, 106, 1, 49, 92, 64, 9, 76, 23, 53, 8, 32, 38, 75, 120, 100, 66, 7, 56, 21, 97, 79, 39, 104, 85, 26, 91, 18, 109, 93, 11, 82, 70, 55, 101, 87, 30, 58, 125, 98, 17, 78, 127, 31, 81, 72, 96, 84, 14, 77, 42, 28, 95, 34, 43, 25, 103, 102, 37, 47, 46, 36, 50], [41, 99, 0, 2, 44, 67, 86, 80, 13, 4, 65, 115, 118, 69, 71, 119, 116, 113, 73, 114, 111, 112, 122, 54, 19, 52, 51, 105, 124, 110, 126, 22, 121, 117, 57, 35, 74, 62, 29, 12, 107, 3, 48, 68, 61, 60, 63, 89, 33, 123, 64, 15, 6, 108, 90, 40, 5, 97, 94, 88, 16, 70, 27, 106, 20, 83, 100, 98, 17, 75, 79, 38, 49, 81, 45, 84, 11, 1, 39, 24, 76, 92, 56, 9, 91, 120, 104, 55, 96, 28, 18, 10, 26, 8, 14, 78, 82, 125, 109, 42, 25, 30, 85, 53, 59, 93, 7, 23, 37, 95, 101, 36, 58, 72, 46, 21, 127, 87, 31, 47, 34, 32, 102, 43, 50, 77, 66, 103], [106, 99, 59, 26, 118, 124, 94, 49, 42, 21, 62, 19, 115, 86, 17, 113, 60, 121, 40, 109, 55, 30, 78, 32, 80, 56, 117, 107, 73, 122, 28, 123, 47, 12, 24, 96, 98, 29, 120, 74, 23, 43, 50, 53, 11, 69, 51, 37, 125, 61, 63, 114, 52, 27, 79, 108, 116, 103, 57, 126, 110, 58, 127, 112, 119, 48, 84, 89, 105, 92, 39, 54, 46, 111, 36, 104, 4, 22, 8, 71, 101, 45, 3, 72, 67, 7, 90, 44, 41, 34, 38, 15, 77, 33, 102, 93, 97, 31, 100, 70, 14, 66, 64, 95, 88, 85, 76, 0, 1, 82, 91, 25, 81, 16, 87, 18, 5, 83, 20, 13, 6, 35, 65, 9, 68, 10, 75, 2], [106, 61, 99, 118, 115, 59, 121, 26, 94, 42, 107, 62, 113, 21, 19, 86, 125, 29, 11, 109, 28, 79, 50, 22, 55, 60, 25, 117, 40, 17, 126, 47, 90, 56, 123, 23, 119, 124, 24, 58, 52, 93, 80, 57, 78, 114, 39, 98, 13, 74, 66, 32, 16, 73, 116, 101, 85, 34, 65, 104, 53, 63, 103, 49, 12, 70, 122, 35, 36, 82, 120, 108, 45, 46, 4, 51, 71, 95, 30, 44, 20, 91, 37, 38, 105, 87, 72, 48, 112, 43, 27, 100, 77, 6, 7, 111, 97, 31, 41, 3, 89, 64, 110, 54, 127, 88, 96, 102, 92, 18, 84, 83, 33, 5, 69, 81, 76, 15, 14, 0, 67, 10, 75, 8, 9, 2, 1, 68], [106, 59, 115, 99, 118, 62, 50, 112, 26, 55, 94, 125, 86, 19, 117, 121, 124, 42, 24, 116, 21, 53, 80, 17, 29, 52, 122, 107, 44, 78, 49, 123, 30, 32, 40, 28, 45, 11, 57, 74, 61, 98, 63, 58, 120, 101, 113, 23, 46, 12, 110, 60, 104, 27, 48, 96, 51, 108, 109, 114, 47, 72, 126, 119, 69, 4, 127, 111, 54, 97, 73, 56, 71, 43, 36, 39, 95, 37, 103, 38, 102, 105, 41, 84, 33, 100, 7, 22, 34, 91, 93, 90, 35, 31, 79, 8, 66, 92, 87, 67, 88, 82, 20, 16, 25, 18, 68, 85, 76, 70, 15, 81, 64, 89, 3, 83, 1, 0, 14, 77, 6, 5, 9, 10, 65, 13, 75, 2], [106, 99, 107, 115, 59, 26, 94, 113, 86, 17, 19, 42, 21, 12, 121, 80, 30, 62, 40, 119, 117, 55, 96, 49, 78, 74, 60, 69, 61, 125, 112, 24, 72, 11, 118, 53, 114, 56, 124, 70, 120, 4, 32, 123, 116, 109, 108, 58, 5, 44, 104, 73, 110, 68, 52, 126, 93, 54, 89, 122, 67, 48, 45, 57, 1, 101, 63, 51, 37, 29, 47, 28, 71, 36, 127, 16, 43, 41, 111, 25, 105, 46, 22, 102, 13, 88, 9, 98, 83, 50, 27, 0, 84, 39, 8, 20, 95, 90, 14, 87, 7, 38, 77, 33, 64, 100, 6, 82, 92, 34, 79, 23, 76, 35, 3, 103, 31, 65, 18, 91, 85, 66, 97, 81, 10, 15, 2, 75]], "model.layers.29.self_attn.k_proj": [[109, 45, 99, 93, 22, 64, 31, 83, 80, 14, 25, 18, 11, 91, 67, 8, 2, 7, 126, 119, 1, 124, 6, 127, 70, 122, 112, 77, 13, 115, 60, 16, 21, 55, 9, 43, 114, 23, 10, 51, 4, 29, 90, 12, 52, 110, 63, 116, 41, 42, 36, 111, 57, 69, 118, 28, 59, 49, 120, 84, 103, 72, 123, 56, 61, 39, 121, 88, 113, 62, 54, 105, 44, 47, 53, 27, 40, 82, 58, 50, 117, 85, 102, 96, 48, 38, 92, 33, 100, 5, 24, 87, 108, 107, 106, 94, 101, 125, 98, 32, 46, 26, 34, 104, 15, 37, 30, 76, 97, 89, 79, 20, 95, 78, 75, 17, 35, 73, 81, 74, 66, 19, 86, 68, 71, 3, 0, 65], [105, 34, 57, 18, 53, 46, 51, 125, 78, 86, 117, 25, 58, 120, 16, 45, 64, 31, 127, 67, 95, 109, 98, 50, 12, 119, 29, 69, 59, 111, 108, 61, 66, 10, 112, 9, 19, 74, 87, 7, 70, 113, 41, 60, 92, 80, 13, 49, 15, 55, 91, 52, 114, 88, 21, 121, 62, 76, 2, 107, 65, 75, 116, 103, 38, 83, 63, 68, 100, 93, 104, 39, 0, 40, 72, 56, 20, 35, 106, 90, 110, 115, 73, 54, 118, 27, 36, 48, 6, 42, 44, 126, 81, 33, 122, 124, 47, 102, 43, 26, 123, 84, 77, 37, 1, 101, 28, 30, 96, 94, 23, 24, 99, 79, 85, 97, 32, 17, 71, 89, 11, 8, 22, 82, 5, 4, 14, 3], [63, 103, 22, 33, 92, 87, 13, 65, 18, 120, 10, 50, 4, 119, 126, 111, 54, 0, 61, 109, 49, 6, 62, 58, 81, 51, 117, 48, 127, 47, 125, 123, 122, 118, 59, 124, 46, 8, 45, 60, 56, 121, 115, 44, 30, 110, 113, 53, 116, 114, 112, 29, 2, 57, 71, 40, 55, 42, 43, 102, 52, 108, 37, 107, 25, 105, 73, 41, 98, 36, 20, 106, 104, 38, 11, 3, 34, 5, 39, 101, 91, 14, 100, 12, 7, 16, 78, 89, 67, 99, 83, 79, 31, 80, 24, 90, 76, 15, 32, 70, 95, 27, 93, 96, 35, 88, 21, 74, 94, 23, 17, 9, 85, 84, 26, 28, 86, 69, 75, 82, 19, 64, 1, 68, 97, 72, 66, 77], [47, 111, 64, 86, 95, 16, 84, 28, 25, 18, 74, 4, 1, 2, 72, 7, 97, 78, 67, 65, 6, 69, 15, 17, 29, 123, 12, 3, 124, 23, 76, 103, 75, 54, 31, 14, 0, 39, 77, 120, 55, 53, 27, 113, 85, 79, 61, 110, 70, 35, 81, 56, 13, 63, 122, 36, 105, 50, 126, 9, 19, 33, 101, 87, 91, 121, 57, 71, 125, 114, 32, 117, 62, 116, 24, 73, 48, 99, 109, 38, 127, 100, 108, 106, 118, 43, 115, 46, 52, 37, 44, 41, 119, 21, 42, 51, 26, 102, 60, 104, 30, 45, 107, 59, 34, 58, 90, 40, 98, 80, 94, 88, 49, 92, 82, 112, 89, 8, 96, 93, 20, 83, 11, 22, 10, 5, 66, 68], [49, 38, 97, 91, 25, 93, 117, 23, 80, 20, 92, 82, 15, 9, 64, 77, 11, 85, 2, 30, 68, 65, 87, 3, 45, 14, 108, 21, 5, 22, 17, 95, 96, 94, 50, 71, 120, 112, 36, 35, 18, 61, 37, 119, 8, 6, 118, 53, 70, 48, 46, 126, 106, 109, 55, 104, 59, 57, 34, 58, 122, 63, 111, 107, 33, 54, 123, 40, 62, 103, 52, 78, 105, 116, 124, 121, 39, 110, 99, 47, 28, 44, 24, 31, 101, 115, 42, 56, 41, 43, 89, 51, 127, 114, 125, 10, 60, 90, 83, 32, 19, 26, 98, 29, 1, 13, 67, 7, 100, 72, 113, 88, 84, 86, 0, 102, 12, 75, 81, 79, 16, 27, 76, 69, 74, 66, 73, 4], [110, 117, 102, 86, 33, 30, 52, 92, 25, 99, 58, 62, 119, 22, 125, 123, 54, 122, 114, 126, 53, 112, 55, 14, 81, 49, 111, 116, 51, 124, 47, 121, 48, 59, 61, 44, 103, 56, 63, 120, 75, 41, 108, 113, 115, 57, 118, 50, 83, 109, 16, 127, 39, 38, 107, 105, 35, 43, 31, 104, 46, 106, 28, 42, 45, 94, 91, 34, 32, 85, 23, 40, 37, 72, 100, 24, 101, 93, 26, 36, 18, 12, 29, 90, 5, 97, 9, 27, 98, 60, 84, 67, 0, 96, 95, 21, 87, 6, 20, 19, 82, 89, 80, 8, 78, 15, 2, 1, 7, 10, 88, 76, 68, 13, 77, 11, 79, 3, 17, 74, 65, 71, 64, 70, 69, 66, 73, 4], [105, 64, 35, 1, 118, 0, 116, 108, 67, 51, 119, 71, 19, 13, 86, 69, 73, 66, 4, 80, 65, 49, 2, 124, 48, 57, 12, 61, 122, 113, 47, 46, 114, 115, 43, 111, 44, 54, 3, 63, 99, 117, 126, 50, 74, 53, 58, 62, 121, 123, 52, 107, 60, 5, 56, 70, 106, 89, 6, 93, 55, 15, 103, 110, 59, 29, 94, 20, 88, 26, 112, 104, 17, 7, 75, 85, 72, 41, 33, 109, 76, 79, 97, 68, 45, 120, 90, 92, 9, 18, 101, 11, 125, 102, 84, 91, 30, 96, 32, 8, 77, 82, 87, 27, 23, 100, 38, 34, 78, 24, 14, 40, 21, 31, 37, 39, 127, 25, 98, 95, 42, 28, 81, 10, 83, 36, 22, 16], [42, 30, 99, 118, 35, 86, 26, 51, 21, 19, 78, 113, 61, 17, 64, 45, 11, 121, 66, 63, 111, 117, 106, 125, 28, 73, 124, 114, 123, 102, 71, 126, 53, 80, 108, 49, 122, 62, 12, 105, 41, 4, 32, 43, 74, 116, 16, 7, 38, 23, 56, 40, 119, 44, 55, 52, 3, 46, 104, 54, 65, 101, 115, 24, 47, 127, 110, 57, 109, 79, 69, 95, 70, 89, 36, 103, 58, 120, 48, 39, 29, 112, 33, 107, 98, 100, 25, 1, 59, 60, 50, 97, 37, 34, 94, 87, 15, 27, 91, 92, 20, 6, 72, 82, 31, 96, 85, 88, 93, 8, 10, 84, 67, 77, 13, 68, 18, 5, 75, 76, 90, 0, 22, 83, 81, 9, 2, 14]], "model.layers.29.self_attn.qk_proj": [[49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 99, 22, 57, 106, 118, 41, 113, 86, 64, 0, 89, 51, 35, 119, 38, 16, 80, 53, 93, 61, 116, 95, 52, 28, 115, 82, 120, 124, 121, 94, 114, 62, 122, 25, 27, 46, 58, 125, 19, 44, 77, 102, 98, 18, 103, 83, 13, 54, 126, 108, 65, 50, 55, 33, 1, 39, 48, 10, 127, 3, 92, 71, 68, 78, 43, 112, 123, 7, 4, 97, 59, 14, 67, 107, 31, 56, 2, 66, 9, 90, 87, 5, 69, 74, 84, 73, 6, 21, 60, 26, 85, 29, 30, 11, 12, 23, 20, 75, 76, 79, 104, 81, 15, 8, 34, 72, 37, 17, 40, 101, 91, 36, 32, 24, 96, 88, 100, 70], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 41, 106, 99, 22, 0, 118, 64, 113, 86, 51, 89, 52, 35, 38, 119, 61, 16, 115, 95, 80, 121, 120, 124, 28, 116, 94, 62, 93, 114, 55, 53, 122, 59, 65, 82, 25, 18, 98, 127, 46, 44, 58, 77, 50, 27, 123, 13, 126, 83, 102, 54, 2, 43, 68, 1, 108, 33, 125, 103, 19, 48, 39, 4, 7, 3, 74, 71, 87, 67, 14, 97, 92, 112, 5, 78, 10, 31, 73, 66, 56, 60, 9, 90, 107, 21, 11, 6, 29, 76, 84, 69, 75, 12, 23, 20, 30, 104, 17, 85, 34, 40, 8, 72, 79, 81, 15, 26, 91, 36, 101, 88, 37, 24, 70, 32, 96, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 99, 22, 113, 106, 64, 41, 86, 0, 118, 89, 35, 51, 119, 38, 115, 52, 124, 116, 16, 61, 80, 93, 120, 95, 62, 94, 122, 18, 28, 98, 102, 123, 25, 59, 53, 82, 65, 127, 121, 19, 50, 27, 67, 48, 46, 54, 83, 126, 125, 114, 77, 1, 44, 7, 58, 33, 43, 56, 108, 103, 2, 3, 13, 68, 112, 14, 39, 66, 10, 92, 97, 4, 55, 78, 90, 71, 9, 74, 84, 69, 87, 60, 73, 5, 21, 75, 31, 29, 107, 15, 20, 30, 12, 23, 11, 76, 85, 104, 6, 26, 70, 79, 81, 34, 40, 37, 101, 72, 17, 8, 91, 36, 32, 96, 88, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 57, 106, 41, 99, 22, 113, 86, 64, 0, 118, 35, 89, 122, 51, 119, 115, 52, 16, 124, 62, 116, 80, 38, 120, 61, 25, 94, 93, 95, 53, 28, 123, 46, 18, 121, 114, 82, 55, 54, 126, 102, 19, 1, 44, 50, 77, 39, 66, 127, 13, 3, 98, 125, 43, 33, 56, 48, 59, 58, 4, 27, 78, 83, 71, 7, 65, 68, 67, 14, 103, 97, 74, 69, 60, 108, 112, 11, 9, 31, 73, 2, 107, 87, 92, 70, 21, 10, 5, 76, 90, 85, 26, 29, 20, 79, 104, 84, 23, 12, 75, 15, 37, 81, 32, 72, 30, 36, 40, 101, 8, 6, 91, 34, 17, 88, 100, 24, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 99, 86, 22, 64, 118, 0, 113, 89, 51, 119, 124, 61, 35, 122, 16, 115, 52, 80, 62, 121, 120, 38, 54, 28, 46, 116, 25, 93, 18, 82, 95, 94, 53, 125, 114, 48, 44, 58, 126, 98, 1, 102, 83, 19, 27, 13, 59, 123, 50, 77, 43, 55, 78, 127, 7, 112, 39, 56, 103, 4, 68, 108, 67, 71, 73, 33, 65, 74, 66, 87, 14, 70, 2, 107, 10, 92, 69, 31, 97, 84, 3, 9, 60, 90, 30, 5, 21, 20, 11, 26, 76, 81, 23, 29, 15, 12, 85, 79, 91, 75, 104, 40, 8, 34, 37, 101, 100, 32, 17, 72, 88, 96, 24, 36, 6], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 106, 57, 22, 86, 118, 0, 99, 89, 64, 51, 113, 119, 35, 80, 52, 16, 116, 38, 115, 122, 124, 95, 61, 18, 44, 62, 28, 82, 94, 46, 53, 121, 93, 25, 125, 120, 27, 13, 126, 58, 19, 54, 114, 98, 50, 55, 48, 77, 65, 83, 71, 112, 102, 4, 123, 108, 107, 33, 59, 39, 68, 1, 56, 127, 103, 66, 78, 97, 92, 3, 87, 67, 74, 70, 14, 73, 10, 2, 43, 7, 60, 11, 90, 5, 31, 84, 21, 20, 9, 30, 69, 76, 23, 15, 34, 79, 29, 37, 75, 12, 8, 85, 40, 17, 26, 91, 81, 72, 104, 36, 32, 88, 101, 6, 24, 100, 96], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 106, 22, 57, 99, 110, 86, 118, 64, 89, 0, 51, 35, 113, 38, 80, 119, 16, 95, 52, 115, 61, 116, 120, 94, 28, 46, 124, 122, 121, 126, 18, 125, 82, 53, 77, 93, 62, 25, 65, 13, 50, 27, 83, 1, 114, 112, 44, 127, 54, 123, 59, 43, 58, 48, 19, 103, 2, 102, 3, 67, 55, 98, 14, 108, 31, 71, 39, 4, 68, 74, 78, 10, 87, 90, 92, 5, 107, 7, 33, 97, 66, 29, 56, 60, 21, 9, 30, 84, 23, 70, 11, 73, 76, 20, 26, 85, 69, 12, 15, 34, 37, 81, 72, 75, 79, 17, 40, 91, 104, 101, 8, 24, 88, 100, 96, 32, 36, 6], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 99, 22, 106, 86, 57, 0, 113, 118, 64, 51, 38, 35, 89, 119, 52, 95, 28, 61, 115, 120, 93, 124, 80, 122, 62, 116, 16, 94, 25, 53, 121, 125, 82, 58, 123, 77, 102, 27, 103, 126, 44, 98, 54, 1, 83, 18, 114, 65, 112, 13, 127, 39, 48, 67, 59, 50, 19, 46, 56, 71, 108, 33, 68, 2, 66, 3, 97, 43, 92, 7, 55, 31, 14, 87, 90, 74, 10, 78, 4, 5, 23, 9, 107, 73, 20, 29, 85, 60, 21, 30, 69, 11, 84, 70, 26, 12, 40, 75, 79, 76, 15, 81, 17, 34, 37, 72, 101, 36, 91, 88, 104, 32, 6, 96, 24, 100, 8], [49, 63, 117, 105, 111, 47, 45, 109, 42, 99, 41, 110, 57, 22, 106, 118, 86, 64, 113, 89, 51, 0, 35, 38, 95, 119, 120, 52, 93, 16, 61, 116, 115, 123, 28, 80, 62, 124, 121, 94, 127, 126, 44, 59, 114, 82, 122, 58, 83, 48, 1, 98, 18, 25, 102, 103, 125, 65, 108, 13, 53, 19, 56, 67, 46, 112, 39, 54, 55, 3, 68, 33, 77, 27, 4, 43, 71, 66, 10, 31, 60, 50, 92, 97, 107, 90, 7, 74, 78, 30, 87, 69, 29, 14, 2, 85, 20, 23, 5, 26, 73, 21, 84, 76, 11, 75, 9, 12, 17, 79, 37, 81, 40, 15, 91, 104, 36, 101, 6, 70, 72, 34, 88, 96, 8, 32, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 57, 99, 118, 106, 41, 113, 22, 119, 64, 0, 86, 51, 35, 89, 38, 61, 115, 62, 52, 124, 116, 16, 55, 28, 80, 121, 120, 123, 25, 58, 46, 95, 65, 122, 126, 127, 82, 94, 53, 93, 18, 39, 83, 13, 98, 112, 114, 59, 44, 67, 125, 102, 19, 33, 77, 48, 56, 50, 92, 27, 4, 54, 31, 7, 68, 9, 2, 60, 43, 1, 71, 78, 3, 103, 108, 66, 74, 69, 97, 87, 14, 84, 10, 90, 11, 5, 107, 73, 29, 30, 76, 6, 20, 21, 23, 79, 26, 104, 40, 75, 85, 12, 91, 15, 34, 17, 81, 101, 8, 72, 36, 37, 70, 88, 100, 32, 96, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 99, 64, 22, 118, 0, 106, 57, 86, 51, 113, 89, 119, 35, 52, 38, 16, 122, 80, 61, 116, 25, 82, 28, 95, 121, 120, 93, 115, 46, 124, 18, 94, 126, 62, 125, 127, 53, 13, 19, 48, 65, 50, 1, 44, 83, 123, 59, 71, 4, 27, 3, 78, 114, 54, 77, 2, 55, 39, 108, 7, 58, 9, 102, 67, 14, 97, 10, 66, 103, 74, 68, 6, 73, 5, 33, 60, 56, 92, 98, 90, 69, 20, 23, 11, 87, 43, 112, 29, 84, 21, 30, 75, 31, 15, 85, 107, 12, 76, 79, 26, 81, 72, 104, 91, 17, 34, 32, 8, 37, 40, 101, 70, 36, 96, 24, 88, 100], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 110, 22, 57, 106, 99, 86, 64, 118, 0, 89, 113, 35, 122, 52, 51, 38, 119, 16, 80, 61, 120, 28, 116, 95, 115, 93, 62, 82, 25, 124, 94, 18, 121, 19, 126, 53, 13, 83, 125, 103, 58, 1, 59, 3, 77, 48, 44, 55, 46, 4, 27, 123, 54, 102, 33, 114, 50, 2, 71, 43, 65, 78, 98, 68, 74, 67, 127, 108, 39, 14, 112, 60, 10, 66, 92, 97, 7, 107, 31, 56, 20, 90, 75, 9, 21, 6, 85, 23, 69, 87, 5, 29, 12, 26, 73, 76, 11, 81, 84, 30, 40, 17, 79, 34, 91, 15, 8, 37, 72, 36, 88, 24, 104, 100, 32, 101, 70, 96], [49, 63, 105, 117, 111, 47, 45, 109, 42, 110, 57, 99, 106, 41, 118, 113, 22, 86, 51, 64, 89, 120, 119, 0, 35, 52, 38, 61, 115, 116, 80, 95, 94, 124, 25, 122, 28, 16, 62, 46, 125, 121, 93, 43, 44, 53, 59, 58, 126, 123, 98, 102, 83, 18, 114, 33, 39, 82, 55, 65, 27, 48, 103, 77, 127, 19, 56, 54, 50, 13, 108, 112, 3, 67, 92, 1, 97, 71, 4, 90, 31, 60, 23, 2, 66, 68, 10, 78, 87, 73, 74, 107, 7, 26, 21, 40, 5, 29, 69, 85, 30, 20, 84, 14, 76, 9, 75, 6, 91, 17, 15, 12, 79, 34, 81, 11, 104, 101, 88, 100, 8, 37, 24, 96, 72, 36, 70, 32], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 99, 41, 106, 22, 57, 64, 118, 0, 86, 113, 51, 89, 35, 119, 80, 52, 61, 120, 38, 121, 95, 115, 16, 124, 122, 28, 116, 62, 25, 93, 53, 94, 59, 58, 123, 1, 48, 46, 65, 125, 50, 83, 13, 127, 114, 18, 92, 19, 98, 82, 71, 33, 4, 103, 66, 3, 126, 67, 77, 108, 54, 102, 27, 55, 60, 39, 14, 112, 2, 44, 43, 73, 10, 7, 107, 78, 56, 74, 68, 97, 31, 69, 84, 5, 87, 90, 20, 29, 9, 23, 76, 12, 75, 11, 26, 21, 30, 85, 40, 6, 34, 70, 79, 15, 81, 17, 37, 72, 104, 8, 100, 91, 101, 24, 88, 36, 32, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 118, 22, 99, 0, 64, 86, 89, 51, 113, 119, 38, 35, 52, 80, 95, 120, 16, 122, 116, 61, 28, 121, 126, 25, 94, 124, 82, 46, 53, 93, 123, 115, 13, 48, 114, 125, 65, 55, 83, 19, 62, 18, 39, 102, 3, 2, 1, 127, 71, 9, 98, 108, 54, 44, 50, 59, 73, 67, 4, 78, 43, 77, 7, 68, 92, 27, 14, 84, 60, 58, 74, 103, 66, 70, 10, 112, 33, 97, 21, 5, 20, 69, 75, 31, 107, 56, 90, 11, 23, 29, 76, 87, 30, 15, 26, 12, 79, 34, 91, 17, 85, 8, 40, 81, 104, 6, 101, 72, 37, 88, 100, 32, 24, 36, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 57, 22, 99, 118, 106, 0, 86, 64, 113, 89, 51, 38, 35, 116, 119, 16, 80, 95, 52, 121, 124, 122, 46, 61, 123, 28, 120, 62, 25, 93, 126, 115, 18, 94, 82, 53, 83, 114, 13, 125, 102, 50, 33, 127, 1, 44, 48, 19, 4, 98, 108, 59, 103, 3, 68, 7, 65, 27, 78, 67, 77, 74, 39, 56, 58, 54, 55, 2, 71, 66, 73, 60, 97, 14, 43, 9, 70, 21, 90, 92, 69, 10, 87, 107, 5, 75, 85, 29, 84, 31, 30, 76, 20, 23, 12, 112, 104, 11, 79, 26, 81, 8, 37, 17, 15, 40, 34, 91, 72, 101, 32, 36, 100, 88, 96, 6, 24], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 22, 110, 106, 99, 57, 118, 86, 51, 113, 64, 0, 35, 89, 38, 52, 80, 95, 16, 120, 28, 119, 61, 94, 115, 93, 25, 124, 116, 125, 123, 53, 121, 122, 46, 82, 102, 18, 114, 62, 83, 39, 126, 19, 27, 58, 13, 55, 1, 59, 48, 92, 33, 103, 127, 50, 98, 43, 44, 77, 87, 108, 54, 56, 4, 14, 65, 74, 90, 67, 31, 78, 66, 97, 3, 71, 73, 7, 10, 85, 68, 112, 2, 60, 29, 21, 23, 69, 9, 30, 76, 11, 107, 84, 26, 20, 5, 70, 81, 75, 12, 34, 104, 17, 91, 37, 15, 40, 32, 36, 24, 101, 79, 8, 72, 100, 96, 88, 6], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 86, 41, 106, 99, 22, 113, 118, 64, 0, 51, 89, 35, 95, 16, 115, 38, 119, 80, 122, 94, 28, 120, 116, 18, 62, 121, 93, 52, 124, 61, 123, 25, 53, 98, 83, 102, 125, 19, 82, 1, 27, 114, 3, 13, 48, 65, 50, 126, 67, 103, 39, 58, 59, 55, 127, 92, 54, 66, 68, 33, 77, 2, 71, 7, 74, 56, 46, 44, 14, 112, 78, 4, 108, 73, 10, 97, 60, 43, 5, 29, 20, 31, 9, 69, 90, 85, 30, 70, 87, 84, 23, 76, 107, 21, 26, 12, 75, 11, 15, 34, 81, 79, 91, 40, 17, 104, 101, 88, 8, 36, 72, 37, 32, 96, 6, 24, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 41, 106, 57, 118, 99, 22, 113, 64, 86, 0, 51, 89, 122, 38, 123, 119, 115, 120, 35, 121, 116, 61, 16, 28, 94, 80, 58, 55, 95, 25, 62, 124, 102, 125, 52, 127, 93, 53, 82, 19, 46, 39, 114, 13, 126, 43, 59, 18, 44, 83, 1, 54, 65, 66, 98, 33, 50, 92, 56, 7, 77, 48, 3, 27, 112, 108, 71, 14, 67, 78, 9, 73, 4, 103, 10, 31, 2, 23, 68, 29, 74, 60, 97, 107, 104, 5, 21, 69, 84, 30, 90, 87, 85, 12, 76, 20, 75, 11, 15, 26, 70, 91, 34, 72, 81, 6, 79, 36, 17, 40, 37, 8, 96, 88, 101, 32, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 106, 57, 99, 22, 118, 86, 64, 113, 51, 0, 89, 35, 119, 38, 61, 52, 16, 28, 94, 120, 124, 116, 122, 80, 121, 62, 115, 82, 48, 125, 55, 25, 95, 93, 102, 127, 123, 54, 53, 39, 58, 46, 44, 83, 98, 114, 27, 59, 19, 18, 65, 1, 13, 112, 126, 33, 14, 2, 50, 77, 74, 3, 108, 68, 103, 73, 92, 43, 97, 71, 4, 7, 66, 78, 31, 60, 9, 10, 21, 56, 67, 6, 5, 90, 75, 23, 87, 76, 84, 69, 11, 29, 107, 20, 30, 85, 12, 79, 104, 26, 17, 72, 81, 36, 34, 37, 15, 101, 8, 70, 88, 40, 91, 96, 24, 32, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 106, 22, 41, 99, 86, 118, 113, 0, 51, 89, 64, 35, 124, 61, 120, 62, 38, 119, 116, 16, 125, 93, 28, 121, 115, 94, 52, 80, 95, 122, 53, 25, 102, 18, 50, 44, 48, 83, 123, 82, 27, 98, 67, 114, 59, 1, 55, 54, 66, 13, 33, 77, 46, 108, 19, 103, 58, 4, 126, 112, 2, 71, 43, 14, 39, 3, 127, 7, 92, 65, 10, 97, 74, 9, 78, 69, 73, 60, 31, 68, 107, 56, 20, 87, 75, 23, 6, 84, 21, 90, 30, 29, 11, 5, 15, 76, 40, 26, 37, 104, 79, 12, 91, 34, 72, 81, 17, 85, 36, 24, 8, 32, 101, 96, 70, 100, 88], [49, 63, 117, 111, 105, 47, 45, 109, 42, 57, 41, 64, 110, 106, 22, 0, 99, 113, 118, 86, 51, 119, 124, 89, 62, 38, 80, 116, 35, 52, 121, 122, 95, 120, 16, 93, 115, 61, 28, 123, 46, 25, 102, 94, 48, 98, 127, 50, 1, 18, 58, 125, 83, 65, 53, 33, 2, 13, 59, 19, 3, 114, 77, 82, 54, 66, 4, 71, 27, 44, 112, 55, 103, 39, 67, 108, 7, 97, 56, 78, 126, 60, 92, 31, 6, 9, 68, 43, 14, 73, 10, 74, 107, 5, 29, 87, 30, 90, 69, 84, 75, 23, 72, 21, 20, 11, 85, 76, 104, 37, 26, 91, 79, 40, 12, 15, 17, 34, 81, 101, 36, 96, 8, 32, 88, 24, 70, 100], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 99, 22, 118, 113, 64, 86, 51, 0, 52, 89, 35, 119, 121, 62, 120, 16, 38, 80, 95, 61, 28, 124, 93, 58, 25, 122, 94, 83, 115, 125, 116, 98, 48, 82, 123, 102, 39, 46, 55, 27, 53, 103, 18, 126, 127, 108, 114, 44, 13, 19, 77, 43, 54, 50, 33, 1, 59, 7, 65, 3, 71, 56, 4, 92, 31, 66, 78, 97, 10, 112, 29, 2, 67, 68, 9, 74, 69, 73, 87, 90, 14, 107, 5, 30, 76, 60, 84, 6, 20, 26, 23, 21, 11, 79, 75, 85, 40, 12, 104, 34, 91, 37, 72, 15, 17, 81, 101, 8, 70, 88, 32, 36, 96, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 22, 99, 118, 0, 86, 51, 89, 113, 52, 35, 64, 16, 119, 62, 38, 28, 121, 95, 116, 115, 80, 122, 124, 82, 25, 120, 94, 61, 93, 125, 53, 58, 27, 46, 18, 98, 55, 126, 48, 50, 102, 83, 114, 19, 33, 39, 3, 77, 123, 71, 13, 108, 127, 54, 1, 103, 68, 44, 78, 74, 65, 59, 7, 97, 112, 67, 43, 66, 14, 4, 92, 60, 56, 10, 73, 69, 9, 21, 84, 76, 31, 2, 30, 11, 87, 107, 29, 5, 6, 85, 23, 90, 75, 20, 26, 12, 34, 72, 37, 79, 15, 40, 101, 8, 36, 104, 81, 17, 91, 70, 88, 32, 96, 24, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 41, 106, 57, 118, 22, 110, 99, 86, 113, 64, 0, 89, 119, 51, 35, 124, 62, 52, 80, 28, 38, 61, 16, 121, 116, 120, 115, 95, 93, 122, 82, 53, 25, 94, 83, 125, 18, 54, 114, 48, 27, 98, 77, 46, 55, 127, 108, 102, 59, 58, 39, 50, 65, 126, 13, 123, 44, 66, 19, 1, 43, 74, 92, 33, 78, 2, 3, 67, 103, 97, 71, 68, 7, 4, 112, 73, 14, 31, 84, 21, 10, 87, 60, 20, 9, 29, 23, 11, 90, 5, 75, 69, 107, 56, 30, 76, 26, 40, 12, 34, 70, 85, 6, 104, 91, 15, 37, 101, 81, 79, 8, 72, 17, 32, 96, 88, 24, 100, 36], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 99, 110, 57, 118, 22, 106, 86, 0, 120, 113, 64, 89, 51, 35, 119, 124, 62, 61, 52, 95, 16, 38, 80, 28, 115, 94, 116, 93, 122, 126, 121, 25, 102, 123, 58, 48, 83, 18, 53, 125, 114, 82, 55, 27, 127, 54, 98, 46, 33, 65, 59, 44, 77, 13, 103, 108, 2, 39, 19, 97, 1, 60, 92, 10, 43, 7, 50, 112, 67, 3, 71, 74, 78, 87, 31, 14, 4, 73, 68, 90, 66, 69, 56, 29, 9, 107, 11, 21, 30, 20, 23, 5, 85, 70, 76, 40, 84, 12, 26, 37, 75, 91, 17, 104, 15, 81, 36, 34, 8, 101, 100, 72, 79, 88, 32, 96, 6, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 118, 41, 22, 99, 57, 106, 86, 64, 0, 89, 51, 113, 35, 119, 38, 52, 124, 61, 120, 116, 80, 115, 16, 95, 126, 25, 102, 121, 93, 28, 122, 82, 114, 53, 62, 94, 54, 125, 83, 108, 58, 1, 123, 18, 39, 103, 19, 127, 27, 44, 50, 3, 55, 33, 66, 46, 67, 7, 92, 59, 13, 77, 65, 48, 43, 98, 4, 74, 112, 107, 97, 56, 68, 87, 78, 71, 9, 60, 90, 31, 10, 5, 14, 73, 2, 29, 26, 84, 85, 69, 20, 21, 11, 30, 70, 23, 12, 37, 17, 76, 75, 15, 34, 81, 104, 91, 40, 36, 79, 72, 101, 32, 8, 88, 96, 100, 24, 6], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 118, 57, 41, 106, 22, 99, 64, 86, 113, 0, 51, 89, 119, 124, 52, 61, 120, 38, 62, 115, 122, 28, 35, 95, 16, 116, 82, 94, 80, 53, 25, 121, 93, 58, 98, 18, 114, 46, 19, 44, 126, 83, 125, 33, 123, 77, 1, 13, 127, 27, 59, 54, 55, 102, 9, 108, 39, 50, 3, 60, 7, 65, 78, 68, 71, 43, 4, 92, 66, 48, 10, 67, 73, 2, 14, 103, 31, 56, 97, 74, 5, 84, 70, 112, 87, 29, 23, 21, 69, 90, 107, 11, 15, 30, 85, 20, 76, 75, 79, 26, 37, 8, 12, 104, 17, 40, 101, 91, 34, 72, 36, 81, 32, 88, 6, 24, 96, 100], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 57, 106, 99, 118, 22, 64, 0, 113, 86, 51, 119, 35, 89, 61, 38, 16, 120, 116, 46, 52, 80, 124, 53, 115, 25, 95, 94, 122, 28, 121, 62, 18, 125, 55, 126, 93, 114, 82, 44, 58, 65, 33, 77, 13, 108, 54, 98, 4, 48, 102, 27, 59, 1, 19, 83, 127, 60, 39, 123, 2, 78, 103, 43, 9, 56, 50, 92, 73, 67, 14, 71, 68, 70, 3, 66, 74, 7, 10, 112, 97, 29, 11, 87, 31, 5, 69, 84, 90, 20, 21, 107, 30, 75, 26, 76, 85, 79, 23, 8, 40, 34, 17, 91, 104, 15, 12, 81, 6, 36, 72, 101, 37, 100, 32, 88, 96, 24], [49, 63, 117, 111, 105, 47, 45, 109, 42, 41, 57, 110, 22, 99, 106, 86, 118, 51, 64, 113, 89, 0, 80, 35, 16, 120, 38, 119, 61, 95, 122, 52, 115, 124, 94, 25, 93, 121, 116, 62, 46, 28, 125, 82, 18, 126, 53, 27, 102, 58, 83, 44, 114, 103, 65, 13, 55, 54, 77, 108, 98, 127, 19, 123, 50, 39, 33, 3, 48, 59, 60, 7, 92, 10, 71, 67, 68, 4, 1, 56, 90, 9, 43, 78, 74, 14, 66, 87, 69, 97, 31, 112, 21, 29, 73, 84, 2, 23, 85, 107, 5, 11, 20, 70, 30, 26, 79, 34, 12, 37, 6, 15, 40, 76, 81, 75, 104, 91, 72, 17, 8, 101, 88, 36, 24, 96, 32, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 99, 41, 22, 0, 118, 106, 86, 64, 51, 113, 89, 35, 80, 119, 62, 52, 38, 124, 28, 16, 120, 95, 116, 121, 115, 61, 93, 122, 53, 18, 123, 94, 46, 25, 83, 98, 125, 77, 102, 48, 58, 54, 108, 82, 4, 55, 126, 1, 44, 27, 7, 59, 114, 127, 68, 39, 66, 103, 65, 50, 92, 71, 43, 10, 13, 33, 73, 2, 3, 74, 112, 90, 9, 67, 19, 60, 14, 78, 11, 87, 20, 29, 31, 97, 12, 21, 56, 6, 107, 5, 104, 69, 84, 40, 30, 75, 23, 85, 34, 76, 79, 26, 91, 70, 8, 37, 101, 81, 72, 17, 24, 88, 15, 32, 100, 36, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 57, 22, 118, 106, 99, 86, 113, 51, 35, 119, 89, 64, 0, 61, 115, 38, 16, 52, 80, 62, 28, 53, 120, 95, 124, 121, 116, 122, 94, 25, 125, 93, 77, 114, 18, 55, 82, 102, 44, 27, 59, 98, 83, 48, 50, 126, 33, 19, 54, 123, 13, 7, 58, 65, 108, 10, 46, 3, 39, 127, 97, 71, 92, 103, 112, 1, 68, 67, 43, 78, 74, 14, 31, 73, 2, 87, 66, 4, 5, 90, 60, 11, 21, 85, 56, 23, 84, 107, 6, 26, 20, 12, 9, 69, 75, 29, 76, 34, 30, 104, 40, 79, 81, 15, 91, 17, 72, 8, 101, 37, 36, 32, 24, 96, 88, 100, 70]], "model.layers.30.self_attn.q_proj": [[61, 38, 48, 50, 97, 49, 111, 123, 83, 64, 89, 1, 127, 108, 79, 41, 86, 28, 94, 114, 14, 116, 102, 106, 17, 119, 67, 57, 54, 4, 40, 13, 59, 65, 85, 73, 109, 39, 52, 37, 58, 126, 12, 68, 60, 92, 84, 45, 120, 11, 125, 76, 8, 72, 44, 55, 87, 105, 80, 121, 43, 10, 2, 62, 110, 63, 74, 46, 118, 51, 81, 42, 101, 23, 3, 24, 9, 56, 71, 6, 112, 124, 5, 31, 15, 53, 35, 107, 0, 34, 117, 115, 21, 18, 88, 36, 113, 100, 7, 27, 47, 104, 16, 69, 93, 26, 90, 19, 95, 77, 103, 122, 20, 82, 32, 96, 66, 91, 99, 30, 29, 75, 22, 98, 33, 70, 78, 25], [61, 38, 50, 48, 97, 89, 112, 83, 55, 108, 17, 11, 60, 114, 72, 57, 85, 70, 28, 14, 111, 15, 62, 101, 3, 92, 123, 66, 25, 127, 34, 6, 126, 113, 36, 106, 86, 59, 52, 87, 69, 124, 39, 102, 45, 58, 0, 120, 94, 125, 63, 37, 29, 118, 104, 35, 43, 109, 47, 115, 10, 64, 119, 121, 56, 122, 51, 49, 40, 22, 27, 95, 53, 4, 46, 116, 90, 103, 21, 24, 32, 107, 81, 110, 23, 117, 26, 44, 19, 99, 16, 76, 42, 105, 41, 98, 88, 77, 30, 67, 20, 54, 73, 31, 93, 100, 65, 96, 18, 8, 84, 82, 12, 5, 80, 1, 78, 2, 91, 75, 7, 33, 71, 79, 13, 68, 74, 9], [50, 61, 38, 111, 48, 55, 97, 89, 106, 45, 86, 127, 60, 41, 125, 123, 124, 120, 17, 85, 102, 108, 70, 121, 62, 112, 92, 59, 52, 58, 51, 43, 119, 114, 49, 126, 66, 87, 11, 57, 83, 14, 53, 46, 39, 28, 40, 101, 115, 113, 104, 118, 117, 47, 56, 116, 122, 105, 63, 54, 44, 110, 103, 91, 3, 25, 72, 107, 109, 32, 36, 42, 0, 64, 34, 15, 6, 100, 30, 35, 94, 9, 33, 22, 37, 81, 90, 10, 31, 21, 95, 27, 96, 13, 12, 24, 99, 88, 26, 78, 84, 93, 19, 69, 67, 4, 98, 8, 16, 23, 18, 76, 29, 82, 77, 65, 73, 20, 80, 2, 7, 5, 79, 75, 1, 71, 68, 74], [61, 50, 111, 38, 112, 55, 60, 114, 124, 48, 110, 125, 97, 123, 109, 92, 53, 127, 89, 58, 52, 51, 62, 121, 44, 28, 118, 59, 41, 108, 119, 120, 45, 102, 46, 63, 126, 122, 116, 47, 104, 113, 56, 117, 54, 43, 101, 105, 57, 107, 37, 42, 29, 115, 88, 86, 106, 103, 40, 80, 100, 39, 34, 91, 49, 35, 30, 84, 36, 85, 83, 93, 25, 98, 32, 95, 99, 24, 16, 94, 17, 31, 23, 18, 22, 15, 69, 96, 21, 76, 82, 11, 90, 33, 87, 77, 20, 27, 81, 26, 4, 13, 19, 12, 68, 7, 73, 2, 71, 9, 5, 79, 75, 14, 10, 72, 74, 66, 70, 6, 0, 78, 1, 64, 67, 8, 65, 3], [46, 110, 34, 92, 119, 86, 24, 90, 83, 113, 61, 49, 54, 95, 82, 94, 31, 38, 70, 85, 74, 0, 121, 13, 7, 120, 127, 114, 12, 8, 101, 9, 59, 29, 79, 17, 47, 112, 23, 1, 48, 98, 53, 80, 51, 52, 4, 63, 2, 109, 67, 97, 55, 26, 42, 125, 58, 116, 36, 117, 60, 40, 41, 126, 108, 18, 62, 56, 124, 111, 122, 43, 115, 123, 73, 57, 105, 87, 118, 91, 81, 45, 25, 102, 28, 11, 19, 39, 37, 106, 64, 107, 100, 44, 104, 66, 93, 16, 27, 103, 20, 33, 88, 50, 96, 32, 68, 89, 35, 22, 84, 14, 5, 99, 21, 69, 30, 65, 78, 3, 10, 15, 6, 77, 71, 75, 76, 72], [46, 110, 34, 90, 86, 92, 24, 79, 18, 113, 31, 95, 83, 17, 119, 9, 70, 54, 4, 85, 42, 74, 55, 56, 61, 12, 120, 59, 58, 107, 98, 117, 7, 63, 67, 115, 13, 44, 50, 49, 116, 88, 114, 38, 16, 36, 40, 96, 29, 73, 8, 94, 91, 20, 48, 111, 52, 23, 97, 118, 37, 81, 2, 0, 109, 100, 57, 122, 125, 43, 45, 1, 51, 39, 127, 103, 93, 104, 47, 41, 27, 106, 5, 123, 33, 126, 101, 112, 124, 80, 26, 62, 28, 14, 102, 15, 32, 60, 87, 77, 108, 11, 78, 75, 105, 99, 84, 53, 76, 25, 69, 121, 35, 89, 30, 82, 21, 19, 72, 10, 68, 64, 6, 22, 66, 3, 65, 71], [46, 34, 110, 113, 120, 92, 24, 83, 90, 42, 31, 86, 17, 94, 8, 79, 14, 54, 118, 112, 117, 15, 52, 9, 66, 100, 45, 85, 36, 55, 38, 70, 13, 76, 84, 58, 11, 60, 10, 51, 98, 3, 56, 4, 63, 0, 29, 95, 49, 65, 32, 12, 48, 127, 119, 124, 122, 39, 59, 116, 47, 104, 41, 23, 43, 53, 126, 19, 44, 107, 61, 75, 7, 62, 1, 111, 67, 78, 106, 22, 114, 68, 109, 105, 35, 108, 18, 80, 82, 37, 123, 115, 96, 91, 102, 88, 97, 28, 93, 26, 6, 103, 50, 33, 40, 27, 121, 69, 87, 5, 20, 25, 57, 71, 101, 81, 30, 2, 89, 21, 125, 16, 77, 74, 73, 99, 72, 64], [46, 110, 34, 92, 90, 24, 86, 49, 83, 69, 13, 113, 114, 95, 61, 119, 31, 23, 85, 56, 94, 53, 80, 100, 17, 98, 54, 108, 9, 125, 57, 52, 103, 79, 27, 115, 74, 63, 50, 11, 18, 126, 111, 7, 41, 99, 122, 25, 28, 120, 1, 67, 55, 88, 29, 4, 26, 87, 62, 101, 45, 39, 33, 117, 104, 36, 43, 60, 102, 58, 38, 116, 118, 127, 59, 40, 51, 96, 44, 124, 123, 109, 73, 42, 121, 15, 70, 107, 5, 97, 106, 30, 112, 91, 78, 48, 47, 37, 89, 35, 32, 76, 105, 12, 0, 20, 21, 81, 75, 93, 16, 71, 19, 14, 8, 84, 64, 68, 10, 82, 72, 2, 65, 3, 22, 77, 6, 66], [42, 63, 117, 97, 27, 86, 106, 89, 16, 78, 31, 84, 115, 52, 11, 99, 81, 76, 53, 119, 96, 56, 49, 13, 10, 88, 100, 26, 73, 92, 121, 55, 118, 7, 24, 51, 50, 61, 114, 57, 8, 102, 127, 90, 20, 122, 41, 23, 46, 19, 116, 60, 28, 95, 82, 38, 6, 25, 36, 58, 94, 33, 120, 91, 62, 22, 35, 21, 54, 110, 83, 48, 9, 59, 15, 18, 66, 5, 125, 111, 126, 109, 17, 124, 72, 47, 39, 103, 112, 107, 108, 85, 113, 44, 123, 87, 40, 93, 14, 69, 80, 29, 45, 4, 77, 70, 30, 105, 43, 32, 74, 34, 101, 104, 98, 3, 12, 79, 37, 75, 68, 0, 71, 65, 67, 2, 1, 64], [42, 63, 117, 97, 35, 16, 56, 84, 27, 83, 86, 102, 101, 94, 92, 18, 19, 57, 11, 64, 106, 121, 49, 116, 93, 52, 100, 4, 37, 103, 24, 29, 98, 81, 72, 85, 65, 77, 99, 127, 9, 50, 76, 88, 70, 54, 55, 122, 61, 31, 73, 32, 96, 51, 119, 118, 58, 120, 68, 115, 78, 79, 95, 114, 82, 26, 74, 90, 109, 111, 28, 20, 125, 124, 23, 25, 41, 60, 7, 59, 48, 34, 3, 10, 80, 47, 46, 43, 36, 113, 67, 53, 5, 108, 126, 66, 1, 22, 44, 110, 105, 89, 15, 39, 21, 112, 38, 12, 30, 91, 8, 45, 62, 40, 104, 33, 123, 14, 6, 69, 13, 0, 107, 2, 17, 87, 71, 75], [42, 63, 11, 81, 78, 7, 97, 117, 84, 1, 86, 106, 27, 102, 94, 4, 55, 0, 65, 114, 6, 116, 98, 16, 9, 21, 13, 2, 100, 66, 58, 50, 92, 119, 89, 14, 30, 19, 101, 115, 93, 8, 64, 67, 52, 35, 18, 77, 76, 71, 82, 73, 75, 88, 31, 108, 12, 38, 70, 122, 103, 68, 72, 36, 20, 83, 10, 3, 91, 17, 23, 74, 80, 24, 79, 15, 32, 26, 5, 22, 29, 28, 90, 85, 120, 46, 96, 53, 39, 61, 69, 25, 34, 99, 49, 95, 118, 59, 87, 104, 121, 123, 57, 111, 47, 109, 126, 56, 37, 54, 124, 62, 45, 110, 125, 127, 48, 33, 60, 51, 105, 44, 41, 40, 43, 113, 107, 112], [63, 42, 55, 57, 49, 56, 51, 121, 112, 52, 122, 54, 116, 118, 127, 61, 120, 50, 48, 111, 59, 125, 47, 124, 108, 113, 115, 126, 60, 117, 114, 110, 97, 62, 44, 53, 85, 58, 123, 94, 46, 109, 38, 45, 24, 43, 86, 107, 41, 119, 27, 33, 102, 29, 106, 40, 39, 100, 28, 92, 104, 105, 18, 36, 26, 103, 22, 31, 90, 99, 101, 35, 88, 91, 21, 81, 37, 96, 84, 77, 30, 87, 93, 32, 15, 98, 95, 34, 20, 17, 82, 83, 19, 79, 25, 23, 80, 13, 16, 89, 12, 73, 75, 76, 11, 70, 78, 68, 74, 1, 10, 14, 0, 4, 8, 5, 67, 72, 9, 7, 6, 69, 71, 65, 66, 64, 2, 3], [111, 44, 61, 125, 116, 84, 108, 20, 124, 97, 52, 75, 100, 60, 11, 120, 117, 48, 115, 46, 47, 106, 56, 50, 49, 123, 121, 119, 28, 112, 87, 70, 63, 62, 55, 53, 59, 122, 127, 118, 110, 57, 113, 42, 90, 126, 114, 39, 51, 92, 45, 88, 54, 58, 93, 107, 109, 105, 32, 41, 98, 104, 91, 40, 79, 102, 25, 43, 15, 17, 86, 30, 37, 103, 27, 38, 31, 26, 101, 19, 35, 81, 33, 34, 36, 94, 99, 74, 23, 95, 83, 96, 22, 29, 89, 10, 2, 82, 21, 6, 16, 13, 85, 66, 24, 72, 64, 67, 18, 0, 76, 14, 65, 77, 68, 80, 5, 71, 9, 78, 4, 1, 12, 8, 73, 3, 7, 69], [44, 111, 116, 88, 18, 97, 61, 16, 78, 9, 115, 90, 51, 108, 89, 30, 109, 100, 71, 47, 4, 125, 102, 85, 26, 76, 2, 20, 120, 83, 10, 86, 79, 80, 73, 56, 68, 92, 23, 82, 94, 93, 112, 27, 55, 95, 11, 58, 66, 6, 40, 64, 118, 14, 87, 70, 52, 63, 53, 42, 127, 122, 126, 65, 7, 124, 67, 41, 50, 49, 12, 19, 24, 119, 91, 113, 28, 29, 74, 117, 81, 101, 59, 25, 1, 114, 103, 13, 38, 43, 21, 31, 17, 32, 107, 57, 123, 48, 96, 98, 0, 121, 72, 62, 34, 60, 84, 105, 106, 77, 46, 22, 8, 35, 54, 104, 5, 3, 45, 39, 36, 99, 33, 110, 37, 15, 75, 69], [44, 61, 108, 125, 97, 106, 115, 85, 88, 100, 116, 47, 51, 49, 120, 117, 126, 46, 124, 52, 87, 28, 93, 112, 59, 111, 50, 48, 119, 55, 17, 122, 92, 76, 57, 127, 62, 90, 123, 113, 118, 114, 56, 63, 60, 121, 21, 26, 53, 32, 54, 107, 58, 105, 45, 91, 40, 30, 42, 12, 104, 109, 86, 41, 39, 79, 27, 74, 110, 94, 103, 5, 34, 102, 43, 19, 23, 69, 38, 98, 83, 36, 37, 33, 31, 15, 101, 81, 95, 82, 99, 22, 35, 67, 72, 25, 89, 84, 96, 13, 2, 10, 29, 64, 18, 24, 16, 71, 20, 7, 77, 80, 3, 0, 66, 8, 78, 14, 65, 70, 4, 9, 11, 1, 68, 6, 73, 75], [44, 125, 111, 108, 61, 116, 115, 97, 51, 117, 87, 52, 122, 88, 106, 112, 49, 55, 126, 60, 100, 120, 93, 47, 59, 124, 58, 50, 113, 57, 48, 119, 118, 62, 127, 28, 46, 45, 92, 56, 63, 121, 114, 123, 90, 42, 17, 53, 74, 39, 41, 107, 104, 109, 19, 54, 86, 32, 79, 30, 91, 102, 40, 105, 13, 98, 110, 103, 26, 94, 15, 43, 95, 38, 36, 83, 35, 25, 101, 27, 37, 34, 89, 81, 96, 22, 23, 31, 33, 99, 29, 77, 67, 21, 85, 72, 84, 82, 10, 16, 2, 64, 24, 0, 9, 18, 70, 20, 80, 65, 8, 76, 5, 71, 78, 3, 14, 66, 12, 7, 11, 69, 75, 1, 73, 4, 68, 6], [113, 103, 122, 124, 52, 33, 55, 93, 119, 107, 47, 24, 42, 26, 48, 31, 112, 43, 54, 104, 90, 117, 123, 56, 45, 85, 41, 57, 108, 118, 116, 53, 87, 83, 115, 121, 37, 62, 120, 63, 35, 61, 99, 22, 29, 125, 114, 91, 59, 23, 39, 40, 21, 28, 60, 105, 49, 44, 102, 111, 101, 98, 51, 92, 127, 126, 106, 110, 58, 32, 36, 50, 38, 88, 46, 34, 97, 95, 100, 27, 11, 109, 96, 84, 30, 89, 18, 94, 82, 8, 10, 16, 72, 19, 14, 20, 80, 13, 71, 25, 75, 86, 77, 74, 81, 15, 78, 70, 5, 7, 68, 65, 2, 76, 6, 69, 0, 66, 64, 79, 12, 67, 17, 4, 3, 73, 1, 9], [122, 103, 33, 52, 93, 31, 117, 119, 91, 83, 22, 24, 54, 112, 111, 124, 90, 85, 26, 60, 105, 43, 27, 125, 42, 107, 45, 39, 97, 127, 29, 113, 20, 114, 96, 123, 73, 92, 118, 53, 78, 46, 61, 120, 110, 44, 58, 86, 116, 40, 115, 102, 48, 51, 21, 49, 47, 38, 62, 50, 55, 109, 101, 108, 25, 121, 126, 95, 35, 81, 56, 99, 37, 36, 63, 19, 30, 41, 28, 15, 57, 13, 34, 104, 89, 84, 87, 94, 32, 100, 106, 9, 82, 80, 23, 75, 98, 18, 14, 16, 59, 72, 88, 77, 5, 6, 64, 70, 67, 8, 1, 11, 0, 69, 10, 65, 68, 2, 17, 71, 4, 3, 79, 66, 7, 76, 12, 74], [57, 122, 103, 52, 113, 110, 105, 112, 22, 33, 54, 38, 101, 59, 116, 125, 43, 111, 60, 26, 109, 118, 50, 48, 97, 55, 3, 34, 39, 47, 126, 92, 78, 45, 127, 24, 40, 124, 31, 114, 61, 64, 120, 90, 75, 65, 119, 49, 123, 108, 83, 115, 18, 67, 20, 68, 71, 99, 36, 107, 106, 104, 44, 62, 63, 46, 100, 53, 96, 102, 25, 41, 42, 117, 19, 95, 51, 58, 74, 84, 121, 93, 56, 37, 21, 89, 87, 27, 94, 30, 32, 13, 35, 73, 86, 88, 82, 85, 98, 72, 28, 91, 10, 14, 23, 4, 77, 29, 9, 6, 8, 2, 1, 7, 0, 69, 66, 80, 70, 11, 16, 12, 15, 5, 79, 81, 17, 76], [103, 113, 93, 122, 24, 33, 15, 81, 76, 29, 83, 17, 57, 85, 10, 70, 104, 88, 21, 36, 34, 48, 19, 7, 22, 84, 97, 26, 66, 107, 5, 35, 12, 27, 124, 87, 42, 54, 30, 38, 55, 90, 23, 43, 16, 82, 74, 102, 75, 8, 20, 121, 63, 86, 49, 119, 94, 77, 45, 91, 68, 127, 14, 71, 62, 117, 73, 0, 79, 61, 13, 25, 32, 28, 2, 106, 80, 111, 11, 69, 116, 18, 47, 98, 59, 52, 31, 58, 89, 3, 72, 67, 92, 4, 78, 37, 96, 40, 99, 123, 100, 95, 118, 110, 39, 120, 112, 6, 46, 101, 65, 9, 64, 105, 60, 53, 126, 108, 44, 125, 114, 1, 41, 50, 115, 51, 56, 109], [56, 39, 60, 53, 117, 122, 94, 127, 109, 123, 116, 92, 58, 50, 124, 61, 103, 105, 59, 51, 55, 52, 113, 33, 111, 90, 114, 119, 115, 62, 97, 110, 54, 42, 121, 63, 47, 96, 57, 98, 34, 43, 95, 25, 36, 44, 45, 26, 108, 46, 112, 19, 126, 28, 49, 91, 85, 125, 30, 107, 118, 40, 38, 120, 102, 37, 106, 104, 88, 48, 41, 100, 23, 35, 101, 99, 27, 31, 29, 81, 20, 32, 83, 86, 87, 84, 89, 22, 93, 24, 76, 21, 78, 14, 17, 15, 75, 16, 66, 18, 79, 71, 74, 9, 0, 5, 4, 12, 67, 11, 80, 65, 82, 73, 7, 69, 70, 1, 2, 72, 77, 8, 68, 64, 3, 10, 6, 13], [53, 39, 60, 123, 61, 124, 55, 56, 50, 109, 51, 116, 54, 127, 120, 57, 122, 63, 97, 115, 47, 52, 59, 121, 90, 112, 119, 26, 58, 113, 117, 62, 114, 108, 126, 28, 125, 44, 49, 118, 107, 43, 94, 48, 110, 46, 19, 45, 105, 106, 111, 23, 38, 42, 41, 103, 101, 29, 100, 40, 85, 33, 104, 87, 102, 34, 25, 78, 35, 99, 22, 92, 95, 37, 36, 86, 30, 96, 98, 31, 91, 32, 83, 93, 21, 24, 17, 18, 27, 81, 20, 15, 14, 84, 75, 88, 89, 11, 79, 72, 76, 82, 16, 80, 12, 5, 77, 9, 74, 70, 66, 13, 67, 7, 6, 71, 8, 2, 69, 3, 10, 4, 65, 73, 0, 68, 1, 64], [39, 53, 56, 25, 18, 77, 16, 23, 74, 71, 4, 60, 97, 70, 64, 9, 0, 91, 52, 29, 31, 76, 34, 92, 67, 65, 122, 1, 100, 117, 30, 36, 15, 84, 13, 124, 40, 68, 2, 12, 82, 125, 99, 20, 10, 5, 17, 69, 98, 8, 94, 55, 6, 104, 79, 19, 81, 73, 14, 85, 42, 61, 88, 32, 7, 50, 86, 11, 90, 26, 3, 72, 24, 66, 21, 83, 75, 27, 80, 37, 107, 93, 38, 118, 87, 22, 114, 89, 113, 35, 78, 28, 41, 95, 63, 46, 126, 109, 96, 127, 51, 101, 58, 119, 33, 116, 123, 120, 59, 105, 45, 108, 102, 47, 112, 106, 54, 115, 62, 43, 57, 121, 44, 111, 110, 49, 48, 103], [39, 56, 53, 25, 18, 16, 23, 34, 97, 30, 83, 77, 52, 60, 122, 76, 29, 100, 74, 86, 93, 124, 61, 11, 94, 104, 10, 9, 71, 15, 22, 21, 13, 36, 55, 127, 90, 20, 26, 108, 91, 27, 119, 19, 80, 89, 98, 35, 40, 117, 79, 88, 92, 82, 59, 70, 107, 31, 12, 65, 51, 85, 84, 28, 41, 17, 4, 78, 102, 14, 7, 64, 87, 67, 75, 114, 72, 99, 8, 69, 81, 0, 37, 46, 32, 57, 24, 1, 112, 2, 125, 96, 73, 95, 54, 58, 115, 33, 126, 50, 118, 3, 101, 113, 38, 116, 42, 105, 6, 111, 63, 45, 120, 62, 44, 123, 49, 106, 68, 66, 103, 48, 110, 109, 5, 47, 43, 121], [123, 41, 34, 120, 115, 107, 58, 60, 12, 48, 51, 90, 127, 125, 82, 70, 23, 109, 73, 116, 32, 47, 77, 121, 62, 29, 106, 84, 126, 14, 59, 55, 86, 24, 108, 80, 46, 26, 45, 57, 63, 52, 54, 122, 37, 110, 4, 36, 92, 38, 50, 61, 105, 56, 117, 79, 113, 111, 112, 119, 104, 103, 118, 42, 69, 124, 15, 43, 94, 49, 53, 67, 66, 101, 85, 44, 93, 99, 114, 7, 89, 33, 88, 39, 97, 1, 95, 31, 28, 30, 91, 100, 35, 40, 64, 102, 21, 22, 74, 10, 13, 19, 83, 78, 8, 98, 9, 20, 76, 6, 16, 18, 87, 17, 25, 96, 27, 2, 81, 0, 71, 65, 11, 75, 5, 3, 72, 68], [41, 123, 34, 107, 65, 66, 0, 29, 90, 121, 14, 37, 1, 74, 2, 11, 59, 93, 70, 80, 124, 72, 81, 69, 68, 105, 26, 47, 4, 10, 60, 75, 58, 12, 84, 3, 24, 86, 32, 109, 17, 82, 103, 7, 115, 77, 111, 127, 125, 35, 13, 21, 5, 15, 120, 73, 46, 110, 51, 71, 45, 8, 67, 94, 52, 97, 126, 118, 55, 87, 48, 56, 106, 116, 42, 62, 61, 57, 108, 101, 63, 112, 38, 33, 98, 28, 50, 122, 31, 44, 117, 113, 43, 53, 6, 18, 64, 79, 100, 119, 88, 92, 104, 54, 114, 49, 16, 99, 30, 27, 85, 20, 39, 83, 78, 91, 36, 76, 40, 95, 19, 25, 89, 9, 96, 23, 102, 22], [123, 41, 109, 34, 59, 29, 115, 90, 58, 23, 127, 61, 120, 51, 125, 43, 126, 111, 103, 108, 46, 54, 113, 122, 47, 121, 57, 110, 116, 56, 86, 107, 48, 50, 124, 52, 118, 63, 38, 117, 12, 55, 112, 80, 53, 119, 84, 42, 89, 93, 60, 21, 62, 49, 45, 114, 102, 106, 104, 99, 70, 44, 101, 26, 77, 6, 7, 4, 82, 25, 14, 36, 39, 32, 40, 37, 31, 94, 88, 35, 65, 33, 100, 97, 73, 92, 10, 105, 22, 74, 3, 27, 83, 96, 87, 0, 67, 20, 30, 19, 17, 91, 15, 64, 85, 98, 95, 28, 24, 72, 66, 79, 68, 81, 13, 18, 75, 11, 69, 9, 71, 78, 16, 76, 1, 8, 2, 5], [123, 41, 109, 34, 107, 59, 58, 121, 23, 47, 60, 127, 29, 115, 84, 55, 90, 120, 125, 106, 56, 104, 43, 112, 12, 80, 126, 88, 61, 46, 50, 62, 103, 53, 122, 48, 116, 124, 57, 51, 32, 54, 45, 52, 111, 110, 63, 93, 113, 26, 38, 73, 114, 117, 42, 118, 82, 99, 108, 36, 101, 119, 86, 14, 37, 35, 89, 49, 24, 102, 105, 70, 100, 74, 17, 77, 65, 25, 44, 39, 21, 85, 97, 83, 31, 4, 94, 40, 67, 95, 7, 3, 92, 30, 76, 91, 79, 33, 0, 16, 9, 19, 28, 98, 20, 27, 96, 18, 22, 6, 87, 11, 78, 72, 15, 69, 81, 64, 13, 75, 66, 1, 71, 10, 68, 8, 5, 2], [127, 126, 63, 38, 120, 82, 20, 97, 91, 80, 9, 11, 13, 21, 56, 25, 69, 14, 30, 15, 29, 24, 90, 26, 75, 8, 94, 86, 102, 99, 124, 2, 88, 37, 93, 39, 95, 76, 103, 84, 78, 3, 7, 74, 43, 98, 71, 46, 68, 114, 17, 54, 121, 18, 47, 67, 125, 108, 19, 105, 0, 101, 85, 119, 58, 77, 44, 5, 49, 10, 55, 111, 42, 34, 51, 89, 96, 83, 31, 27, 62, 92, 107, 123, 52, 57, 35, 53, 48, 28, 110, 61, 87, 6, 72, 12, 60, 116, 16, 33, 64, 32, 59, 22, 118, 73, 70, 23, 66, 115, 79, 122, 100, 117, 112, 50, 41, 109, 36, 113, 104, 65, 81, 4, 40, 45, 106, 1], [63, 38, 126, 123, 119, 54, 56, 120, 62, 110, 47, 124, 61, 59, 125, 118, 60, 122, 51, 121, 115, 117, 111, 58, 48, 53, 50, 46, 49, 55, 23, 127, 116, 102, 109, 112, 97, 113, 114, 57, 52, 44, 87, 29, 45, 81, 93, 43, 26, 105, 92, 86, 108, 94, 88, 107, 41, 106, 17, 42, 104, 99, 103, 22, 40, 33, 21, 90, 39, 84, 101, 37, 95, 15, 83, 18, 35, 98, 79, 30, 74, 36, 31, 96, 100, 89, 70, 34, 27, 25, 28, 76, 0, 32, 80, 20, 6, 85, 19, 91, 65, 10, 24, 82, 78, 16, 12, 13, 1, 75, 14, 8, 67, 66, 64, 4, 69, 73, 68, 3, 9, 11, 7, 71, 2, 5, 77, 72], [126, 127, 63, 38, 123, 54, 119, 56, 62, 110, 125, 120, 61, 47, 60, 124, 59, 118, 46, 122, 49, 121, 51, 111, 115, 53, 55, 48, 58, 50, 116, 117, 113, 112, 97, 52, 44, 23, 114, 102, 29, 57, 43, 109, 45, 105, 93, 81, 86, 94, 26, 108, 104, 92, 88, 87, 107, 41, 42, 40, 99, 90, 103, 17, 22, 106, 33, 39, 31, 37, 74, 98, 101, 30, 15, 21, 95, 83, 36, 100, 35, 70, 96, 6, 34, 84, 27, 32, 20, 28, 79, 18, 25, 89, 0, 65, 16, 76, 19, 85, 80, 14, 91, 12, 82, 10, 24, 67, 78, 1, 4, 13, 64, 8, 75, 66, 68, 9, 3, 7, 73, 77, 69, 72, 5, 71, 2, 11], [126, 63, 38, 127, 16, 120, 20, 88, 78, 97, 82, 91, 75, 56, 30, 47, 73, 99, 21, 29, 54, 32, 13, 125, 39, 7, 15, 86, 119, 95, 62, 5, 90, 124, 25, 10, 8, 66, 80, 51, 69, 76, 0, 123, 9, 67, 84, 102, 111, 24, 77, 49, 58, 98, 53, 103, 109, 43, 114, 28, 26, 61, 44, 27, 79, 60, 50, 34, 18, 104, 14, 117, 118, 71, 59, 107, 19, 115, 70, 122, 57, 65, 121, 48, 17, 12, 85, 87, 89, 31, 113, 55, 83, 101, 112, 92, 100, 72, 52, 116, 23, 4, 110, 94, 22, 74, 35, 11, 46, 6, 45, 96, 33, 37, 105, 41, 108, 93, 68, 106, 40, 36, 81, 3, 1, 42, 2, 64]], "model.layers.30.self_attn.k_proj": [[61, 102, 50, 33, 48, 89, 86, 112, 92, 85, 14, 83, 60, 0, 58, 52, 113, 2, 108, 94, 11, 17, 114, 40, 120, 121, 47, 62, 65, 59, 55, 45, 123, 126, 72, 57, 53, 43, 116, 127, 56, 119, 124, 125, 63, 118, 51, 69, 98, 42, 38, 122, 117, 67, 46, 115, 44, 77, 49, 109, 41, 106, 80, 110, 54, 68, 107, 103, 100, 104, 29, 37, 105, 101, 9, 39, 10, 87, 91, 7, 6, 35, 64, 15, 31, 99, 111, 12, 36, 71, 4, 5, 84, 93, 70, 30, 90, 96, 32, 88, 27, 95, 23, 26, 34, 73, 82, 18, 28, 13, 66, 22, 76, 24, 16, 20, 21, 3, 74, 75, 1, 78, 25, 8, 19, 79, 97, 81], [110, 46, 98, 90, 0, 92, 24, 13, 31, 67, 18, 86, 2, 70, 113, 74, 1, 15, 83, 52, 17, 8, 4, 12, 16, 80, 51, 105, 9, 49, 84, 59, 21, 79, 41, 123, 116, 11, 64, 68, 94, 85, 7, 23, 54, 127, 119, 109, 29, 96, 124, 58, 115, 22, 122, 34, 100, 97, 101, 30, 38, 48, 14, 53, 45, 60, 126, 106, 118, 104, 39, 40, 35, 117, 56, 108, 120, 62, 63, 107, 47, 93, 37, 114, 5, 44, 87, 50, 102, 33, 43, 78, 111, 91, 27, 125, 42, 10, 89, 103, 112, 55, 36, 99, 32, 73, 69, 121, 61, 76, 26, 57, 95, 66, 71, 25, 28, 65, 75, 19, 20, 77, 81, 72, 6, 3, 88, 82], [106, 63, 86, 33, 117, 27, 11, 77, 78, 9, 4, 81, 7, 0, 84, 1, 55, 18, 49, 57, 31, 122, 16, 121, 95, 30, 72, 42, 116, 85, 56, 50, 59, 114, 119, 52, 51, 87, 58, 110, 54, 53, 43, 120, 34, 111, 44, 48, 118, 61, 124, 112, 115, 90, 113, 125, 103, 46, 36, 47, 62, 24, 29, 66, 41, 107, 105, 127, 37, 64, 45, 65, 109, 123, 3, 28, 126, 60, 104, 6, 67, 102, 10, 100, 108, 68, 32, 38, 99, 39, 35, 92, 40, 80, 94, 12, 5, 101, 73, 26, 23, 69, 93, 25, 76, 15, 8, 88, 20, 89, 70, 98, 91, 96, 83, 82, 14, 19, 74, 79, 13, 71, 2, 21, 17, 22, 75, 97], [108, 36, 44, 33, 86, 115, 47, 94, 111, 127, 17, 61, 52, 59, 26, 28, 41, 57, 96, 114, 126, 119, 49, 112, 104, 125, 117, 122, 118, 37, 48, 56, 120, 42, 62, 107, 60, 13, 55, 63, 50, 21, 123, 58, 124, 121, 88, 46, 43, 113, 39, 51, 106, 53, 102, 45, 40, 110, 109, 116, 54, 79, 101, 100, 105, 90, 31, 103, 72, 38, 5, 95, 98, 85, 93, 99, 76, 19, 14, 34, 24, 35, 27, 84, 87, 30, 91, 16, 12, 92, 89, 80, 32, 67, 97, 11, 3, 78, 74, 25, 65, 29, 23, 9, 75, 22, 81, 82, 18, 0, 20, 77, 2, 70, 83, 8, 73, 71, 6, 15, 7, 66, 1, 10, 68, 64, 4, 69], [39, 122, 113, 97, 49, 29, 57, 112, 26, 22, 88, 27, 73, 124, 24, 19, 93, 78, 48, 102, 59, 30, 127, 81, 123, 21, 10, 17, 47, 105, 118, 117, 85, 1, 83, 40, 54, 15, 111, 61, 107, 119, 63, 76, 43, 115, 95, 100, 60, 58, 109, 114, 52, 106, 126, 31, 12, 62, 55, 5, 53, 87, 41, 110, 37, 0, 121, 66, 120, 46, 7, 108, 4, 92, 104, 116, 51, 44, 103, 36, 125, 89, 79, 84, 101, 99, 96, 50, 42, 34, 25, 3, 32, 45, 13, 94, 56, 98, 14, 35, 38, 82, 77, 90, 75, 91, 80, 8, 69, 28, 20, 9, 18, 23, 16, 11, 33, 72, 71, 68, 86, 67, 6, 64, 70, 65, 2, 74], [103, 56, 53, 25, 33, 18, 16, 94, 77, 74, 60, 90, 23, 4, 70, 71, 87, 76, 22, 119, 0, 65, 109, 124, 55, 89, 122, 50, 120, 63, 52, 75, 123, 51, 86, 85, 67, 91, 93, 113, 44, 54, 35, 125, 127, 117, 114, 100, 47, 61, 116, 29, 9, 118, 36, 28, 19, 101, 59, 107, 106, 57, 98, 43, 62, 32, 72, 111, 49, 121, 42, 48, 58, 110, 8, 115, 45, 78, 108, 126, 11, 81, 41, 104, 46, 96, 105, 112, 79, 102, 2, 34, 5, 95, 92, 17, 31, 7, 37, 40, 38, 84, 66, 99, 30, 24, 12, 20, 6, 73, 27, 68, 88, 13, 14, 15, 39, 3, 80, 83, 1, 82, 64, 21, 97, 69, 26, 10], [105, 123, 98, 86, 59, 93, 64, 127, 50, 125, 60, 90, 67, 116, 121, 46, 80, 117, 124, 126, 51, 119, 73, 96, 56, 122, 101, 58, 57, 62, 48, 63, 14, 115, 45, 61, 11, 53, 112, 118, 52, 120, 82, 110, 55, 42, 43, 111, 113, 47, 106, 12, 114, 54, 44, 69, 1, 40, 109, 84, 49, 70, 108, 66, 39, 102, 24, 81, 37, 68, 3, 23, 100, 95, 19, 104, 36, 91, 71, 28, 8, 15, 77, 32, 21, 94, 35, 74, 17, 72, 103, 97, 85, 10, 5, 89, 99, 30, 38, 29, 25, 107, 33, 20, 87, 27, 83, 31, 92, 79, 41, 34, 6, 7, 88, 75, 65, 22, 26, 2, 13, 76, 18, 78, 4, 0, 9, 16], [126, 102, 63, 33, 86, 93, 127, 56, 95, 26, 120, 47, 124, 111, 112, 121, 54, 114, 46, 119, 53, 23, 61, 117, 51, 118, 57, 107, 48, 15, 122, 60, 58, 45, 55, 44, 59, 105, 52, 17, 104, 62, 20, 115, 49, 82, 94, 103, 13, 123, 50, 106, 108, 40, 125, 113, 116, 30, 41, 43, 42, 109, 110, 88, 39, 91, 79, 31, 28, 37, 38, 27, 87, 101, 18, 77, 92, 81, 89, 99, 29, 36, 96, 100, 34, 35, 83, 21, 11, 74, 32, 7, 10, 98, 75, 19, 25, 90, 84, 71, 65, 9, 6, 85, 16, 8, 14, 12, 76, 5, 97, 22, 78, 80, 24, 4, 73, 67, 1, 72, 66, 0, 69, 68, 70, 3, 2, 64]], "model.layers.30.self_attn.qk_proj": [[63, 61, 123, 56, 110, 126, 53, 46, 122, 113, 44, 50, 106, 127, 111, 117, 108, 60, 48, 22, 42, 57, 102, 59, 103, 52, 124, 125, 39, 105, 33, 49, 120, 119, 116, 38, 51, 90, 112, 86, 115, 55, 54, 26, 93, 29, 58, 97, 118, 114, 41, 109, 121, 47, 34, 24, 88, 30, 94, 92, 43, 62, 82, 25, 31, 45, 98, 28, 81, 89, 36, 23, 18, 107, 77, 83, 91, 101, 87, 13, 40, 80, 14, 17, 21, 85, 100, 78, 11, 16, 19, 27, 37, 79, 9, 84, 104, 75, 95, 15, 73, 76, 64, 0, 20, 96, 12, 71, 10, 7, 8, 35, 74, 67, 6, 4, 70, 68, 5, 99, 32, 65, 1, 72, 3, 69, 2, 66], [63, 61, 123, 126, 56, 110, 46, 53, 122, 113, 50, 44, 111, 106, 42, 117, 127, 108, 60, 48, 22, 57, 102, 120, 125, 103, 105, 124, 59, 52, 38, 112, 39, 86, 51, 33, 29, 115, 58, 54, 49, 26, 47, 55, 118, 97, 41, 109, 93, 116, 90, 119, 114, 30, 121, 45, 24, 94, 62, 88, 28, 25, 107, 92, 43, 31, 34, 98, 82, 91, 23, 89, 13, 17, 100, 18, 40, 104, 36, 81, 77, 14, 11, 19, 16, 78, 83, 80, 85, 87, 21, 37, 95, 9, 79, 99, 101, 76, 27, 0, 12, 75, 15, 70, 73, 68, 74, 64, 10, 20, 71, 35, 32, 84, 8, 7, 4, 3, 96, 1, 65, 67, 69, 6, 5, 72, 66, 2], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 127, 117, 42, 48, 108, 22, 57, 102, 60, 103, 124, 105, 120, 59, 52, 51, 86, 33, 38, 39, 115, 54, 116, 49, 125, 55, 112, 58, 93, 119, 26, 47, 118, 121, 114, 29, 90, 97, 107, 24, 94, 109, 30, 62, 28, 88, 41, 34, 92, 25, 43, 40, 89, 45, 98, 18, 83, 31, 36, 81, 23, 101, 82, 100, 13, 16, 77, 80, 14, 91, 21, 19, 0, 11, 70, 17, 87, 85, 104, 95, 78, 20, 9, 37, 99, 75, 79, 15, 27, 74, 76, 71, 10, 84, 12, 32, 73, 64, 7, 4, 35, 67, 1, 96, 72, 68, 65, 3, 8, 5, 69, 2, 66, 6], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 117, 108, 42, 127, 48, 52, 57, 22, 120, 124, 103, 102, 60, 58, 125, 39, 105, 38, 116, 121, 47, 49, 112, 51, 54, 86, 26, 33, 59, 93, 118, 114, 97, 119, 55, 43, 29, 109, 90, 115, 30, 24, 62, 41, 94, 45, 107, 34, 28, 92, 40, 25, 31, 88, 82, 104, 13, 98, 81, 89, 14, 80, 23, 36, 83, 78, 19, 18, 101, 100, 21, 91, 17, 77, 64, 11, 37, 87, 16, 76, 85, 95, 70, 99, 75, 79, 84, 15, 9, 73, 0, 27, 10, 71, 12, 20, 68, 32, 7, 3, 35, 1, 74, 96, 72, 65, 4, 8, 67, 69, 5, 66, 2, 6], [63, 61, 123, 56, 126, 53, 110, 46, 122, 113, 44, 50, 111, 127, 106, 108, 117, 42, 22, 48, 102, 52, 60, 57, 124, 120, 103, 125, 59, 51, 86, 116, 105, 39, 119, 58, 54, 47, 49, 55, 33, 112, 90, 26, 115, 118, 38, 93, 114, 29, 97, 41, 121, 109, 24, 62, 43, 107, 34, 31, 83, 92, 25, 89, 88, 94, 36, 30, 82, 81, 23, 45, 28, 18, 80, 21, 13, 98, 77, 14, 11, 17, 78, 40, 87, 101, 85, 27, 16, 100, 0, 19, 95, 79, 91, 104, 76, 15, 73, 12, 9, 75, 70, 64, 20, 84, 74, 37, 10, 1, 7, 72, 3, 71, 4, 65, 68, 35, 67, 96, 32, 8, 6, 5, 99, 69, 66, 2], [63, 61, 126, 56, 123, 110, 53, 46, 122, 113, 44, 50, 108, 111, 106, 42, 127, 22, 57, 48, 117, 105, 103, 102, 120, 86, 60, 52, 59, 38, 47, 112, 125, 124, 39, 54, 58, 116, 118, 51, 93, 33, 114, 29, 26, 45, 49, 90, 97, 55, 119, 41, 24, 109, 121, 115, 92, 25, 28, 43, 62, 88, 94, 81, 82, 23, 30, 21, 98, 36, 34, 14, 95, 31, 18, 13, 83, 107, 87, 89, 91, 80, 78, 17, 77, 27, 85, 40, 11, 19, 15, 104, 16, 101, 100, 12, 64, 9, 76, 79, 75, 37, 73, 70, 0, 4, 72, 74, 84, 20, 10, 7, 35, 6, 68, 65, 71, 32, 96, 3, 99, 67, 66, 69, 5, 8, 1, 2], [63, 61, 110, 123, 126, 56, 53, 122, 46, 44, 113, 50, 42, 111, 106, 108, 48, 22, 127, 102, 117, 59, 105, 125, 103, 124, 60, 57, 52, 120, 86, 112, 39, 47, 90, 116, 26, 38, 29, 93, 119, 115, 41, 109, 45, 55, 33, 121, 97, 114, 58, 51, 118, 43, 92, 54, 88, 62, 25, 24, 28, 49, 94, 23, 31, 81, 18, 107, 21, 82, 87, 85, 30, 80, 36, 89, 83, 98, 91, 17, 27, 77, 40, 95, 100, 11, 34, 14, 13, 78, 19, 16, 20, 15, 101, 12, 76, 6, 79, 73, 9, 75, 74, 104, 84, 10, 35, 37, 64, 0, 32, 7, 67, 71, 4, 72, 99, 65, 68, 70, 8, 1, 96, 3, 2, 5, 69, 66], [63, 61, 123, 56, 110, 126, 53, 122, 46, 113, 44, 50, 111, 106, 108, 42, 127, 48, 22, 117, 57, 102, 59, 60, 103, 124, 120, 52, 125, 115, 55, 105, 119, 112, 33, 90, 93, 38, 58, 116, 39, 49, 86, 51, 26, 47, 29, 97, 41, 121, 109, 114, 118, 92, 45, 54, 88, 25, 62, 24, 43, 30, 31, 89, 28, 107, 91, 94, 34, 18, 21, 36, 81, 23, 82, 17, 98, 40, 83, 100, 87, 13, 27, 14, 101, 16, 80, 78, 11, 85, 77, 20, 37, 95, 64, 75, 79, 0, 76, 19, 104, 15, 6, 9, 73, 84, 35, 10, 12, 96, 67, 7, 1, 99, 74, 72, 32, 65, 4, 71, 68, 3, 8, 69, 2, 66, 5, 70], [63, 61, 56, 110, 123, 53, 126, 46, 122, 113, 44, 50, 106, 111, 127, 42, 48, 108, 52, 60, 102, 57, 124, 117, 103, 22, 120, 58, 59, 116, 86, 125, 105, 39, 38, 51, 93, 119, 55, 112, 109, 97, 33, 29, 90, 26, 115, 49, 45, 47, 118, 43, 25, 41, 94, 92, 114, 88, 24, 54, 31, 107, 121, 30, 62, 23, 28, 18, 89, 34, 98, 91, 13, 40, 82, 27, 87, 83, 80, 21, 78, 100, 17, 36, 19, 77, 81, 95, 85, 11, 104, 20, 16, 14, 76, 84, 101, 79, 75, 37, 73, 6, 15, 12, 68, 99, 0, 32, 7, 4, 72, 9, 10, 74, 64, 71, 35, 1, 3, 96, 65, 5, 70, 8, 67, 2, 69, 66], [63, 61, 123, 126, 56, 110, 53, 122, 46, 113, 44, 50, 111, 106, 108, 127, 48, 60, 117, 124, 125, 120, 52, 102, 57, 22, 42, 103, 55, 38, 59, 49, 51, 58, 105, 33, 39, 47, 119, 121, 97, 86, 116, 115, 90, 93, 26, 43, 109, 54, 112, 29, 114, 34, 62, 118, 92, 24, 94, 88, 45, 30, 98, 107, 31, 41, 13, 40, 23, 28, 25, 83, 100, 82, 18, 91, 81, 37, 89, 104, 16, 77, 17, 85, 87, 80, 11, 21, 19, 64, 78, 6, 14, 36, 101, 75, 95, 27, 99, 74, 84, 73, 7, 68, 10, 4, 0, 20, 9, 71, 79, 12, 15, 76, 35, 72, 3, 65, 67, 96, 32, 1, 8, 70, 5, 66, 69, 2], [63, 61, 123, 126, 56, 110, 46, 122, 53, 113, 44, 50, 111, 106, 127, 108, 117, 48, 42, 22, 60, 52, 125, 57, 102, 103, 59, 124, 105, 112, 33, 39, 93, 49, 55, 120, 119, 86, 114, 121, 51, 116, 47, 38, 26, 118, 90, 54, 97, 29, 62, 115, 41, 109, 58, 24, 88, 34, 43, 92, 94, 30, 25, 31, 13, 89, 82, 14, 107, 17, 45, 28, 40, 98, 85, 36, 18, 77, 78, 19, 23, 91, 101, 81, 80, 83, 21, 104, 16, 11, 95, 73, 74, 9, 79, 87, 76, 12, 15, 0, 75, 64, 100, 27, 20, 6, 84, 7, 71, 4, 10, 72, 35, 8, 65, 67, 70, 3, 37, 68, 99, 32, 69, 96, 1, 66, 5, 2], [63, 61, 110, 123, 126, 56, 53, 46, 122, 113, 44, 50, 106, 108, 127, 111, 42, 22, 102, 48, 60, 124, 125, 52, 105, 103, 86, 117, 57, 112, 120, 59, 116, 33, 39, 29, 58, 93, 55, 119, 26, 41, 38, 115, 97, 47, 90, 114, 49, 51, 109, 118, 25, 121, 92, 24, 43, 54, 31, 62, 88, 30, 82, 23, 17, 94, 13, 81, 18, 98, 40, 28, 45, 14, 91, 77, 78, 101, 85, 16, 107, 34, 83, 89, 19, 21, 87, 80, 36, 27, 11, 95, 73, 100, 79, 75, 76, 9, 12, 84, 104, 37, 20, 71, 64, 35, 15, 4, 74, 0, 10, 68, 70, 69, 32, 72, 7, 8, 3, 96, 67, 99, 1, 6, 65, 5, 66, 2], [63, 61, 56, 110, 123, 126, 53, 122, 46, 113, 44, 50, 108, 127, 111, 106, 22, 42, 48, 124, 60, 102, 103, 38, 57, 120, 125, 55, 52, 58, 86, 119, 59, 33, 49, 105, 117, 47, 90, 29, 26, 97, 112, 39, 115, 116, 93, 109, 92, 118, 51, 54, 45, 114, 41, 24, 25, 43, 94, 88, 23, 121, 28, 91, 31, 62, 30, 18, 34, 89, 107, 17, 100, 82, 85, 83, 27, 87, 98, 95, 81, 19, 40, 13, 80, 78, 36, 21, 77, 37, 75, 14, 15, 16, 101, 104, 11, 0, 79, 70, 35, 76, 84, 73, 20, 99, 10, 9, 64, 12, 4, 96, 32, 68, 7, 74, 71, 72, 3, 1, 67, 8, 65, 66, 2, 69, 6, 5], [63, 61, 123, 56, 110, 126, 46, 53, 122, 113, 44, 50, 108, 111, 42, 106, 48, 127, 22, 124, 120, 117, 60, 57, 55, 102, 105, 103, 59, 39, 52, 58, 125, 38, 119, 33, 49, 51, 121, 47, 86, 116, 93, 112, 90, 26, 115, 41, 118, 109, 114, 29, 97, 24, 54, 62, 94, 43, 45, 92, 31, 25, 28, 88, 98, 107, 23, 82, 30, 21, 91, 89, 18, 34, 40, 83, 87, 17, 13, 78, 80, 36, 14, 81, 85, 75, 77, 104, 101, 16, 100, 64, 0, 15, 27, 70, 11, 95, 19, 37, 76, 12, 73, 79, 84, 7, 68, 74, 35, 8, 9, 71, 32, 67, 20, 99, 10, 4, 65, 69, 1, 3, 5, 72, 96, 66, 6, 2], [63, 61, 126, 123, 56, 110, 46, 53, 122, 44, 113, 50, 111, 106, 108, 42, 22, 127, 48, 117, 105, 57, 124, 102, 52, 103, 125, 55, 60, 120, 38, 115, 49, 39, 33, 121, 26, 97, 59, 116, 119, 86, 93, 54, 41, 47, 109, 90, 43, 51, 29, 114, 112, 34, 118, 58, 24, 45, 98, 62, 88, 92, 94, 25, 30, 31, 82, 28, 13, 23, 81, 89, 107, 40, 14, 78, 0, 80, 21, 87, 77, 18, 91, 83, 16, 17, 19, 100, 101, 36, 85, 76, 95, 75, 11, 70, 64, 104, 84, 9, 74, 73, 79, 15, 20, 12, 8, 37, 27, 35, 71, 10, 99, 7, 1, 68, 67, 4, 69, 65, 32, 3, 72, 96, 66, 2, 5, 6], [63, 61, 126, 123, 56, 110, 53, 46, 122, 113, 44, 50, 111, 106, 127, 108, 42, 22, 48, 117, 102, 52, 124, 57, 103, 105, 38, 60, 33, 86, 125, 116, 58, 59, 120, 118, 115, 114, 49, 39, 55, 112, 119, 93, 29, 97, 51, 26, 47, 41, 30, 94, 90, 54, 43, 121, 24, 109, 92, 25, 31, 89, 45, 34, 88, 40, 28, 107, 82, 98, 14, 91, 23, 81, 77, 62, 18, 80, 78, 100, 16, 85, 83, 87, 95, 13, 17, 21, 19, 101, 11, 64, 36, 76, 27, 75, 104, 79, 9, 73, 20, 12, 37, 84, 8, 3, 10, 70, 74, 0, 68, 4, 15, 7, 71, 1, 35, 96, 67, 99, 65, 5, 32, 69, 72, 2, 6, 66], [63, 61, 110, 126, 53, 56, 123, 46, 122, 44, 113, 50, 111, 106, 42, 108, 127, 22, 124, 57, 102, 48, 117, 60, 52, 105, 86, 103, 38, 116, 59, 125, 58, 33, 55, 47, 39, 112, 26, 29, 119, 120, 118, 115, 97, 90, 93, 49, 51, 121, 94, 109, 114, 43, 41, 62, 92, 31, 88, 45, 24, 25, 30, 34, 54, 23, 28, 40, 81, 83, 18, 89, 82, 17, 91, 98, 13, 100, 77, 16, 107, 87, 36, 21, 101, 95, 78, 14, 80, 27, 19, 84, 11, 85, 37, 20, 75, 104, 79, 15, 10, 12, 99, 9, 64, 76, 0, 4, 73, 96, 35, 7, 74, 8, 67, 68, 6, 3, 65, 71, 70, 32, 1, 5, 72, 66, 69, 2], [63, 61, 110, 56, 126, 123, 53, 46, 122, 113, 44, 50, 111, 106, 127, 42, 108, 48, 117, 57, 22, 102, 52, 105, 103, 60, 124, 59, 86, 116, 115, 33, 55, 38, 120, 39, 47, 125, 109, 112, 49, 97, 26, 51, 93, 62, 41, 29, 90, 119, 114, 118, 121, 58, 54, 94, 24, 92, 30, 107, 34, 88, 28, 45, 89, 43, 25, 18, 98, 31, 91, 23, 81, 83, 40, 13, 36, 16, 82, 19, 17, 78, 75, 77, 37, 14, 87, 100, 21, 85, 27, 101, 80, 104, 95, 9, 0, 64, 79, 20, 6, 15, 76, 73, 84, 35, 74, 11, 99, 12, 7, 96, 8, 32, 10, 1, 67, 71, 3, 4, 69, 72, 70, 66, 68, 65, 5, 2], [63, 61, 123, 126, 110, 56, 46, 53, 122, 113, 50, 44, 111, 106, 42, 127, 52, 108, 48, 57, 102, 117, 124, 120, 22, 103, 60, 51, 105, 39, 125, 33, 38, 47, 116, 59, 55, 86, 115, 49, 121, 118, 26, 112, 29, 58, 119, 97, 93, 109, 45, 43, 90, 54, 62, 41, 114, 30, 31, 107, 94, 24, 34, 40, 92, 98, 88, 23, 28, 36, 25, 77, 100, 81, 101, 89, 82, 17, 18, 91, 13, 21, 75, 78, 83, 95, 19, 14, 80, 16, 87, 37, 104, 9, 85, 11, 99, 6, 27, 12, 35, 76, 7, 84, 73, 71, 20, 64, 15, 79, 74, 10, 0, 68, 1, 32, 96, 8, 69, 3, 4, 65, 2, 72, 67, 5, 70, 66], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 48, 127, 117, 22, 38, 124, 57, 52, 102, 105, 103, 120, 60, 33, 125, 59, 47, 39, 86, 26, 112, 97, 119, 51, 49, 58, 114, 116, 118, 29, 55, 115, 93, 90, 109, 121, 41, 54, 92, 30, 24, 45, 94, 34, 88, 31, 28, 62, 43, 107, 25, 23, 91, 98, 77, 36, 18, 81, 104, 40, 89, 87, 82, 17, 13, 19, 78, 101, 21, 85, 80, 83, 14, 79, 95, 16, 9, 10, 75, 6, 100, 76, 20, 11, 0, 15, 74, 37, 12, 71, 73, 68, 84, 64, 8, 27, 7, 32, 99, 4, 3, 72, 65, 35, 1, 96, 67, 66, 5, 69, 2, 70], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 111, 106, 42, 108, 117, 57, 48, 127, 22, 60, 124, 52, 105, 102, 103, 59, 109, 33, 120, 125, 115, 114, 112, 116, 55, 86, 49, 38, 51, 118, 47, 39, 62, 90, 58, 93, 119, 26, 29, 97, 41, 121, 54, 107, 94, 45, 24, 28, 88, 92, 43, 98, 34, 30, 89, 36, 25, 31, 82, 95, 91, 83, 13, 80, 40, 77, 18, 104, 21, 87, 17, 81, 101, 23, 14, 78, 16, 75, 19, 11, 85, 64, 27, 37, 100, 15, 9, 76, 79, 6, 73, 0, 10, 35, 20, 99, 84, 4, 74, 65, 67, 32, 7, 71, 68, 12, 8, 1, 72, 69, 3, 96, 70, 2, 5, 66], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 117, 42, 106, 108, 127, 124, 52, 48, 102, 57, 60, 120, 103, 125, 22, 105, 112, 55, 58, 39, 59, 114, 38, 33, 115, 121, 119, 51, 118, 47, 116, 86, 29, 49, 26, 90, 109, 93, 54, 41, 107, 97, 94, 88, 62, 45, 30, 43, 31, 28, 23, 34, 92, 24, 91, 25, 83, 87, 36, 98, 82, 100, 17, 81, 89, 18, 77, 104, 64, 40, 37, 14, 19, 13, 85, 21, 95, 27, 101, 75, 80, 16, 78, 0, 76, 15, 20, 79, 11, 4, 9, 3, 73, 74, 99, 35, 10, 12, 96, 84, 6, 1, 70, 71, 7, 67, 8, 32, 65, 72, 69, 5, 68, 2, 66], [63, 61, 123, 110, 126, 56, 53, 46, 122, 113, 44, 50, 111, 127, 106, 108, 42, 124, 117, 48, 102, 22, 120, 60, 52, 125, 58, 38, 116, 33, 57, 103, 59, 119, 105, 115, 86, 118, 112, 49, 39, 55, 26, 47, 29, 93, 90, 121, 51, 97, 114, 41, 54, 94, 109, 88, 43, 28, 34, 45, 24, 62, 25, 30, 31, 92, 107, 89, 36, 98, 91, 82, 18, 23, 83, 87, 81, 100, 77, 17, 13, 40, 21, 101, 16, 14, 95, 85, 80, 78, 19, 0, 11, 27, 75, 104, 64, 9, 15, 84, 76, 71, 20, 73, 4, 74, 37, 79, 72, 65, 70, 12, 35, 99, 3, 1, 96, 7, 10, 68, 67, 2, 32, 8, 5, 66, 6, 69], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 106, 111, 108, 42, 48, 22, 127, 120, 117, 124, 105, 52, 102, 57, 103, 38, 125, 33, 60, 59, 116, 118, 55, 112, 86, 97, 26, 93, 49, 90, 39, 121, 47, 119, 29, 51, 58, 114, 41, 115, 54, 109, 24, 92, 43, 62, 94, 28, 30, 88, 34, 25, 45, 36, 89, 107, 40, 82, 17, 98, 31, 23, 78, 91, 18, 81, 13, 104, 77, 83, 80, 19, 14, 95, 85, 87, 21, 16, 11, 75, 15, 79, 27, 76, 100, 70, 10, 9, 101, 37, 12, 73, 35, 4, 99, 20, 64, 72, 84, 32, 74, 71, 68, 7, 0, 67, 5, 1, 65, 3, 8, 96, 69, 6, 66, 2], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 22, 48, 117, 57, 124, 52, 127, 60, 105, 120, 102, 59, 103, 116, 112, 55, 114, 115, 38, 125, 119, 86, 33, 121, 49, 39, 26, 109, 90, 47, 51, 118, 93, 29, 54, 43, 97, 41, 107, 58, 94, 28, 92, 25, 24, 30, 62, 23, 45, 88, 31, 36, 17, 34, 77, 85, 83, 98, 89, 40, 81, 87, 82, 101, 21, 18, 78, 16, 19, 91, 100, 27, 95, 13, 11, 104, 80, 75, 15, 70, 79, 9, 14, 73, 64, 84, 76, 12, 37, 7, 35, 74, 20, 0, 10, 68, 71, 32, 72, 67, 3, 99, 65, 8, 1, 96, 4, 5, 66, 69, 6, 2], [63, 61, 110, 126, 123, 56, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 22, 48, 127, 124, 120, 60, 52, 102, 117, 57, 103, 105, 55, 125, 116, 33, 59, 115, 38, 119, 86, 49, 29, 39, 112, 118, 47, 90, 26, 58, 93, 121, 109, 97, 114, 94, 54, 51, 41, 24, 28, 62, 30, 92, 34, 25, 88, 43, 107, 31, 89, 98, 23, 91, 45, 82, 18, 100, 40, 36, 37, 83, 77, 19, 87, 17, 81, 85, 101, 27, 80, 95, 16, 13, 104, 21, 11, 75, 14, 64, 78, 35, 12, 79, 84, 9, 20, 15, 73, 99, 76, 10, 70, 0, 71, 74, 96, 68, 65, 32, 72, 3, 1, 7, 67, 5, 4, 69, 8, 2, 66, 6], [63, 61, 110, 123, 126, 56, 53, 46, 122, 113, 50, 44, 106, 111, 42, 108, 48, 117, 127, 22, 57, 52, 60, 102, 103, 105, 124, 59, 116, 38, 112, 125, 120, 109, 115, 121, 39, 55, 33, 119, 49, 86, 51, 93, 97, 118, 114, 54, 58, 26, 90, 41, 29, 62, 47, 88, 24, 94, 30, 91, 89, 34, 43, 31, 92, 25, 107, 28, 45, 101, 36, 98, 40, 18, 81, 82, 87, 83, 17, 85, 23, 19, 80, 11, 21, 13, 78, 100, 16, 27, 14, 77, 95, 104, 84, 37, 79, 35, 75, 12, 73, 0, 99, 76, 15, 9, 8, 64, 20, 74, 70, 32, 68, 10, 71, 7, 65, 67, 96, 4, 72, 3, 69, 1, 5, 6, 2, 66], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 108, 117, 48, 127, 42, 22, 124, 120, 57, 52, 102, 103, 60, 125, 38, 105, 116, 33, 59, 39, 121, 49, 55, 112, 51, 47, 119, 26, 114, 115, 86, 93, 41, 43, 54, 90, 97, 58, 29, 109, 118, 62, 45, 34, 88, 92, 24, 94, 31, 107, 25, 89, 28, 30, 36, 91, 82, 81, 23, 98, 18, 77, 21, 11, 83, 14, 17, 40, 80, 0, 87, 78, 37, 16, 13, 19, 85, 100, 79, 9, 104, 84, 73, 95, 75, 12, 10, 64, 27, 15, 101, 76, 35, 71, 74, 99, 20, 6, 67, 68, 7, 1, 4, 65, 96, 70, 3, 32, 2, 72, 5, 8, 69, 66], [63, 61, 123, 126, 110, 53, 56, 46, 122, 113, 50, 44, 111, 106, 127, 117, 108, 42, 48, 125, 52, 124, 59, 22, 102, 57, 103, 38, 112, 60, 105, 33, 120, 116, 51, 119, 39, 115, 55, 97, 54, 43, 86, 62, 93, 49, 29, 26, 114, 41, 47, 58, 90, 118, 121, 34, 94, 24, 92, 109, 45, 89, 30, 88, 28, 31, 40, 95, 18, 101, 98, 25, 23, 14, 91, 107, 80, 82, 17, 16, 11, 85, 36, 77, 13, 78, 19, 100, 81, 104, 0, 87, 15, 83, 64, 73, 71, 6, 21, 79, 9, 75, 37, 76, 12, 10, 74, 27, 35, 20, 84, 4, 65, 68, 7, 8, 3, 67, 1, 96, 99, 70, 32, 72, 69, 66, 5, 2], [63, 61, 126, 123, 56, 110, 46, 53, 122, 113, 50, 44, 111, 42, 106, 57, 108, 117, 124, 22, 127, 48, 59, 52, 102, 125, 60, 120, 103, 105, 116, 38, 86, 39, 55, 112, 119, 58, 33, 26, 49, 114, 97, 29, 93, 51, 47, 41, 118, 90, 43, 115, 54, 45, 24, 109, 62, 88, 28, 92, 25, 94, 121, 18, 30, 34, 91, 31, 23, 100, 82, 81, 98, 89, 77, 21, 36, 85, 78, 87, 17, 19, 40, 14, 107, 80, 101, 13, 83, 11, 16, 104, 95, 73, 27, 15, 6, 9, 76, 79, 84, 75, 12, 37, 0, 4, 35, 74, 20, 68, 10, 71, 99, 64, 8, 7, 3, 67, 32, 1, 72, 69, 65, 5, 70, 96, 66, 2], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 111, 124, 108, 117, 106, 42, 57, 60, 127, 22, 48, 103, 52, 102, 38, 116, 120, 59, 105, 49, 86, 55, 125, 39, 33, 47, 121, 114, 112, 51, 90, 115, 97, 29, 119, 26, 54, 118, 41, 93, 109, 58, 43, 25, 28, 62, 94, 30, 34, 24, 92, 31, 23, 88, 45, 36, 95, 78, 81, 82, 98, 19, 18, 83, 107, 91, 89, 13, 77, 21, 87, 6, 100, 14, 80, 85, 11, 40, 64, 73, 17, 104, 16, 15, 101, 9, 0, 71, 37, 4, 79, 7, 27, 75, 10, 74, 12, 84, 68, 67, 76, 8, 35, 3, 20, 1, 99, 65, 96, 66, 32, 72, 2, 69, 5, 70], [63, 61, 123, 110, 56, 126, 53, 122, 46, 113, 44, 50, 111, 106, 42, 108, 127, 22, 48, 60, 117, 102, 103, 124, 57, 125, 120, 52, 59, 38, 33, 105, 86, 55, 39, 29, 97, 49, 51, 121, 119, 26, 47, 90, 112, 115, 116, 93, 54, 114, 30, 109, 118, 41, 62, 43, 94, 25, 92, 31, 24, 88, 58, 34, 28, 91, 18, 81, 45, 100, 98, 82, 23, 107, 85, 11, 14, 13, 89, 37, 17, 36, 21, 80, 19, 83, 16, 40, 87, 77, 95, 27, 79, 78, 104, 73, 101, 6, 15, 9, 75, 35, 12, 10, 76, 99, 74, 8, 20, 71, 0, 4, 84, 96, 7, 32, 68, 72, 67, 69, 1, 3, 70, 65, 5, 64, 2, 66]], "model.layers.31.self_attn.q_proj": [[40, 121, 98, 46, 18, 25, 111, 9, 30, 21, 13, 51, 15, 11, 48, 105, 49, 67, 23, 60, 55, 53, 4, 6, 58, 116, 114, 124, 72, 107, 31, 88, 115, 95, 41, 109, 22, 108, 61, 17, 37, 59, 92, 50, 45, 19, 87, 43, 127, 20, 5, 97, 1, 14, 123, 27, 126, 69, 75, 78, 81, 90, 16, 120, 102, 101, 8, 24, 112, 93, 85, 52, 26, 63, 118, 0, 7, 33, 117, 62, 122, 96, 76, 94, 74, 113, 35, 36, 54, 28, 12, 77, 79, 57, 39, 56, 82, 70, 73, 42, 89, 84, 3, 80, 32, 106, 110, 103, 83, 99, 100, 2, 125, 65, 119, 68, 38, 29, 86, 71, 44, 47, 10, 104, 91, 64, 66, 34], [40, 121, 98, 46, 9, 13, 111, 21, 25, 18, 51, 67, 15, 49, 4, 95, 11, 108, 60, 50, 48, 117, 45, 68, 107, 88, 61, 116, 37, 64, 122, 41, 30, 58, 6, 92, 1, 72, 2, 105, 43, 123, 65, 109, 55, 0, 28, 87, 42, 3, 53, 5, 126, 23, 27, 39, 127, 54, 74, 19, 101, 114, 73, 85, 14, 62, 70, 113, 22, 115, 66, 31, 63, 34, 20, 93, 16, 78, 120, 33, 118, 69, 110, 124, 36, 90, 7, 75, 52, 104, 82, 57, 80, 77, 94, 32, 29, 24, 81, 10, 103, 12, 83, 17, 79, 76, 89, 26, 59, 8, 35, 112, 38, 44, 99, 119, 84, 125, 56, 71, 86, 100, 96, 102, 106, 91, 97, 47], [40, 111, 46, 121, 98, 37, 49, 21, 25, 117, 54, 114, 18, 95, 23, 60, 11, 15, 107, 51, 61, 53, 13, 116, 105, 4, 78, 62, 16, 48, 67, 30, 9, 113, 124, 80, 24, 72, 39, 41, 123, 17, 19, 6, 1, 120, 108, 88, 42, 90, 92, 70, 83, 52, 63, 8, 109, 0, 50, 58, 20, 85, 27, 87, 55, 122, 56, 84, 26, 65, 127, 115, 45, 38, 22, 103, 59, 112, 43, 96, 125, 29, 71, 126, 75, 118, 91, 57, 100, 7, 119, 36, 102, 101, 10, 68, 33, 79, 76, 2, 14, 34, 35, 81, 44, 74, 86, 94, 12, 97, 82, 110, 32, 47, 89, 73, 104, 5, 106, 93, 31, 69, 28, 99, 77, 66, 64, 3], [40, 121, 98, 46, 117, 111, 13, 21, 18, 61, 37, 9, 25, 15, 4, 51, 48, 30, 105, 11, 122, 107, 72, 49, 58, 87, 108, 24, 6, 23, 123, 67, 116, 63, 1, 110, 90, 88, 60, 0, 124, 14, 31, 127, 101, 39, 126, 45, 32, 95, 43, 41, 27, 109, 85, 52, 26, 92, 93, 64, 80, 50, 75, 5, 97, 22, 36, 78, 55, 100, 74, 16, 57, 70, 53, 103, 89, 19, 82, 73, 102, 28, 115, 113, 120, 83, 59, 35, 38, 42, 62, 12, 118, 17, 106, 20, 2, 114, 119, 79, 76, 71, 33, 125, 81, 66, 54, 112, 8, 68, 44, 91, 7, 77, 56, 96, 86, 94, 69, 65, 99, 29, 3, 84, 10, 104, 47, 34], [117, 50, 105, 108, 123, 49, 99, 44, 97, 25, 31, 87, 120, 21, 112, 16, 93, 82, 58, 78, 34, 29, 62, 118, 48, 114, 51, 38, 30, 43, 109, 32, 90, 42, 55, 7, 106, 10, 59, 103, 36, 53, 107, 52, 12, 61, 96, 45, 33, 124, 47, 46, 60, 26, 56, 23, 35, 63, 88, 39, 95, 113, 17, 125, 77, 28, 115, 121, 54, 19, 83, 57, 24, 122, 98, 94, 84, 68, 119, 116, 104, 15, 110, 92, 11, 69, 111, 40, 102, 126, 3, 100, 20, 127, 14, 22, 91, 86, 79, 27, 101, 8, 75, 37, 71, 89, 65, 74, 81, 85, 2, 72, 18, 64, 6, 76, 66, 73, 9, 80, 70, 67, 4, 13, 1, 5, 41, 0], [105, 123, 49, 117, 31, 99, 118, 2, 114, 109, 10, 16, 7, 82, 25, 78, 64, 69, 3, 68, 65, 50, 12, 8, 21, 77, 44, 97, 87, 67, 89, 115, 58, 41, 0, 66, 4, 71, 1, 111, 23, 98, 59, 61, 93, 104, 120, 127, 72, 63, 5, 39, 56, 28, 51, 70, 42, 91, 108, 112, 24, 57, 95, 122, 20, 107, 74, 15, 36, 75, 121, 106, 14, 34, 83, 52, 86, 29, 9, 54, 6, 37, 79, 92, 84, 88, 30, 18, 27, 101, 22, 13, 73, 110, 103, 46, 80, 11, 17, 96, 62, 26, 19, 90, 119, 33, 76, 113, 43, 85, 60, 102, 45, 38, 48, 53, 81, 116, 47, 40, 55, 125, 32, 124, 94, 35, 126, 100], [105, 49, 123, 108, 31, 58, 118, 10, 25, 16, 117, 82, 99, 3, 69, 21, 8, 87, 12, 65, 78, 77, 68, 109, 94, 7, 29, 122, 114, 1, 120, 42, 5, 55, 97, 64, 4, 44, 50, 111, 43, 6, 36, 98, 115, 23, 73, 95, 33, 13, 125, 38, 2, 93, 11, 79, 48, 76, 63, 62, 66, 88, 61, 72, 89, 30, 67, 91, 47, 54, 46, 51, 27, 18, 56, 39, 32, 9, 92, 110, 90, 101, 100, 45, 112, 75, 86, 70, 113, 103, 116, 0, 127, 17, 34, 106, 74, 80, 71, 81, 14, 52, 40, 20, 104, 22, 121, 53, 60, 41, 84, 19, 83, 59, 96, 24, 107, 26, 37, 119, 15, 28, 57, 126, 85, 102, 124, 35], [118, 49, 108, 105, 58, 44, 42, 99, 21, 117, 87, 97, 94, 31, 12, 39, 25, 93, 29, 82, 104, 46, 51, 56, 52, 33, 38, 95, 43, 103, 113, 119, 107, 26, 109, 45, 73, 50, 16, 59, 115, 91, 125, 123, 111, 47, 60, 57, 112, 40, 127, 114, 20, 121, 54, 110, 77, 53, 101, 63, 37, 122, 126, 23, 61, 48, 1, 120, 124, 116, 32, 19, 102, 36, 55, 100, 96, 81, 78, 75, 4, 98, 62, 8, 85, 106, 92, 79, 34, 5, 17, 30, 70, 72, 10, 89, 9, 7, 90, 69, 35, 6, 22, 66, 76, 68, 24, 65, 28, 27, 18, 15, 3, 88, 83, 84, 13, 86, 2, 0, 41, 14, 67, 11, 80, 74, 71, 64], [40, 109, 121, 34, 59, 89, 53, 120, 17, 45, 55, 15, 86, 83, 114, 12, 29, 113, 71, 95, 28, 93, 4, 73, 94, 31, 69, 125, 67, 46, 122, 56, 106, 42, 33, 11, 92, 2, 77, 107, 61, 44, 88, 65, 108, 23, 38, 6, 58, 115, 8, 30, 50, 110, 26, 116, 0, 118, 126, 16, 99, 90, 21, 119, 87, 80, 62, 117, 20, 48, 101, 37, 49, 3, 68, 32, 85, 24, 41, 81, 84, 13, 52, 76, 19, 36, 82, 18, 1, 43, 91, 100, 112, 97, 74, 103, 78, 14, 96, 79, 35, 5, 64, 22, 75, 27, 39, 54, 57, 25, 47, 123, 63, 10, 124, 7, 127, 70, 104, 111, 51, 105, 9, 72, 102, 60, 66, 98], [40, 109, 121, 34, 15, 12, 86, 17, 73, 83, 53, 4, 89, 55, 71, 67, 59, 120, 45, 42, 92, 114, 2, 106, 95, 69, 122, 113, 0, 77, 6, 46, 29, 68, 28, 103, 56, 118, 11, 8, 33, 116, 72, 31, 18, 125, 65, 111, 37, 38, 88, 1, 61, 100, 85, 23, 108, 30, 101, 115, 64, 107, 98, 119, 20, 112, 82, 48, 3, 74, 126, 66, 70, 16, 13, 14, 81, 5, 93, 21, 84, 110, 19, 27, 94, 79, 80, 87, 96, 22, 102, 76, 58, 35, 7, 97, 25, 10, 91, 24, 36, 26, 62, 104, 75, 63, 117, 57, 47, 39, 78, 49, 41, 32, 90, 44, 9, 51, 43, 52, 127, 60, 99, 123, 105, 50, 124, 54], [40, 109, 121, 34, 89, 17, 86, 15, 55, 83, 12, 53, 45, 95, 59, 67, 122, 28, 92, 2, 29, 0, 65, 73, 69, 71, 114, 56, 4, 21, 42, 116, 6, 62, 120, 61, 125, 119, 31, 77, 46, 49, 10, 11, 93, 16, 1, 19, 26, 96, 88, 98, 36, 126, 106, 113, 118, 64, 30, 23, 57, 108, 44, 48, 101, 13, 103, 18, 123, 58, 22, 66, 37, 14, 27, 72, 76, 20, 87, 70, 80, 112, 127, 8, 115, 78, 94, 54, 68, 33, 84, 32, 81, 82, 24, 25, 7, 91, 39, 79, 90, 107, 74, 3, 85, 104, 52, 35, 5, 60, 51, 110, 75, 99, 9, 43, 100, 38, 47, 111, 97, 124, 102, 63, 117, 41, 50, 105], [121, 109, 40, 59, 34, 53, 111, 44, 114, 39, 124, 118, 116, 115, 88, 20, 61, 58, 55, 93, 117, 60, 126, 56, 62, 49, 108, 50, 72, 122, 48, 54, 63, 43, 46, 41, 57, 51, 119, 92, 127, 123, 47, 86, 125, 42, 31, 110, 105, 120, 98, 38, 52, 107, 113, 112, 45, 106, 83, 97, 89, 103, 104, 27, 36, 102, 28, 96, 26, 100, 14, 101, 32, 35, 37, 99, 10, 17, 95, 29, 33, 30, 94, 18, 85, 91, 87, 68, 23, 78, 25, 90, 13, 84, 73, 15, 4, 77, 70, 21, 74, 82, 8, 24, 6, 16, 75, 12, 80, 2, 71, 11, 19, 22, 66, 1, 64, 9, 67, 3, 81, 7, 69, 5, 76, 79, 0, 65], [107, 100, 58, 32, 21, 112, 78, 19, 61, 127, 87, 93, 115, 12, 43, 9, 114, 120, 116, 119, 16, 56, 82, 55, 42, 96, 111, 113, 25, 72, 38, 125, 54, 92, 79, 48, 29, 5, 66, 47, 88, 97, 110, 105, 57, 123, 4, 124, 35, 50, 49, 2, 18, 27, 0, 24, 63, 36, 62, 28, 53, 71, 91, 39, 77, 75, 68, 1, 86, 11, 3, 26, 69, 46, 20, 7, 15, 104, 126, 14, 95, 117, 106, 83, 73, 74, 98, 13, 23, 6, 70, 64, 41, 84, 45, 89, 67, 80, 30, 44, 99, 90, 85, 81, 10, 108, 40, 76, 59, 51, 94, 37, 31, 34, 103, 22, 65, 101, 102, 122, 8, 60, 52, 17, 109, 33, 121, 118], [107, 58, 100, 63, 127, 32, 112, 25, 56, 61, 47, 39, 115, 114, 116, 79, 21, 105, 113, 54, 108, 104, 96, 19, 119, 87, 42, 55, 120, 126, 93, 45, 43, 50, 41, 110, 125, 35, 106, 17, 122, 82, 111, 57, 97, 20, 44, 118, 60, 52, 48, 88, 123, 30, 124, 117, 49, 59, 16, 53, 38, 27, 46, 109, 62, 36, 74, 91, 78, 33, 51, 5, 121, 102, 101, 28, 24, 103, 40, 71, 75, 92, 99, 89, 31, 11, 37, 86, 94, 90, 72, 95, 12, 26, 98, 34, 15, 22, 9, 23, 84, 10, 14, 13, 77, 81, 18, 65, 29, 85, 70, 6, 3, 67, 66, 7, 83, 80, 69, 0, 68, 1, 8, 76, 4, 64, 2, 73], [43, 93, 58, 2, 68, 65, 107, 67, 0, 39, 87, 12, 70, 9, 69, 78, 64, 1, 21, 25, 63, 72, 96, 10, 16, 66, 110, 19, 112, 103, 100, 8, 61, 114, 97, 56, 7, 32, 37, 73, 29, 79, 5, 75, 55, 82, 120, 116, 14, 38, 20, 17, 49, 44, 91, 42, 3, 80, 57, 115, 113, 51, 109, 119, 101, 45, 74, 4, 104, 23, 40, 85, 28, 125, 98, 33, 127, 124, 47, 46, 31, 6, 53, 105, 59, 99, 48, 71, 35, 83, 11, 24, 121, 15, 26, 84, 106, 81, 60, 77, 86, 22, 89, 108, 123, 34, 95, 18, 30, 13, 76, 27, 92, 126, 118, 88, 50, 90, 41, 54, 94, 122, 36, 52, 111, 102, 62, 117], [107, 100, 58, 127, 112, 32, 12, 19, 78, 61, 25, 16, 9, 82, 119, 114, 21, 87, 43, 93, 113, 115, 54, 120, 66, 56, 0, 55, 125, 39, 49, 4, 38, 42, 105, 29, 116, 72, 97, 111, 68, 27, 63, 5, 81, 110, 96, 47, 69, 48, 124, 62, 51, 6, 71, 57, 35, 10, 92, 123, 3, 83, 28, 91, 44, 84, 126, 95, 79, 76, 7, 89, 23, 98, 75, 20, 18, 64, 88, 80, 90, 37, 101, 106, 22, 8, 52, 15, 85, 30, 73, 67, 50, 103, 59, 117, 11, 74, 70, 53, 118, 2, 13, 31, 14, 34, 1, 45, 109, 24, 46, 65, 77, 94, 33, 86, 108, 26, 104, 99, 41, 40, 60, 17, 122, 121, 102, 36], [55, 58, 115, 50, 52, 39, 122, 62, 63, 59, 121, 106, 34, 54, 103, 53, 47, 48, 95, 120, 119, 118, 117, 124, 56, 126, 113, 107, 127, 123, 49, 105, 96, 111, 116, 51, 112, 93, 42, 61, 84, 46, 109, 60, 114, 57, 44, 125, 43, 110, 45, 102, 91, 27, 22, 108, 38, 29, 99, 86, 41, 104, 76, 97, 24, 89, 30, 35, 40, 37, 82, 33, 32, 101, 100, 25, 36, 92, 16, 18, 10, 8, 20, 88, 31, 98, 94, 90, 28, 65, 21, 85, 23, 81, 15, 74, 14, 87, 80, 26, 4, 12, 78, 3, 17, 11, 1, 72, 5, 19, 2, 77, 68, 71, 79, 83, 6, 69, 73, 70, 64, 67, 66, 0, 75, 13, 9, 7], [50, 58, 39, 62, 52, 34, 122, 55, 45, 120, 118, 115, 106, 24, 108, 105, 81, 47, 64, 51, 126, 63, 71, 124, 93, 84, 103, 56, 109, 95, 113, 59, 96, 44, 53, 125, 117, 57, 27, 119, 54, 25, 48, 5, 112, 116, 111, 38, 42, 99, 127, 60, 91, 49, 22, 121, 114, 3, 86, 30, 2, 32, 61, 0, 20, 82, 123, 104, 35, 4, 46, 73, 98, 107, 43, 41, 110, 77, 66, 65, 101, 6, 15, 31, 76, 102, 14, 97, 28, 40, 88, 29, 67, 10, 37, 92, 21, 83, 18, 36, 26, 87, 8, 80, 89, 33, 100, 23, 78, 16, 85, 11, 94, 90, 12, 7, 69, 70, 79, 68, 74, 17, 72, 13, 9, 1, 19, 75], [58, 50, 115, 39, 34, 83, 77, 15, 2, 73, 11, 3, 71, 109, 24, 27, 6, 0, 81, 64, 122, 93, 79, 1, 51, 52, 85, 44, 68, 4, 22, 20, 67, 45, 113, 82, 9, 54, 99, 5, 25, 26, 66, 72, 8, 80, 106, 111, 65, 56, 19, 88, 114, 7, 12, 95, 116, 13, 78, 62, 104, 59, 63, 107, 21, 35, 89, 75, 10, 17, 119, 105, 70, 98, 74, 69, 28, 30, 32, 18, 61, 29, 76, 33, 14, 120, 91, 117, 118, 86, 41, 57, 92, 46, 127, 87, 16, 96, 126, 108, 31, 43, 23, 97, 94, 90, 38, 36, 103, 100, 102, 101, 37, 53, 110, 84, 124, 121, 47, 49, 125, 55, 40, 48, 123, 60, 112, 42], [58, 115, 62, 39, 55, 52, 122, 106, 34, 108, 107, 53, 120, 61, 63, 103, 105, 95, 118, 47, 54, 124, 59, 109, 49, 91, 117, 60, 57, 119, 48, 114, 112, 111, 93, 96, 99, 121, 84, 126, 46, 45, 116, 127, 123, 125, 41, 104, 51, 110, 42, 43, 19, 56, 22, 113, 44, 24, 33, 86, 7, 40, 38, 71, 32, 102, 101, 76, 25, 82, 97, 87, 37, 27, 16, 29, 35, 10, 100, 36, 83, 94, 50, 31, 98, 21, 30, 18, 90, 92, 20, 81, 11, 23, 78, 28, 89, 26, 75, 88, 74, 85, 15, 5, 14, 64, 8, 4, 80, 67, 0, 3, 69, 1, 12, 65, 66, 17, 68, 2, 79, 73, 77, 72, 6, 13, 9, 70], [122, 104, 105, 126, 53, 112, 51, 110, 118, 123, 55, 119, 57, 111, 124, 120, 114, 117, 59, 121, 125, 63, 116, 115, 61, 49, 34, 54, 113, 106, 60, 89, 32, 86, 50, 31, 62, 48, 46, 58, 56, 127, 44, 52, 45, 109, 91, 33, 98, 47, 107, 43, 25, 108, 95, 73, 41, 40, 20, 82, 93, 42, 75, 22, 83, 81, 103, 23, 19, 88, 39, 96, 101, 37, 87, 102, 92, 99, 36, 11, 90, 38, 29, 100, 27, 26, 35, 21, 72, 97, 84, 17, 94, 13, 30, 18, 14, 24, 16, 28, 78, 76, 85, 6, 80, 8, 77, 9, 68, 4, 10, 70, 79, 5, 12, 71, 67, 3, 74, 69, 15, 7, 65, 2, 64, 0, 66, 1], [55, 122, 104, 105, 127, 119, 121, 114, 62, 60, 118, 117, 112, 110, 57, 51, 63, 111, 49, 124, 34, 56, 116, 123, 61, 31, 125, 113, 59, 44, 86, 47, 120, 48, 115, 108, 54, 58, 33, 89, 106, 45, 46, 50, 52, 53, 126, 109, 98, 107, 91, 41, 43, 81, 42, 95, 88, 40, 87, 23, 103, 83, 27, 101, 32, 37, 25, 29, 92, 19, 93, 90, 39, 20, 30, 96, 22, 99, 38, 102, 36, 18, 82, 100, 94, 80, 26, 35, 97, 75, 15, 17, 28, 24, 85, 69, 12, 77, 76, 78, 84, 21, 13, 6, 14, 3, 10, 16, 71, 79, 8, 5, 70, 4, 72, 11, 9, 65, 67, 74, 68, 73, 2, 7, 64, 66, 0, 1], [55, 122, 104, 105, 126, 41, 88, 18, 20, 80, 79, 10, 31, 60, 98, 15, 85, 78, 114, 76, 121, 29, 86, 75, 34, 69, 12, 33, 93, 91, 118, 8, 47, 53, 123, 89, 27, 62, 110, 13, 120, 71, 59, 119, 63, 3, 116, 108, 57, 113, 51, 124, 56, 28, 44, 96, 97, 127, 74, 49, 43, 112, 87, 16, 106, 83, 115, 111, 54, 50, 117, 5, 26, 61, 25, 58, 94, 24, 125, 95, 46, 21, 48, 52, 72, 45, 42, 90, 82, 84, 109, 107, 14, 92, 23, 9, 39, 30, 100, 19, 32, 77, 101, 22, 73, 102, 36, 37, 81, 99, 35, 11, 17, 38, 70, 103, 7, 2, 6, 4, 40, 68, 67, 66, 65, 64, 1, 0], [122, 55, 104, 64, 71, 2, 126, 79, 4, 10, 1, 3, 73, 105, 78, 18, 8, 76, 20, 80, 77, 75, 9, 31, 69, 86, 7, 93, 51, 98, 68, 81, 88, 6, 27, 85, 121, 5, 66, 65, 107, 119, 70, 16, 23, 67, 114, 0, 84, 37, 74, 63, 11, 59, 97, 82, 106, 17, 15, 33, 120, 91, 118, 127, 21, 14, 57, 52, 41, 49, 12, 53, 87, 34, 95, 39, 72, 13, 110, 100, 89, 24, 47, 83, 29, 101, 46, 32, 25, 117, 112, 19, 56, 44, 28, 60, 111, 103, 99, 90, 26, 22, 58, 50, 124, 45, 125, 30, 96, 102, 92, 123, 94, 109, 36, 43, 113, 35, 40, 108, 38, 42, 48, 115, 116, 54, 61, 62], [103, 126, 98, 9, 14, 4, 21, 17, 71, 95, 64, 75, 67, 115, 27, 24, 88, 1, 109, 63, 48, 44, 62, 61, 16, 70, 118, 104, 114, 116, 31, 124, 45, 90, 122, 106, 3, 19, 5, 99, 105, 127, 6, 65, 56, 12, 46, 32, 37, 91, 53, 50, 113, 13, 66, 47, 100, 93, 22, 57, 81, 23, 20, 78, 15, 87, 0, 74, 92, 51, 69, 123, 72, 7, 41, 76, 73, 34, 10, 83, 11, 85, 112, 68, 49, 2, 79, 94, 125, 55, 58, 40, 89, 8, 77, 117, 26, 35, 121, 33, 80, 107, 119, 25, 82, 38, 52, 86, 42, 84, 54, 101, 97, 30, 36, 29, 28, 18, 110, 59, 39, 60, 96, 43, 102, 108, 111, 120], [103, 126, 98, 17, 65, 75, 14, 9, 71, 21, 4, 3, 61, 48, 88, 95, 109, 69, 0, 115, 44, 41, 5, 24, 27, 106, 124, 63, 118, 70, 62, 105, 114, 112, 12, 16, 122, 1, 31, 64, 45, 13, 50, 127, 91, 113, 19, 76, 56, 99, 46, 47, 90, 68, 81, 66, 84, 85, 86, 57, 116, 110, 78, 123, 55, 104, 23, 80, 77, 117, 96, 92, 18, 15, 100, 37, 20, 29, 42, 73, 67, 6, 82, 87, 72, 33, 53, 74, 107, 120, 34, 7, 119, 43, 54, 11, 32, 58, 26, 121, 89, 30, 83, 94, 93, 49, 51, 36, 102, 97, 38, 25, 79, 40, 35, 2, 22, 125, 10, 8, 28, 108, 52, 101, 111, 59, 60, 39], [103, 126, 98, 61, 95, 21, 88, 17, 27, 124, 44, 14, 109, 63, 91, 48, 113, 75, 115, 9, 104, 31, 122, 105, 77, 71, 90, 39, 50, 24, 83, 100, 127, 12, 4, 47, 37, 41, 114, 116, 53, 45, 19, 99, 36, 42, 57, 107, 87, 56, 33, 121, 93, 62, 94, 74, 51, 72, 13, 32, 70, 35, 29, 106, 119, 38, 112, 10, 123, 67, 1, 108, 40, 118, 84, 96, 97, 79, 20, 52, 110, 85, 26, 28, 5, 58, 54, 80, 76, 86, 46, 25, 55, 111, 73, 15, 69, 18, 22, 59, 125, 102, 64, 43, 117, 16, 101, 30, 81, 78, 23, 82, 8, 49, 89, 120, 92, 60, 6, 34, 11, 2, 3, 7, 68, 0, 66, 65], [126, 103, 41, 98, 61, 95, 118, 51, 112, 127, 119, 109, 54, 110, 91, 124, 27, 116, 62, 57, 88, 21, 53, 48, 29, 44, 39, 106, 122, 113, 105, 104, 46, 31, 114, 55, 63, 37, 42, 45, 47, 58, 50, 56, 92, 108, 22, 17, 115, 121, 14, 16, 100, 59, 60, 111, 117, 90, 123, 19, 43, 49, 120, 125, 52, 107, 94, 9, 84, 15, 75, 36, 102, 33, 38, 99, 30, 101, 40, 89, 35, 93, 24, 18, 32, 8, 83, 82, 96, 71, 85, 3, 23, 97, 5, 72, 4, 86, 13, 77, 20, 87, 25, 76, 28, 10, 26, 12, 80, 2, 65, 34, 74, 79, 69, 78, 66, 70, 1, 6, 67, 81, 0, 64, 68, 11, 73, 7], [58, 104, 56, 34, 80, 52, 84, 26, 42, 109, 12, 9, 82, 31, 28, 24, 35, 64, 70, 37, 103, 96, 11, 6, 94, 36, 66, 95, 49, 60, 114, 2, 87, 126, 76, 92, 7, 20, 73, 47, 115, 23, 14, 86, 113, 21, 51, 16, 97, 53, 22, 85, 10, 48, 41, 107, 38, 127, 27, 91, 93, 4, 106, 67, 111, 77, 30, 74, 0, 122, 54, 68, 121, 108, 59, 17, 18, 19, 45, 15, 13, 29, 75, 90, 124, 125, 102, 120, 83, 110, 118, 8, 88, 63, 116, 40, 46, 55, 50, 123, 43, 61, 101, 3, 65, 39, 69, 98, 89, 105, 79, 72, 32, 71, 119, 81, 57, 99, 33, 112, 62, 1, 44, 78, 100, 25, 5, 117], [56, 52, 104, 58, 107, 37, 34, 31, 109, 48, 21, 85, 55, 120, 95, 87, 112, 114, 126, 63, 40, 26, 111, 41, 113, 121, 47, 36, 54, 116, 60, 125, 28, 42, 124, 59, 50, 110, 92, 123, 61, 118, 62, 83, 108, 91, 49, 14, 119, 24, 115, 43, 45, 44, 53, 127, 74, 51, 46, 84, 94, 106, 105, 66, 7, 80, 0, 38, 39, 99, 67, 11, 103, 57, 64, 23, 35, 101, 25, 19, 102, 122, 88, 17, 6, 98, 81, 96, 27, 79, 32, 86, 13, 10, 15, 117, 30, 90, 100, 8, 93, 33, 78, 4, 97, 82, 29, 3, 72, 12, 68, 89, 2, 5, 1, 22, 77, 75, 65, 70, 9, 16, 69, 18, 71, 20, 73, 76], [58, 104, 37, 109, 107, 53, 127, 34, 126, 57, 50, 31, 120, 60, 121, 41, 63, 59, 114, 115, 118, 54, 48, 123, 113, 116, 108, 111, 110, 52, 49, 21, 61, 119, 51, 112, 95, 45, 55, 47, 44, 103, 46, 106, 40, 43, 124, 105, 87, 26, 39, 36, 62, 125, 92, 42, 117, 77, 17, 96, 28, 122, 56, 82, 97, 102, 38, 24, 18, 35, 15, 30, 85, 32, 83, 90, 86, 99, 69, 94, 33, 101, 25, 100, 65, 98, 27, 79, 74, 89, 84, 23, 13, 29, 80, 88, 93, 4, 72, 7, 67, 14, 91, 1, 5, 19, 81, 6, 64, 66, 12, 3, 10, 22, 11, 8, 71, 9, 20, 78, 68, 75, 70, 0, 2, 16, 76, 73], [104, 58, 34, 56, 31, 126, 26, 28, 47, 52, 23, 127, 84, 92, 41, 82, 83, 122, 38, 87, 103, 80, 115, 94, 35, 114, 48, 109, 14, 53, 54, 95, 90, 12, 20, 50, 125, 36, 71, 42, 86, 27, 101, 113, 7, 44, 74, 32, 102, 118, 119, 37, 96, 121, 60, 97, 30, 88, 111, 106, 62, 55, 24, 59, 29, 1, 124, 18, 57, 78, 99, 112, 45, 22, 105, 93, 9, 63, 117, 116, 120, 25, 33, 51, 123, 6, 49, 3, 110, 89, 19, 100, 85, 43, 21, 17, 81, 98, 108, 39, 16, 5, 46, 61, 91, 72, 76, 11, 77, 10, 69, 8, 15, 40, 107, 13, 68, 79, 73, 65, 4, 67, 0, 75, 66, 70, 2, 64]], "model.layers.31.self_attn.k_proj": [[104, 121, 34, 25, 47, 1, 72, 6, 21, 110, 51, 0, 15, 11, 13, 18, 112, 113, 4, 9, 64, 60, 44, 41, 111, 31, 87, 54, 30, 116, 66, 45, 107, 114, 67, 120, 123, 19, 10, 105, 109, 117, 53, 80, 58, 71, 92, 61, 98, 57, 17, 62, 23, 124, 55, 27, 42, 14, 22, 103, 63, 56, 3, 7, 65, 102, 69, 5, 84, 115, 16, 78, 50, 106, 49, 46, 43, 90, 122, 28, 100, 101, 119, 94, 99, 91, 125, 24, 79, 93, 52, 59, 26, 36, 33, 118, 81, 76, 97, 108, 96, 88, 68, 74, 83, 75, 12, 39, 95, 126, 8, 32, 89, 38, 70, 48, 29, 127, 37, 40, 35, 86, 20, 82, 2, 73, 77, 85], [41, 49, 123, 117, 64, 50, 113, 25, 87, 95, 44, 53, 93, 7, 65, 33, 35, 82, 69, 16, 21, 3, 45, 108, 10, 118, 8, 77, 54, 78, 12, 121, 68, 66, 127, 63, 115, 43, 120, 58, 47, 83, 106, 91, 42, 57, 111, 2, 0, 99, 110, 36, 124, 11, 55, 109, 94, 97, 125, 112, 102, 116, 61, 107, 101, 59, 37, 31, 32, 51, 119, 19, 126, 62, 6, 60, 52, 100, 84, 40, 34, 46, 39, 114, 98, 26, 9, 15, 122, 24, 38, 105, 103, 70, 28, 88, 22, 56, 73, 48, 27, 29, 5, 89, 86, 104, 75, 17, 30, 13, 96, 79, 90, 85, 76, 92, 20, 1, 81, 14, 4, 67, 80, 23, 18, 72, 71, 74], [104, 121, 45, 98, 109, 0, 86, 65, 83, 89, 15, 67, 73, 56, 6, 17, 69, 50, 12, 53, 120, 93, 114, 55, 92, 61, 125, 110, 59, 31, 49, 71, 118, 2, 62, 58, 116, 126, 124, 11, 26, 57, 52, 88, 37, 123, 119, 47, 122, 48, 42, 108, 106, 30, 111, 54, 100, 51, 43, 115, 113, 77, 105, 102, 4, 127, 63, 60, 112, 97, 44, 29, 117, 103, 66, 28, 85, 7, 107, 64, 46, 20, 27, 33, 16, 90, 23, 41, 13, 101, 99, 96, 39, 38, 21, 95, 32, 18, 36, 74, 94, 91, 76, 68, 10, 35, 75, 70, 25, 82, 87, 72, 3, 80, 81, 24, 84, 14, 22, 19, 78, 79, 5, 9, 1, 8, 34, 40], [43, 93, 96, 48, 87, 16, 12, 127, 61, 78, 9, 36, 49, 19, 21, 66, 51, 0, 25, 119, 68, 58, 82, 65, 39, 5, 114, 54, 72, 50, 111, 10, 79, 3, 47, 56, 8, 106, 110, 120, 75, 116, 70, 105, 53, 107, 4, 33, 6, 63, 20, 55, 115, 108, 125, 44, 71, 32, 35, 91, 46, 17, 62, 27, 76, 126, 37, 123, 31, 102, 24, 97, 42, 57, 118, 121, 117, 98, 40, 7, 26, 18, 52, 113, 1, 59, 73, 11, 99, 88, 109, 122, 34, 41, 92, 60, 64, 74, 112, 124, 2, 45, 101, 95, 67, 103, 38, 94, 104, 13, 29, 89, 30, 77, 80, 86, 22, 83, 90, 28, 81, 84, 23, 15, 14, 100, 85, 69], [103, 58, 98, 115, 50, 45, 77, 83, 73, 81, 91, 114, 6, 11, 15, 24, 108, 29, 25, 99, 27, 4, 64, 106, 2, 31, 65, 30, 71, 127, 3, 95, 110, 28, 82, 54, 117, 57, 113, 35, 63, 107, 93, 96, 105, 49, 42, 90, 88, 84, 125, 22, 111, 119, 120, 43, 48, 126, 53, 59, 60, 118, 124, 102, 51, 46, 47, 39, 21, 68, 123, 26, 112, 122, 61, 14, 109, 5, 44, 41, 40, 52, 97, 80, 23, 69, 87, 86, 85, 37, 56, 33, 100, 116, 38, 8, 36, 101, 34, 92, 104, 94, 89, 62, 79, 55, 74, 121, 12, 16, 32, 70, 17, 10, 76, 66, 19, 20, 78, 18, 72, 67, 1, 13, 75, 7, 9, 0], [40, 126, 122, 95, 55, 34, 91, 86, 20, 29, 65, 88, 64, 33, 23, 2, 119, 41, 82, 89, 107, 114, 113, 98, 63, 127, 115, 4, 110, 121, 1, 61, 71, 118, 106, 75, 18, 78, 42, 73, 0, 49, 125, 80, 79, 103, 51, 117, 116, 59, 62, 124, 97, 47, 56, 45, 36, 85, 123, 52, 43, 48, 108, 46, 50, 68, 60, 76, 54, 111, 10, 57, 101, 109, 22, 44, 58, 81, 112, 25, 120, 7, 93, 13, 77, 8, 92, 53, 37, 90, 83, 102, 30, 39, 32, 38, 67, 17, 96, 19, 74, 3, 21, 100, 14, 99, 69, 105, 35, 28, 94, 72, 5, 24, 6, 27, 66, 26, 12, 70, 15, 87, 16, 84, 9, 31, 11, 104], [126, 34, 39, 64, 17, 88, 21, 75, 9, 71, 14, 31, 4, 112, 3, 63, 108, 45, 114, 1, 27, 124, 70, 105, 115, 65, 103, 12, 109, 111, 49, 61, 5, 106, 2, 36, 51, 91, 58, 16, 53, 46, 18, 116, 118, 122, 95, 127, 69, 24, 94, 104, 55, 41, 54, 76, 42, 50, 113, 35, 90, 107, 13, 19, 96, 62, 22, 32, 20, 40, 48, 79, 125, 57, 110, 83, 87, 59, 99, 102, 60, 119, 89, 52, 92, 26, 100, 123, 117, 47, 67, 77, 29, 93, 86, 25, 23, 38, 97, 30, 10, 28, 101, 56, 72, 66, 84, 43, 15, 37, 120, 82, 33, 121, 8, 80, 44, 98, 74, 68, 0, 78, 7, 81, 11, 73, 6, 85], [40, 58, 56, 98, 95, 87, 92, 52, 90, 26, 30, 101, 84, 32, 6, 66, 45, 80, 11, 64, 21, 41, 106, 18, 51, 28, 9, 54, 82, 60, 48, 59, 126, 112, 118, 100, 61, 12, 25, 79, 116, 88, 125, 122, 63, 110, 111, 113, 78, 39, 3, 8, 62, 127, 20, 43, 74, 119, 49, 114, 13, 72, 29, 47, 24, 50, 121, 108, 46, 97, 16, 76, 17, 124, 53, 123, 44, 104, 120, 109, 89, 115, 55, 102, 105, 27, 96, 57, 14, 1, 38, 4, 91, 73, 7, 117, 85, 65, 10, 94, 77, 22, 33, 83, 15, 42, 0, 93, 36, 103, 35, 19, 99, 37, 75, 5, 86, 81, 23, 107, 67, 70, 69, 68, 71, 2, 31, 34]], "model.layers.31.self_attn.qk_proj": [[121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 123, 103, 56, 104, 117, 45, 93, 89, 40, 61, 98, 105, 85, 21, 107, 25, 127, 23, 34, 41, 51, 114, 64, 118, 39, 95, 111, 0, 31, 53, 73, 9, 48, 108, 52, 4, 29, 87, 78, 27, 82, 12, 14, 44, 120, 24, 113, 28, 54, 16, 65, 79, 15, 76, 18, 81, 68, 66, 88, 80, 112, 63, 116, 17, 124, 83, 71, 2, 75, 7, 67, 119, 3, 19, 1, 110, 11, 60, 22, 5, 106, 62, 59, 46, 47, 6, 96, 13, 8, 99, 32, 72, 77, 125, 69, 42, 70, 91, 57, 26, 74, 86, 30, 36, 100, 20, 10, 37, 84, 97, 33, 90, 35, 101, 102, 94, 38, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 103, 123, 117, 104, 45, 93, 89, 61, 98, 85, 107, 21, 105, 40, 118, 25, 127, 51, 23, 34, 41, 114, 111, 64, 0, 31, 39, 9, 29, 95, 52, 73, 27, 68, 108, 87, 76, 124, 48, 112, 12, 54, 53, 4, 113, 28, 78, 80, 88, 15, 18, 82, 81, 116, 1, 65, 66, 120, 24, 16, 2, 14, 67, 7, 83, 6, 71, 79, 11, 110, 119, 75, 44, 63, 17, 19, 72, 106, 3, 47, 46, 60, 22, 99, 59, 13, 5, 77, 62, 32, 69, 125, 96, 42, 57, 8, 74, 91, 86, 84, 70, 26, 37, 30, 20, 97, 10, 36, 100, 90, 94, 33, 35, 38, 102, 101, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 103, 117, 45, 104, 98, 21, 89, 93, 105, 107, 40, 61, 85, 51, 114, 25, 34, 41, 0, 64, 127, 118, 108, 111, 23, 95, 39, 31, 53, 9, 68, 87, 73, 52, 12, 27, 48, 82, 63, 15, 29, 14, 28, 124, 78, 4, 116, 24, 88, 120, 112, 44, 54, 76, 113, 18, 2, 80, 16, 1, 81, 66, 6, 65, 7, 79, 110, 72, 17, 83, 67, 3, 60, 19, 75, 119, 11, 71, 46, 59, 69, 106, 47, 99, 22, 5, 62, 77, 32, 13, 96, 125, 42, 91, 86, 100, 57, 20, 84, 37, 70, 26, 33, 10, 36, 97, 30, 74, 8, 35, 94, 90, 102, 101, 92, 38], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 105, 89, 21, 107, 98, 93, 61, 40, 118, 85, 51, 41, 34, 25, 23, 127, 31, 0, 64, 52, 114, 108, 73, 39, 111, 95, 68, 29, 9, 113, 53, 116, 16, 54, 27, 124, 28, 78, 87, 12, 4, 76, 14, 24, 80, 120, 66, 79, 15, 88, 1, 112, 81, 72, 82, 65, 44, 48, 18, 63, 71, 83, 7, 2, 67, 6, 46, 3, 110, 106, 59, 17, 60, 19, 75, 11, 42, 69, 13, 32, 47, 99, 5, 125, 77, 26, 22, 57, 119, 62, 84, 74, 91, 96, 33, 20, 86, 10, 100, 94, 90, 37, 70, 36, 8, 30, 97, 102, 101, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 104, 103, 117, 45, 21, 93, 98, 105, 40, 107, 25, 89, 41, 61, 23, 118, 85, 127, 51, 34, 0, 73, 111, 114, 31, 9, 64, 39, 53, 108, 52, 95, 68, 12, 78, 24, 120, 15, 4, 113, 65, 82, 16, 79, 116, 124, 14, 48, 29, 27, 54, 87, 80, 44, 81, 72, 67, 2, 3, 76, 63, 83, 28, 71, 1, 7, 66, 18, 112, 47, 75, 17, 11, 46, 88, 119, 106, 19, 5, 60, 42, 13, 6, 69, 110, 77, 26, 59, 99, 22, 62, 57, 70, 125, 96, 32, 33, 86, 84, 10, 30, 20, 100, 91, 37, 8, 74, 97, 36, 102, 94, 90, 92, 38, 35, 101], [121, 126, 58, 43, 49, 122, 109, 50, 55, 115, 56, 123, 104, 103, 45, 117, 98, 93, 21, 61, 41, 89, 105, 40, 107, 23, 25, 0, 85, 127, 64, 114, 31, 39, 51, 118, 34, 73, 111, 9, 108, 95, 27, 53, 15, 16, 79, 80, 4, 87, 113, 24, 52, 120, 82, 78, 76, 68, 1, 12, 14, 119, 83, 48, 54, 66, 18, 65, 116, 17, 7, 112, 29, 75, 88, 44, 2, 28, 71, 72, 81, 67, 63, 11, 106, 5, 99, 3, 59, 124, 46, 19, 60, 110, 13, 47, 69, 77, 22, 70, 42, 26, 96, 125, 6, 32, 62, 20, 84, 86, 57, 10, 91, 33, 30, 37, 36, 100, 74, 8, 101, 97, 94, 102, 90, 35, 38, 92], [121, 126, 58, 43, 122, 49, 55, 109, 50, 115, 123, 56, 93, 104, 103, 117, 45, 21, 89, 98, 61, 107, 105, 40, 25, 41, 85, 23, 127, 114, 34, 108, 118, 39, 51, 0, 9, 31, 64, 73, 53, 95, 27, 111, 113, 79, 14, 29, 4, 24, 15, 82, 78, 52, 12, 76, 16, 44, 88, 68, 48, 18, 54, 87, 17, 80, 116, 81, 11, 65, 75, 63, 112, 120, 19, 28, 1, 119, 7, 66, 124, 71, 110, 83, 3, 72, 70, 106, 67, 2, 13, 96, 22, 125, 42, 59, 77, 30, 5, 47, 62, 99, 57, 69, 60, 26, 46, 32, 86, 20, 33, 8, 84, 97, 6, 37, 10, 36, 100, 91, 74, 90, 101, 38, 94, 102, 35, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 123, 56, 117, 103, 104, 45, 93, 98, 21, 89, 61, 40, 85, 105, 41, 107, 25, 114, 108, 23, 34, 118, 51, 127, 0, 64, 39, 111, 9, 95, 53, 31, 73, 29, 27, 24, 113, 124, 82, 4, 14, 79, 44, 48, 76, 68, 52, 18, 112, 12, 16, 28, 87, 116, 120, 66, 15, 54, 80, 63, 78, 81, 1, 7, 119, 75, 65, 3, 83, 71, 88, 70, 17, 2, 19, 11, 60, 67, 59, 106, 47, 110, 22, 13, 72, 77, 99, 125, 46, 5, 96, 57, 42, 32, 26, 30, 69, 6, 91, 8, 36, 86, 62, 37, 84, 10, 74, 97, 20, 33, 101, 90, 100, 94, 38, 102, 35, 92], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 56, 103, 123, 45, 117, 93, 104, 40, 85, 21, 89, 98, 105, 41, 61, 107, 25, 23, 114, 34, 51, 108, 111, 39, 127, 9, 95, 29, 31, 118, 73, 27, 113, 53, 52, 4, 64, 87, 0, 12, 24, 124, 44, 16, 48, 82, 76, 63, 78, 116, 18, 17, 88, 112, 120, 28, 68, 119, 60, 54, 15, 79, 14, 80, 81, 83, 65, 110, 19, 11, 7, 1, 46, 59, 75, 70, 106, 71, 3, 32, 42, 2, 66, 67, 125, 77, 47, 26, 13, 96, 99, 57, 91, 72, 22, 8, 62, 5, 86, 30, 97, 6, 74, 20, 69, 84, 37, 10, 36, 33, 94, 90, 100, 101, 35, 102, 92, 38], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 45, 117, 93, 104, 98, 89, 105, 61, 40, 21, 107, 41, 25, 118, 64, 0, 34, 51, 114, 85, 95, 39, 127, 53, 23, 9, 108, 31, 73, 27, 111, 112, 87, 68, 29, 113, 52, 120, 15, 78, 12, 24, 4, 44, 80, 48, 88, 16, 2, 14, 110, 82, 3, 67, 7, 116, 71, 63, 1, 65, 76, 28, 54, 66, 81, 18, 11, 106, 79, 83, 60, 75, 124, 17, 119, 19, 70, 46, 59, 8, 99, 13, 32, 42, 6, 47, 5, 77, 69, 22, 62, 125, 57, 72, 30, 20, 26, 91, 74, 86, 96, 84, 10, 97, 37, 36, 94, 90, 100, 33, 101, 35, 38, 102, 92], [121, 58, 126, 43, 122, 55, 49, 109, 50, 115, 56, 123, 103, 104, 117, 45, 93, 98, 21, 105, 61, 41, 89, 0, 40, 107, 127, 23, 118, 64, 34, 25, 51, 114, 9, 73, 85, 39, 31, 108, 52, 95, 111, 4, 16, 78, 15, 12, 113, 87, 14, 76, 80, 24, 124, 68, 27, 44, 53, 79, 1, 82, 120, 29, 65, 67, 83, 8, 116, 54, 88, 48, 18, 71, 7, 63, 11, 28, 2, 66, 3, 75, 60, 81, 5, 17, 110, 46, 19, 106, 112, 13, 70, 119, 59, 42, 6, 32, 77, 99, 69, 62, 22, 57, 86, 26, 47, 91, 125, 84, 74, 10, 20, 97, 96, 33, 72, 37, 100, 30, 36, 94, 90, 35, 102, 101, 92, 38], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 93, 89, 21, 98, 85, 105, 61, 107, 40, 41, 25, 51, 118, 23, 114, 127, 34, 95, 0, 73, 64, 39, 31, 9, 111, 108, 53, 113, 27, 29, 12, 52, 28, 24, 44, 78, 124, 14, 87, 16, 4, 82, 15, 79, 80, 68, 65, 76, 54, 48, 1, 11, 81, 18, 120, 88, 116, 8, 63, 2, 17, 71, 75, 7, 83, 110, 6, 112, 119, 66, 46, 67, 13, 19, 60, 3, 42, 77, 57, 59, 32, 47, 99, 106, 5, 26, 69, 22, 125, 91, 86, 20, 96, 62, 70, 100, 84, 97, 74, 37, 30, 72, 33, 10, 36, 102, 90, 101, 94, 38, 35, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 89, 98, 40, 21, 107, 25, 61, 105, 85, 95, 51, 41, 127, 114, 118, 39, 34, 108, 29, 23, 111, 27, 31, 53, 64, 112, 73, 113, 0, 87, 9, 44, 119, 76, 28, 4, 78, 82, 54, 24, 79, 120, 18, 80, 14, 71, 16, 52, 68, 15, 12, 48, 17, 124, 2, 88, 8, 1, 65, 63, 19, 6, 11, 7, 81, 110, 67, 83, 75, 116, 3, 60, 59, 106, 66, 47, 46, 32, 22, 125, 13, 42, 77, 57, 99, 30, 69, 96, 5, 26, 84, 97, 70, 91, 10, 86, 74, 36, 62, 20, 100, 37, 33, 90, 72, 94, 38, 35, 101, 102, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 104, 103, 117, 45, 93, 21, 89, 98, 61, 64, 40, 105, 107, 51, 127, 34, 0, 25, 41, 118, 85, 31, 73, 114, 95, 23, 39, 9, 52, 111, 4, 53, 108, 29, 113, 76, 87, 27, 78, 16, 1, 28, 82, 44, 71, 48, 8, 12, 24, 68, 18, 67, 112, 110, 81, 3, 120, 88, 65, 14, 116, 80, 15, 2, 11, 6, 60, 79, 66, 83, 54, 63, 19, 7, 106, 124, 75, 17, 125, 119, 47, 13, 46, 99, 5, 69, 59, 77, 42, 91, 62, 32, 96, 86, 10, 57, 97, 22, 26, 74, 84, 37, 20, 70, 72, 36, 30, 100, 33, 101, 94, 90, 102, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 123, 56, 117, 103, 104, 45, 21, 93, 98, 0, 105, 64, 40, 107, 25, 34, 41, 89, 51, 73, 23, 61, 114, 118, 53, 85, 9, 127, 31, 27, 95, 111, 76, 68, 113, 4, 39, 108, 52, 29, 16, 78, 82, 67, 24, 65, 18, 66, 80, 54, 7, 44, 87, 15, 79, 14, 71, 116, 1, 63, 6, 83, 28, 12, 8, 48, 81, 11, 112, 88, 2, 120, 17, 3, 106, 75, 119, 124, 5, 60, 19, 22, 110, 47, 99, 46, 59, 13, 69, 77, 42, 125, 84, 26, 32, 70, 57, 10, 91, 62, 74, 96, 37, 97, 20, 72, 30, 86, 100, 90, 36, 33, 94, 38, 102, 101, 92, 35], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 56, 103, 123, 45, 117, 104, 93, 98, 21, 89, 85, 118, 107, 114, 61, 25, 23, 34, 105, 41, 40, 127, 73, 51, 31, 108, 95, 9, 0, 111, 64, 39, 29, 52, 4, 53, 87, 27, 76, 14, 79, 44, 24, 1, 16, 78, 120, 68, 28, 82, 112, 12, 15, 124, 48, 116, 65, 113, 80, 81, 71, 63, 18, 54, 7, 88, 83, 67, 11, 66, 106, 119, 110, 60, 75, 19, 17, 8, 2, 47, 32, 22, 59, 57, 13, 3, 26, 77, 6, 99, 70, 46, 125, 5, 42, 69, 62, 72, 91, 96, 33, 20, 84, 74, 10, 30, 86, 97, 37, 100, 36, 101, 90, 102, 35, 38, 94, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 117, 103, 104, 45, 93, 89, 85, 21, 40, 107, 25, 105, 23, 98, 61, 31, 34, 127, 39, 51, 114, 95, 118, 64, 41, 111, 0, 108, 29, 9, 27, 73, 87, 53, 28, 82, 52, 112, 68, 88, 76, 113, 24, 15, 48, 1, 81, 14, 120, 44, 18, 79, 4, 71, 78, 16, 106, 12, 80, 54, 2, 110, 66, 67, 17, 63, 19, 65, 116, 11, 119, 83, 75, 7, 60, 70, 32, 125, 13, 124, 3, 59, 47, 77, 91, 42, 96, 8, 57, 62, 99, 97, 72, 26, 46, 22, 69, 37, 5, 20, 74, 30, 100, 6, 86, 10, 33, 84, 90, 35, 36, 38, 101, 94, 102, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 105, 98, 89, 40, 21, 107, 61, 85, 51, 0, 34, 64, 41, 118, 114, 25, 95, 23, 39, 73, 111, 31, 9, 127, 108, 53, 76, 52, 28, 27, 29, 87, 4, 116, 82, 44, 112, 78, 15, 113, 120, 48, 80, 24, 54, 106, 16, 124, 110, 18, 1, 2, 63, 68, 71, 88, 14, 12, 11, 83, 19, 17, 81, 66, 119, 65, 79, 67, 7, 46, 60, 3, 70, 77, 72, 47, 59, 99, 75, 32, 5, 22, 69, 91, 125, 96, 62, 42, 13, 57, 6, 84, 86, 26, 8, 30, 10, 36, 37, 74, 97, 33, 20, 94, 90, 100, 102, 35, 38, 101, 92], [121, 126, 58, 43, 49, 55, 122, 50, 109, 115, 123, 56, 103, 117, 104, 45, 105, 98, 93, 40, 21, 25, 107, 89, 61, 51, 41, 85, 0, 95, 111, 23, 34, 31, 39, 9, 64, 114, 118, 73, 29, 127, 53, 27, 4, 87, 113, 52, 44, 14, 78, 71, 116, 1, 76, 28, 24, 108, 16, 80, 48, 54, 66, 124, 82, 112, 68, 15, 18, 88, 3, 12, 2, 83, 81, 46, 79, 120, 17, 70, 47, 67, 11, 106, 7, 119, 63, 65, 72, 110, 59, 75, 42, 60, 125, 19, 32, 13, 77, 99, 26, 5, 62, 57, 91, 69, 20, 22, 84, 96, 86, 74, 6, 8, 37, 10, 97, 100, 36, 30, 90, 33, 101, 94, 102, 38, 35, 92], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 56, 123, 104, 117, 103, 45, 93, 98, 21, 105, 25, 89, 40, 61, 107, 85, 41, 23, 34, 64, 51, 127, 31, 118, 0, 9, 39, 114, 95, 73, 111, 27, 68, 16, 79, 108, 4, 53, 82, 29, 52, 76, 87, 48, 44, 24, 12, 65, 78, 88, 28, 116, 14, 66, 46, 71, 18, 80, 113, 3, 11, 15, 112, 7, 2, 72, 1, 54, 124, 70, 106, 119, 81, 17, 120, 83, 67, 75, 63, 47, 60, 19, 99, 110, 5, 32, 77, 13, 22, 69, 59, 42, 26, 125, 57, 62, 84, 74, 10, 96, 91, 86, 30, 6, 33, 20, 37, 97, 8, 90, 100, 36, 38, 102, 94, 35, 101, 92], [121, 126, 58, 43, 109, 49, 122, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 105, 61, 98, 40, 21, 89, 41, 107, 114, 51, 25, 127, 85, 118, 34, 23, 0, 95, 64, 31, 111, 53, 52, 9, 39, 108, 44, 73, 48, 27, 87, 76, 78, 82, 79, 16, 24, 29, 113, 4, 14, 1, 68, 17, 120, 28, 12, 110, 81, 88, 15, 71, 72, 124, 2, 54, 80, 11, 112, 66, 60, 63, 116, 65, 18, 46, 7, 106, 3, 83, 67, 47, 99, 62, 19, 59, 119, 70, 22, 75, 5, 125, 13, 69, 77, 100, 91, 6, 57, 96, 42, 84, 32, 26, 37, 20, 86, 33, 74, 36, 30, 10, 8, 102, 97, 94, 90, 101, 35, 38, 92], [121, 126, 58, 43, 122, 55, 49, 109, 50, 115, 56, 123, 117, 45, 103, 104, 93, 105, 98, 61, 64, 51, 89, 95, 21, 40, 41, 107, 85, 0, 25, 34, 118, 23, 31, 29, 114, 53, 108, 39, 9, 111, 52, 73, 127, 44, 87, 4, 27, 124, 16, 113, 116, 14, 68, 79, 76, 18, 67, 82, 120, 66, 78, 2, 28, 112, 63, 12, 48, 1, 106, 80, 72, 15, 24, 17, 88, 54, 65, 7, 47, 83, 110, 81, 71, 119, 3, 19, 11, 75, 6, 46, 125, 60, 57, 69, 99, 59, 5, 13, 77, 32, 70, 42, 26, 22, 62, 84, 96, 86, 91, 10, 97, 20, 8, 74, 33, 37, 30, 90, 101, 100, 36, 94, 102, 38, 35, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 117, 103, 45, 104, 93, 105, 98, 21, 40, 89, 61, 25, 107, 51, 41, 34, 85, 114, 118, 127, 95, 23, 31, 39, 111, 0, 108, 73, 64, 53, 29, 9, 27, 52, 87, 4, 113, 44, 28, 112, 76, 78, 14, 48, 79, 82, 88, 3, 80, 54, 71, 116, 16, 15, 110, 17, 18, 120, 1, 24, 7, 12, 124, 47, 63, 68, 67, 83, 2, 65, 81, 59, 60, 119, 19, 106, 11, 72, 6, 75, 66, 5, 125, 46, 77, 13, 69, 32, 99, 57, 22, 26, 96, 42, 62, 91, 20, 86, 74, 97, 84, 8, 10, 30, 70, 37, 33, 36, 100, 90, 38, 35, 94, 92, 101, 102], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 123, 56, 103, 117, 104, 45, 93, 21, 98, 61, 40, 105, 23, 89, 25, 41, 107, 118, 34, 85, 127, 31, 51, 0, 111, 108, 114, 64, 95, 73, 39, 9, 52, 27, 53, 24, 113, 4, 76, 18, 15, 12, 54, 80, 79, 87, 120, 44, 16, 68, 78, 82, 48, 29, 28, 112, 6, 1, 14, 81, 71, 63, 88, 2, 65, 7, 17, 75, 66, 11, 124, 83, 116, 119, 72, 106, 47, 19, 46, 3, 67, 110, 69, 22, 59, 60, 99, 42, 125, 77, 32, 13, 26, 62, 5, 91, 74, 96, 8, 57, 86, 84, 100, 20, 33, 97, 10, 36, 30, 37, 70, 90, 94, 101, 102, 35, 38, 92], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 117, 104, 45, 93, 98, 40, 89, 21, 105, 41, 61, 85, 107, 51, 25, 127, 23, 34, 95, 118, 114, 0, 39, 9, 73, 31, 64, 53, 108, 111, 27, 52, 4, 113, 79, 87, 44, 48, 78, 12, 80, 28, 68, 24, 14, 76, 54, 3, 120, 29, 82, 15, 81, 110, 66, 65, 116, 18, 75, 63, 1, 112, 124, 7, 6, 88, 16, 11, 71, 17, 2, 83, 119, 106, 67, 19, 42, 60, 46, 22, 99, 47, 77, 62, 57, 96, 72, 8, 13, 59, 32, 5, 69, 125, 91, 70, 30, 20, 26, 97, 74, 86, 84, 37, 10, 33, 36, 100, 90, 101, 38, 102, 35, 94, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 45, 117, 104, 89, 93, 98, 40, 105, 21, 85, 34, 107, 25, 114, 41, 23, 118, 95, 127, 51, 64, 31, 61, 39, 111, 29, 108, 0, 27, 112, 53, 73, 28, 87, 44, 9, 12, 113, 52, 68, 18, 78, 88, 24, 15, 82, 124, 116, 120, 54, 48, 14, 76, 60, 79, 16, 66, 1, 7, 75, 81, 65, 80, 4, 17, 63, 110, 11, 19, 2, 71, 106, 6, 119, 47, 59, 83, 99, 67, 3, 13, 46, 77, 42, 8, 125, 96, 22, 32, 57, 91, 5, 62, 70, 69, 20, 86, 37, 30, 74, 72, 84, 26, 36, 97, 100, 94, 38, 10, 33, 101, 90, 35, 102, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 104, 45, 93, 40, 98, 21, 89, 105, 127, 61, 25, 107, 23, 85, 34, 41, 111, 114, 51, 108, 73, 118, 39, 31, 95, 0, 29, 9, 64, 48, 28, 52, 68, 53, 27, 54, 112, 87, 24, 82, 12, 120, 113, 4, 78, 76, 44, 88, 16, 14, 79, 17, 80, 1, 15, 18, 116, 110, 81, 83, 63, 65, 2, 75, 7, 47, 59, 46, 66, 19, 71, 60, 8, 67, 119, 11, 106, 3, 13, 6, 42, 32, 99, 62, 125, 124, 5, 77, 26, 70, 69, 57, 86, 96, 37, 30, 91, 33, 72, 22, 97, 20, 84, 74, 100, 10, 36, 102, 94, 35, 90, 38, 101, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 104, 45, 93, 98, 21, 61, 89, 40, 105, 41, 85, 107, 23, 0, 25, 51, 34, 114, 64, 95, 31, 127, 111, 73, 118, 39, 9, 113, 108, 29, 24, 53, 52, 27, 68, 4, 120, 76, 79, 16, 18, 82, 28, 87, 12, 78, 44, 54, 116, 1, 124, 14, 15, 80, 7, 71, 48, 46, 3, 8, 66, 2, 75, 88, 67, 83, 112, 81, 110, 65, 63, 11, 17, 60, 70, 106, 99, 125, 47, 19, 22, 5, 119, 13, 59, 42, 77, 57, 32, 69, 62, 26, 74, 6, 37, 10, 97, 91, 96, 84, 100, 72, 20, 86, 33, 30, 36, 90, 94, 101, 102, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 93, 98, 105, 89, 21, 64, 40, 61, 85, 0, 107, 51, 25, 114, 118, 23, 41, 127, 9, 53, 34, 111, 95, 31, 73, 39, 108, 48, 113, 1, 68, 12, 52, 27, 44, 54, 87, 4, 29, 120, 14, 80, 18, 78, 70, 24, 66, 7, 8, 79, 28, 82, 15, 76, 63, 16, 112, 67, 71, 81, 124, 75, 46, 65, 88, 2, 3, 17, 110, 116, 69, 83, 11, 47, 106, 119, 19, 125, 59, 60, 77, 57, 32, 13, 99, 42, 62, 5, 26, 96, 10, 22, 33, 84, 86, 74, 97, 100, 20, 37, 91, 30, 36, 6, 72, 90, 102, 101, 94, 35, 92, 38], [121, 58, 126, 43, 122, 49, 50, 55, 109, 115, 56, 123, 103, 104, 117, 45, 93, 21, 40, 105, 98, 89, 61, 85, 127, 23, 41, 25, 107, 118, 114, 51, 34, 31, 39, 95, 73, 111, 9, 108, 64, 53, 0, 27, 29, 48, 12, 44, 79, 52, 82, 68, 87, 24, 54, 15, 18, 14, 113, 80, 78, 28, 16, 76, 120, 81, 70, 65, 112, 17, 116, 4, 88, 124, 75, 8, 83, 7, 110, 3, 1, 71, 11, 60, 19, 66, 63, 59, 106, 125, 47, 119, 2, 67, 13, 57, 69, 46, 77, 99, 5, 32, 96, 22, 42, 62, 20, 30, 26, 86, 10, 74, 91, 84, 97, 100, 33, 37, 36, 72, 94, 6, 90, 101, 102, 38, 35, 92], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 104, 103, 117, 45, 98, 93, 89, 105, 61, 21, 40, 107, 41, 23, 25, 51, 127, 64, 34, 85, 0, 114, 39, 118, 31, 108, 73, 111, 9, 95, 4, 12, 53, 15, 87, 44, 29, 27, 113, 28, 79, 68, 52, 1, 78, 48, 14, 112, 80, 24, 16, 120, 116, 63, 76, 18, 65, 70, 2, 7, 54, 71, 67, 66, 81, 8, 82, 110, 11, 83, 124, 17, 3, 19, 47, 88, 75, 106, 46, 119, 125, 13, 60, 69, 59, 32, 5, 77, 99, 57, 62, 22, 96, 91, 6, 42, 86, 26, 30, 97, 72, 37, 20, 74, 84, 10, 90, 36, 94, 33, 100, 101, 102, 38, 35, 92], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 123, 56, 103, 117, 104, 45, 93, 40, 89, 85, 105, 107, 98, 61, 21, 127, 23, 25, 114, 34, 118, 41, 95, 111, 51, 39, 9, 108, 29, 73, 31, 52, 53, 27, 12, 0, 24, 64, 18, 16, 120, 14, 113, 78, 87, 79, 54, 44, 112, 48, 4, 76, 15, 82, 80, 88, 68, 63, 28, 17, 1, 75, 81, 83, 119, 116, 8, 71, 11, 19, 7, 65, 106, 124, 2, 67, 3, 110, 66, 60, 70, 59, 5, 77, 13, 47, 96, 22, 62, 32, 99, 69, 46, 6, 42, 125, 57, 91, 86, 20, 26, 30, 72, 84, 74, 97, 10, 37, 36, 33, 94, 90, 100, 101, 35, 38, 102, 92]]} diff --git a/test/srt/experiment_runner.py b/test/srt/experiment_runner.py new file mode 100644 index 0000000..c4966dc --- /dev/null +++ b/test/srt/experiment_runner.py @@ -0,0 +1,359 @@ +import argparse +import logging +import os +import queue +import re +import subprocess +import threading +import time +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import psutil +import requests +import yaml + + +@dataclass +class ServerConfig: + command: str + process_names: List[str] + default_port: int + + +@dataclass +class TaskConfig: + server_cmd: str + client_cmd: str + name: Optional[str] = None + server_type: Optional[str] = None + + +@dataclass +class TaskResult: + name: str + success: bool + output: str + runtime: float + timestamp: str + + +SERVER_DEFAULTS = { + "sglang": ServerConfig( + command="sglang.launch_server", + process_names=["sglang.launch_server"], + default_port=30000, + ), + "vllm": ServerConfig( + command="vllm.entrypoints.openai.api_server", + process_names=["vllm.entrypoints.openai.api_server"], + default_port=8000, + ), +} + + +def parse_key_info(output: str) -> str: + """Extract and format key information from the output""" + key_info = [] + + # Extract Args namespace + args_match = re.search(r"Namespace\(.*?\)", output, re.DOTALL) + if args_match: + key_info.append(args_match.group(0)) + + # Extract input/output token counts + token_matches = re.findall(r"#(Input|Output) tokens: \d+", output) + key_info.extend(token_matches) + + # Extract benchmark result section + result_match = re.search( + r"============ Serving Benchmark Result ============.*?={50,}", + output, + re.DOTALL, + ) + if result_match: + key_info.append(result_match.group(0)) + + return "\n\n".join(key_info) + + +def extract_port_from_command(cmd: str, server_type: str) -> int: + port_match = re.search(r"--port[= ](\d+)", cmd) + if port_match: + return int(port_match.group(1)) + return SERVER_DEFAULTS.get(server_type, ServerConfig("", [], 8000)).default_port + + +def detect_server_type(cmd: str) -> str: + for server_type, config in SERVER_DEFAULTS.items(): + if config.command in cmd: + return server_type + return "unknown" + + +def stream_output( + process: subprocess.Popen, prefix: str, logger: logging.Logger +) -> queue.Queue: + output_queue = queue.Queue() + + def stream_pipe(pipe, prefix): + for line in iter(pipe.readline, ""): + if prefix == "CLIENT": + output_queue.put(line.rstrip()) + logger.debug(f"{prefix} | {line.rstrip()}") + + stdout_thread = threading.Thread( + target=stream_pipe, args=(process.stdout, prefix), daemon=True + ) + stderr_thread = threading.Thread( + target=stream_pipe, args=(process.stderr, prefix), daemon=True + ) + + stdout_thread.start() + stderr_thread.start() + return output_queue, (stdout_thread, stderr_thread) + + +class ProcessManager: + def __init__(self): + self.server_process: Optional[subprocess.Popen] = None + self.client_process: Optional[subprocess.Popen] = None + self.logger = logging.getLogger(__name__) + + def start_process( + self, command: str, prefix: str + ) -> Tuple[subprocess.Popen, queue.Queue]: + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + output_queue, threads = stream_output(process, prefix, self.logger) + return process, output_queue, threads + + def kill_process_tree(self, process: subprocess.Popen): + try: + parent = psutil.Process(process.pid) + children = parent.children(recursive=True) + + for child in children: + try: + child.kill() + except psutil.NoSuchProcess: + pass + + parent.kill() + gone, alive = psutil.wait_procs(children + [parent], timeout=3) + + for p in alive: + try: + p.kill() + except psutil.NoSuchProcess: + pass + + except psutil.NoSuchProcess: + pass + + def cleanup(self, process_names: List[str]): + if self.client_process: + self.kill_process_tree(self.client_process) + self.client_process = None + + if self.server_process: + self.kill_process_tree(self.server_process) + self.server_process = None + + for proc in psutil.process_iter(["pid", "name", "cmdline"]): + try: + cmdline = " ".join(proc.cmdline()) + if any(name in cmdline for name in process_names): + proc.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + + +class ExperimentRunner: + def __init__(self): + self.process_manager = ProcessManager() + self.logger = logging.getLogger(__name__) + + def wait_for_server(self, port: int, timeout: int = 300) -> bool: + start_time = time.time() + + while time.time() - start_time < timeout: + try: + response = requests.get(f"http://localhost:{port}/health") + if response.status_code == 200: + self.logger.debug(f"Server ready on port {port}") + return True + except requests.RequestException: + time.sleep(2) + return False + + def run_task(self, config: TaskConfig) -> TaskResult: + start_time = time.time() + client_output = [] + + try: + if not config.server_type: + config.server_type = detect_server_type(config.server_cmd) + + server_config = SERVER_DEFAULTS.get(config.server_type) + if not server_config: + raise ValueError(f"Unknown server type: {config.server_type}") + + port = extract_port_from_command(config.server_cmd, config.server_type) + + self.process_manager.cleanup(server_config.process_names) + + self.logger.debug(f"Starting server: {config.name}") + self.process_manager.server_process, _, server_threads = ( + self.process_manager.start_process(config.server_cmd, "SERVER") + ) + + if not self.wait_for_server(port): + raise TimeoutError("Server startup timeout") + + time.sleep(10) + + self.logger.debug("Starting client") + self.process_manager.client_process, output_queue, client_threads = ( + self.process_manager.start_process(config.client_cmd, "CLIENT") + ) + + returncode = self.process_manager.client_process.wait() + + while True: + try: + line = output_queue.get_nowait() + client_output.append(line) + except queue.Empty: + break + + if returncode != 0: + raise RuntimeError(f"Client failed with code {returncode}") + + # Parse and format the output + full_output = "\n".join(client_output) + formatted_output = parse_key_info(full_output) + + return TaskResult( + name=config.name, + success=True, + output=formatted_output, + runtime=time.time() - start_time, + timestamp=datetime.now().isoformat(), + ) + + except Exception as e: + return TaskResult( + name=config.name, + success=False, + output=str(e), + runtime=time.time() - start_time, + timestamp=datetime.now().isoformat(), + ) + + finally: + if config.server_type in SERVER_DEFAULTS: + self.process_manager.cleanup( + SERVER_DEFAULTS[config.server_type].process_names + ) + time.sleep(10) + + +def load_config(config_path: str) -> List[TaskConfig]: + with open(config_path, "r") as f: + config_data = yaml.safe_load(f) + + configs = [] + for idx, entry in enumerate(config_data.get("tasks", [])): + if not isinstance(entry, dict): + raise ValueError(f"Invalid entry at index {idx}") + + config = TaskConfig( + server_cmd=entry.get("server_cmd"), + client_cmd=entry.get("client_cmd"), + name=entry.get("name", f"task-{idx+1}"), + server_type=entry.get("server_type"), + ) + + if not config.server_cmd or not config.client_cmd: + raise ValueError(f"Missing commands in {config.name}") + + configs.append(config) + + return configs + + +def setup_logging(debug: bool = False): + level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(), logging.FileHandler("experiment.log")], + ) + + +def format_results(results: List[TaskResult]) -> str: + """Format experiment results in Markdown for GitHub step summary.""" + output = ["# Experiment Results\n"] + + for result in results: + output.append(f"## {result.name}") + output.append(f"**Status**: {'✅ Success' if result.success else '❌ Failed'}") + output.append(f"**Runtime**: {result.runtime:.2f} seconds") + output.append(f"**Timestamp**: {result.timestamp}") + output.append("\n**Output**:\n```") + output.append(result.output) + output.append("```\n") + + return "\n".join(output) + + +def write_in_github_step_summary(results: List[TaskResult]): + """Write formatted results to GitHub step summary.""" + if not os.environ.get("GITHUB_STEP_SUMMARY"): + logging.warning("GITHUB_STEP_SUMMARY environment variable not set") + return + + formatted_content = format_results(results) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(formatted_content) + + +def main(): + parser = argparse.ArgumentParser(description="Experiment Runner") + parser.add_argument( + "--config", type=str, required=True, help="Path to YAML config file" + ) + parser.add_argument("--debug", action="store_true", help="Enable debug output") + args = parser.parse_args() + + setup_logging(args.debug) + logger = logging.getLogger(__name__) + results = [] + + try: + configs = load_config(args.config) + runner = ExperimentRunner() + + for config in configs: + logger.info(f"Running {config.name}") + result = runner.run_task(config) + results.append(result) + + write_in_github_step_summary(results) + except Exception as e: + logger.error(f"Error: {e}") + raise + + +if __name__ == "__main__": + main() diff --git a/test/srt/kv_cache_scales_llama3_1_8b.json b/test/srt/kv_cache_scales_llama3_1_8b.json new file mode 100644 index 0000000..3e890e5 --- /dev/null +++ b/test/srt/kv_cache_scales_llama3_1_8b.json @@ -0,0 +1,42 @@ +{ + "model_type": "llama", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 1, + "1": 1, + "2": 1, + "3": 1, + "4": 1, + "5": 1, + "6": 1, + "7": 1, + "8": 1, + "9": 1, + "10": 1, + "11": 1, + "12": 1, + "13": 1, + "14": 1, + "15": 1, + "16": 1, + "17": 1, + "18": 1, + "19": 1, + "20": 1, + "21": 1, + "22": 1, + "23": 1, + "24": 1, + "25": 1, + "26": 1, + "27": 1, + "28": 1, + "29": 1, + "30": 1, + "31": 1 + } + } + } +} diff --git a/test/srt/kv_cache_scales_llama3_8b.json b/test/srt/kv_cache_scales_llama3_8b.json new file mode 100644 index 0000000..466b0d0 --- /dev/null +++ b/test/srt/kv_cache_scales_llama3_8b.json @@ -0,0 +1,42 @@ +{ + "model_type": "llama", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 0.0408, + "1": 0.0503, + "2": 0.0667, + "3": 0.0909, + "4": 0.1135, + "5": 0.127, + "6": 0.1768, + "7": 0.1488, + "8": 0.1135, + "9": 0.1203, + "10": 0.1013, + "11": 0.0842, + "12": 0.1231, + "13": 0.1096, + "14": 0.1221, + "15": 0.1013, + "16": 0.1067, + "17": 0.0952, + "18": 0.0899, + "19": 0.097, + "20": 0.087, + "21": 0.0994, + "22": 0.0904, + "23": 0.1013, + "24": 0.1019, + "25": 0.1053, + "26": 0.1, + "27": 0.0894, + "28": 0.1013, + "29": 0.1488, + "30": 0.0766, + "31": 0.0821 + } + } + } +} diff --git a/test/srt/kv_cache_scales_qwen2_1_5b.json b/test/srt/kv_cache_scales_qwen2_1_5b.json new file mode 100644 index 0000000..9847475 --- /dev/null +++ b/test/srt/kv_cache_scales_qwen2_1_5b.json @@ -0,0 +1,38 @@ +{ + "model_type": "qwen", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 0.9846, + "1": 0.0645, + "2": 0.0731, + "3": 0.0800, + "4": 0.0748, + "5": 0.0780, + "6": 0.0702, + "7": 0.0894, + "8": 0.0410, + "9": 0.0758, + "10": 0.0556, + "11": 0.0731, + "12": 0.0899, + "13": 0.0780, + "14": 0.1441, + "15": 0.0914, + "16": 0.5614, + "17": 0.1067, + "18": 0.0537, + "19": 0.0658, + "20": 0.0523, + "21": 0.0533, + "22": 0.0699, + "23": 0.0635, + "24": 0.0588, + "25": 0.0884, + "26": 0.0947, + "27": 0.1032 + } + } + } +} diff --git a/test/srt/models/compare.py b/test/srt/models/compare.py new file mode 100644 index 0000000..2fe3535 --- /dev/null +++ b/test/srt/models/compare.py @@ -0,0 +1,52 @@ +""" +used for debug using tensor comparison +dump {name: tensor} into "log_hf.jsonl" and "log_srt.jsonl" +use the same name for two tensors that supposed to be close +recommend name like: "layer 2 after mlp" +""" + +import json +import sys + +import torch + +if len(sys.argv) > 1: + assert sys.argv[1] == "base" + hf_log = "base_log_hf.jsonl" + srt_log = "base_log_srt.jsonl" +else: + hf_log = "log_hf.jsonl" + srt_log = "log_srt.jsonl" + + +def load_data(filepath): + tensors = {} + with open(filepath, "r") as f: + lines = f.readlines() + for line in lines: + data = json.loads(line) + for k, v in data.items(): + tensors[k] = torch.tensor(v) + return tensors + + +hf_tensors = load_data(hf_log) +srt_tensors = load_data(srt_log) + + +def get_diff(t1, t2): + t1 = t1.reshape(t2.shape) + max_diff = torch.max(abs(t1.reshape(t2.shape) - t2)) + l2_dis = torch.dist(t1, t2, p=2) + return l2_dis, max_diff + + +for k, _ in srt_tensors.items(): + l2_dis, max_diff = get_diff(hf_tensors[k], srt_tensors[k]) + print(f"{k} {l2_dis=} {max_diff=}") + if k == "layer 1 attn": + print(hf_tensors[k]) + print(srt_tensors[k]) + if k == "layer 0 prefill k": + print(srt_tensors[k].shape) + print(hf_tensors[k].shape) diff --git a/test/srt/models/lora/test_lora.py b/test/srt/models/lora/test_lora.py new file mode 100644 index 0000000..6f8a03d --- /dev/null +++ b/test/srt/models/lora/test_lora.py @@ -0,0 +1,304 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase + +LORA_SETS = [ + # { + # "base": "meta-llama/Llama-2-7b-hf", + # "loras": ["RuterNorway/Llama-2-7b-chat-norwegian-LoRa"], + # }, + {"base": "meta-llama/Llama-2-7b-hf", "loras": ["winddude/wizardLM-LlaMA-LoRA-7B"]}, + # {"base": "Qwen/Qwen2.5-14B-Instruct", "loras": ["mssongit/Qwen2.5-14B-SFT-LoRA"]}, + # {"base": "mistralai/Mistral-7B-Instruct-v0.3", "loras": ["/home/ying/test_lora"]}, + # { + # "base": "mistralai/Mistral-7B-Instruct-v0.3", + # "loras": [ + # "/home/ying/test_lora", + # "/home/ying/test_lora_1", + # "/home/ying/test_lora_2", + # "/home/ying/test_lora_3", + # "/home/ying/test_lora_4", + # ], + # }, + # {"base": "meta-llama/Llama-2-7b-hf", "loras": ["yard1/llama-2-7b-sql-lora-test"]}, +] +TORCH_DTYPES = [torch.float16] + +PROMPTS = [ + """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""", + """ +### Instruction: +Tell me about llamas and alpacas +### Response: +Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. +### Question 2: +What do you know about llamas? +### Answer: +""", +] + +# import json +# +# with open("/home/ying/test_prompt/dialogue_choice_prompts.json", "r") as f: +# samples = json.load(f) +# for sample in samples[:5]: +# assert sample[0]["role"] == "user" +# PROMPTS.append(sample[0]["content"][:2000]) + + +class TestLoRA(CustomTestCase): + + def inference(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens): + print("=================== testing inference =======================") + base_path = lora_set["base"] + all_lora_paths = lora_set["loras"] + batch_lora_paths = [None] + i = 0 + for _ in range(len(prompts) - 1): + batch_lora_paths.append(all_lora_paths[i]) + i = (i + 1) % len(all_lora_paths) + + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=tp_size, + lora_paths=all_lora_paths, + max_loras_per_batch=3, + disable_cuda_graph=True, + disable_radix_cache=True, + ) as srt_runner: + srt_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths + ) + srt_outputs_lora_path_none = srt_runner.forward( + prompts, + max_new_tokens=max_new_tokens, + lora_paths=[None] * len(prompts), + ) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths + ) + + with HFRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + ) as hf_runner: + hf_no_lora_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + with SRTRunner( + base_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="generation", + ) as srt_runner: + srt_no_lora_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + for i in range(len(prompts)): + # compare input logprobs + hf_logprobs = torch.Tensor(hf_outputs.top_input_logprobs[i]) + srt_logprobs = torch.Tensor(srt_outputs.top_input_logprobs[i]) + hf_no_lora_logprobs = torch.Tensor(hf_no_lora_outputs.top_input_logprobs[i]) + srt_no_lora_logprobs = torch.Tensor( + srt_no_lora_outputs.top_input_logprobs[i] + ) + print( + "max input diff between hf_lora and srt_lora", + torch.max(abs(hf_logprobs - srt_logprobs)), + ) + print( + "max input diff between srt_base and srt_lora", + torch.max(abs(srt_no_lora_logprobs - srt_logprobs)), + ) + print( + "max input diff between srt_base and hf_base", + torch.max(abs(srt_no_lora_logprobs - hf_no_lora_logprobs)), + ) + print( + "max input diff between hf_lora and hf_base", + torch.max(abs(hf_logprobs - hf_no_lora_logprobs)), + ) + + # compare output logprobs + hf_logprobs = torch.Tensor(hf_outputs.top_output_logprobs[i]) + srt_logprobs = torch.Tensor(srt_outputs.top_output_logprobs[i]) + # print( + # "\noutput logprobs diff", + # [ + # float(torch.max(abs(hf_logprobs[j] - srt_logprobs[j]))) + # for j in range(max_new_tokens) + # ], + # ) + print( + "max output diff between hf_lora and srt_lora", + torch.max(abs(hf_logprobs - srt_logprobs)), + "\n", + ) + + # compare output strings + print(f"{hf_outputs.output_strs=}") + print(f"{srt_outputs.output_strs=}") + print(f"{hf_no_lora_outputs.output_strs=}") + print(f"{srt_no_lora_outputs.output_strs=}") + print(f"{srt_outputs_lora_path_none.output_strs=}") + for i in range(len(prompts)): + assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[ + i + ].strip(" "), ( + srt_outputs.output_strs[i].strip(" "), + hf_outputs.output_strs[i].strip(" "), + ) + assert ( + srt_no_lora_outputs.output_strs[i].strip(" ") + == hf_no_lora_outputs.output_strs[i] + ), ( + srt_no_lora_outputs.output_strs[i].strip(" "), + hf_no_lora_outputs.output_strs[i], + ) + # assert srt_outputs_lora_path_none == srt_no_lora_outputs + + def serving(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens): + print("=================== testing serving =======================") + # test batch forward + base_path = lora_set["base"] + all_lora_paths = lora_set["loras"] + batch_lora_paths = [None] + i = 0 + for _ in range(len(prompts) - 1): + batch_lora_paths.append(all_lora_paths[i]) + i = (i + 1) % len(all_lora_paths) + + with SRTRunner( + base_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="generation", + lora_paths=all_lora_paths, + max_loras_per_batch=3, + disable_cuda_graph=True, + disable_radix_cache=True, + ) as srt_runner: + srt_outputs = srt_runner.batch_forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths + ) + + with HFRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + output_str_only=True, + ) as hf_runner: + hf_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths + ) + + # compare output strings + print(f"{hf_outputs.output_strs=}") + print(f"{srt_outputs.output_strs=}") + for i in range(len(prompts)): + assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i], ( + srt_outputs.output_strs[i].strip(" "), + hf_outputs.output_strs[i], + ) + + def base_inference(self, prompts, lora_set, tp_size, torch_dtype, max_new_tokens): + print("=================== testing base inference =======================") + base_path = lora_set["base"] + all_lora_paths = lora_set["loras"] + batch_lora_paths = [None] * len(prompts) + + with SRTRunner( + base_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="generation", + ) as srt_runner: + srt_no_lora_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + with SRTRunner( + base_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="generation", + lora_paths=all_lora_paths, + ) as srt_runner: + srt_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=batch_lora_paths + ) + + for i in range(len(prompts)): + srt_no_lora_logprobs = torch.Tensor( + srt_no_lora_outputs.top_input_logprobs[i] + ) + srt_logprobs = torch.Tensor(srt_outputs.top_input_logprobs[i]) + print("max_diff", torch.max(abs(srt_no_lora_logprobs - srt_logprobs))) + + print(f"{srt_no_lora_outputs.output_strs=}") + print(f"{srt_outputs.output_strs=}") + + for i in range(len(prompts)): + assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i], ( + srt_outputs.output_strs[i].strip(" "), + hf_outputs.output_strs[i], + ) + assert ( + srt_no_lora_outputs[i].output_strs.strip(" ") + == hf_no_lora_outputs[i].output_strs + ) + + def test_all(self): + for lora_set in LORA_SETS: + # self.load_lora_adapter(lora_set, 1) + for torch_dtype in TORCH_DTYPES: + tp_size = 1 + max_new_tokens = 32 + self.inference(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens) + # self.serving(PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens) + # self.base_inference( + # PROMPTS, lora_set, tp_size, torch_dtype, max_new_tokens + # ) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/models/lora/test_lora_backend.py b/test/srt/models/lora/test_lora_backend.py new file mode 100644 index 0000000..79d7dbe --- /dev/null +++ b/test/srt/models/lora/test_lora_backend.py @@ -0,0 +1,255 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import unittest +from typing import List + +import torch +from utils import BACKENDS, TORCH_DTYPES, LoRAAdaptor, LoRAModelCase + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, calculate_rouge_l, is_in_ci + +CI_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + ), + ], + max_loras_per_batch=1, + ), + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + prefill_tolerance=1e-1, + ), + ], + max_loras_per_batch=1, + ), +] + +ALL_OTHER_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-2-7b-hf", + adaptors=[LoRAAdaptor(name="winddude/wizardLM-LlaMA-LoRA-7B")], + max_loras_per_batch=2, + ), +] + +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. + ### Question 2: + What do you know about llamas? + ### Answer: + """, +] + + +class TestLoRABackend(CustomTestCase): + def run_backend( + self, + prompt: str, + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + backend: str, + ): + """ + Run backend tests for a single prompt and model case. + """ + base_path = model_case.base + adaptor = model_case.adaptors[0] + print( + f"\n========== Testing backend '{backend}' for base '{base_path}' --- " + f"Prompt '{prompt[:50]}...' using adaptor '{adaptor.name}' ---" + ) + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + lora_paths=[adaptor.name for adaptor in model_case.adaptors], + max_loras_per_batch=model_case.max_loras_per_batch, + lora_backend=backend, + disable_cuda_graph=True, + disable_radix_cache=True, + mem_fraction_static=0.88, + disable_custom_all_reduce=False, + ) as srt_runner: + srt_outputs = srt_runner.forward( + [prompt], max_new_tokens=max_new_tokens, lora_paths=[adaptor.name] + ) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_outputs = hf_runner.forward( + [prompt], max_new_tokens=max_new_tokens, lora_paths=[adaptor.name] + ) + + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + mem_fraction_static=0.88, + disable_custom_all_reduce=False, + ) as srt_runner: + srt_no_lora_outputs = srt_runner.forward( + [prompt], max_new_tokens=max_new_tokens + ) + + with HFRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + ) as hf_runner: + hf_no_lora_outputs = hf_runner.forward( + [prompt], max_new_tokens=max_new_tokens + ) + + # Use individual adapter tolerances if set, otherwise use model defaults + prefill_tol = ( + adaptor.prefill_tolerance + if adaptor.prefill_tolerance is not None + else model_case.prefill_tolerance + ) + decode_tol = ( + adaptor.decode_tolerance + if adaptor.decode_tolerance is not None + else model_case.decode_tolerance + ) + rouge_tol = ( + adaptor.rouge_l_tolerance + if adaptor.rouge_l_tolerance is not None + else model_case.rouge_l_tolerance + ) + + # Compare prefill stage logprobs (HF vs SRTRunner with LoRA) + hf_prefill = torch.tensor(hf_outputs.top_input_logprobs[0]) + srt_prefill = torch.tensor(srt_outputs.top_input_logprobs[0]) + max_prefill_diff = torch.max(torch.abs(hf_prefill - srt_prefill)) + print("Max prefill diff (HF vs SRT):", max_prefill_diff) + + # Compare decode stage logprobs + hf_decode = torch.tensor(hf_outputs.top_output_logprobs[0]) + srt_decode = torch.tensor(srt_outputs.top_output_logprobs[0]) + max_decode_diff = torch.max(torch.abs(hf_decode - srt_decode)) + print("Max decode diff (HF vs SRT):", max_decode_diff) + + srt_output_str = srt_outputs.output_strs[0].strip() + hf_output_str = hf_outputs.output_strs[0].strip() + rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0] + print("ROUGE-L score:", rouge_score) + print("SRT output:", srt_output_str) + print("HF output:", hf_output_str) + + # Additional: compare prefill outputs between base model (no LoRA) and LoRA model for reference + hf_no_lora_prefill = torch.tensor(hf_no_lora_outputs.top_input_logprobs[0]) + srt_no_lora_prefill = torch.tensor(srt_no_lora_outputs.top_input_logprobs[0]) + print( + "Max diff (SRT base vs SRT LoRA prefill):", + torch.max(torch.abs(srt_no_lora_prefill - srt_prefill)), + ) + print( + "Max diff (HF base vs HF LoRA prefill):", + torch.max(torch.abs(hf_no_lora_prefill - hf_prefill)), + ) + + if hf_prefill.shape[0] <= 100: + assert torch.all(torch.abs(hf_prefill - srt_prefill) < prefill_tol), ( + f"Prefill logprobs mismatch for base '{base_path}', adaptor '{adaptor.name}', " + f"backend '{backend}', prompt: '{prompt[:50]}...'" + ) + + if hf_decode.shape[0] <= 100: + assert torch.all(torch.abs(hf_decode - srt_decode) < decode_tol), ( + f"Decode logprobs mismatch for base '{base_path}', adaptor '{adaptor.name}', " + f"backend '{backend}', prompt: '{prompt[:50]}...'" + ) + + if rouge_score < rouge_tol: + + raise AssertionError( + f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} " + f"for base '{base_path}', adaptor '{adaptor.name}', backend '{backend}', prompt: '{prompt[:50]}...'" + ) + + def run_backend_batch( + self, + prompts: List[str], + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + backend: str, + ): + # TODO: Implement batch processing version of run_backend + raise NotImplementedError( + "Batch processing version of run_backend is not implemented yet." + ) + + def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + PROMPTS + if not model_case.skip_long_prompt + else [p for p in PROMPTS if len(p) < 1000] + ) + for torch_dtype in TORCH_DTYPES: + for backend in BACKENDS: + for prompt in prompts: + self.run_backend( + prompt, + model_case, + torch_dtype, + max_new_tokens=32, + backend=backend, + ) + + def test_ci_lora_models(self): + self._run_backend_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_backend_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/models/lora/test_lora_tp.py b/test/srt/models/lora/test_lora_tp.py new file mode 100644 index 0000000..925a34b --- /dev/null +++ b/test/srt/models/lora/test_lora_tp.py @@ -0,0 +1,257 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import unittest +from typing import List + +import torch +from utils import TORCH_DTYPES, LoRAAdaptor, LoRAModelCase + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, calculate_rouge_l, is_in_ci + +CI_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + ), + ], + max_loras_per_batch=1, + ), + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + prefill_tolerance=1e-1, + ), + ], + max_loras_per_batch=1, + ), +] + +ALL_OTHER_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-2-7b-hf", + adaptors=[LoRAAdaptor(name="winddude/wizardLM-LlaMA-LoRA-7B")], + max_loras_per_batch=2, + ), +] + +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. + ### Question 2: + What do you know about llamas? + ### Answer: + """, +] + +BACKEND = "triton" + + +class TestLoRATP(CustomTestCase): + def run_tp( + self, + prompt: str, + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + ): + """ + Run triton backend tests with specified TP size for a single prompt and model case. + """ + base_path = model_case.base + adaptor = model_case.adaptors[0] + tp_size = model_case.tp_size + print( + f"\n========== Testing triton backend with TP size {tp_size} for base '{base_path}' --- " + f"Prompt '{prompt[:50]}...' using adaptor '{adaptor.name}' ---" + ) + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=tp_size, + lora_paths=[adaptor.name for adaptor in model_case.adaptors], + max_loras_per_batch=model_case.max_loras_per_batch, + lora_backend=BACKEND, + disable_cuda_graph=True, + disable_radix_cache=True, + mem_fraction_static=0.88, + disable_custom_all_reduce=True, + ) as srt_runner: + srt_outputs = srt_runner.forward( + [prompt], max_new_tokens=max_new_tokens, lora_paths=[adaptor.name] + ) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_outputs = hf_runner.forward( + [prompt], max_new_tokens=max_new_tokens, lora_paths=[adaptor.name] + ) + + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=tp_size, + mem_fraction_static=0.88, + disable_custom_all_reduce=True, + ) as srt_runner: + srt_no_lora_outputs = srt_runner.forward( + [prompt], max_new_tokens=max_new_tokens + ) + + with HFRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + ) as hf_runner: + hf_no_lora_outputs = hf_runner.forward( + [prompt], max_new_tokens=max_new_tokens + ) + + # Use individual adapter tolerances if set, otherwise use model defaults + prefill_tol = ( + adaptor.prefill_tolerance + if adaptor.prefill_tolerance is not None + else model_case.prefill_tolerance + ) + decode_tol = ( + adaptor.decode_tolerance + if adaptor.decode_tolerance is not None + else model_case.decode_tolerance + ) + rouge_tol = ( + adaptor.rouge_l_tolerance + if adaptor.rouge_l_tolerance is not None + else model_case.rouge_l_tolerance + ) + + # Compare prefill stage logprobs (HF vs SRTRunner with LoRA) + hf_prefill = torch.tensor(hf_outputs.top_input_logprobs[0]) + srt_prefill = torch.tensor(srt_outputs.top_input_logprobs[0]) + max_prefill_diff = torch.max(torch.abs(hf_prefill - srt_prefill)) + print("Max prefill diff (HF vs SRT):", max_prefill_diff) + + # Compare decode stage logprobs + hf_decode = torch.tensor(hf_outputs.top_output_logprobs[0]) + srt_decode = torch.tensor(srt_outputs.top_output_logprobs[0]) + max_decode_diff = torch.max(torch.abs(hf_decode - srt_decode)) + print("Max decode diff (HF vs SRT):", max_decode_diff) + + srt_output_str = srt_outputs.output_strs[0].strip() + hf_output_str = hf_outputs.output_strs[0].strip() + rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0] + print("ROUGE-L score:", rouge_score) + print("SRT output:", srt_output_str) + print("HF output:", hf_output_str) + + # Additional: compare prefill outputs between base model (no LoRA) and LoRA model for reference + hf_no_lora_prefill = torch.tensor(hf_no_lora_outputs.top_input_logprobs[0]) + srt_no_lora_prefill = torch.tensor(srt_no_lora_outputs.top_input_logprobs[0]) + print( + "Max diff (SRT base vs SRT LoRA prefill):", + torch.max(torch.abs(srt_no_lora_prefill - srt_prefill)), + ) + print( + "Max diff (HF base vs HF LoRA prefill):", + torch.max(torch.abs(hf_no_lora_prefill - hf_prefill)), + ) + + if hf_prefill.shape[0] <= 100: + assert torch.all(torch.abs(hf_prefill - srt_prefill) < prefill_tol), ( + f"Prefill logprobs mismatch for base '{base_path}', adaptor '{adaptor.name}', " + f"triton backend with TP {tp_size}, prompt: '{prompt[:50]}...'" + ) + + if hf_decode.shape[0] <= 100: + assert torch.all(torch.abs(hf_decode - srt_decode) < decode_tol), ( + f"Decode logprobs mismatch for base '{base_path}', adaptor '{adaptor.name}', " + f"triton backend with TP {tp_size}, prompt: '{prompt[:50]}...'" + ) + + if rouge_score < rouge_tol: + raise AssertionError( + f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} " + f"for base '{base_path}', adaptor '{adaptor.name}', triton backend with TP {tp_size}, prompt: '{prompt[:50]}...'" + ) + + def run_tp_batch( + self, + prompts: List[str], + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + tp_size: int, + ): + # TODO: Implement batch processing version of run_tp + raise NotImplementedError( + "Batch processing version of run_tp is not implemented yet." + ) + + def _run_tp_on_model_cases(self, model_cases: List[LoRAModelCase]): + tp_list = [2] # Define TP sizes to iterate over + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + PROMPTS + if not model_case.skip_long_prompt + else [p for p in PROMPTS if len(p) < 1000] + ) + for tp_size in tp_list: + model_case.tp_size = tp_size + for torch_dtype in TORCH_DTYPES: + for prompt in prompts: + self.run_tp( + prompt, + model_case, + torch_dtype, + max_new_tokens=32, + ) + + def test_ci_lora_models(self): + self._run_tp_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_tp_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/models/lora/test_multi_lora_backend.py b/test/srt/models/lora/test_multi_lora_backend.py new file mode 100644 index 0000000..68e78c2 --- /dev/null +++ b/test/srt/models/lora/test_multi_lora_backend.py @@ -0,0 +1,238 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest +from typing import List + +import torch +from utils import BACKENDS, TORCH_DTYPES, LoRAAdaptor, LoRAModelCase + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, calculate_rouge_l, is_in_ci + +MULTI_LORA_MODELS = [ + # multi-rank case + LoRAModelCase( + base="meta-llama/Llama-2-7b-hf", + adaptors=[ + LoRAAdaptor( + name="winddude/wizardLM-LlaMA-LoRA-7B", + prefill_tolerance=1e-1, + ), + LoRAAdaptor( + name="RuterNorway/Llama-2-7b-chat-norwegian-LoRa", + prefill_tolerance=3e-1, + ), + ], + max_loras_per_batch=2, + ), + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + prefill_tolerance=1e-1, + ), + LoRAAdaptor( + name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + prefill_tolerance=1e-1, + ), + ], + max_loras_per_batch=2, + ), +] + +# All prompts are used at once in a batch. +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids. + ### Question: + What do you know about llamas? + ### Answer: + """, +] + + +class TestMultiLoRABackend(CustomTestCase): + def run_backend_batch( + self, + prompts: List[str], + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + backend: str, + ): + """ + The multi-LoRA backend test functionality is not supported yet. + This function uses all prompts at once and prints a message indicating that support is pending. + """ + base_path = model_case.base + adaptor_names = [adaptor.name for adaptor in model_case.adaptors] + print( + f"\n========== Testing multi-LoRA backend '{backend}' for base '{model_case.base}' --- " + f"Using prompts {[p[:50] for p in prompts]} with adaptors: {adaptor_names} ---" + ) + print( + "run_backend_batch: Multi-LoRA backend test functionality is pending support." + ) + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + lora_paths=[adaptor.name for adaptor in model_case.adaptors], + max_loras_per_batch=model_case.max_loras_per_batch, + lora_backend=backend, + disable_cuda_graph=True, + disable_radix_cache=True, + mem_fraction_static=0.88, + ) as srt_runner: + srt_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names + ) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names + ) + + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + mem_fraction_static=0.88, + ) as srt_runner: + srt_no_lora_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + with HFRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + ) as hf_runner: + hf_no_lora_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + # Compare prefill stage logprobs (HF vs SRTRunner with LoRA) + for i in range(len(prompts)): + adaptor = model_case.adaptors[i] + # Use individual adapter tolerances if set, otherwise use model defaults + prefill_tol = ( + adaptor.prefill_tolerance + if adaptor.prefill_tolerance is not None + else model_case.prefill_tolerance + ) + decode_tol = ( + adaptor.decode_tolerance + if adaptor.decode_tolerance is not None + else model_case.decode_tolerance + ) + rouge_tol = ( + adaptor.rouge_l_tolerance + if adaptor.rouge_l_tolerance is not None + else model_case.rouge_l_tolerance + ) + # Compare prefill stage logprobs (HF vs SRTRunner with LoRA) + hf_prefill = torch.tensor(hf_outputs.top_input_logprobs[i]) + srt_prefill = torch.tensor(srt_outputs.top_input_logprobs[i]) + max_prefill_diff = torch.max(torch.abs(hf_prefill - srt_prefill)) + print("Max prefill diff (HF vs SRT):", max_prefill_diff) + + # Compare decode stage logprobs + hf_decode = torch.tensor(hf_outputs.top_output_logprobs[i]) + srt_decode = torch.tensor(srt_outputs.top_output_logprobs[i]) + max_decode_diff = torch.max(torch.abs(hf_decode - srt_decode)) + print("Max decode diff (HF vs SRT):", max_decode_diff) + + srt_output_str = srt_outputs.output_strs[i].strip() + hf_output_str = hf_outputs.output_strs[i].strip() + rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0] + print("ROUGE-L score:", rouge_score) + print("SRT output:", srt_output_str) + print("HF output:", hf_output_str) + + # Additional: compare prefill outputs between base model (no LoRA) and LoRA model for reference + hf_no_lora_prefill = torch.tensor(hf_no_lora_outputs.top_input_logprobs[i]) + srt_no_lora_prefill = torch.tensor( + srt_no_lora_outputs.top_input_logprobs[i] + ) + print( + "Max diff (SRT base vs SRT LoRA prefill):", + torch.max(torch.abs(srt_no_lora_prefill - srt_prefill)), + ) + print( + "Max diff (HF base vs HF LoRA prefill):", + torch.max(torch.abs(hf_no_lora_prefill - hf_prefill)), + ) + + if hf_prefill.shape[0] <= 100: + assert torch.all(torch.abs(hf_prefill - srt_prefill) < prefill_tol), ( + f"Prefill logprobs mismatch for base '{base_path}', adaptor '{adaptor_names}', " + f"backend '{backend}', prompt: '{prompts[0][:50]}...'" + ) + + if hf_decode.shape[0] <= 100: + assert torch.all(torch.abs(hf_decode - srt_decode) < decode_tol), ( + f"Decode logprobs mismatch for base '{base_path}', adaptor '{adaptor_names}', " + f"backend '{backend}', prompt: '{prompts[0][:50]}...'" + ) + + if rouge_score < rouge_tol: + raise AssertionError( + f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} " + f"for base '{base_path}', adaptor '{adaptor_names}', backend '{backend}', prompt: '{prompts[0][:50]}...'" + ) + + def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters. + batch_prompts = ( + PROMPTS + if not model_case.skip_long_prompt + else [p for p in PROMPTS if len(p) < 1000] + ) + for torch_dtype in TORCH_DTYPES: + for backend in BACKENDS: + self.run_backend_batch( + batch_prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend=backend, + ) + + def test_multi_lora_models(self): + # Optionally skip tests in CI environments. + if is_in_ci(): + return + self._run_backend_on_model_cases(MULTI_LORA_MODELS) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/models/lora/utils.py b/test/srt/models/lora/utils.py new file mode 100644 index 0000000..116389a --- /dev/null +++ b/test/srt/models/lora/utils.py @@ -0,0 +1,49 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import dataclasses +from typing import List + +import torch + + +@dataclasses.dataclass +class LoRAAdaptor: + name: str + prefill_tolerance: float = None + decode_tolerance: float = None + rouge_l_tolerance: float = None + + +@dataclasses.dataclass +class LoRAModelCase: + base: str + adaptors: List[LoRAAdaptor] + tp_size: int = 1 + prefill_tolerance: float = 1e-1 + decode_tolerance: float = 1e-1 + rouge_l_tolerance: float = 1.0 + max_loras_per_batch: int = 1 + skip_long_prompt: bool = False + + def __post_init__(self): + if len(self.adaptors) > self.max_loras_per_batch: + raise ValueError( + f"For base '{self.base}', number of adaptors ({len(self.adaptors)}) " + f"must be <= max_loras_per_batch ({self.max_loras_per_batch})" + ) + + +TORCH_DTYPES = [torch.float16] +BACKENDS = ["triton"] diff --git a/test/srt/models/test_clip_models.py b/test/srt/models/test_clip_models.py new file mode 100644 index 0000000..8a79656 --- /dev/null +++ b/test/srt/models/test_clip_models.py @@ -0,0 +1,80 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest + +import torch +from transformers import AutoProcessor + +from sglang.srt.utils import load_image +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import get_similarities + +TEXTS = "two Subway Series sandwiches with meats, cheese, lettuce, tomatoes, and onions on a black background, accompanied by the Subway Series logo, highlighting a new sandwich series." +IMAGES = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" +MODELS = [ + ("openai/clip-vit-large-patch14-336", 1e-5), +] +TORCH_DTYPES = [torch.float16] + + +class TestClipModels(unittest.TestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_embeddings(self, model, prefill_tolerance, torch_dtype): + + with HFRunner( + model, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + hf_text_embeds = hf_runner.forward(prompts=TEXTS) + hf_image_embeds = hf_runner.forward(image_data=IMAGES) + + with SRTRunner( + model, + tp_size=1, + torch_dtype=torch_dtype, + model_type="embedding", + ) as srt_runner: + text_embeds = srt_runner.forward(prompts=TEXTS) + image_embeds = srt_runner.forward(prompts="padding", image_data=IMAGES) + + text_similarity = get_similarities( + text_embeds.embed_logits[0], hf_text_embeds.embed_logits[0] + ) + image_similarity = get_similarities( + image_embeds.embed_logits[0], hf_image_embeds.embed_logits[0] + ) + print("text similarity diff", abs(text_similarity - 1)) + print("image similarity diff", abs(image_similarity - 1)) + assert torch.all( + abs(text_similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + assert torch.all( + abs(image_similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_accuracy(self): + for model, prefill_tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_embeddings(model, prefill_tolerance, torch_dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_embedding_models.py b/test/srt/models/test_embedding_models.py new file mode 100644 index 0000000..517ee83 --- /dev/null +++ b/test/srt/models/test_embedding_models.py @@ -0,0 +1,109 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import random +import unittest + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci + +MODELS = [ + ("Alibaba-NLP/gte-Qwen2-1.5B-instruct", 1, 1e-5), + ("intfloat/e5-mistral-7b-instruct", 1, 1e-5), + ("marco/mcdse-2b-v1", 1, 1e-5), + ("jason9693/Qwen2.5-1.5B-apeach", 1, 1e-5), +] +TORCH_DTYPES = [torch.float16] + + +class TestEmbeddingModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def _truncate_prompts(self, prompts, model_path): + config = AutoConfig.from_pretrained(model_path) + max_length = getattr(config, "max_position_embeddings", 2048) + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + truncated_prompts = [] + for prompt in prompts: + tokens = tokenizer(prompt, return_tensors="pt", truncation=False) + if len(tokens.input_ids[0]) > max_length: + truncated_text = tokenizer.decode( + tokens.input_ids[0][: max_length - 1], skip_special_tokens=True + ) + truncated_prompts.append(truncated_text) + else: + truncated_prompts.append(prompt) + return truncated_prompts + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + prefill_tolerance, + ) -> None: + truncated_prompts = self._truncate_prompts(prompts, model_path) + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + hf_outputs = hf_runner.forward(truncated_prompts) + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="embedding", + ) as srt_runner: + srt_outputs = srt_runner.forward(truncated_prompts) + + for i in range(len(prompts)): + hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) + srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) + + similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) + print("similarity diff", abs(similarity - 1)) + + if len(prompts[i]) <= 1000: + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for torch_dtype in TORCH_DTYPES: + self.assert_close_prefill_logits( + DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_generation_models.py b/test/srt/models/test_generation_models.py new file mode 100644 index 0000000..237af60 --- /dev/null +++ b/test/srt/models/test_generation_models.py @@ -0,0 +1,156 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Usage: + +To test a specific model: +1. Add it to ALL_OTHER_MODELS +2. Run `ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others` +""" + +import dataclasses +import multiprocessing as mp +import os +import unittest +from typing import List + +import torch + +from sglang.test.runners import ( + DEFAULT_PROMPTS, + HFRunner, + SRTRunner, + check_close_model_outputs, +) +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +@dataclasses.dataclass +class ModelCase: + model_path: str + tp_size: int = 1 + prefill_tolerance: float = 5e-2 + decode_tolerance: float = 5e-2 + rouge_l_tolerance: float = 1 + skip_long_prompt: bool = False + trust_remote_code: bool = False + + +# Popular models that run on the CI +CI_MODELS = [ + ModelCase("meta-llama/Llama-3.1-8B-Instruct"), + ModelCase("google/gemma-2-2b"), +] + +# All other models that do not run on the CI +ALL_OTHER_MODELS = [ + ModelCase("Qwen/Qwen2-1.5B"), + ModelCase("Qwen/Qwen2.5-14B-Instruct"), + ModelCase("HuggingFaceTB/SmolLM-135M-Instruct", skip_long_prompt=True), + ModelCase("allenai/OLMo-1B-0724-hf", decode_tolerance=8e-2, skip_long_prompt=True), + ModelCase( + "THUDM/glm-4-9b-chat", tp_size=2, trust_remote_code=True, skip_long_prompt=True + ), + ModelCase("openai-community/gpt2"), + ModelCase("microsoft/Phi-3-small-8k-instruct"), + ModelCase("allenai/OLMo-2-1124-7B-Instruct", skip_long_prompt=True), + ModelCase("ibm-granite/granite-3.0-2b-instruct", skip_long_prompt=True), +] + +TORCH_DTYPES = [torch.float16] + + +class TestGenerationModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_logits_and_output_strs( + self, + prompts: List[str], + model_case: ModelCase, + torch_dtype: torch.dtype, + ) -> None: + model_path = model_case.model_path + prefill_tolerance, decode_tolerance, rouge_l_tolerance = ( + model_case.prefill_tolerance, + model_case.decode_tolerance, + model_case.rouge_l_tolerance, + ) + max_new_tokens = 32 + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + ) as hf_runner: + hf_outputs = hf_runner.forward(prompts, max_new_tokens=max_new_tokens) + + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + ) as srt_runner: + srt_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens) + + check_close_model_outputs( + hf_outputs=hf_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=model_case.prefill_tolerance, + decode_tolerance=model_case.decode_tolerance, + rouge_l_tolerance=model_case.rouge_l_tolerance, + debug_text=f"model_path={model_path} prompts={prompts}", + ) + + def test_ci_models(self): + for model_case in CI_MODELS: + for torch_dtype in TORCH_DTYPES: + + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs( + prompts, model_case, torch_dtype + ) + + def test_others(self): + if is_in_ci(): + return + + for model_case in ALL_OTHER_MODELS: + # Only run a specified model + if ( + "ONLY_RUN" in os.environ + and os.environ["ONLY_RUN"] != model_case.model_path + ): + continue + + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs(prompts, model_case, torch.float16) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_gme_qwen_models.py b/test/srt/models/test_gme_qwen_models.py new file mode 100644 index 0000000..265bf8a --- /dev/null +++ b/test/srt/models/test_gme_qwen_models.py @@ -0,0 +1,85 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + + +import multiprocessing as mp +import unittest + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, get_similarities + +TEXTS = "two Subway Series sandwiches with meats, cheese, lettuce, tomatoes, and onions on a black background, accompanied by the Subway Series logo, highlighting a new sandwich series." +IMAGES = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + + +MODELS = [ + ("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", 1e-3), +] +TORCH_DTYPES = [torch.float16] + + +class TestQmeQwenModels(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_embeddings(self, model, prefill_tolerance, torch_dtype): + + prompts_no_image = f"<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\n{TEXTS}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>" + prompts_with_image = f"<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>assistant\n<|endoftext|>" + with HFRunner( + model, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + hf_text_embeddings = hf_runner.forward(prompts=[prompts_no_image]) + hf_image_embeddings = hf_runner.forward( + prompts=[prompts_with_image], image_data=[IMAGES] + ) + with SRTRunner( + model, + tp_size=1, + torch_dtype=torch_dtype, + model_type="embedding", + ) as srt_runner: + srt_text_embeddings = srt_runner.forward(prompts=prompts_no_image) + srt_image_embeddings = srt_runner.forward( + prompts=prompts_with_image, image_data=IMAGES + ) + + similarity = get_similarities( + hf_text_embeddings.embed_logits[0], srt_text_embeddings.embed_logits[0] + ) + print("texts similarity diff", abs(similarity - 1)) + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + similarity = get_similarities( + hf_image_embeddings.embed_logits[0], srt_image_embeddings.embed_logits[0] + ) + print("images similarity diff", abs(similarity - 1)) + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_accuracy(self): + for model, prefill_tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_embeddings(model, prefill_tolerance, torch_dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_grok_models.py b/test/srt/models/test_grok_models.py new file mode 100644 index 0000000..625fa1a --- /dev/null +++ b/test/srt/models/test_grok_models.py @@ -0,0 +1,53 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestGrok(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmzheng/grok-1" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--load-format", + "dummy", + "--json-model-override-args", + '{"num_hidden_layers": 2}', + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=64, + max_new_tokens=256, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + + # It is dummy weights so we only assert the output throughput instead of accuracy. + self.assertGreater(metrics["output_throughput"], 1000) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_qwen_models.py b/test/srt/models/test_qwen_models.py new file mode 100644 index 0000000..567e19f --- /dev/null +++ b/test/srt/models/test_qwen_models.py @@ -0,0 +1,77 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestQwen2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +class TestQwen2FP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "neuralmagic/Qwen2-7B-Instruct-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_reward_models.py b/test/srt/models/test_reward_models.py new file mode 100644 index 0000000..5592ce2 --- /dev/null +++ b/test/srt/models/test_reward_models.py @@ -0,0 +1,92 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase + +MODELS = [ + ("LxzGordon/URM-LLaMa-3.1-8B", 1, 4e-2), + ("Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", 1, 4e-2), +] +TORCH_DTYPES = [torch.float16] + +# PROMPT = "Jane has 12 apples. She gives 4 apples to her friend Mark, then buys 1 more apple, and finally splits all her apples equally among herself and her 2 siblings. How many apples does each person get?" +# RESPONSE1 = "1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys 1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among herself and her 2 siblings (3 people in total). 9 ÷ 3 = 3 apples each. Each person gets 3 apples." +# RESPONSE2 = "1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys 1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among her 2 siblings (2 people in total). 9 ÷ 2 = 4.5 apples each. Each person gets 4 apples." + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +CONVS = [ + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}], + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}], +] + + +class TestRewardModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_reward_scores( + self, + convs, + model_path, + tp_size, + torch_dtype, + tolerance, + ) -> None: + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="reward", + ) as hf_runner: + hf_outputs = hf_runner.forward(convs) + + with SRTRunner( + model_path, + torch_dtype=torch_dtype, + model_type="reward", + ) as srt_runner: + prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False) + srt_outputs = srt_runner.forward(prompts) + + hf_scores = torch.tensor(hf_outputs.scores) + srt_scores = torch.tensor(srt_outputs.scores) + print(f"{hf_scores=}") + print(f"{srt_scores=}") + + assert torch.all( + abs(hf_scores - srt_scores) < tolerance + ), "reward scores are not all close" + + def test_reward_scores(self): + for model, tp_size, tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_reward_scores( + CONVS, model, tp_size, torch_dtype, tolerance + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py new file mode 100644 index 0000000..625d551 --- /dev/null +++ b/test/srt/run_suite.py @@ -0,0 +1,188 @@ +import argparse +import glob +from dataclasses import dataclass + +from sglang.test.test_utils import run_unittest_files + + +@dataclass +class TestFile: + name: str + estimated_time: float = 60 + + +suites = { + "per-commit": [ + TestFile("models/lora/test_lora.py", 76), + TestFile("models/lora/test_lora_backend.py", 420), + TestFile("models/lora/test_multi_lora_backend.py", 144), + TestFile("models/test_embedding_models.py", 119), + TestFile("models/test_generation_models.py", 103), + TestFile("models/test_grok_models.py", 60), + TestFile("models/test_qwen_models.py", 82), + TestFile("models/test_reward_models.py", 83), + TestFile("models/test_gme_qwen_models.py", 45), + TestFile("models/test_clip_models.py", 100), + TestFile("test_abort.py", 51), + TestFile("test_block_int8.py", 22), + TestFile("test_chunked_prefill.py", 336), + TestFile("test_eagle_infer.py", 447), + TestFile("test_ebnf_constrained.py"), + TestFile("test_fp8_kernel.py", 2), + TestFile("test_embedding_openai_server.py", 36), + TestFile("test_hidden_states.py", 55), + TestFile("test_int8_kernel.py", 1), + TestFile("test_input_embeddings.py", 38), + TestFile("test_json_constrained.py", 98), + TestFile("test_large_max_new_tokens.py", 41), + TestFile("test_metrics.py", 32), + TestFile("test_mla.py", 92), + TestFile("test_mla_deepseek_v3.py", 221), + TestFile("test_mla_int8_deepseek_v3.py", 421), + TestFile("test_mla_flashinfer.py", 395), + TestFile("test_mla_fp8.py", 93), + TestFile("test_no_chunked_prefill.py", 126), + TestFile("test_no_overlap_scheduler.py", 262), + TestFile("test_openai_server.py", 124), + TestFile("test_penalty.py", 41), + TestFile("test_page_size.py", 60), + TestFile("test_pytorch_sampling_backend.py", 66), + TestFile("test_radix_attention.py", 167), + TestFile("test_reasoning_content.py", 89), + TestFile("test_regex_constrained.py", 64), + TestFile("test_release_memory_occupation.py", 44), + TestFile("test_request_length_validation.py", 31), + TestFile("test_retract_decode.py", 54), + TestFile("test_server_args.py", 1), + TestFile("test_skip_tokenizer_init.py", 72), + TestFile("test_srt_engine.py", 237), + TestFile("test_srt_endpoint.py", 94), + TestFile("test_torch_compile.py", 76), + TestFile("test_torch_compile_moe.py", 85), + TestFile("test_torch_native_attention_backend.py", 149), + TestFile("test_torchao.py", 70), + TestFile("test_triton_attention_kernels.py", 4), + TestFile("test_triton_attention_backend.py", 134), + TestFile("test_update_weights_from_disk.py", 114), + TestFile("test_update_weights_from_tensor.py", 48), + TestFile("test_vertex_endpoint.py", 31), + TestFile("test_vision_chunked_prefill.py", 223), + TestFile("test_vlm_accuracy.py", 60), + TestFile("test_vision_openai_server.py", 344), + TestFile("test_fim_completion.py", 120), + TestFile("test_w8a8_quantization.py", 46), + TestFile("test_eval_fp8_accuracy.py", 172), + TestFile("test_create_kvindices.py", 2), + TestFile("test_hicache.py", 60), + TestFile("test_hicache_mla.py", 90), + ], + "nightly": [ + TestFile("test_nightly_gsm8k_eval.py"), + ], + "vllm_dependency_test": [ + TestFile("test_vllm_dependency.py"), + TestFile("test_awq.py"), + TestFile("test_gguf.py", 78), + TestFile("test_gptqmodel_dynamic.py", 72), + ], +} + + +def auto_partition(files, rank, size): + """ + Partition files into size sublists with approximately equal sums of estimated times + using stable sorting, and return the partition for the specified rank. + + Args: + files (list): List of file objects with estimated_time attribute + rank (int): Index of the partition to return (0 to size-1) + size (int): Number of partitions + + Returns: + list: List of file objects in the specified rank's partition + """ + weights = [f.estimated_time for f in files] + + if not weights or size <= 0 or size > len(weights): + return [] + + # Create list of (weight, original_index) tuples + # Using negative index as secondary key to maintain original order for equal weights + indexed_weights = [(w, -i) for i, w in enumerate(weights)] + # Stable sort in descending order by weight + # If weights are equal, larger (negative) index comes first (i.e., earlier original position) + indexed_weights = sorted(indexed_weights, reverse=True) + + # Extract original indices (negate back to positive) + indexed_weights = [(w, -i) for w, i in indexed_weights] + + # Initialize partitions and their sums + partitions = [[] for _ in range(size)] + sums = [0.0] * size + + # Greedy approach: assign each weight to partition with smallest current sum + for weight, idx in indexed_weights: + # Find partition with minimum sum + min_sum_idx = sums.index(min(sums)) + partitions[min_sum_idx].append(idx) + sums[min_sum_idx] += weight + + # Return the files corresponding to the indices in the specified rank's partition + indices = partitions[rank] + return [files[i] for i in indices] + + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--timeout-per-file", + type=int, + default=1800, + help="The time limit for running one file in seconds.", + ) + arg_parser.add_argument( + "--suite", + type=str, + default=list(suites.keys())[0], + choices=list(suites.keys()) + ["all"], + help="The suite to run", + ) + arg_parser.add_argument( + "--range-begin", + type=int, + default=0, + help="The begin index of the range of the files to run.", + ) + arg_parser.add_argument( + "--range-end", + type=int, + default=None, + help="The end index of the range of the files to run.", + ) + arg_parser.add_argument( + "--auto-partition-id", + type=int, + help="Use auto load balancing. The part id.", + ) + arg_parser.add_argument( + "--auto-partition-size", + type=int, + help="Use auto load balancing. The number of parts.", + ) + args = arg_parser.parse_args() + print(f"{args=}") + + if args.suite == "all": + files = glob.glob("**/test_*.py", recursive=True) + else: + files = suites[args.suite] + + if args.auto_partition_size: + files = auto_partition(files, args.auto_partition_id, args.auto_partition_size) + else: + files = files[args.range_begin : args.range_end] + + print("The running tests are ", [f.name for f in files]) + + exit_code = run_unittest_files(files, args.timeout_per_file) + exit(exit_code) diff --git a/test/srt/test_abort.py b/test/srt/test_abort.py new file mode 100644 index 0000000..d2ab4d0 --- /dev/null +++ b/test/srt/test_abort.py @@ -0,0 +1,54 @@ +import multiprocessing +import time +import unittest +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.test.test_utils import CustomTestCase, run_and_check_memory_leak + + +class TestAbort(CustomTestCase): + def workload_func(self, base_url, model): + def process_func(): + def run_one(_): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 2048, + }, + }, + ) + ret = response.json() + + with ThreadPoolExecutor(16) as executor: + list(executor.map(run_one, list(range(16)))) + + p = multiprocessing.Process(target=process_func) + p.start() + time.sleep(0.5) + p.terminate() + time.sleep(10) + + def test_memory_leak(self): + run_and_check_memory_leak( + self.workload_func, + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=8192, + assert_has_abort=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_awq.py b/test/srt/test_awq.py new file mode 100644 index 0000000..99ca3f8 --- /dev/null +++ b/test/srt/test_awq.py @@ -0,0 +1,45 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestAWQ(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--trust-remote-code"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.64) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_bench_one_batch.py b/test/srt/test_bench_one_batch.py new file mode 100644 index 0000000..9973b1f --- /dev/null +++ b/test/srt/test_bench_one_batch.py @@ -0,0 +1,58 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + CustomTestCase, + get_bool_env_var, + is_in_ci, + run_bench_one_batch, + write_github_step_summary, +) + + +class TestBenchOneBatch(CustomTestCase): + def test_bs1(self): + output_throughput = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"] + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_bs1\n" + f"output_throughput : {output_throughput:.2f} token/s\n" + ) + self.assertGreater(output_throughput, 135) + + def test_moe_tp2_bs1(self): + output_throughput = run_bench_one_batch( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"] + ) + + use_vllm_custom_allreduce = get_bool_env_var( + "USE_VLLM_CUSTOM_ALLREDUCE", default="false" + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_tp2_bs1 ({use_vllm_custom_allreduce=})\n" + f"output_throughput : {output_throughput:.2f} token/s\n" + ) + self.assertGreater(output_throughput, 124) + + def test_torch_compile_tp2_bs1(self): + output_throughput = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_torch_compile_tp2_bs1\n" + f"output_throughput : {output_throughput:.2f} token/s\n" + ) + self.assertGreater(output_throughput, 235) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_bench_serving.py b/test/srt/test_bench_serving.py new file mode 100644 index 0000000..811b5d7 --- /dev/null +++ b/test/srt/test_bench_serving.py @@ -0,0 +1,204 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_FP8_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_ci, + run_bench_serving, + write_github_step_summary, +) + + +class TestBenchServing(CustomTestCase): + + def test_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_default\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 3350) + + def test_offline_throughput_non_stream_small_batch_size(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=200, + request_rate=float("inf"), + other_server_args=["--max-running-requests", "10"], + dataset_name="sharegpt", + random_input_len=None, + random_output_len=None, + disable_stream=True, + need_warmup=True, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_non_stream_small_batch_size\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + # There is a regression with torch 2.5 + # This number was 950 for torch 2.4 + self.assertGreater(res["output_throughput"], 1000) + + def test_offline_throughput_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=["--disable-radix-cache"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_without_radix_cache\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 3350) + + def test_offline_throughput_without_chunked_prefill(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=["--chunked-prefill-size", "-1"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_without_chunked_prefill\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 2600) + + def test_offline_throughput_with_triton_attention_backend(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[ + "--attention-backend", + "triton", + "--context-length", + "8192", + ], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_with_triton_attention_backend\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 3450) + + def test_offline_throughput_default_fp8(self): + res = run_bench_serving( + model=DEFAULT_FP8_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_default_fp8\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 3900) + + def test_online_latency_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=100, + request_rate=1, + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_online_latency_default\n" + f'median_e2e_latency_ms : {res["median_e2e_latency_ms"]:.2f} ms\n' + ) + self.assertLess(res["median_e2e_latency_ms"], 11000) + self.assertLess(res["median_ttft_ms"], 86) + self.assertLess(res["median_itl_ms"], 10) + + def test_online_latency_eagle(self): + res = run_bench_serving( + model=DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + num_prompts=300, + request_rate=8, + sharegpt_context_len=3072, + disable_ignore_eos=True, + dataset_name="sharegpt", + other_server_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "16", + "--mem-fraction-static", + "0.7", + ], + need_warmup=True, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_online_latency_eagle\n" + f'median_e2e_latency_ms : {res["median_e2e_latency_ms"]:.2f} ms\n' + f'accept_length : {res["accept_length"]:.2f} \n' + ) + self.assertLess(res["median_e2e_latency_ms"], 900) + self.assertGreater(res["accept_length"], 2.99) + + def test_moe_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_offline_throughput_default\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 2200) + + def test_moe_offline_throughput_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2", "--disable-radix-cache"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_offline_throughput_without_radix_cache\n" + f'Output throughput: {res["output_throughput"]:.2f} token/s\n' + ) + self.assertGreater(res["output_throughput"], 2200) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_block_int8.py b/test/srt/test_block_int8.py new file mode 100644 index 0000000..2b8b841 --- /dev/null +++ b/test/srt/test_block_int8.py @@ -0,0 +1,222 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.test.test_utils import CustomTestCase + + +# For test +def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8): + """Function to perform per-token-group quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Note that only `torch.float8_e4m3fn` is supported for now. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + iinfo = torch.iinfo(dtype) + int8_min = iinfo.min + int8_max = iinfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / int8_max + x_q = (x_ / x_s).clamp(min=int8_min, max=int8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +# For test +def native_w8a8_block_int8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +# For test +def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): + """This function performs fused moe with block-wise quantization using native torch.""" + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + _, block_k = block_shape[0], block_shape[1] + a_q, a_s = native_per_token_group_quant_int8(a, block_k) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + inter_out = native_w8a8_block_int8_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype + ) + act_out = SiluAndMul().forward_native(inter_out) + act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k) + act_out = act_out.to(torch.float32) + out[mask] = native_w8a8_block_int8_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8BlockINT8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33, 64, 222] + N = [128, 1024] + K = [256, 4096] + E = [8, 24] + TOP_KS = [2, 6] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_int8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + int8_info = torch.iinfo(torch.int8) + int8_max, int8_min = int8_info.max, int8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 * int8_max + w1 = w1_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 * int8_max + w2 = w2_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + out = fused_moe( + a, + w1, + w2, + score, + topk, + renormalize=False, + use_int8_w8a8=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=block_size, + ) + ref_out = torch_w8a8_block_int8_moe( + a, w1, w2, w1_s, w2_s, score, topk, block_size + ) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.02 + ) + + def test_w8a8_block_int8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_block_int8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/test_cache_report.py b/test/srt/test_cache_report.py new file mode 100644 index 0000000..2dd5619 --- /dev/null +++ b/test/srt/test_cache_report.py @@ -0,0 +1,211 @@ +import asyncio +import unittest + +import openai +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestCacheReport(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.min_cached = 5 + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=300, + other_args=[ + "--chunked-prefill-size=40", + "--enable-cache-report", + ], + ) + cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") + cls.aclient = openai.AsyncClient(api_key="EMPTY", base_url=f"{cls.base_url}/v1") + + usage = cls.run_openai(cls, "1").usage + # we can assume that our request is of size 1, plus the total template size + # ideally we would like to know the begin size / end size of the template to be more precise + total_template_size = usage.prompt_tokens - 1 + print(f"template size: {total_template_size}") + usage2 = cls.run_openai(cls, "2").usage + assert usage2.prompt_tokens_details.cached_tokens <= total_template_size + cls.min_cached = max( + usage2.prompt_tokens_details.cached_tokens, + total_template_size - usage2.prompt_tokens_details.cached_tokens, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, return_logprob=False, top_logprobs_num=0, n=1): + response = requests.post( + self.base_url + "/generate", + # we use an uncommon start to minimise the chance that the cache is hit by chance + json={ + "text": "_ The capital of France is", + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "stop_token_ids": [119690], + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + return response + + def run_openai(self, message): + response = self.client.chat.completions.create( + model=self.model, + messages=[ + # {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": message}, + ], + temperature=0, + max_tokens=100, + ) + return response + + async def run_openai_async(self, message): + response = await self.aclient.chat.completions.create( + model=self.model, + messages=[ + {"role": "user", "content": message}, + ], + temperature=0, + max_tokens=100, + ) + return response + + def cache_report_openai(self, message): + response = self.run_openai(message) + print( + f"openai first request cached_tokens: {int(response.usage.prompt_tokens_details.cached_tokens)}" + ) + first_cached_tokens = int(response.usage.prompt_tokens_details.cached_tokens) + # assert int(response.usage.cached_tokens) == 0 + assert first_cached_tokens < self.min_cached + response = self.run_openai(message) + cached_tokens = int(response.usage.prompt_tokens_details.cached_tokens) + print(f"openai second request cached_tokens: {cached_tokens}") + assert cached_tokens > 0 + assert cached_tokens == int(response.usage.prompt_tokens) - 1 + return first_cached_tokens + + async def cache_report_openai_async(self, message): + response = await self.run_openai_async(message) + cached_tokens = int(response.usage.prompt_tokens_details.cached_tokens) + prompt_tokens = int(response.usage.prompt_tokens) + return cached_tokens, prompt_tokens + + def test_generate(self): + print("=" * 100) + response = self.run_decode() + # print(response.json()) + cached_tokens = int(response.json()["meta_info"]["cached_tokens"]) + print(f"sglang first request cached_tokens: {cached_tokens}") + print( + f"sglang first request prompt_tokens: {int(response.json()['meta_info']['prompt_tokens'])}" + ) + # can't assure to be 0: depends on the initialisation request / if a template is used with the model + assert cached_tokens < self.min_cached + response = self.run_decode() + cached_tokens = int(response.json()["meta_info"]["cached_tokens"]) + print(f"sglang second request cached_tokens: {cached_tokens}") + print( + f"sglang second request prompt_tokens: {int(response.json()['meta_info']['prompt_tokens'])}" + ) + assert cached_tokens == int(response.json()["meta_info"]["prompt_tokens"]) - 1 + + def test_cache_split_prefill_openai(self): + print("=" * 100) + self.cache_report_openai( + "€ This is a very long and unique text that should not be already cached, the twist is" + " that it should be longer than the chunked-prefill-size, so it should be split among" + " several prefill requests. Still, it shouldn't be cached" + ) + + def test_cache_report_openai(self): + print("=" * 100) + # warm up the cache, for the template + self.run_openai("Introduce the capital of France.") + + first_cached_tokens_1 = self.run_openai( + "How many sparrow do you need to lift a coconut?" + ).usage.prompt_tokens_details.cached_tokens + + usage_2 = self.run_openai("* sing something about cats").usage + first_cached_tokens_2 = usage_2.prompt_tokens_details.cached_tokens + # first request may not have 0 cached tokens, but if they only have the template in common they + # should be the same once the cache is warmed up + assert first_cached_tokens_1 == first_cached_tokens_2 + + resp = self.run_openai("* sing something about cats and dogs") + print(resp.usage) + + resp = self.run_openai("* sing something about cats, please") + print(resp.usage) + assert ( + resp.usage.prompt_tokens_details.cached_tokens + >= usage_2.prompt_tokens - self.min_cached + ) + + def test_cache_report_openai_async(self): + print("=" * 100) + + async def run_test(): + task0 = asyncio.create_task( + self.cache_report_openai_async( + "first request, to start the inference and let the next two request be started in the same batch" + ) + ) + await asyncio.sleep(0.05) # to force the first request to be started first + task1 = asyncio.create_task( + self.cache_report_openai_async( + "> can the same batch parallel request use the cache?" + ) + ) + task2 = asyncio.create_task( + self.cache_report_openai_async( + "> can the same batch parallel request use the cache?" + ) + ) + result0, result1, result2 = await asyncio.gather(task0, task1, task2) + + cached_tokens0, prompt_tokens0 = result0 + cached_tokens1, prompt_tokens1 = result1 + cached_tokens2, prompt_tokens2 = result2 + + print( + f"Async request 0 - Cached tokens: {cached_tokens0}, Prompt tokens: {prompt_tokens0}" + ) + print( + f"Async request 1 - Cached tokens: {cached_tokens1}, Prompt tokens: {prompt_tokens1}" + ) + print( + f"Async request 2 - Cached tokens: {cached_tokens2}, Prompt tokens: {prompt_tokens2}" + ) + + # Assert that no requests used the cache (becausefirst is alone, and the next two are in the same batch) + # If a new optimisation limiting starting request with same prefix at the same time was added + # to maximise the cache hit, this would not be true + assert cached_tokens1 == cached_tokens2 == cached_tokens0 + + asyncio.run(run_test()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_chunked_prefill.py b/test/srt/test_chunked_prefill.py new file mode 100644 index 0000000..fbdc87c --- /dev/null +++ b/test/srt/test_chunked_prefill.py @@ -0,0 +1,31 @@ +""" +python3 -m unittest test_chunked_prefill.TestChunkedPrefill.test_mixed_chunked_prefill_without_radix_cache +""" + +import unittest + +from sglang.test.test_utils import CustomTestCase, run_mmlu_test, run_mulit_request_test + + +class TestChunkedPrefill(CustomTestCase): + def test_chunked_prefill(self): + run_mmlu_test(disable_radix_cache=False, enable_mixed_chunk=False) + + def test_mixed_chunked_prefill(self): + run_mmlu_test(disable_radix_cache=False, enable_mixed_chunk=True) + + def test_chunked_prefill_without_radix_cache(self): + run_mmlu_test(disable_radix_cache=True, enable_mixed_chunk=False) + + def test_mixed_chunked_prefill_without_radix_cache(self): + run_mmlu_test(disable_radix_cache=True, enable_mixed_chunk=True) + + def test_mixed_chunked_prefill_multi_requests(self): + run_mulit_request_test( + enable_mixed_chunk=True, + chunked_prefill_size=2048, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_create_kvindices.py b/test/srt/test_create_kvindices.py new file mode 100644 index 0000000..4196eb2 --- /dev/null +++ b/test/srt/test_create_kvindices.py @@ -0,0 +1,75 @@ +import itertools +import unittest + +import numpy as np +import torch + +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.test.test_utils import CustomTestCase + + +class TestCreateKvIndices(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_test(self, batch, max_batch, max_context_len): + req_to_token = torch.arange( + max_batch * max_context_len, dtype=torch.int32, device="cuda" + ).reshape((max_batch, max_context_len)) + req_pool_indices = torch.tensor( + torch.from_numpy( + np.random.choice(range(max_batch), size=batch, replace=False) + ), + dtype=torch.int32, + device="cuda", + ) + paged_kernel_lens = torch.tensor( + torch.from_numpy( + np.random.choice(range(max_context_len), size=batch, replace=False) + ), + dtype=torch.int32, + device="cuda", + ) + + kv_indptr = torch.zeros((batch + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1:] = torch.cumsum(paged_kernel_lens, dim=0) + + # ref + req_pool_indices_cpu = req_pool_indices.cpu().numpy() + paged_kernel_lens_cpu = paged_kernel_lens.cpu().numpy() + kv_indices_ref = torch.cat( + [ + req_to_token[req_pool_indices_cpu[i], : paged_kernel_lens_cpu[i]] + for i in range(batch) + ], + dim=0, + ).contiguous() + + # triton + kv_indices_triton = torch.empty(kv_indptr[-1], dtype=torch.int32, device="cuda") + create_flashinfer_kv_indices_triton[(batch,)]( + req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + None, + kv_indices_triton, + req_to_token.size(1), + ) + + # Check + self.assertTrue(torch.equal(kv_indices_ref, kv_indices_triton)) + + def test_create_kvindices(self): + BATCH = [1, 37, 1786] + MAX_BATCH = 4096 + MAX_CONTEXT_LEN = 4096 + for batch in BATCH: + self._run_test(batch, MAX_BATCH, MAX_CONTEXT_LEN) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_custom_allreduce.py b/test/srt/test_custom_allreduce.py new file mode 100644 index 0000000..38600ae --- /dev/null +++ b/test/srt/test_custom_allreduce.py @@ -0,0 +1,166 @@ +import os +import random +import socket +import unittest +from typing import Any + +import ray +import torch +import torch.distributed as dist + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.communication_op import ( # noqa + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, +) +from sglang.test.test_utils import CustomTestCase + + +def get_open_port() -> int: + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, + cls: Any, + test_target: Any, +) -> None: + + # Using ray helps debugging the error when it failed + # as compared to multiprocessing. + # NOTE: We need to set working_dir for distributed tests, + # otherwise we may get import errors on ray workers + + ray.init(log_to_driver=True) + + distributed_init_port = get_open_port() + refs = [] + for rank in range(world_size): + refs.append(test_target.remote(cls, world_size, rank, distributed_init_port)) + ray.get(refs) + + ray.shutdown() + + +class TestCustomAllReduce(CustomTestCase): + @classmethod + def setUpClass(cls): + random.seed(42) + # 512B to 32MB + cls.test_sizes = [512, 4096, 32768, 262144, 2097152, 16777216, 33554432] + cls.world_sizes = [2, 4, 6, 8] + cls.test_loop = 10 + + def test_graph_allreduce(self): + for world_size in self.world_sizes: + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.graph_allreduce) + + def test_eager_allreduce(self): + for world_size in self.world_sizes: + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.eager_allreduce) + + @ray.remote(num_gpus=1, max_calls=1) + def graph_allreduce(self, world_size, rank, distributed_init_port): + del os.environ["CUDA_VISIBLE_DEVICES"] + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + # A small all_reduce for warmup. + # this is needed because device communicators might be created lazily + # (e.g. NCCL). This will ensure that the communicator is initialized + # before any communication happens, so that this group can be used for + # graph capture immediately. + data = torch.zeros(1) + data = data.to(device=device) + torch.distributed.all_reduce(data, group=group) + torch.cuda.synchronize() + del data + + for sz in self.test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(self.test_loop): + with graph_capture() as graph_capture_context: + # use integers so result matches NCCL exactly + inp1 = torch.randint( + 1, + 16, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + inp2 = torch.randint( + 1, + 16, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph( + graph, stream=graph_capture_context.stream + ): + out1 = tensor_model_parallel_all_reduce(inp1) + # the input buffer is immediately modified to test + # synchronization + dist.all_reduce(inp1, group=group) + out2 = tensor_model_parallel_all_reduce(inp2) + dist.all_reduce(inp2, group=group) + graph.replay() + torch.testing.assert_close(out1, inp1) + torch.testing.assert_close(out2, inp2) + + @ray.remote(num_gpus=1, max_calls=1) + def eager_allreduce(self, world_size, rank, distributed_init_port): + del os.environ["CUDA_VISIBLE_DEVICES"] + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + for sz in self.test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(self.test_loop): + inp1 = torch.randint( + 1, 16, (sz,), dtype=dtype, device=torch.cuda.current_device() + ) + out1 = tensor_model_parallel_all_reduce(inp1) + dist.all_reduce(inp1, group=group) + torch.testing.assert_close(out1, inp1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_data_parallelism.py b/test/srt/test_data_parallelism.py new file mode 100644 index 0000000..e54e3a5 --- /dev/null +++ b/test/srt/test_data_parallelism.py @@ -0,0 +1,78 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDataParallelism(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dp", 2], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_update_weight(self): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": DEFAULT_MODEL_NAME_FOR_TEST}, + ) + + # check if the response is 200 + assert response.status_code == 200 + + # pause a few seconds then send again + time.sleep(1) + + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": DEFAULT_MODEL_NAME_FOR_TEST}, + ) + + # check if the response is 200 + assert response.status_code == 200 + + def test_get_memory_pool_size(self): + # use `get_server_info` instead since `get_memory_pool_size` is merged into `get_server_info` + response = requests.get(self.base_url + "/get_server_info") + assert response.status_code == 200 + + time.sleep(1) + + response = requests.get(self.base_url + "/get_server_info") + assert response.status_code == 200 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_double_sparsity.py b/test/srt/test_double_sparsity.py new file mode 100644 index 0000000..c936e79 --- /dev/null +++ b/test/srt/test_double_sparsity.py @@ -0,0 +1,65 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDoubleSparsity(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + dirpath = os.path.dirname(__file__) + config_file = os.path.join( + dirpath, "double-sparsity-config-Llama-3.1-8B-Instruct.json" + ) + # NOTE: Generate the config file by running https://github.com/andy-yang-1/DoubleSparse/blob/main/evaluation/group_channel_config.py + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-double-sparsity", + "--ds-channel-config-path", + config_file, + "--ds-heavy-channel-num", + "32", + "--ds-heavy-channel-type", + "k", + "--ds-heavy-token-num", + "512", + "--ds-sparse-decode-threshold", + "0", + "--max-total-tokens", + "200000", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_dp_attention.py b/test/srt/test_dp_attention.py new file mode 100644 index 0000000..622f9b6 --- /dev/null +++ b/test/srt/test_dp_attention.py @@ -0,0 +1,62 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDPAttentionDP2TP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--enable-dp-attention", + "--dp", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) diff --git a/test/srt/test_eagle_infer.py b/test/srt/test_eagle_infer.py new file mode 100644 index 0000000..e29b097 --- /dev/null +++ b/test/srt/test_eagle_infer.py @@ -0,0 +1,539 @@ +import json +import multiprocessing as mp +import os +import random +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from types import SimpleNamespace +from typing import List, Optional + +import numpy as np +import requests +import torch + +import sglang as sgl +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_logprob_check, +) + +torch_dtype = torch.float16 +prefill_tolerance = 5e-2 +decode_tolerance: float = 5e-2 + + +class TestEAGLEEngine(CustomTestCase): + BASE_CONFIG = { + "model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + "speculative_draft_model_path": DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "speculative_algorithm": "EAGLE", + "speculative_num_steps": 5, + "speculative_eagle_topk": 4, + "speculative_num_draft_tokens": 8, + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 4, + } + NUM_CONFIGS = 2 + + def setUp(self): + self.prompt = "Today is a sunny day and I like" + self.sampling_params = {"temperature": 0, "max_new_tokens": 8} + + ref_engine = sgl.Engine( + model_path=self.BASE_CONFIG["model_path"], cuda_graph_max_bs=1 + ) + self.ref_output = ref_engine.generate(self.prompt, self.sampling_params)["text"] + ref_engine.shutdown() + + def test_correctness(self): + configs = [ + # Basic config + self.BASE_CONFIG, + # Chunked prefill + {**self.BASE_CONFIG, "chunked_prefill_size": 4}, + ] + + for i, config in enumerate(configs[: self.NUM_CONFIGS]): + with self.subTest(i=i): + print(f"{config=}") + engine = sgl.Engine(**config, log_level="info", decode_log_interval=10) + try: + self._test_single_generation(engine) + self._test_batch_generation(engine) + self._test_eos_token(engine) + self._test_acc_length(engine) + finally: + engine.shutdown() + print("=" * 100) + + def _test_single_generation(self, engine): + output = engine.generate(self.prompt, self.sampling_params)["text"] + print(f"{output=}, {self.ref_output=}") + self.assertEqual(output, self.ref_output) + + def _test_batch_generation(self, engine): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + params = {"temperature": 0, "max_new_tokens": 50} + + outputs = engine.generate(prompts, params) + for prompt, output in zip(prompts, outputs): + print(f"Prompt: {prompt}") + print(f"Generated: {output['text']}") + print("-" * 40) + + print(f"{engine.get_server_info()=}") + + avg_spec_accept_length = engine.get_server_info()["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 1.9) + + def _test_eos_token(self, engine): + prompt = "[INST] <>\nYou are a helpful assistant.\n<>\nToday is a sunny day and I like [/INST]" + params = { + "temperature": 0.1, + "max_new_tokens": 1024, + "skip_special_tokens": False, + } + + tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST) + output = engine.generate(prompt, params)["text"] + print(f"{output=}") + + tokens = tokenizer.encode(output, truncation=False) + self.assertNotIn(tokenizer.eos_token_id, tokens) + + def _test_acc_length(self, engine): + prompt = [ + "Human: Give me a fully functional FastAPI server. Show the python code.\n\nAssistant:", + ] * 5 # test batched generation + sampling_params = {"temperature": 0, "max_new_tokens": 512} + output = engine.generate(prompt, sampling_params) + output = output[0] + + if "spec_verify_ct" in output["meta_info"]: + acc_length = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["spec_verify_ct"] + ) + else: + acc_length = 1.0 + + speed = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["e2e_latency"] + ) + print(f"{acc_length=}") + + if engine.server_args.model_path == DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST: + self.assertGreater(acc_length, 3.6) + else: + self.assertGreater(acc_length, 2.6) + + +class TestEAGLEEngineTokenMap(TestEAGLEEngine): + BASE_CONFIG = { + "model_path": "meta-llama/Meta-Llama-3-8B-Instruct", + "speculative_draft_model_path": "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B", + "speculative_algorithm": "EAGLE", + "speculative_num_steps": 5, + "speculative_eagle_topk": 4, + "speculative_num_draft_tokens": 8, + "speculative_token_map": "thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt", + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 4, + "dtype": "float16", + } + NUM_CONFIGS = 1 + + +class TestEAGLE3Engine(TestEAGLEEngine): + BASE_CONFIG = { + "model_path": "meta-llama/Llama-3.1-8B-Instruct", + "speculative_draft_model_path": "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B", + "speculative_algorithm": "EAGLE3", + "speculative_num_steps": 5, + "speculative_eagle_topk": 16, + "speculative_num_draft_tokens": 64, + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 4, + "dtype": "float16", + } + NUM_CONFIGS = 1 + + +class TestEAGLEServer(CustomTestCase): + PROMPTS = [ + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nToday is a sunny day and I like[/INST]" + '[INST] <>\\nYou are a helpful assistant.\\n<>\\nWhat are the mental triggers in Jeff Walker\'s Product Launch Formula and "Launch" book?[/INST]', + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nSummarize Russell Brunson's Perfect Webinar Script...[/INST]", + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nwho are you?[/INST]", + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nwhere are you from?[/INST]", + ] + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def send_request(self): + time.sleep(random.uniform(0, 2)) + for prompt in self.PROMPTS: + url = self.base_url + "/generate" + data = { + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024, + }, + } + response = requests.post(url, json=data) + assert response.status_code == 200 + + def send_requests_abort(self): + for prompt in self.PROMPTS: + try: + time.sleep(random.uniform(0, 2)) + url = self.base_url + "/generate" + data = { + "model": "base", + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024, + }, + } + # set timeout = 1s, mock disconnected + requests.post(url, json=data, timeout=1) + except Exception as e: + print(e) + pass + + def test_request_abort(self): + concurrency = 4 + threads = [ + threading.Thread(target=self.send_request) for _ in range(concurrency) + ] + [ + threading.Thread(target=self.send_requests_abort) + for _ in range(concurrency) + ] + for worker in threads: + worker.start() + for p in threads: + p.join() + + def test_max_token_one(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=1, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + + # Just run and check it does not hang + metrics = run_eval(args) + self.assertGreater(metrics["output_throughput"], 50) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.20) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 3.5) + + # Wait a little bit so that the memory check happens. + time.sleep(4) + + def test_logprob_start_len(self): + logprob_start_len = 4 + new_tokens = 4 + prompts = [ + "I have a very good idea on", + "Today is a sunndy day and", + ] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "logprob_start_len": logprob_start_len, + }, + ) + response_json = response.json() + print(json.dumps(response_json, indent=2)) + + for res in response_json: + self.assertEqual( + res["meta_info"]["prompt_tokens"], + logprob_start_len + len(res["meta_info"]["input_token_logprobs"]), + ) + + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + + def test_logprob_match(self): + """Test the output logprobs are close to the input logprobs if we run a prefill again.""" + + def run_generate( + prompt, return_logprob=False, max_new_tokens=512, logprob_start_len=-1 + ): + + if isinstance(prompt, str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + response = requests.post( + self.base_url + "/generate", + json={ + **prompt_kwargs, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + return response.json() + + prompt = "I have a very good idea on how to" + + gen = run_generate(prompt, return_logprob=True, logprob_start_len=0) + output_logprobs = np.array( + [x[0] for x in gen["meta_info"]["output_token_logprobs"]] + ) + num_prompts_tokens = gen["meta_info"]["prompt_tokens"] + + input_tokens = [x[1] for x in gen["meta_info"]["input_token_logprobs"]] + output_tokens = [x[1] for x in gen["meta_info"]["output_token_logprobs"]] + + new_prompt = input_tokens + output_tokens + score = run_generate( + new_prompt, return_logprob=True, logprob_start_len=0, max_new_tokens=0 + ) + output_logprobs_score = np.array( + [ + x[0] + for x in score["meta_info"]["input_token_logprobs"][num_prompts_tokens:] + ] + ) + + print(f"{output_logprobs[-10:]=}") + print(f"{output_logprobs_score[-10:]=}") + + diff = np.abs(output_logprobs - output_logprobs_score) + max_diff = np.max(diff) + self.assertLess(max_diff, 0.25) + + def test_logprob_mixed(self): + args = [] + temperature = 0 + # input_len, output_len, temperature, logprob_start_len, return_logprob, top_logprobs_num + # Llama 2 context length seems to be only 2k, so we can only test small length. + for input_len in [200, 500, 1000, 2000]: + for output_len in [4, 8]: + for logprob_start_len in [0, 100, 300, 800, 1998]: + for return_logprob in [True, False]: + for top_logprobs_num in [0, 5]: + + if logprob_start_len >= input_len: + continue + + args.append( + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) + ) + + random.shuffle(args) + + func = partial(run_logprob_check, self) + with ThreadPoolExecutor(8) as executor: + list(executor.map(func, args)) + + def run_decode(self, sampling_params): + return_logprob = True + top_logprobs_num = 5 + return_text = True + n = 1 + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "Human: Write a travel blog post to Hawaii.\n\nAssistant:", + "sampling_params": { + "max_new_tokens": 48, + "n": n, + "temperature": 0.7, + **sampling_params, + }, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + self.assertEqual(response.status_code, 200) + print(json.dumps(response.json())) + print("=" * 100) + + def test_penalty_mixed(self): + args = [ + {}, + {}, + {}, + {"frequency_penalty": 2}, + {"presence_penalty": 1}, + {"min_new_tokens": 16}, + {"frequency_penalty": 0.2}, + {"presence_penalty": 0.4}, + {"min_new_tokens": 8}, + {"frequency_penalty": 0.4, "presence_penalty": 0.8}, + {"frequency_penalty": 0.4, "min_new_tokens": 12}, + {"presence_penalty": 0.8, "min_new_tokens": 12}, + {"presence_penalty": -0.3, "frequency_penalty": 1.3, "min_new_tokens": 32}, + {"presence_penalty": 0.3, "frequency_penalty": -1.3, "min_new_tokens": 32}, + ] + random.shuffle(args * 5) + with ThreadPoolExecutor(8) as executor: + list(executor.map(self.run_decode, args)) + + +class TestEAGLERetract(TestEAGLEServer): + @classmethod + def setUpClass(cls): + # These config helps find a leak. + os.environ["SGLANG_CI_SMALL_KV_SIZE"] = "4500" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 64, + ], + ) + + +class TestEAGLEServerTriton(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--attention-backend", + "triton", + "--max-running-requests", + 8, + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_ebnf_constrained.py b/test/srt/test_ebnf_constrained.py new file mode 100644 index 0000000..e0771b9 --- /dev/null +++ b/test/srt/test_ebnf_constrained.py @@ -0,0 +1,242 @@ +""" +python3 -m unittest test_ebnf_constrained.TestEBNFConstrained.test_ebnf_generate_email +python3 -m unittest test_ebnf_constrained.TestEBNFConstrained.test_ebnf_generate_greeting +python3 -m unittest test_ebnf_constrained.TestEBNFConstrainedLLGuidance.test_ebnf_generate_email +python3 -m unittest test_ebnf_constrained.TestEBNFConstrainedLLGuidance.test_ebnf_generate_greeting +""" + +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def setup_class(cls, backend: str, disable_overlap: bool): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.ebnf_grammar = 'root ::= "test"' # Default grammar + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + +class TestEBNFConstrained(CustomTestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, "xgrammar", disable_overlap=False) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + ebnf, + expected_patterns, + prompt, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "ebnf": ebnf, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + + ret = response.json() + print(json.dumps(ret, indent=2)) + print("=" * 100) + + if not isinstance(ret, list): + self.fail(f"Expected response to be a list, but got {type(ret)}") + + for item in ret: + text = item.get("text", "").strip() + if not text: + self.fail("Generated text is empty.") + + match = False + for pattern in expected_patterns: + if self.regex_match(text, pattern): + match = True + break + if not match: + self.fail(f"Text '{text}' does not match any of the allowed patterns.") + + def regex_match(self, text, pattern): + import re + + return re.match(pattern, text) is not None + + def test_ebnf_generate_email(self): + self.__class__.ebnf_grammar = 'root ::= "user@example.com"' + allowed_patterns = [r"^user@example\.com$"] + prompt = "Generate an email address:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_greeting(self): + self.__class__.ebnf_grammar = 'root ::= "Hello" | "Hi" | "Hey"' + allowed_patterns = [r"^(Hello|Hi|Hey)$"] + prompt = "Generate a greeting:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_number(self): + self.__class__.ebnf_grammar = """ + root ::= digit digit digit + digit ::= [0-9] + """ + allowed_patterns = [r"^\d{3}$"] + prompt = "Generate a three-digit number:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_phone(self): + self.__class__.ebnf_grammar = """ + root ::= "(" area ")" " " prefix "-" line + area ::= [0-9] [0-9] [0-9] + prefix ::= [0-9] [0-9] [0-9] + line ::= [0-9] [0-9] [0-9] [0-9] + """ + allowed_patterns = [r"^\(\d{3}\) \d{3}-\d{4}$"] + prompt = "Generate a phone number:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_date(self): + self.__class__.ebnf_grammar = """ + root ::= year "-" month "-" day + year ::= "2024" + month ::= "01" | "02" | "03" | "04" | "05" | "06" | "07" | "08" | "09" | "10" | "11" | "12" + day ::= "01" | "02" | "03" | "04" | "05" | "06" | "07" | "08" | "09" | "10" | + "11" | "12" | "13" | "14" | "15" | "16" | "17" | "18" | "19" | "20" | + "21" | "22" | "23" | "24" | "25" | "26" | "27" | "28" | "29" | "30" | "31" + """ + allowed_patterns = [r"^2024-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$"] + prompt = "Generate a date in YYYY-MM-DD format:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_hex_color(self): + self.__class__.ebnf_grammar = """ + root ::= "#" hex hex hex hex hex hex + hex ::= [0-9] | [A-F] + """ + allowed_patterns = [r"^#[0-9A-F]{6}$"] + prompt = "Generate a hex color code:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_complex_json(self): + self.__class__.ebnf_grammar = """ + root ::= object + object ::= "{" ws pair (ws "," ws pair)* ws "}" + pair ::= "\\"name\\"" ws ":" ws value | + "\\"age\\"" ws ":" ws number | + "\\"city\\"" ws ":" ws string + value ::= string | number + string ::= "\\"" [a-zA-Z0-9 ]+ "\\"" + number ::= [1-9] [0-9]* + ws ::= [ ]* + """ + allowed_patterns = [ + r'^{\s*"name"\s*:\s*"[a-zA-Z0-9 ]+"\s*,\s*"age"\s*:\s*[1-9][0-9]*\s*,\s*"city"\s*:\s*"[a-zA-Z0-9 ]+"\s*}$', + ] + prompt = "Generate a simple JSON with name, age, and city:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_custom_log_format(self): + self.__class__.ebnf_grammar = """ + root ::= logentry + logentry ::= "[" datetime "] " level ": System.process - " message + datetime ::= "2024-01-01T12:00:00Z" + level ::= "INFO" + message ::= "Operation " [a-z]+ " successfully" + """ + allowed_patterns = [ + r"^\[2024-01-01T12:00:00Z\] INFO: System\.process - Operation [a-z]+ successfully$" + ] + prompt = "Generate a log entry:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_embedding_openai_server.py b/test/srt/test_embedding_openai_server.py new file mode 100644 index 0000000..e7d19d4 --- /dev/null +++ b/test/srt/test_embedding_openai_server.py @@ -0,0 +1,87 @@ +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "intfloat/e5-mistral-7b-instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_embedding(self, use_list_input, token_input): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input] * 2 + num_prompts = len(prompt_arg) + num_prompt_tokens *= num_prompts + else: + prompt_arg = prompt_input + num_prompts = 1 + + response = client.embeddings.create( + input=prompt_arg, + model=self.model, + ) + + assert len(response.data) == num_prompts + assert isinstance(response.data, list) + assert response.data[0].embedding + assert response.data[0].index is not None + assert response.data[0].object == "embedding" + assert response.model == self.model + assert response.object == "list" + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert ( + response.usage.total_tokens == num_prompt_tokens + ), f"{response.usage.total_tokens} vs {num_prompt_tokens}" + + def run_batch(self): + # FIXME: not implemented + pass + + def test_embedding(self): + # TODO: the fields of encoding_format, dimensions, user are skipped + # TODO: support use_list_input + for use_list_input in [False, True]: + for token_input in [False, True]: + self.run_embedding(use_list_input, token_input) + + def test_batch(self): + self.run_batch() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_eval_accuracy_large.py b/test/srt/test_eval_accuracy_large.py new file mode 100644 index 0000000..efb2024 --- /dev/null +++ b/test/srt/test_eval_accuracy_large.py @@ -0,0 +1,92 @@ +""" +Usage: +python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--log-level-http", "warning"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') + + self.assertGreater(metrics["score"], 0.70) + + def test_human_eval(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary( + f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' + ) + + self.assertGreater(metrics["score"], 0.64) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary( + f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' + ) + + self.assertGreater(metrics["score"], 0.835) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_eval_fp8_accuracy.py b/test/srt/test_eval_fp8_accuracy.py new file mode 100644 index 0000000..12b2499 --- /dev/null +++ b/test/srt/test_eval_fp8_accuracy.py @@ -0,0 +1,159 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_FP8_MODEL_NAME_FOR_ACCURACY_TEST, + DEFAULT_FP8_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST, + DEFAULT_FP8_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST, + DEFAULT_FP8_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_REVISION, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEvalFP8Accuracy(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_FP8_MODEL_NAME_FOR_ACCURACY_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.61) + + +class TestEvalFP8DynamicQuantAccuracy(CustomTestCase): + + def _run_test(self, model, other_args, expected_score): + base_url = DEFAULT_URL_FOR_TEST + other_args = other_args or [] + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], expected_score) + finally: + kill_process_tree(process.pid) + + def test_mmlu_offline_only(self): + """Test with offline quantization only.""" + self._run_test( + model=DEFAULT_FP8_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST, + other_args=[], + expected_score=0.64, + ) + + def test_mmlu_offline_and_online_override(self): + """Test with both offline and online quantization.""" + self._run_test( + model=DEFAULT_FP8_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST, + other_args=["--quantization", "w8a8_fp8"], + # inference will use sgl kernel w/ online quant override + # we observed that the accuracy is higher then offline only + expected_score=0.64, + ) + + def test_mmlu_online_only(self): + """Test with online quantization only.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_TEST, + # inference will use sgl kernel w/ online quantization only + # we observed that the accuracy is higher then offline only + other_args=["--quantization", "w8a8_fp8"], + expected_score=0.64, + ) + + def test_mmlu_fp16_baseline(self): + """Test with unquantized fp16 baseline.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_TEST, + other_args=[], + expected_score=0.64, + ) + + +class TestEvalFP8ModelOptQuantAccuracy(CustomTestCase): + + def _run_test(self, model, other_args, expected_score): + base_url = DEFAULT_URL_FOR_TEST + other_args = other_args or [] + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], expected_score) + finally: + kill_process_tree(process.pid) + + @unittest.skipIf( + torch.version.hip is not None, "modelopt quantization unsupported on ROCm" + ) + def test_mmlu_offline_only(self): + """Test with offline quantization only.""" + self._run_test( + model=DEFAULT_FP8_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST, + other_args=[ + "--quantization", + "modelopt", + "--revision", + DEFAULT_FP8_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_REVISION, + ], + expected_score=0.64, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_expert_distribution.py b/test/srt/test_expert_distribution.py new file mode 100644 index 0000000..e382630 --- /dev/null +++ b/test/srt/test_expert_distribution.py @@ -0,0 +1,124 @@ +import csv +import glob +import os +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestExpertDistribution(CustomTestCase): + def setUp(self): + # Clean up any existing expert distribution files before each test + for f in glob.glob("expert_distribution_*.csv"): + os.remove(f) + + def tearDown(self): + # Clean up any expert distribution files after each test + for f in glob.glob("expert_distribution_*.csv"): + os.remove(f) + + def test_expert_distribution_record(self): + """Test expert distribution record endpoints""" + process = popen_launch_server( + # The feature is only implemented in deepseek_v2.py + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct", + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + ], + ) + + try: + # Start recording + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/start_expert_distribution_record" + ) + self.assertEqual(response.status_code, 200) + + # Make some requests to generate expert distribution data + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + + # Stop recording + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/stop_expert_distribution_record" + ) + self.assertEqual(response.status_code, 200) + + # Dump the recorded data + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/dump_expert_distribution_record" + ) + self.assertEqual(response.status_code, 200) + + # Verify the dumped file exists and has correct format + csv_files = glob.glob("expert_distribution_*.csv") + self.assertEqual( + len(csv_files), + 1, + f"Expected exactly one expert distribution CSV file {csv_files=}", + ) + + # Check CSV file format + with open(csv_files[0], "r") as f: + csv_reader = csv.reader(f) + + # Check header + header = next(csv_reader) + self.assertEqual( + header, + ["layer_id", "expert_id", "count"], + "CSV header should be 'layer_id,expert_id,count'", + ) + + # Check data rows + rows = list(csv_reader) + self.assertGreater(len(rows), 0, "CSV file should contain data rows") + + for row in rows: + # Verify each row has 3 columns + self.assertEqual( + len(row), + 3, + "Each row should have layer_id, expert_id and count", + ) + + # Verify data types + layer_id, expert_id, count = row + self.assertTrue( + layer_id.isdigit(), + f"layer_id should be an integer {row=} {rows=}", + ) + self.assertTrue( + expert_id.isdigit(), + f"expert_id should be an integer {row=} {rows=}", + ) + self.assertTrue( + count.isdigit(), f"count should be an integer {row=} {rows=}" + ) + + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fim_completion.py b/test/srt/test_fim_completion.py new file mode 100644 index 0000000..09db1d4 --- /dev/null +++ b/test/srt/test_fim_completion.py @@ -0,0 +1,72 @@ +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFimCompletion(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "deepseek-ai/deepseek-coder-1.3b-base" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + other_args = ["--completion-template", "deepseek_coder"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_fim_completion(self, number_of_completion): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "function sum(a: number, b: number): number{\n" + suffix = "}" + + prompt_input = self.tokenizer.encode(prompt) + self.tokenizer.encode(suffix) + num_prompt_tokens = len(prompt_input) + 2 + + response = client.completions.create( + model=self.model, + prompt=prompt, + suffix=suffix, + temperature=0.3, + max_tokens=32, + stream=False, + n=number_of_completion, + ) + + print(response) + print(len(response.choices)) + assert len(response.choices) == number_of_completion + assert response.id + assert response.created + assert response.object == "text_completion" + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_fim_completion(self): + for number_of_completion in [1, 3]: + self.run_fim_completion(number_of_completion) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fp8_kernel.py b/test/srt/test_fp8_kernel.py new file mode 100644 index 0000000..1f8d94b --- /dev/null +++ b/test/srt/test_fp8_kernel.py @@ -0,0 +1,128 @@ +import unittest + +import torch + +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_fp8, + w8a8_block_fp8_matmul, +) +from sglang.test.test_utils import CustomTestCase + + +class TestFP8Base(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.M = 256 + # test non-aligned + cls.N = 1024 + 64 + cls.K = 512 + cls.group_size = 128 + cls.quant_type = torch.float8_e4m3fn + cls.output_type = torch.bfloat16 + + @staticmethod + def _make_A(M, K, group_size, out_dtype): + quant_A = torch.rand( + M, K // group_size, group_size, dtype=torch.float32, device="cuda" + ) + # -1 ~ 1 + quant_A = quant_A * 2 - 1 + # scaling abs max to fmax + finfo = torch.finfo(out_dtype) + fmax = finfo.max + scaling = fmax / quant_A.abs().amax(-1, keepdim=True) + quant_A *= scaling + quant_A = quant_A.to(out_dtype).to(torch.float32) + + # create scale and A + scale = torch.rand(M, K // group_size, dtype=torch.float32, device="cuda") + scale /= fmax + A = quant_A * scale[..., None] + + A = A.reshape(M, K) + quant_A = quant_A.reshape(M, K).to(out_dtype) + return A, quant_A, scale + + @staticmethod + def _make_B(K, N, group_size, out_dtype): + def _aligned_size(a, b): + return (a + b - 1) // b * b + + K_aligned = _aligned_size(K, group_size) + N_aligned = _aligned_size(N, group_size) + + quant_B = torch.rand( + K_aligned // group_size, + group_size, + N_aligned // group_size, + group_size, + dtype=torch.float32, + device="cuda", + ) + quant_B = quant_B * 2 - 1 + + # scaling abs max to fmax + finfo = torch.finfo(out_dtype) + fmax = finfo.max + scaling = fmax / quant_B.abs().amax((1, 3), keepdim=True) + quant_B *= scaling + quant_B = quant_B.to(out_dtype).to(torch.float32) + + scale = torch.rand( + K_aligned // group_size, + 1, + N_aligned // group_size, + 1, + dtype=torch.float32, + device="cuda", + ) + scale /= fmax + + B = quant_B * scale + + B = B.reshape(K_aligned, N_aligned)[:K, :N] + quant_B = quant_B.reshape(K_aligned, N_aligned).to(out_dtype)[:K, :N] + scale = scale.reshape(K_aligned // group_size, N_aligned // group_size) + return B, quant_B, scale + + +class TestPerTokenGroupQuantFP8(TestFP8Base): + def test_per_token_group_quant_fp8(self): + if torch.cuda.get_device_capability()[0] < 9: + return + A, A_quant_gt, scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + A_quant, scale = per_token_group_quant_fp8( + x=A, group_size=self.group_size, dtype=self.quant_type + ) + torch.testing.assert_close(scale, scale_gt) + diff = (A_quant.to(torch.float16) - A_quant_gt.to(torch.float16)).abs() + diff_count = (diff > 1e-5).count_nonzero() + assert diff_count / diff.numel() < 1e-4 + + +class TestW8A8BlockFP8Matmul(TestFP8Base): + def test_w8a8_block_fp8_matmul(self): + if torch.cuda.get_device_capability()[0] < 9: + return + A, A_quant_gt, A_scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + B, B_quant_gt, B_scale_gt = self._make_B( + K=self.K, N=self.N, group_size=self.group_size, out_dtype=self.quant_type + ) + C_gt = A.to(self.output_type) @ B.to(self.output_type) + C = w8a8_block_fp8_matmul( + A=A_quant_gt, + B=B_quant_gt.T.contiguous(), + As=A_scale_gt, + Bs=B_scale_gt.T.contiguous(), + block_size=[128, 128], + output_dtype=self.output_type, + ) + torch.testing.assert_close(C, C_gt, atol=0.5, rtol=1e-4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fp8_kvcache.py b/test/srt/test_fp8_kvcache.py new file mode 100644 index 0000000..669926f --- /dev/null +++ b/test/srt/test_fp8_kvcache.py @@ -0,0 +1,114 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFp8KvcacheBase(CustomTestCase): + model_config = None + + @classmethod + def setUpClass(cls): + if cls.model_config is None: + raise NotImplementedError("model_config must be specified in subclass") + + cls.model = cls.model_config["model_name"] + cls.base_url = DEFAULT_URL_FOR_TEST + dirpath = os.path.dirname(__file__) + config_file = os.path.join(dirpath, cls.model_config["config_filename"]) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--kv-cache-dtype", + "fp8_e4m3", + "--quantization-param-path", + config_file, + ], + ) + + +class TestFp8KvcacheLlama(TestFp8KvcacheBase): + model_config = { + "model_name": DEFAULT_MODEL_NAME_FOR_TEST, + "config_filename": "kv_cache_scales_llama3_8b.json", + } + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.80) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestFp8KvcacheQwen(TestFp8KvcacheBase): + model_config = { + "model_name": DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + "config_filename": "kv_cache_scales_qwen2_1_5b.json", + } + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.01) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_function_calling.py b/test/srt/test_function_calling.py new file mode 100644 index 0000000..9eaa930 --- /dev/null +++ b/test/srt/test_function_calling.py @@ -0,0 +1,299 @@ +import json +import time +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServerFunctionCalling(CustomTestCase): + @classmethod + def setUpClass(cls): + # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + # If your server needs extra parameters to test function calling, please add them here. + "--tool-call-parser", + "llama3", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_function_calling_format(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "int", + "description": "A number", + }, + "b": { + "type": "int", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Compute (3+5)"}] + response = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + content = response.choices[0].message.content + tool_calls = response.choices[0].message.tool_calls + + assert content is None, ( + "When function call is successful, message.content should be None, " + f"but got: {content}" + ) + assert ( + isinstance(tool_calls, list) and len(tool_calls) > 0 + ), "tool_calls should be a non-empty list" + + function_name = tool_calls[0].function.name + assert function_name == "add", "Function name should be 'add'" + + def test_function_calling_streaming_simple(self): + """ + Test: Whether the function name can be correctly recognized in streaming mode. + - Expect a function call to be found, and the function name to be correct. + - Verify that streaming mode returns at least multiple chunks. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "What is the temperature in Paris?"}] + + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_function_name = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + if tool_call.function.name: + self.assertEqual( + tool_call.function.name, + "get_current_weather", + "Function name should be 'get_current_weather'", + ) + found_function_name = True + break + + self.assertTrue( + found_function_name, + "Target function name 'get_current_weather' was not found in the streaming chunks", + ) + + def test_function_calling_streaming_args_parsing(self): + """ + Test: Whether the function call arguments returned in streaming mode can be correctly concatenated into valid JSON. + - The user request requires multiple parameters. + - AI may return the arguments in chunks that need to be concatenated. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two integers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "int", + "description": "First integer", + }, + "b": { + "type": "int", + "description": "Second integer", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [ + {"role": "user", "content": "Please sum 5 and 7, just call the function."} + ] + + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.9, + top_p=0.9, + stream=True, + tools=tools, + ) + + argument_fragments = [] + function_name = None + for chunk in response_stream: + choice = chunk.choices[0] + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + # Record the function name on first occurrence + function_name = tool_call.function.name or function_name + # In case of multiple chunks, JSON fragments may need to be concatenated + if tool_call.function.arguments: + argument_fragments.append(tool_call.function.arguments) + + self.assertEqual(function_name, "add", "Function name should be 'add'") + joined_args = "".join(argument_fragments) + self.assertTrue( + len(joined_args) > 0, + "No parameter fragments were returned in the function call", + ) + + # Check whether the concatenated JSON is valid + try: + args_obj = json.loads(joined_args) + except json.JSONDecodeError: + self.fail( + "The concatenated tool call arguments are not valid JSON, parsing failed" + ) + + self.assertIn("a", args_obj, "Missing parameter 'a'") + self.assertIn("b", args_obj, "Missing parameter 'b'") + self.assertEqual(str(args_obj["a"]), "5", "Parameter a should be 5") + self.assertEqual(str(args_obj["b"]), "7", "Parameter b should be 7") + + def test_function_call_strict(self): + """ + Test: Whether the strict mode of function calling works as expected. + - When strict mode is enabled, the AI should not return a function call if the function name is not recognized. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "int", + "description": "First integer", + }, + "int_b": { + "type": "int", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + } + ] + + messages = [ + {"role": "user", "content": "Please compute 5 - 7, using your tool."} + ] + response = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual(function_name, "sub", "Function name should be 'sub'") + self.assertEqual(str(args_obj["int_a"]), "5", "Parameter int_a should be 5") + self.assertEqual(str(args_obj["int_b"]), "7", "Parameter int_b should be 7") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fused_moe.py b/test/srt/test_fused_moe.py new file mode 100644 index 0000000..fcff74d --- /dev/null +++ b/test/srt/test_fused_moe.py @@ -0,0 +1,178 @@ +import unittest + +import torch +import torch.nn.functional as F +from tqdm import tqdm +from vllm.model_executor.layers.fused_moe import fused_moe as fused_moe_vllm + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.test.test_utils import CustomTestCase + + +class TestFusedMOE(CustomTestCase): + NUM_EXPERTS = [8, 64] + TOP_KS = [2, 6] + + @staticmethod + def create_random_cuda_tensor(shape, dtype, mean=0, std=0.01): + """Create a random CUDA tensor + + Args: + shape: Tensor shape + dtype: Data type + mean: Mean value + std: Standard deviation + + Returns: + torch.Tensor: Randomly initialized CUDA tensor + """ + return torch.empty(shape, dtype=dtype, device="cuda").normal_(mean, std) + + def get_tolerance(self, dtype): + """Get tolerance values for different data types + + Args: + dtype: Data type + + Returns: + tuple: (relative tolerance, absolute tolerance) + """ + if dtype == torch.float32: + return 1e-3, 1e-5 + elif dtype in [torch.float16, torch.bfloat16]: + return 1e-1, 1e-2 + else: + return 1e-2, 1e-2 # Default values for other types + + def torch_naive_moe(self, a, w1, w2, score, topk): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()(a[mask] @ w1[i].transpose(0, 1)) @ w2[ + i + ].transpose(0, 1) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + def _test_case(self, m, n, k, e, topk, dtype, use_fp8_w8a8=False): + rtol, atol = self.get_tolerance(dtype) + + if use_fp8_w8a8: + # AssertionError: fp8e4nv data type is not supported on CUDA arch < 89 + capability = torch.cuda.get_device_capability() + if not (capability[0] >= 9 or capability == (8, 9)): + return + + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + score = self.create_random_cuda_tensor((m, e), dtype) + + w1_scale = self.create_random_cuda_tensor(e, torch.float32) + w2_scale = self.create_random_cuda_tensor(e, torch.float32) + a1_scale = self.create_random_cuda_tensor(1, torch.float32) + a2_scale = self.create_random_cuda_tensor(1, torch.float32) + + sglang_output = fused_moe( + a, + w1, + w2, + score, + topk, + renormalize=False, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + vllm_output = fused_moe_vllm( + a, + w1, + w2, + score, + topk, + renormalize=False, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + torch.testing.assert_close(sglang_output, vllm_output, rtol=rtol, atol=atol) + + else: + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + score = self.create_random_cuda_tensor((m, e), dtype) + + triton_output = fused_moe(a, w1, w2, score, topk, renormalize=False) + torch_output = self.torch_naive_moe(a, w1, w2, score, topk) + torch.testing.assert_close( + triton_output, torch_output, rtol=rtol, atol=atol + ) + + def test_various_configurations(self): + m_values = [1, 33, 64, 222, 1024 * 128] + n_values = [128, 1024, 2048] + k_values = [128, 511, 1024] + dtypes = [torch.float16, torch.bfloat16] + fp8_modes = [False, True] + + # Calculate total number of tests + total_tests = ( + len(m_values) + * len(n_values) + * len(k_values) + * len(self.NUM_EXPERTS) + * len(self.TOP_KS) + * len(dtypes) + * len(fp8_modes) + ) + + # Create progress bar + with tqdm(total=total_tests, desc="Running MoE tests") as pbar: + for m in m_values: + for n in n_values: + for k in k_values: + for e in self.NUM_EXPERTS: + for topk in self.TOP_KS: + for dtype in dtypes: + for use_fp8_w8a8 in fp8_modes: + with self.subTest( + m=m, + n=n, + k=k, + e=e, + topk=topk, + dtype=dtype, + fp8=use_fp8_w8a8, + ): + self._test_case( + m, + n, + k, + e, + topk, + dtype, + use_fp8_w8a8=use_fp8_w8a8, + ) + pbar.update(1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_get_weights_by_name.py b/test/srt/test_get_weights_by_name.py new file mode 100644 index 0000000..3d404df --- /dev/null +++ b/test/srt/test_get_weights_by_name.py @@ -0,0 +1,183 @@ +import gc +import unittest + +import numpy as np +import requests +import torch +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) +from sglang.utils import terminate_process + + +def _process_return(ret): + if isinstance(ret, list) and len(ret) == 2: + print(f"running assert_allclose on data parallel") + np.testing.assert_allclose(ret[0], ret[1]) + return np.array(ret[0]) + return np.array(ret) + + +class TestGetWeightsByName(CustomTestCase): + + def init_hf_model(self, model_name, tie_word_embeddings): + self.hf_model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="bfloat16", tie_word_embeddings=tie_word_embeddings + ).to("cuda:0") + + def init_backend(self, backend, dp, tp, model_name): + self.backend = backend + self.dp = dp + self.tp = tp + if backend == "Engine": + self.engine = sgl.Engine( + model_path=model_name, + random_seed=42, + tp_size=tp, + dp_size=dp, + ) + else: + self.process = popen_launch_server( + model_name, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--tp-size", + str(tp), + "--dp-size", + str(dp), + ), + ) + + def clean_up(self): + del self.hf_model + gc.collect() + torch.cuda.empty_cache() + if self.backend == "Engine": + self.engine.shutdown() + else: + terminate_process(self.process) + + def assert_tie_word_embeddings(self, truncate_size): + print("assert_tie_word_embeddings") + if self.backend == "Engine": + backend_ret = _process_return( + self.engine.get_weights_by_name("lm_head.weight", truncate_size) + ) + else: + backend_ret = _process_return( + requests.get( + f"{DEFAULT_URL_FOR_TEST}/get_weights_by_name", + json={"name": "lm_head.weight", "truncate_size": truncate_size}, + ).json() + ) + print("assert_tie_word_embeddings of hf and backend") + assert np.allclose( + self.hf_model.get_parameter("model.embed_tokens.weight") + .cpu() + .detach() + .float() + .numpy()[:truncate_size], + backend_ret, + ) + assert np.allclose( + self.hf_model.get_parameter("lm_head.weight") + .cpu() + .detach() + .float() + .numpy()[:truncate_size], + self.hf_model.get_parameter("model.embed_tokens.weight") + .cpu() + .detach() + .float() + .numpy()[:truncate_size], + ) + + def assert_weights_all_close(self, param_name, truncate_size): + print( + f"param_name: {param_name}, backend: {self.backend}, dp: {self.dp}, tp: {self.tp}" + ) + param = self.hf_model.get_parameter(param_name)[:truncate_size] + param_np = param.cpu().detach().float().numpy() + + if self.backend == "Engine": + engine_ret = self.engine.get_weights_by_name(param_name, truncate_size) + engine_ret = _process_return(engine_ret) + np.testing.assert_allclose(engine_ret, param_np, rtol=1e-5, atol=1e-5) + + if self.backend == "Runtime": + runtime_ret = requests.get( + f"{DEFAULT_URL_FOR_TEST}/get_weights_by_name", + json={"name": param_name, "truncate_size": truncate_size}, + ).json() + runtime_ret = _process_return(runtime_ret) + np.testing.assert_allclose(runtime_ret, param_np, rtol=1e-5, atol=1e-5) + + def test_get_weights_by_name(self): + if is_in_ci(): + test_suits = [ + ("Engine", 1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST), + ] + else: + test_suits = [ + ("Runtime", 1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST), + ("Engine", 1, 1, DEFAULT_MODEL_NAME_FOR_TEST), + ] + if torch.cuda.device_count() >= 2: + test_suits.append(("Engine", 1, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST)) + test_suits.append(("Runtime", 2, 1, DEFAULT_MODEL_NAME_FOR_TEST)) + + if torch.cuda.device_count() >= 4: + test_suits.extend( + [ + ("Engine", 2, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST), + ("Runtime", 2, 2, DEFAULT_MODEL_NAME_FOR_TEST), + ] + ) + + parameters = [ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "model.layers.1.self_attn.q_proj.weight", + "model.layers.2.self_attn.k_proj.weight", + "model.layers.3.self_attn.v_proj.weight", + "model.layers.4.self_attn.o_proj.weight", + "model.layers.5.mlp.gate_proj.weight", + "model.layers.6.mlp.up_proj.weight", + "model.layers.7.mlp.down_proj.weight", + "model.layers.8.post_attention_layernorm.weight", + "model.norm.weight", + "lm_head.weight", + ] + + truncate_size = 100 + + for test_suit in test_suits: + if test_suit[-1] == DEFAULT_MODEL_NAME_FOR_TEST: + tie_word_embeddings = False + else: + tie_word_embeddings = True + + self.init_hf_model(test_suit[-1], tie_word_embeddings) + self.init_backend(*test_suit) + + for param_name in parameters: + self.assert_weights_all_close(param_name, truncate_size) + + if tie_word_embeddings: + self.assert_tie_word_embeddings(truncate_size) + + self.clean_up() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_gguf.py b/test/srt/test_gguf.py new file mode 100644 index 0000000..e977606 --- /dev/null +++ b/test/srt/test_gguf.py @@ -0,0 +1,27 @@ +import unittest + +from huggingface_hub import hf_hub_download + +import sglang as sgl +from sglang.test.test_utils import CustomTestCase + + +class TestGGUF(CustomTestCase): + def test_models(self): + prompt = "Today is a sunny day and I like" + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + model_path = hf_hub_download( + "Qwen/Qwen2-1.5B-Instruct-GGUF", + filename="qwen2-1_5b-instruct-q4_k_m.gguf", + ) + + engine = sgl.Engine(model_path=model_path, random_seed=42, cuda_graph_max_bs=2) + outputs = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + self.assertEqual(outputs, " it. I have a lot of work") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_gptqmodel_dynamic.py b/test/srt/test_gptqmodel_dynamic.py new file mode 100644 index 0000000..54dbaf4 --- /dev/null +++ b/test/srt/test_gptqmodel_dynamic.py @@ -0,0 +1,214 @@ +import time +import unittest + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def check_quant_method(model_path: str, use_marlin_kernel: bool): + from sglang.srt.configs.device_config import DeviceConfig + from sglang.srt.configs.load_config import LoadConfig + from sglang.srt.configs.model_config import AttentionArch, ModelConfig + from sglang.srt.distributed import ( + get_tp_group, + init_distributed_environment, + initialize_model_parallel, + set_custom_all_reduce, + ) + from sglang.srt.distributed.parallel_state import monkey_patch_vllm_parallel_state + from sglang.srt.layers.quantization import get_dynamic_override + from sglang.srt.model_loader import get_model + from sglang.srt.server_args import PortArgs, ServerArgs + + try: + init_distributed_environment( + backend="nccl", + world_size=1, + rank=0, + local_rank=0, + distributed_init_method="tcp://127.0.0.1:2646", + ) + initialize_model_parallel(tensor_model_parallel_size=1) + monkey_patch_vllm_parallel_state() + except AssertionError: + # ignore this error: tensor model parallel group is already initialized + pass + + server_args = ServerArgs(model_path=model_path, dtype=torch.float16) + model_config = ModelConfig( + server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + revision=server_args.revision, + context_length=server_args.context_length, + model_override_args=server_args.json_model_override_args, + is_embedding=server_args.is_embedding, + dtype=server_args.dtype, + quantization=server_args.quantization, + ) + + load_config = LoadConfig() + device_config = DeviceConfig("cuda") + model = get_model( + model_config=model_config, load_config=load_config, device_config=device_config + ) + + from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod + from vllm.model_executor.layers.quantization.gptq_marlin import ( + GPTQMarlinLinearMethod, + ) + + from sglang.srt.layers.linear import UnquantizedLinearMethod + + linear_method_cls = ( + GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod) + ) + + for name, submodule in model.named_modules(): + if name == "lm_head": + assert isinstance(submodule.quant_method, linear_method_cls) + elif name == "model.layers.0.self_attn.qkv_proj": + # The first layer is quantized using bits=4, group_size=128 + # desc_act=True + assert isinstance(submodule.quant_method, linear_method_cls) + config = submodule.quant_method.quant_config + assert config.weight_bits == 4 + assert config.group_size == 128 + assert config.desc_act + elif name == "model.layers.1.self_attn.qkv_proj": + # The second layer is quantized using bits=8, group_size=32 + # desc_act=False + assert isinstance(submodule.quant_method, linear_method_cls) + config = submodule.quant_method.quant_config + assert get_dynamic_override(config, layer_name=name, key="bits") == 8 + assert get_dynamic_override(config, layer_name=name, key="group_size") == 32 + assert not get_dynamic_override(config, layer_name=name, key="desc_act") + elif ( + name == "model.layers.2.self_attn.qkv_proj" + or name == "model.layers.2.mlp.gate_up_proj" + ): + # All other layers (layer index >= 2) are not quantized + assert isinstance(submodule.quant_method, UnquantizedLinearMethod) + + del model + + +# GPTQ with Dynamic Per/Module Quantization Control +# Leverages GPTQModel (pypi) to produce the `dynamic` models +# Test GPTQ fallback kernel that is not Marlin +class TestGPTQModelDynamic(CustomTestCase): + MODEL_PATH = ( + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse" + ) + + @classmethod + def setUpClass(cls): + cls.model = cls.MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "float16"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.001, + }, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.time() + result = self.run_decode(max_tokens) + tok = time.time() + + print(f"result = `{result}`") + + self.assertIn("paris", result["text"].lower()) + + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + self.assertGreaterEqual(throughput, 140) + + def test_gptq_module(self): + check_quant_method(self.MODEL_PATH, use_marlin_kernel=False) + + +# GPTQ with Dynamic Per/Module Quantization Control +# Leverages GPTQModel (pypi) to produce the `dynamic` models +# Test Marlin kernel +class TestGPTQModelDynamicWithMarlin(CustomTestCase): + MODEL_PATH = ( + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue" + ) + + @classmethod + def setUpClass(cls): + cls.model = cls.MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "float16"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.001, + }, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.time() + result = self.run_decode(max_tokens) + tok = time.time() + + print(f"result = `{result}`") + + assert "paris" in result["text"].lower() + + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 140 + + def test_gptq_marlin_module(self): + check_quant_method(self.MODEL_PATH, use_marlin_kernel=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_health_check.py b/test/srt/test_health_check.py new file mode 100644 index 0000000..1f101e4 --- /dev/null +++ b/test/srt/test_health_check.py @@ -0,0 +1,28 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestHealthCheck(CustomTestCase): + def test_health_check(self): + """Test that metrics endpoint returns data when enabled""" + with self.assertRaises(TimeoutError): + popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=60, + other_args=[ + "--disable-cuda-graph", + "--json-model-override-args", + '{"architectures": ["LlamaForCausalLMForHealthTest"]}', + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_hicache.py b/test/srt/test_hicache.py new file mode 100644 index 0000000..ac7a27f --- /dev/null +++ b/test/srt/test_hicache.py @@ -0,0 +1,45 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPageSize(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-hierarchical-cache"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_hicache_mla.py b/test/srt/test_hicache_mla.py new file mode 100644 index 0000000..8396615 --- /dev/null +++ b/test/srt/test_hicache_mla.py @@ -0,0 +1,57 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestHierarchicalMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--trust-remote-code", "--enable-hierarchical-cache"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_hidden_states.py b/test/srt/test_hidden_states.py new file mode 100644 index 0000000..81e42f7 --- /dev/null +++ b/test/srt/test_hidden_states.py @@ -0,0 +1,137 @@ +import unittest + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestHiddenState(CustomTestCase): + def test_return_hidden_states(self): + prompts = ["Today is", "Today is a sunny day and I like"] + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + tokenizer = AutoTokenizer.from_pretrained(model_path) + input_ids = tokenizer(prompts).input_ids + + sampling_params = { + "temperature": 0, + "max_new_tokens": 8, + } + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + skip_tokenizer_init=True, + ) + outputs = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + engine.shutdown() + + for output in outputs: + self.assertEqual(len(output["meta_info"]["hidden_states"]), 8) + for i in range(len(output["meta_info"]["hidden_states"])): + assert isinstance(output["meta_info"]["hidden_states"][i], list) + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + # Checks that splicing of the batch was done correctly + self.assertGreater( + outputs[1]["meta_info"]["hidden_states"][0].shape[0], + outputs[0]["meta_info"]["hidden_states"][0].shape[0], + ) + + model = AutoModelForCausalLM.from_pretrained( + model_path, torch_dtype=torch.bfloat16, device_map="cuda" + ) + + for input_id, output in zip(input_ids, outputs): + with torch.inference_mode(): + hf_out = model( + torch.tensor( + [input_id + output["output_ids"][:-1]], device=model.device + ), + output_hidden_states=True, + ) + print("=== HF Hiddens ===") + print(hf_out["hidden_states"][-1][0]) + sg_hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ).to("cuda") + print("=== SRT Hiddens ===") + print(sg_hidden_states) + + print( + f"Max diff: {torch.max(torch.abs(hf_out['hidden_states'][-1][0] - sg_hidden_states))}" + ) + + atol = 0.8 + self.assertTrue( + torch.allclose( + hf_out["hidden_states"][-1][0], + sg_hidden_states, + atol=atol, + rtol=0, + ) + ) + + def test_repeatedly_changes_hidden_states(self): + prompts = ["Today is", "Today is a sunny day and I like"] + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + tokenizer = AutoTokenizer.from_pretrained(model_path) + input_ids = tokenizer(prompts).input_ids + + sampling_params = { + "temperature": 0, + "max_new_tokens": 8, + } + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + skip_tokenizer_init=True, + ) + outputs_completion_first_round = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + outputs_hidden_state = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=False, + ) + + outputs_completion_last_round = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + engine.shutdown() + + for ( + output_completion_first_round, + output_hidden_state, + output_completion_last_round, + ) in zip( + outputs_completion_first_round, + outputs_hidden_state, + outputs_completion_last_round, + ): + self.assertEqual( + len(output_completion_first_round["meta_info"]["hidden_states"]), 8 + ) + self.assertNotIn("hidden_states", output_hidden_state["meta_info"]) + self.assertEqual( + len(output_completion_last_round["meta_info"]["hidden_states"]), 8 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_input_embeddings.py b/test/srt/test_input_embeddings.py new file mode 100644 index 0000000..7efd665 --- /dev/null +++ b/test/srt/test_input_embeddings.py @@ -0,0 +1,151 @@ +import json +import os +import tempfile +import unittest + +import requests +from transformers import AutoModelForCausalLM, AutoTokenizer + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestInputEmbeds(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model) + cls.ref_model = AutoModelForCausalLM.from_pretrained(cls.model) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--disable-radix", "--cuda-graph-max-bs", 4], + ) + cls.texts = [ + "The capital of France is", + "What is the best time of year to visit Japan for cherry blossoms?", + ] + + def generate_input_embeddings(self, text): + """Generate input embeddings for a given text.""" + input_ids = self.tokenizer(text, return_tensors="pt")["input_ids"] + embeddings = self.ref_model.get_input_embeddings()(input_ids) + return embeddings.squeeze().tolist() # Convert tensor to a list for API use + + def send_request(self, payload): + """Send a POST request to the /generate endpoint and return the response.""" + response = requests.post( + self.base_url + "/generate", + json=payload, + timeout=30, # Set a reasonable timeout for the API request + ) + if response.status_code == 200: + return response.json() + return { + "error": f"Request failed with status {response.status_code}: {response.text}" + } + + def send_file_request(self, file_path): + """Send a POST request to the /generate_from_file endpoint with a file.""" + with open(file_path, "rb") as f: + response = requests.post( + self.base_url + "/generate_from_file", + files={"file": f}, + timeout=30, # Set a reasonable timeout for the API request + ) + if response.status_code == 200: + return response.json() + return { + "error": f"Request failed with status {response.status_code}: {response.text}" + } + + def test_text_based_response(self): + """Test and print API responses using text-based input.""" + for text in self.texts: + payload = { + "model": self.model, + "text": text, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + response = self.send_request(payload) + print( + f"Text Input: {text}\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + + def test_embedding_based_response(self): + """Test and print API responses using input embeddings.""" + for text in self.texts: + embeddings = self.generate_input_embeddings(text) + payload = { + "model": self.model, + "input_embeds": embeddings, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + response = self.send_request(payload) + print( + f"Embeddings Input (for text '{text}'):\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + + def test_compare_text_vs_embedding(self): + """Test and compare responses for text-based and embedding-based inputs.""" + for text in self.texts: + # Text-based payload + text_payload = { + "model": self.model, + "text": text, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + # Embedding-based payload + embeddings = self.generate_input_embeddings(text) + embed_payload = { + "model": self.model, + "input_embeds": embeddings, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + # Get responses + text_response = self.send_request(text_payload) + embed_response = self.send_request(embed_payload) + # Print responses + print( + f"Text Input: {text}\nText-Based Response: {json.dumps(text_response, indent=2)}\n" + ) + print( + f"Embeddings Input (for text '{text}'):\nEmbedding-Based Response: {json.dumps(embed_response, indent=2)}\n{'-' * 80}" + ) + # This is flaky, so we skip this temporarily + # self.assertEqual(text_response["text"], embed_response["text"]) + + def test_generate_from_file(self): + """Test the /generate_from_file endpoint using tokenized embeddings.""" + for text in self.texts: + embeddings = self.generate_input_embeddings(text) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as tmp_file: + json.dump(embeddings, tmp_file) + tmp_file_path = tmp_file.name + + try: + response = self.send_file_request(tmp_file_path) + print( + f"Text Input: {text}\nResponse from /generate_from_file: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + finally: + # Ensure the temporary file is deleted + os.remove(tmp_file_path) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_int8_kernel.py b/test/srt/test_int8_kernel.py new file mode 100644 index 0000000..959aab9 --- /dev/null +++ b/test/srt/test_int8_kernel.py @@ -0,0 +1,164 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 +from sglang.test.test_utils import CustomTestCase + + +def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + return C.reshape(origin_C_shape).to(output_dtype) + + +def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + # Calculate routing + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], output_dtype=a.dtype + ) + # Activation function + act_out = SiluAndMul().forward_native(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = per_token_quant_int8(act_out) + + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype + ) + # Apply routing weights and sum + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8Int8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33] + N = [128, 1024] + K = [256, 4096] + E = [8] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_int8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # Initialize int8 quantization parameters + factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / 10 + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk) + out = fused_moe( + a, + w1, + w2, + score, + topk, + renormalize=False, + use_fp8_w8a8=False, # Not using fp8 + use_int8_w8a16=False, # Not using int8-w8a16 + use_int8_w8a8=True, # Using int8-w8a8 + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=None, # Not using block quantization + ) + + # Check results + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.05 + ) + + def test_w8a8_int8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_int8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/test_json_constrained.py b/test/srt/test_json_constrained.py new file mode 100644 index 0000000..7b42319 --- /dev/null +++ b/test/srt/test_json_constrained.py @@ -0,0 +1,147 @@ +""" +python3 -m unittest test_json_constrained.TestJSONConstrainedOutlinesBackend.test_json_generate +python3 -m unittest test_json_constrained.TestJSONConstrainedXGrammarBackend.test_json_generate +python3 -m unittest test_json_constrained.TestJSONConstrainedLLGuidanceBackend.test_json_generate +""" + +import json +import unittest +from concurrent.futures import ThreadPoolExecutor + +import openai +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def setup_class(cls, backend: str): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], + "additionalProperties": False, + } + ) + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + +class TestJSONConstrainedOutlinesBackend(CustomTestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="outlines") + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, json_schema, return_logprob=False, top_logprobs_num=0, n=1): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "stop_token_ids": [119690], + "json_schema": json_schema, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + ret = response.json() + print(json.dumps(ret)) + print("=" * 100) + + if not json_schema: + return + + # Make sure the json output is valid + try: + js_obj = json.loads(ret["text"]) + except (TypeError, json.decoder.JSONDecodeError): + raise + + self.assertIsInstance(js_obj["name"], str) + self.assertIsInstance(js_obj["population"], int) + + def test_json_generate(self): + self.run_decode(json_schema=self.json_schema) + + def test_json_openai(self): + client = openai.Client(api_key="EMPTY", base_url=f"{self.base_url}/v1") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=128, + response_format={ + "type": "json_schema", + "json_schema": {"name": "foo", "schema": json.loads(self.json_schema)}, + }, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + + self.assertIsInstance(js_obj["name"], str) + self.assertIsInstance(js_obj["population"], int) + + def test_mix_json_and_other(self): + json_schemas = [None, None, self.json_schema, self.json_schema] * 10 + + with ThreadPoolExecutor(len(json_schemas)) as executor: + list(executor.map(self.run_decode, json_schemas)) + + +class TestJSONConstrainedXGrammarBackend(TestJSONConstrainedOutlinesBackend): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="xgrammar") + + +class TestJSONConstrainedLLGuidanceBackend(TestJSONConstrainedOutlinesBackend): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="llguidance") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_large_max_new_tokens.py b/test/srt/test_large_max_new_tokens.py new file mode 100644 index 0000000..a6d6bae --- /dev/null +++ b/test/srt/test_large_max_new_tokens.py @@ -0,0 +1,103 @@ +""" +python3 -m unittest test_large_max_new_tokens.TestLargeMaxNewTokens.test_chat_completion +""" + +import os +import time +import unittest +from concurrent.futures import ThreadPoolExecutor + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + popen_launch_server, +) + + +class TestLargeMaxNewTokens(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=( + "--max-total-token", + "1536", + "--context-len", + "8192", + "--decode-log-interval", + "2", + ), + env={"SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "256", **os.environ}, + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def run_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "Please repeat the world 'hello' for 10000 times.", + }, + ], + temperature=0, + ) + return response + + def test_chat_completion(self): + num_requests = 4 + + futures = [] + with ThreadPoolExecutor(num_requests) as executor: + # Send multiple requests + for i in range(num_requests): + futures.append(executor.submit(self.run_chat_completion)) + + # Ensure that they are running concurrently + pt = 0 + while pt >= 0: + time.sleep(5) + lines = open(STDERR_FILENAME).readlines() + for line in lines[pt:]: + print(line, end="", flush=True) + if f"#running-req: {num_requests}" in line: + all_requests_running = True + pt = -1 + break + pt += 1 + + assert all_requests_running + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_matched_stop.py b/test/srt/test_matched_stop.py new file mode 100644 index 0000000..357b07f --- /dev/null +++ b/test/srt/test_matched_stop.py @@ -0,0 +1,140 @@ +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +MANY_NEW_TOKENS_PROMPT = """ +Please write an extremely detailed and vivid fantasy story, set in a world full of intricate magic systems, political intrigue, and complex characters. +Ensure that you thoroughly describe every scene, character's motivations, and the environment. Include long, engaging dialogues and elaborate on the inner thoughts of the characters. +Each section should be as comprehensive as possible to create a rich and immersive experience for the reader. +The story should span multiple events, challenges, and character developments over time. Aim to make the story at least 3,000 words long. +""" + + +class TestMatchedStop(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=300, + other_args=["--max-running-requests", "10"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_completions_generation( + self, + prompt=MANY_NEW_TOKENS_PROMPT, + max_tokens=1, + stop=None, + finish_reason=None, + matched_stop=None, + ): + payload = { + "prompt": prompt, + "model": self.model, + "temperature": 0, + "top_p": 1, + "max_tokens": max_tokens, + } + + if stop is not None: + payload["stop"] = stop + + response_completions = requests.post( + self.base_url + "/v1/completions", + json=payload, + ) + print(json.dumps(response_completions.json())) + print("=" * 100) + + assert ( + response_completions.json()["choices"][0]["finish_reason"] == finish_reason + ) + assert response_completions.json()["choices"][0]["matched_stop"] == matched_stop + + def run_chat_completions_generation( + self, + prompt=MANY_NEW_TOKENS_PROMPT, + max_tokens=1, + stop=None, + finish_reason=None, + matched_stop=None, + ): + chat_payload = { + "model": self.model, + "messages": [ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": prompt}, + ], + "temperature": 0, + "top_p": 1, + "max_tokens": max_tokens, + } + + if stop is not None: + chat_payload["stop"] = stop + + response_chat = requests.post( + self.base_url + "/v1/chat/completions", + json=chat_payload, + ) + print(json.dumps(response_chat.json())) + print("=" * 100) + + assert response_chat.json()["choices"][0]["finish_reason"] == finish_reason + assert response_chat.json()["choices"][0]["matched_stop"] == matched_stop + + def test_finish_stop_str(self): + self.run_completions_generation( + max_tokens=1000, stop="\n", finish_reason="stop", matched_stop="\n" + ) + self.run_chat_completions_generation( + max_tokens=1000, stop="\n", finish_reason="stop", matched_stop="\n" + ) + + def test_finish_stop_eos(self): + llama_format_prompt = """ + <|begin_of_text|><|start_header_id|>system<|end_header_id|> + You are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|> + + What is 2 + 2?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + """ + eos_token_id = 128009 + self.run_completions_generation( + prompt=llama_format_prompt, + max_tokens=1000, + finish_reason="stop", + matched_stop=eos_token_id, + ) + self.run_chat_completions_generation( + prompt="What is 2 + 2?", + max_tokens=1000, + finish_reason="stop", + matched_stop=eos_token_id, + ) + + def test_finish_length(self): + self.run_completions_generation( + max_tokens=5, finish_reason="length", matched_stop=None + ) + self.run_chat_completions_generation( + max_tokens=5, finish_reason="length", matched_stop=None + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_metrics.py b/test/srt/test_metrics.py new file mode 100644 index 0000000..fadd427 --- /dev/null +++ b/test/srt/test_metrics.py @@ -0,0 +1,88 @@ +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEnableMetrics(CustomTestCase): + def test_metrics_enabled(self): + """Test that metrics endpoint returns data when enabled""" + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-metrics", "--cuda-graph-max-bs", 2], + ) + + try: + # Make some requests to generate some metrics + response = requests.get(f"{DEFAULT_URL_FOR_TEST}/health_generate") + self.assertEqual(response.status_code, 200) + + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + "stream": True, + }, + stream=True, + ) + for _ in response.iter_lines(decode_unicode=False): + pass + + # Get metrics + metrics_response = requests.get(f"{DEFAULT_URL_FOR_TEST}/metrics") + self.assertEqual(metrics_response.status_code, 200) + metrics_content = metrics_response.text + + print(f"metrics_content=\n{metrics_content}") + + # Verify essential metrics are present + essential_metrics = [ + "sglang:num_running_reqs", + "sglang:num_used_tokens", + "sglang:token_usage", + "sglang:gen_throughput", + "sglang:num_queue_reqs", + "sglang:cache_hit_rate", + "sglang:spec_accept_length", + "sglang:prompt_tokens_total", + "sglang:generation_tokens_total", + "sglang:cached_tokens_total", + "sglang:num_requests_total", + "sglang:time_to_first_token_seconds", + "sglang:time_per_output_token_seconds", + "sglang:inter_token_latency_seconds", + "sglang:e2e_request_latency_seconds", + ] + + for metric in essential_metrics: + self.assertIn(metric, metrics_content, f"Missing metric: {metric}") + + # Verify model name label is present and correct + expected_model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + self.assertIn(f'model_name="{expected_model_name}"', metrics_content) + + # Verify metrics have values (not empty) + self.assertIn("_sum{", metrics_content) + self.assertIn("_count{", metrics_content) + self.assertIn("_bucket{", metrics_content) + + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla.py b/test/srt/test_mla.py new file mode 100644 index 0000000..40f82e0 --- /dev/null +++ b/test/srt/test_mla.py @@ -0,0 +1,62 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--enable-torch-compile", + "--cuda-graph-max-bs", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_deepseek_v3.py b/test/srt/test_mla_deepseek_v3.py new file mode 100644 index 0000000..096c5a9 --- /dev/null +++ b/test/srt/test_mla_deepseek_v3.py @@ -0,0 +1,111 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLADeepseekV3(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestDeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft", + "lmsys/sglang-ci-dsv3-test-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_flashinfer.py b/test/srt/test_mla_flashinfer.py new file mode 100644 index 0000000..4fa3eb5 --- /dev/null +++ b/test/srt/test_mla_flashinfer.py @@ -0,0 +1,166 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFlashinferMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--enable-torch-compile", + "--cuda-graph-max-bs", + "2", + "--enable-flashinfer-mla", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestFlashinferMLANoRagged(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--enable-torch-compile", + "--disable-cuda-graph", + "--cuda-graph-max-bs", + "4", + "--enable-flashinfer-mla", + "--flashinfer-mla-disable-ragged", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestFlashinferMLAMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft", + "lmsys/sglang-ci-dsv3-test-NextN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--enable-flashinfer-mla", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_fp8.py b/test/srt/test_mla_fp8.py new file mode 100644 index 0000000..a2fac98 --- /dev/null +++ b/test/srt/test_mla_fp8.py @@ -0,0 +1,49 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--kv-cache-dtype", + "fp8_e5m2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + assert metrics["score"] >= 0.8 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_int8_deepseek_v3.py b/test/srt/test_mla_int8_deepseek_v3.py new file mode 100644 index 0000000..6437000 --- /dev/null +++ b/test/srt/test_mla_int8_deepseek_v3.py @@ -0,0 +1,210 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLADeepseekV3ChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreaterEqual(metrics["accuracy"], 0.61) + + +class TestDeepseekV3MTPChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft", + "sgl-project/sglang-ci-dsv3-channel-int8-test-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +class TestMLADeepseekV3BlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestDeepseekV3MTPBlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft", + "sgl-project/sglang-ci-dsv3-block-int8-test-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["avg_spec_accept_length"] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_tp.py b/test/srt/test_mla_tp.py new file mode 100644 index 0000000..e957cf2 --- /dev/null +++ b/test/srt/test_mla_tp.py @@ -0,0 +1,66 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDeepseekTP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreater(metrics["accuracy"], 0.62) + + def test_gsm8k_bs1(self): + # test torch compile accuracy for bs=1 + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=10, + max_new_tokens=512, + parallel=1, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreater(metrics["accuracy"], 0.62) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_modelopt_fp8kvcache.py b/test/srt/test_modelopt_fp8kvcache.py new file mode 100644 index 0000000..a4704c2 --- /dev/null +++ b/test/srt/test_modelopt_fp8kvcache.py @@ -0,0 +1,30 @@ +import unittest + +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod + +from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptFp8Config, + ModelOptFp8KVCacheMethod, +) +from sglang.test.test_utils import CustomTestCase + + +class TestModelOptFp8KVCacheMethod(CustomTestCase): + def test_kv_cache_method_initialization(self): + """Test that ModelOptFp8KVCacheMethod can be instantiated and + inherits from BaseKVCacheMethod.""" + # Create a ModelOptFp8Config object + quant_config = ModelOptFp8Config(is_checkpoint_fp8_serialized=True) + + # Instantiate the KV cache method + kv_cache_method = ModelOptFp8KVCacheMethod(quant_config) + + # Check inheritance + self.assertIsInstance(kv_cache_method, BaseKVCacheMethod) + + # Check that the quant_config is stored + self.assertEqual(kv_cache_method.quant_config, quant_config) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_models_from_modelscope.py b/test/srt/test_models_from_modelscope.py new file mode 100644 index 0000000..dacaae3 --- /dev/null +++ b/test/srt/test_models_from_modelscope.py @@ -0,0 +1,40 @@ +import os +import shutil +import subprocess +import unittest +from unittest import mock + +from sglang.srt.utils import prepare_model_and_tokenizer +from sglang.test.test_utils import CustomTestCase + + +class TestDownloadFromModelScope(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = "iic/nlp_lstmcrf_word-segmentation_chinese-news" + stat, output = subprocess.getstatusoutput("pip install modelscope") + + cls.with_modelscope_environ = {k: v for k, v in os.environ.items()} + cls.with_modelscope_environ["SGLANG_USE_MODELSCOPE"] = "True" + + @classmethod + def tearDownClass(cls): + pass + + def test_prepare_model_and_tokenizer(self): + from modelscope.utils.file_utils import get_model_cache_root + + model_cache_root = get_model_cache_root() + if os.path.exists(model_cache_root): + shutil.rmtree(model_cache_root) + with mock.patch.dict(os.environ, self.with_modelscope_environ, clear=True): + model_path, tokenizer_path = prepare_model_and_tokenizer( + self.model, self.model + ) + assert os.path.exists(os.path.join(model_path, "pytorch_model.bin")) + assert os.path.exists(os.path.join(tokenizer_path, "config.json")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_moe_deepep.py b/test/srt/test_moe_deepep.py new file mode 100644 index 0000000..a25146e --- /dev/null +++ b/test/srt/test_moe_deepep.py @@ -0,0 +1,89 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPureTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--enable-deepep-moe", + "--disable-cuda-graph", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + +class TestDPAttn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--dp", + "2", + "--enable-dp-attention", + "--enable-deepep-moe", + "--disable-cuda-graph", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_moe_ep.py b/test/srt/test_moe_ep.py new file mode 100644 index 0000000..284dcba --- /dev/null +++ b/test/srt/test_moe_ep.py @@ -0,0 +1,114 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEpMoE(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + "--enable-ep-moe", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +class TestEpMoEFP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + "--enable-ep-moe", + "--quantization", + "fp8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_moe_eval_accuracy_large.py b/test/srt/test_moe_eval_accuracy_large.py new file mode 100644 index 0000000..26bbd24 --- /dev/null +++ b/test/srt/test_moe_eval_accuracy_large.py @@ -0,0 +1,94 @@ +""" +Usage: +python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestMoEEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--log-level-http", + "warning", + "--tp", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.62) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') + + def test_human_eval(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.40) + + if is_in_ci(): + write_github_step_summary( + f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' + ) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.61) + + if is_in_ci(): + write_github_step_summary( + f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_gsm8k_eval.py b/test/srt/test_nightly_gsm8k_eval.py new file mode 100644 index 0000000..5250159 --- /dev/null +++ b/test/srt/test_nightly_gsm8k_eval.py @@ -0,0 +1,188 @@ +import json +import os +import unittest +import warnings +from datetime import datetime +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +MODEL_SCORE_THRESHOLDS = { + "meta-llama/Llama-3.1-8B-Instruct": 0.82, + "mistralai/Mistral-7B-Instruct-v0.3": 0.58, + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85, + "google/gemma-2-27b-it": 0.92, + "meta-llama/Llama-3.1-70B-Instruct": 0.95, + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.64, + "Qwen/Qwen2-57B-A14B-Instruct": 0.86, + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83, + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84, + # The threshold of neuralmagic/gemma-2-2b-it-FP8 should be 0.6, but this model has some accuracy regression. + # The fix is tracked at https://github.com/sgl-project/sglang/issues/4324, we set it to 0.50, for now, to make CI green. + "neuralmagic/gemma-2-2b-it-FP8": 0.50, + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65, + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94, + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82, +} + + +def parse_models(model_string): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def popen_launch_server_wrapper(base_url, model, is_fp8, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_fp8: + if "Llama-3" in model or "gemma-2" in model: + other_args.extend(["--kv-cache-dtype", "fp8_e5m2"]) + elif "Qwen2-72B-Instruct-FP8" in model: + other_args.extend(["--quantization", "fp8"]) + elif "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" in model: + other_args.extend([]) + else: + other_args.extend(["--quantization", "fp8", "--kv-cache-dtype", "fp8_e5m2"]) + if is_tp2: + other_args.extend(["--tp", "2"]) + if "DeepSeek" in model: + other_args.extend(["--mem-frac", "0.85"]) + if "AWQ" in model: + other_args.extend(["--quantization", "awq"]) + elif "GPTQ" in model: + other_args.extend(["--quantization", "gptq"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def write_results_to_json(model, metrics, mode="a"): + result = { + "timestamp": datetime.now().isoformat(), + "model": model, + "metrics": metrics, + "score": metrics["score"], + } + + existing_results = [] + if mode == "a" and os.path.exists("results.json"): + try: + with open("results.json", "r") as f: + existing_results = json.load(f) + except json.JSONDecodeError: + existing_results = [] + + if isinstance(existing_results, list): + existing_results.append(result) + else: + existing_results = [result] + + with open("results.json", "w") as f: + json.dump(existing_results, f, indent=2) + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary(f"### TestNightlyGsm8KEval\n{summary}") + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + process = popen_launch_server_wrapper( + self.base_url, model, is_fp8, is_tp2 + ) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_human_eval.py b/test/srt/test_nightly_human_eval.py new file mode 100644 index 0000000..2a1ea3b --- /dev/null +++ b/test/srt/test_nightly_human_eval.py @@ -0,0 +1,105 @@ +import os +import shutil +import signal +import subprocess +import unittest + +from test_nightly_gsm8k_eval import parse_models, popen_launch_server_wrapper + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, +) + + +class TestNightlyHumanEval(CustomTestCase): + @classmethod + def setUpClass(cls): + if is_in_ci(): + cls.model_groups = [([DEFAULT_MODEL_NAME_FOR_TEST], False, False)] + else: + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), + ( + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), + True, + False, + ), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = None + cls.eval_process = None + + @classmethod + def tearDownClass(cls): + if cls.process: + kill_process_tree(cls.process.pid) + if cls.eval_process: + kill_process_tree(cls.eval_process.pid) + + def run_evalplus(self, model): + print("Delete evalplus results") + shutil.rmtree("evalplus_results", ignore_errors=True) + cmd = [ + "evalplus.evaluate", + "--model", + model, + "--dataset", + "humaneval", + "--backend", + "openai", + "--base-url", + "http://localhost:6157/v1", + "--greedy", + ] + + try: + self.eval_process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + preexec_fn=os.setsid, + ) + + stdout, stderr = self.eval_process.communicate(timeout=600) + + if self.eval_process.returncode != 0: + print(f"Fail to human eval model={model} err={stderr}") + + print("=" * 42) + print(stdout) + print("=" * 42) + except subprocess.TimeoutExpired: + if self.eval_process: + os.killpg(os.getpgid(self.eval_process.pid), signal.SIGTERM) + print(f"Timeout during evaluation for model={model}") + except Exception as e: + print(f"Error running evalplus for model={model} {str(e)}") + if self.eval_process: + os.killpg(os.getpgid(self.eval_process.pid), signal.SIGTERM) + + def test_human_eval_all_models(self): + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + # NOTE: only Llama for now + if "Llama" in model: + with self.subTest(model=model): + self.process = popen_launch_server_wrapper( + self.base_url, model, is_fp8, is_tp2 + ) + self.run_evalplus(model) + self.tearDownClass() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_math_eval.py b/test/srt/test_nightly_math_eval.py new file mode 100644 index 0000000..20db124 --- /dev/null +++ b/test/srt/test_nightly_math_eval.py @@ -0,0 +1,47 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--log-level-http", "warning"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_math(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="math", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreaterEqual( + metrics["score"], 0.519 - 0.02 + ) # -2% to account for sampling variance + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_no_chunked_prefill.py b/test/srt/test_no_chunked_prefill.py new file mode 100644 index 0000000..59869ff --- /dev/null +++ b/test/srt/test_no_chunked_prefill.py @@ -0,0 +1,30 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_serving, + run_mmlu_test, +) + + +class TestNoChunkedPrefill(CustomTestCase): + + def test_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, enable_mixed_chunk=False, chunked_prefill_size=-1 + ) + + def test_no_chunked_prefill_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=10, + request_rate=float("inf"), + other_server_args=["--disable-radix-cache", "--chunked-prefill-size", "-1"], + ) + + assert res["completed"] == 10 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_no_overlap_scheduler.py b/test/srt/test_no_overlap_scheduler.py new file mode 100644 index 0000000..d236819 --- /dev/null +++ b/test/srt/test_no_overlap_scheduler.py @@ -0,0 +1,35 @@ +""" +Usage: +python3 -m unittest test_overlap_schedule.TestOverlapSchedule.test_radix_attention_chunked_prefill +python3 test_overlap_schedule.py +""" + +import unittest + +from sglang.test.test_utils import CustomTestCase, run_mmlu_test + + +class TestOverlapSchedule(CustomTestCase): + def test_no_radix_attention_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=True, chunked_prefill_size=32, disable_overlap=True + ) + + def test_no_radix_attention_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=True, chunked_prefill_size=-1, disable_overlap=True + ) + + def test_radix_attention_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, chunked_prefill_size=32, disable_overlap=True + ) + + def test_radix_attention_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, chunked_prefill_size=-1, disable_overlap=True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_openai_server.py b/test/srt/test_openai_server.py new file mode 100644 index 0000000..fe6646d --- /dev/null +++ b/test/srt/test_openai_server.py @@ -0,0 +1,680 @@ +""" +python3 -m unittest test_openai_server.TestOpenAIServer.test_batch +python3 -m unittest test_openai_server.TestOpenAIServer.test_completion + +""" + +import json +import re +import time +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_completion( + self, echo, logprobs, use_list_input, parallel_sample_num, token_input + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + response = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + echo=echo, + logprobs=logprobs, + n=parallel_sample_num, + ) + + assert len(response.choices) == num_choices * parallel_sample_num + + if echo: + text = response.choices[0].text + assert text.startswith(prompt) + + if logprobs: + assert response.choices[0].logprobs + assert isinstance(response.choices[0].logprobs.tokens[0], str) + assert isinstance(response.choices[0].logprobs.top_logprobs[1], dict) + ret_num_top_logprobs = len(response.choices[0].logprobs.top_logprobs[1]) + + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0 + + # when echo=True and request.logprobs>0, logprob_start_len is 0, so the first token's logprob would be None. + if not echo: + assert response.choices[0].logprobs.token_logprobs[0] + + assert response.id + assert response.created + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_completion_stream( + self, echo, logprobs, use_list_input, parallel_sample_num, token_input + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + generator = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + echo=echo, + logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0 + assert usage.completion_tokens > 0 + assert usage.total_tokens > 0 + continue + + index = response.choices[0].index + is_first = is_firsts.get(index, True) + + if logprobs: + assert response.choices[0].logprobs + assert isinstance(response.choices[0].logprobs.tokens[0], str) + if not (is_first and echo): + assert isinstance( + response.choices[0].logprobs.top_logprobs[0], dict + ) + ret_num_top_logprobs = len( + response.choices[0].logprobs.top_logprobs[0] + ) + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0 + + if is_first: + if echo: + assert response.choices[0].text.startswith( + prompt + ), f"{response.choices[0].text} and all args {echo} {logprobs} {token_input} {is_first}" + is_firsts[index] = False + assert response.id + assert response.created + + for index in [i for i in range(parallel_sample_num * num_choices)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + def run_chat_completion(self, logprobs, parallel_sample_num): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + n=parallel_sample_num, + ) + + if logprobs: + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ) + + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert len(response.choices) == parallel_sample_num + assert response.choices[0].message.role == "assistant" + assert isinstance(response.choices[0].message.content, str) + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_chat_completion_stream(self, logprobs, parallel_sample_num=1): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0 + assert usage.completion_tokens > 0 + assert usage.total_tokens > 0 + continue + + index = response.choices[0].index + data = response.choices[0].delta + + if is_firsts.get(index, True): + assert data.role == "assistant" + is_firsts[index] = False + continue + + if logprobs: + assert response.choices[0].logprobs + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ) + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs, list + ) + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert ( + isinstance(data.content, str) + or isinstance(data.reasoning_content, str) + or len(data.tool_calls) > 0 + or response.choices[0].finish_reason + ) + assert response.id + assert response.created + + for index in [i for i in range(parallel_sample_num)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + def _create_batch(self, mode, client): + if mode == "completion": + input_file_path = "complete_input.jsonl" + # write content to input file + content = [ + { + "custom_id": "request-1", + "method": "POST", + "url": "/v1/completions", + "body": { + "model": "gpt-3.5-turbo-instruct", + "prompt": "List 3 names of famous soccer player: ", + "max_tokens": 20, + }, + }, + { + "custom_id": "request-2", + "method": "POST", + "url": "/v1/completions", + "body": { + "model": "gpt-3.5-turbo-instruct", + "prompt": "List 6 names of famous basketball player: ", + "max_tokens": 40, + }, + }, + { + "custom_id": "request-3", + "method": "POST", + "url": "/v1/completions", + "body": { + "model": "gpt-3.5-turbo-instruct", + "prompt": "List 6 names of famous tenniss player: ", + "max_tokens": 40, + }, + }, + ] + + else: + input_file_path = "chat_input.jsonl" + content = [ + { + "custom_id": "request-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "gpt-3.5-turbo-0125", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + { + "role": "user", + "content": "Hello! List 3 NBA players and tell a story", + }, + ], + "max_tokens": 30, + }, + }, + { + "custom_id": "request-2", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": "gpt-3.5-turbo-0125", + "messages": [ + {"role": "system", "content": "You are an assistant. "}, + { + "role": "user", + "content": "Hello! List three capital and tell a story", + }, + ], + "max_tokens": 50, + }, + }, + ] + + with open(input_file_path, "w") as file: + for line in content: + file.write(json.dumps(line) + "\n") + + with open(input_file_path, "rb") as file: + uploaded_file = client.files.create(file=file, purpose="batch") + if mode == "completion": + endpoint = "/v1/completions" + elif mode == "chat": + endpoint = "/v1/chat/completions" + completion_window = "24h" + batch_job = client.batches.create( + input_file_id=uploaded_file.id, + endpoint=endpoint, + completion_window=completion_window, + ) + + return batch_job, content, uploaded_file + + def run_batch(self, mode): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + batch_job, content, uploaded_file = self._create_batch(mode=mode, client=client) + + while batch_job.status not in ["completed", "failed", "cancelled"]: + time.sleep(3) + print( + f"Batch job status: {batch_job.status}...trying again in 3 seconds..." + ) + batch_job = client.batches.retrieve(batch_job.id) + assert ( + batch_job.status == "completed" + ), f"Batch job status is not completed: {batch_job.status}" + assert batch_job.request_counts.completed == len(content) + assert batch_job.request_counts.failed == 0 + assert batch_job.request_counts.total == len(content) + + result_file_id = batch_job.output_file_id + file_response = client.files.content(result_file_id) + result_content = file_response.read().decode("utf-8") # Decode bytes to string + results = [ + json.loads(line) + for line in result_content.split("\n") + if line.strip() != "" + ] + assert len(results) == len(content) + for delete_fid in [uploaded_file.id, result_file_id]: + del_pesponse = client.files.delete(delete_fid) + assert del_pesponse.deleted + + def run_cancel_batch(self, mode): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + batch_job, _, uploaded_file = self._create_batch(mode=mode, client=client) + + assert batch_job.status not in ["cancelling", "cancelled"] + + batch_job = client.batches.cancel(batch_id=batch_job.id) + assert batch_job.status == "cancelling" + + while batch_job.status not in ["failed", "cancelled"]: + batch_job = client.batches.retrieve(batch_job.id) + print( + f"Batch job status: {batch_job.status}...trying again in 3 seconds..." + ) + time.sleep(3) + + assert batch_job.status == "cancelled" + del_response = client.files.delete(uploaded_file.id) + assert del_response.deleted + + def test_completion(self): + for echo in [False, True]: + for logprobs in [None, 5]: + for use_list_input in [True, False]: + for parallel_sample_num in [1, 2]: + for token_input in [False, True]: + self.run_completion( + echo, + logprobs, + use_list_input, + parallel_sample_num, + token_input, + ) + + def test_completion_stream(self): + # parallel sampling adn list input are not supported in streaming mode + for echo in [False, True]: + for logprobs in [None, 5]: + for use_list_input in [True, False]: + for parallel_sample_num in [1, 2]: + for token_input in [False, True]: + self.run_completion_stream( + echo, + logprobs, + use_list_input, + parallel_sample_num, + token_input, + ) + + def test_chat_completion(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion(logprobs, parallel_sample_num) + + def test_chat_completion_stream(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(logprobs, parallel_sample_num) + + def test_batch(self): + for mode in ["completion", "chat"]: + self.run_batch(mode) + + def test_cancel_batch(self): + for mode in ["completion", "chat"]: + self.run_cancel_batch(mode) + + def test_regex(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + regex = ( + r"""\{\n""" + + r""" "name": "[\w]+",\n""" + + r""" "population": [\d]+\n""" + + r"""\}""" + ) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=128, + extra_body={"regex": regex}, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + def test_penalty(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=32, + frequency_penalty=1.0, + ) + text = response.choices[0].message.content + assert isinstance(text, str) + + def test_response_prefill(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + { + "role": "assistant", + "content": "{\n", + }, + ], + temperature=0, + ) + + assert ( + response.choices[0] + .message.content.strip() + .startswith('"name": "SmartHome Mini",') + ) + + def test_model_list(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + models = list(client.models.list()) + assert len(models) == 1 + assert isinstance(getattr(models[0], "max_model_len", None), int) + + +# ------------------------------------------------------------------------- +# EBNF Test Class: TestOpenAIServerEBNF +# Launches the server with xgrammar, has only EBNF tests +# ------------------------------------------------------------------------- +class TestOpenAIServerEBNF(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # passing xgrammar specifically + other_args = ["--grammar-backend", "xgrammar"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_ebnf(self): + """ + Ensure we can pass `ebnf` to the local openai server + and that it enforces the grammar. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "Hello" | "Hi" | "Hey" + """ + pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful EBNF test bot."}, + {"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."}, + ], + temperature=0, + max_tokens=32, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + print("EBNF test output:", repr(text)) + self.assertTrue(len(text) > 0, "Got empty text from EBNF generation") + self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices") + + def test_ebnf_strict_json(self): + """ + A stricter EBNF that produces exactly {"name":"Alice"} format + with no trailing punctuation or extra fields. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "{" pair "}" + pair ::= "\"name\"" ":" string + string ::= "\"" [A-Za-z]+ "\"" + """ + pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$') + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "EBNF mini-JSON generator."}, + { + "role": "user", + "content": "Generate single key JSON with only letters.", + }, + ], + temperature=0, + max_tokens=64, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + print("EBNF strict JSON test output:", repr(text)) + self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test") + self.assertRegex( + text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape" + ) + + +class TestOpenAIEmbedding(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Configure embedding-specific args + other_args = ["--is-embedding", "--enable-metrics"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_embedding_single(self): + """Test single embedding request""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create(model=self.model, input="Hello world") + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_embedding_batch(self): + """Test batch embedding request""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, input=["Hello world", "Test text"] + ) + self.assertEqual(len(response.data), 2) + self.assertTrue(len(response.data[0].embedding) > 0) + self.assertTrue(len(response.data[1].embedding) > 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_page_size.py b/test/srt/test_page_size.py new file mode 100644 index 0000000..f56e1ed --- /dev/null +++ b/test/srt/test_page_size.py @@ -0,0 +1,47 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPageSize(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_DEBUG_MEMORY_POOL"] = "1" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--page-size", 4, "--chunked-prefill-size", 128], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_patch_torch.py b/test/srt/test_patch_torch.py new file mode 100644 index 0000000..a2c0450 --- /dev/null +++ b/test/srt/test_patch_torch.py @@ -0,0 +1,133 @@ +import os +import traceback +import unittest +from typing import Dict, List + +import torch +import torch.multiprocessing as mp + +from sglang.srt.patch_torch import monkey_patch_torch_reductions + + +class TestReleaseMemoryOccupation(unittest.TestCase): + def test_monkey_patch_torch_reductions(self): + mp.set_start_method("spawn", force=True) + + for enable_patch in [False, True]: + for params in [ + # Same visible devices + dict( + sender_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + receiver_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + ), + # Different visible devices + dict( + sender_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + receiver_info=dict( + visible_devices=[1, 0], + # If enable patch, this should be fixed, and cuda:1 becomes cuda:0 + tensor_device=0 if enable_patch else 1, + ), + ), + ]: + with self.subTest(f"{enable_patch=} {params=}"): + self._test_monkey_patch_torch_reductions_core( + enable_patch=enable_patch, **params + ) + + def _test_monkey_patch_torch_reductions_core( + self, + sender_info: Dict, + receiver_info: Dict, + enable_patch: bool, + ): + print( + f'test_monkey_patch_torch_reductions_core {os.environ.get("CUDA_VISIBLE_DEVICES")=}' + ) + cuda_visible_devices_list: List[int] = [ + int(x) + for x in os.environ.get("CUDA_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7").split( + "," + ) + ] + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + queue = mp.Queue() + for role, info in [ + ("sender", sender_info), + ("receiver", receiver_info), + ]: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join( + str(cuda_visible_devices_list[device]) + for device in info["visible_devices"] + ) + p = mp.Process( + target=_run_subprocess, + kwargs=dict( + role=role, + queue=queue, + output_writer=output_writer, + tensor_device=info["tensor_device"], + enable_patch=enable_patch, + ), + ) + p.start() + processes.append(p) + + for _ in range(len(processes)): + self.assertTrue( + output_reader.recv(), f"Subprocess has error, please see logs above." + ) + + for p in processes: + p.join() + + +def _run_subprocess( + role: str, queue: mp.Queue, output_writer, tensor_device: int, enable_patch: bool +): + print( + f'subprocess[{role}] start {os.environ.get("CUDA_VISIBLE_DEVICES")=}', + flush=True, + ) + + if enable_patch: + print(f"subprocess[{role}] execute monkey_patch_torch_reductions", flush=True) + monkey_patch_torch_reductions() + + try: + if role == "sender": + tensor = torch.tensor([1.0, 2.0], device=f"cuda:{tensor_device}") + print(f"sender queue.put {tensor=} {tensor.device=}") + queue.put(tensor) + assert queue.get() == "done" + elif role == "receiver": + tensor = queue.get() + print(f"receiver queue.get {tensor=} {tensor.device=}") + assert str(tensor.device) == f"cuda:{tensor_device}" + queue.put("done") + else: + raise NotImplementedError + + execution_ok = True + except Exception as e: + print(f"subprocess[{role}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_penalty.py b/test/srt/test_penalty.py new file mode 100644 index 0000000..bfbd277 --- /dev/null +++ b/test/srt/test_penalty.py @@ -0,0 +1,95 @@ +import json +import random +import unittest +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPenalty(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, sampling_params): + return_logprob = True + top_logprobs_num = 5 + return_text = True + n = 1 + + response = requests.post( + self.base_url + "/generate", + json={ + # prompt that is supposed to generate < 32 tokens + "text": "<|start_header_id|>user<|end_header_id|>\n\nWhat is the answer for 1 + 1 = ?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "sampling_params": { + "max_new_tokens": 48, + "n": n, + **sampling_params, + }, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + self.assertEqual(response.status_code, 200) + print(json.dumps(response.json())) + print("=" * 100) + + def test_default_values(self): + self.run_decode({}) + + def test_frequency_penalty(self): + self.run_decode({"frequency_penalty": 2}) + + def test_min_new_tokens(self): + self.run_decode({"min_new_tokens": 16}) + + def test_presence_penalty(self): + self.run_decode({"presence_penalty": 2}) + + def test_penalty_mixed(self): + args = [ + {}, + {}, + {}, + {"frequency_penalty": 2}, + {"presence_penalty": 1}, + {"min_new_tokens": 16}, + {"frequency_penalty": 0.2}, + {"presence_penalty": 0.4}, + {"min_new_tokens": 8}, + {"frequency_penalty": 0.4, "presence_penalty": 0.8}, + {"frequency_penalty": 0.4, "min_new_tokens": 12}, + {"presence_penalty": 0.8, "min_new_tokens": 12}, + {"presence_penalty": -0.3, "frequency_penalty": 1.3, "min_new_tokens": 32}, + {"presence_penalty": 0.3, "frequency_penalty": -1.3, "min_new_tokens": 32}, + ] + random.shuffle(args * 5) + with ThreadPoolExecutor(8) as executor: + list(executor.map(self.run_decode, args)) + + +if __name__ == "__main__": + unittest.main(verbosity=3) diff --git a/test/srt/test_pytorch_sampling_backend.py b/test/srt/test_pytorch_sampling_backend.py new file mode 100644 index 0000000..8a36831 --- /dev/null +++ b/test/srt/test_pytorch_sampling_backend.py @@ -0,0 +1,90 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPyTorchSamplingBackend(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--sampling-backend", "pytorch", "--disable-radix-cache"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_greedy(self): + + first_text = None + + # ensure the answer is identical across single response + for _ in range(5): + response_single = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of Germany is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + text = response_single["text"] + if first_text is None: + first_text = text + + self.assertEqual(text, first_text) + + first_text = None + + response_batch = requests.post( + self.base_url + "/generate", + json={ + "text": ["The capital of Germany is"] * 10, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + + # ensure the answer is identical among the batch + for i in range(10): + text = response_batch[i]["text"] + if first_text is None: + first_text = text + self.assertEqual(text, first_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_radix_attention.py b/test/srt/test_radix_attention.py new file mode 100644 index 0000000..554801d --- /dev/null +++ b/test/srt/test_radix_attention.py @@ -0,0 +1,134 @@ +import os +import random +import unittest + +import requests + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + kill_process_tree, + popen_launch_server, +) + + +def gen_radix_tree(num_nodes=400, chunk_len=256): + num0 = num_nodes // 2 + num1 = num_nodes - num0 + nodes = [{"input_ids": [37] * 117, "decode_len": 217}] + for _ in range(num0): + parent = random.choice(nodes) + unique_len = random.randint(0, chunk_len) + decode_len = random.randint(0, chunk_len) + token_id = random.randint(0, 32000) + child = { + "input_ids": parent["input_ids"] + [token_id] * unique_len, + "decode_len": decode_len, + } + nodes.append(child) + + while num1 > 0: + num_branch = random.randint(1, min(num1, 10)) + parent = random.choice(nodes) + for _ in range(num_branch): + unique_len = random.randint(0, chunk_len) + decode_len = random.randint(0, chunk_len) + token_id = random.randint(0, 32000) + child = { + "input_ids": parent["input_ids"] + [token_id] * unique_len, + "decode_len": decode_len, + } + nodes.append(child) + + num1 -= num_branch + + random.shuffle(nodes) + return nodes + + +def run_test(base_url, nodes): + data = { + "input_ids": [node["input_ids"] for node in nodes], + "sampling_params": [ + {"max_new_tokens": node["decode_len"], "temperature": 0} for node in nodes + ], + } + + res = requests.post(base_url + "/generate", json=data) + assert res.status_code == 200 + + +class TestRadixCacheFCFS(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "fcfs", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_radix_attention(self): + nodes = gen_radix_tree() + run_test(self.base_url, nodes) + + +class TestRadixCacheLPM(TestRadixCacheFCFS): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "lpm", + ], + ) + + +class TestRadixCacheNonOverlapLPM(TestRadixCacheFCFS): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--disable-overlap-schedule", + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "lpm", + ], + ) + + +if __name__ == "__main__": + os.environ["SGLANG_TEST_RETRACT"] = "true" + unittest.main() diff --git a/test/srt/test_reasoning_content.py b/test/srt/test_reasoning_content.py new file mode 100644 index 0000000..695dabf --- /dev/null +++ b/test/srt/test_reasoning_content.py @@ -0,0 +1,343 @@ +""" +Usage: +python3 -m unittest test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_false +python3 -m unittest test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true +python3 -m unittest test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true_stream_reasoning_false +python3 -m unittest test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_false +python3 -m unittest test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_true +python3 -m unittest test_reasoning_content.TestReasoningContentStartup.test_nonstreaming +python3 -m unittest test_reasoning_content.TestReasoningContentStartup.test_streaming +""" + +import json +import unittest + +import openai +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_REASONING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestReasoningContentAPI(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_REASONING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--reasoning-parser", + "deepseek-r1", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert len(response.choices[0].message.reasoning_content) > 0 + assert len(response.choices[0].message.content) > 0 + + +class TestReasoningContentWithoutParser(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_REASONING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[], # No reasoning parser + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert not reasoning_content or len(reasoning_content) == 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_regex_constrained.py b/test/srt/test_regex_constrained.py new file mode 100644 index 0000000..b0fb497 --- /dev/null +++ b/test/srt/test_regex_constrained.py @@ -0,0 +1,187 @@ +""" +python3 -m unittest test_regex_constrained.TestRegexConstrained.test_regex_generate_email +python3 -m unittest test_regex_constrained.TestRegexConstrained.test_regex_generate_greeting +python3 -m unittest test_regex_constrained.TestRegexConstrainedLLGuidance.test_regex_generate_email +python3 -m unittest test_regex_constrained.TestRegexConstrainedLLGuidance.test_regex_generate_greeting +""" + +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def setup_class(cls, backend: str, disable_overlap: bool): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + +class TestRegexConstrained(CustomTestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, "xgrammar", disable_overlap=False) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + regex, + prompt, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "regex": regex, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + + ret = response.json() + print(json.dumps(ret, indent=2)) + print("=" * 100) + + if not isinstance(ret, list): + self.fail(f"Expected response to be a list, but got {type(ret)}") + + for item in ret: + text = item.get("text", "").strip() + if not text: + self.fail("Generated text is empty.") + + if not self.regex_match(text, regex): + self.fail(f"Text '{text}' does not match regex pattern.") + + def regex_match(self, text, pattern): + import re + + return re.match(pattern, text) is not None + + def test_regex_generate_email(self): + pattern = r"^user@example\.com$" + prompt = "Generate an email address:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_greeting(self): + pattern = r"^(Hello|Hi|Hey)$" + prompt = "Generate a greeting:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_number(self): + pattern = r"^\d{3}$" + prompt = "Generate a three-digit number:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_phone(self): + pattern = r"^\(\d{3}\) \d{3}-\d{4}$" + prompt = "Generate a phone number:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_date(self): + pattern = r"^2024-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$" + prompt = "Generate a date in YYYY-MM-DD format:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_hex_color(self): + pattern = r"^#[0-9A-F]{6}$" + prompt = "Generate a hex color code:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_complex_json(self): + pattern = r'^\{\s*"name"\s*:\s*"[a-zA-Z0-9 ]+"\s*,\s*"age"\s*:\s*[1-9][0-9]*\s*,\s*"city"\s*:\s*"[a-zA-Z0-9 ]+"\s*\}$' + prompt = "Generate a simple JSON with name, age, and city:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_custom_log_format(self): + pattern = r"^\[2024-01-01T12:00:00Z\] INFO: System\.process - Operation [a-z]+ successfully$" + prompt = "Generate a log entry:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + +class TestRegexConstrainedLLGuidance(TestRegexConstrained): + @classmethod + def setUpClass(cls): + setup_class(cls, "llguidance", disable_overlap=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_release_memory_occupation.py b/test/srt/test_release_memory_occupation.py new file mode 100644 index 0000000..7ccd9f1 --- /dev/null +++ b/test/srt/test_release_memory_occupation.py @@ -0,0 +1,98 @@ +import time +import unittest + +import torch +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + +# (temporarily) set to true to observe memory usage in nvidia-smi more clearly +_DEBUG_EXTRA = True + + +class TestReleaseMemoryOccupation(CustomTestCase): + def test_release_and_resume_occupation(self): + prompt = "Today is a sunny day and I like" + sampling_params = {"temperature": 0, "max_new_tokens": 8} + model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + expect_output = " to spend it outdoors. I decided to" + + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + # disable_cuda_graph=True, # for debugging only + ) + hf_model_new = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="bfloat16" + ) + + print("generate (#1)") + outputs = engine.generate(prompt, sampling_params)["text"] + self.assertEqual(outputs, expect_output) + + if _DEBUG_EXTRA: + time.sleep(3) + + self.assertEqual( + _try_allocate_big_tensor(), + False, + "Should not be able to allocate big tensors before releasing", + ) + + print("release_memory_occupation start") + t = time.time() + engine.release_memory_occupation() + if _DEBUG_EXTRA: + print("release_memory_occupation", time.time() - t) + + if _DEBUG_EXTRA: + time.sleep(5) + + self.assertEqual( + _try_allocate_big_tensor(), + True, + "Should be able to allocate big tensors aftre releasing", + ) + + if _DEBUG_EXTRA: + time.sleep(5) + + print("resume_memory_occupation start") + t = time.time() + engine.resume_memory_occupation() + if _DEBUG_EXTRA: + print("resume_memory_occupation", time.time() - t) + + self.assertEqual( + _try_allocate_big_tensor(), + False, + "Should not be able to allocate big tensors after resuming", + ) + + print("update_weights_from_tensor") + # As if: PPO has updated hf model's weights, and now we sync it to SGLang + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + print("generate (#2)") + outputs = engine.generate(prompt, sampling_params)["text"] + self.assertEqual(outputs, expect_output) + + if _DEBUG_EXTRA: + time.sleep(4) + + engine.shutdown() + + +def _try_allocate_big_tensor(size: int = 20_000_000_000): + try: + torch.empty((size,), dtype=torch.uint8, device="cuda") + torch.cuda.empty_cache() + return True + except torch.cuda.OutOfMemoryError: + return False + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_request_length_validation.py b/test/srt/test_request_length_validation.py new file mode 100644 index 0000000..b3c202f --- /dev/null +++ b/test/srt/test_request_length_validation.py @@ -0,0 +1,88 @@ +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestRequestLengthValidation(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start server with auto truncate disabled + cls.process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=("--max-total-tokens", "1000", "--context-length", "1000"), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_input_length_longer_than_context_length(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " * 1200 # Will tokenize to more than context length + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + ) + + self.assertIn("is longer than the model's context length", str(cm.exception)) + + def test_input_length_longer_than_maximum_allowed_length(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " * 999 # the maximum allowed length is 994 tokens + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + ) + + self.assertIn("is longer than the model's context length", str(cm.exception)) + + def test_max_tokens_validation(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + max_tokens=1200, + ) + + self.assertIn( + "Requested token count exceeds the model's maximum context", + str(cm.exception), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_retract_decode.py b/test/srt/test_retract_decode.py new file mode 100644 index 0000000..92f5ab9 --- /dev/null +++ b/test/srt/test_retract_decode.py @@ -0,0 +1,60 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestRetractDecode(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_TEST_RETRACT"] = "1" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestRetractDecodeChunkCache(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_TEST_RETRACT"] = "1" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--disable-radix-cache", "--chunked-prefill-size", 128], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_sagemaker_server.py b/test/srt/test_sagemaker_server.py new file mode 100644 index 0000000..68688c1 --- /dev/null +++ b/test/srt/test_sagemaker_server.py @@ -0,0 +1,179 @@ +""" +python3 -m unittest test_sagemaker_server.TestSageMakerServer.test_chat_completion +""" + +import json +import unittest + +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestSageMakerServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_chat_completion(self, logprobs, parallel_sample_num): + data = { + "model": self.model, + "messages": [ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + "temperature": 0, + "logprobs": logprobs is not None and logprobs > 0, + "top_logprobs": logprobs, + "n": parallel_sample_num, + } + + headers = {"Authorization": f"Bearer {self.api_key}"} + + response = requests.post( + f"{self.base_url}/invocations", json=data, headers=headers + ).json() + + if logprobs: + assert isinstance( + response["choices"][0]["logprobs"]["content"][0]["top_logprobs"][0][ + "token" + ], + str, + ) + + ret_num_top_logprobs = len( + response["choices"][0]["logprobs"]["content"][0]["top_logprobs"] + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert len(response["choices"]) == parallel_sample_num + assert response["choices"][0]["message"]["role"] == "assistant" + assert isinstance(response["choices"][0]["message"]["content"], str) + assert response["id"] + assert response["created"] + assert response["usage"]["prompt_tokens"] > 0 + assert response["usage"]["completion_tokens"] > 0 + assert response["usage"]["total_tokens"] > 0 + + def run_chat_completion_stream(self, logprobs, parallel_sample_num=1): + data = { + "model": self.model, + "messages": [ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + "temperature": 0, + "logprobs": logprobs is not None and logprobs > 0, + "top_logprobs": logprobs, + "stream": True, + "stream_options": {"include_usage": True}, + "n": parallel_sample_num, + } + + headers = {"Authorization": f"Bearer {self.api_key}"} + + response = requests.post( + f"{self.base_url}/invocations", json=data, stream=True, headers=headers + ) + + is_firsts = {} + for line in response.iter_lines(): + line = line.decode("utf-8").replace("data: ", "") + if len(line) < 1 or line == "[DONE]": + continue + print(f"value: {line}") + line = json.loads(line) + usage = line.get("usage") + if usage is not None: + assert usage["prompt_tokens"] > 0 + assert usage["completion_tokens"] > 0 + assert usage["total_tokens"] > 0 + continue + + index = line.get("choices")[0].get("index") + data = line.get("choices")[0].get("delta") + + if is_firsts.get(index, True): + assert data["role"] == "assistant" + is_firsts[index] = False + continue + + if logprobs: + assert line.get("choices")[0].get("logprobs") + assert isinstance( + line.get("choices")[0] + .get("logprobs") + .get("content")[0] + .get("top_logprobs")[0] + .get("token"), + str, + ) + assert isinstance( + line.get("choices")[0] + .get("logprobs") + .get("content")[0] + .get("top_logprobs"), + list, + ) + ret_num_top_logprobs = len( + line.get("choices")[0] + .get("logprobs") + .get("content")[0] + .get("top_logprobs") + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert isinstance(data["content"], str) + assert line["id"] + assert line["created"] + + for index in [i for i in range(parallel_sample_num)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + def test_chat_completion(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion(logprobs, parallel_sample_num) + + def test_chat_completion_stream(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(logprobs, parallel_sample_num) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_schedule_policy.py b/test/srt/test_schedule_policy.py new file mode 100644 index 0000000..4a4f57b --- /dev/null +++ b/test/srt/test_schedule_policy.py @@ -0,0 +1,65 @@ +import unittest + +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.schedule_policy import ( + CacheAgnosticPolicy, + CacheAwarePolicy, + SchedulePolicy, +) +from sglang.srt.mem_cache.radix_cache import RadixCache, TreeNode +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.test.test_utils import CustomTestCase + + +class TestSchedulePolicy(CustomTestCase): + + def setUp(self): + self.tree_cache = RadixCache(None, None, False) + + def test_init_with_cache_aware_policy(self): + policy = SchedulePolicy( + policy="lpm", tree_cache=self.tree_cache, enable_hierarchical_cache=True + ) + self.assertEqual(policy.policy, CacheAwarePolicy.LPM) + + def test_init_with_cache_agnostic_policy(self): + policy = SchedulePolicy( + policy="fcfs", tree_cache=self.tree_cache, enable_hierarchical_cache=True + ) + self.assertEqual(policy.policy, CacheAgnosticPolicy.FCFS) + + def test_init_with_unknown_policy(self): + with self.assertRaises(ValueError): + SchedulePolicy( + policy="invalid", + tree_cache=self.tree_cache, + enable_hierarchical_cache=True, + ) + + def test_init_with_disabled_cache(self): + disabled_tree_cache = RadixCache(None, None, disable=True, page_size=1) + policy = SchedulePolicy( + policy="lpm", tree_cache=disabled_tree_cache, enable_hierarchical_cache=True + ) + self.assertEqual(policy.policy, CacheAgnosticPolicy.FCFS) + + def test_calc_priority_fcfs(self): + tree_cache = RadixCache(None, None, False) + waiting_queue = [ + Req(1, "a b", [1, 2], SamplingParams()), + Req(3, "a b c", [1, 2, 3], SamplingParams()), + Req(2, "a", [1], SamplingParams()), + ] + + policy = SchedulePolicy( + policy="fcfs", tree_cache=tree_cache, enable_hierarchical_cache=True + ) + policy.calc_priority(waiting_queue) + # Check if FCFS keeps the original order + self.assertEqual(waiting_queue[0].rid, 1) + self.assertEqual(waiting_queue[1].rid, 3) + self.assertEqual(waiting_queue[2].rid, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_server_args.py b/test/srt/test_server_args.py new file mode 100644 index 0000000..42ccb49 --- /dev/null +++ b/test/srt/test_server_args.py @@ -0,0 +1,262 @@ +import json +import unittest +from unittest.mock import MagicMock, patch + +from sglang.srt.server_args import PortArgs, prepare_server_args +from sglang.test.test_utils import CustomTestCase + + +class TestPrepareServerArgs(CustomTestCase): + def test_prepare_server_args(self): + server_args = prepare_server_args( + [ + "--model-path", + "model_path", + "--json-model-override-args", + '{"rope_scaling": {"factor": 2.0, "rope_type": "linear"}}', + ] + ) + self.assertEqual(server_args.model_path, "model_path") + self.assertEqual( + json.loads(server_args.json_model_override_args), + {"rope_scaling": {"factor": 2.0, "rope_type": "linear"}}, + ) + + +class TestPortArgs(unittest.TestCase): + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.tempfile.NamedTemporaryFile") + def test_init_new_standard_case(self, mock_temp_file, mock_is_port_available): + + mock_is_port_available.return_value = True + mock_temp_file.return_value.name = "temp_file" + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = False + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("ipc://")) + self.assertTrue(port_args.scheduler_input_ipc_name.startswith("ipc://")) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("ipc://")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_single_node_dp_attention(self, mock_is_port_available): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 1 + server_args.dist_init_addr = None + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://127.0.0.1:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://127.0.0.1:") + ) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://127.0.0.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_dp_rank(self, mock_is_port_available): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 1 + server_args.dist_init_addr = "192.168.1.1:25000" + + port_args = PortArgs.init_new(server_args, dp_rank=2) + + print(f"{port_args=}") + self.assertTrue(port_args.scheduler_input_ipc_name.endswith(":25007")) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_ipv4_address(self, mock_is_port_available): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1:25000" + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://192.168.1.1:") + ) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv4_address(self, mock_is_port_available): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1" + + with self.assertRaises(AssertionError) as context: + PortArgs.init_new(server_args) + + self.assertIn( + "please provide --dist-init-addr as host:port", str(context.exception) + ) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv4_address_invalid_port( + self, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1:abc" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_ipv6_address( + self, mock_is_valid_ipv6, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]:25000" + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://[2001:db8::1]:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://[2001:db8::1]:") + ) + self.assertTrue( + port_args.detokenizer_ipc_name.startswith("tcp://[2001:db8::1]:") + ) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=False) + def test_init_new_with_invalid_ipv6_address( + self, mock_is_valid_ipv6, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[invalid-ipv6]:25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid IPv6 address", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv6_address_missing_bracket( + self, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1:25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid IPv6 address format", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_missing_port( + self, mock_is_valid_ipv6, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn( + "a port must be specified in IPv6 address", str(context.exception) + ) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_invalid_port( + self, mock_is_valid_ipv6, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]:abcde" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid port in IPv6 address", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_wrong_separator( + self, mock_is_valid_ipv6, mock_is_port_available + ): + + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]#25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("expected ':' after ']'", str(context.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_session_control.py b/test/srt/test_session_control.py new file mode 100644 index 0000000..7a68b2b --- /dev/null +++ b/test/srt/test_session_control.py @@ -0,0 +1,778 @@ +""" +Usage: +python3 -m unittest test_session_control.TestSessionControl.test_session_control +python3 -m unittest test_session_control.TestSessionControl.test_session_control_with_branching +python3 -m unittest test_session_control.TestSessionControl.test_session_control_backtrack_with_abort +python3 -m unittest test_session_control.TestSessionControlVision.test_session_control +""" + +import asyncio +import json +import unittest + +import aiohttp +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def remove_prefix(text: str, prefix: str) -> str: + return text[len(prefix) :] if text.startswith(prefix) else text + + +class TestSessionControl(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_session_control(self, gen_len=12): + chunks = [ + "Let me tell you something about France.", + "The capital of France is", + "The population of the city is", + "A brief history about that city is", + ] + tokenizer = get_tokenizer(self.model) + chunks_ids = [tokenizer.encode(x) for x in chunks] + for i in range(1, len(chunks_ids)): + if chunks_ids[i][0] == tokenizer.bos_token_id: + chunks_ids[i] = chunks_ids[i][1:] + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + rid = None + + # open an existing session, should get session_id as None + response = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000, "session_id": session_id}, + ).json() + assert isinstance(response, dict) and "error" in response + + first_rid = None + outputs_from_session = [] + logprobs_from_session = [] + cur_logprob_start_len = 0 + for i, chunk_ids in enumerate(chunks_ids): + max_new_tokens = gen_len if i > 0 else 1 # prefill only for the first chunk + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunk_ids, + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + "logprob_start_len": cur_logprob_start_len - 1, + }, + ).json() + rid = response["meta_info"]["id"] + if i == 0: + first_rid = rid + if i > 0: + outputs_from_session.append(response["text"]) + logprobs_from_session.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + cur_logprob_start_len += len(chunk_ids) + max_new_tokens + + # query with a logprob_start_len longer than the request, should see error + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunk_ids, + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + "logprob_start_len": cur_logprob_start_len + len(chunk_ids), + }, + ).json() + assert "Request with a lower logprob_start_len" in response["error"]["message"] + + # backtrack to the first request and regenerate + cur_logprob_start_len = 0 + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunks_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + "logprob_start_len": cur_logprob_start_len, + }, + ).json() + outputs_from_session.append(response["text"]) + logprobs_from_session.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + + # query with a non-existing rid (the last one should be disappeared becuase of backtrack), should see abort + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunks_ids[-1], + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ).json() + assert response["meta_info"]["finish_reason"]["type"] == "abort" + + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + assert ret.status_code == 200 + + # send a request to a closed session, should see abort + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunks_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ).json() + assert response["meta_info"]["finish_reason"]["type"] == "abort" + + # 2. not use session control + requests.post(self.base_url + "/flush_cache") + + input_ids_first_req = None + input_ids = [] + outputs_normal = [] + logprobs_normal = [] + for i, chunk_ids in enumerate(chunks_ids): + input_ids += chunk_ids + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": ( + gen_len if i > 0 else 1 + ), # prefill only for the first chunk + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ).json() + if i > 0: + output_ids = tokenizer.encode(response["text"]) + if output_ids[0] == tokenizer.bos_token_id: + output_ids = output_ids[1:] + input_ids += output_ids[:-1] + outputs_normal.append(response["text"]) + logprobs_normal.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + if i == 0: + input_ids_first_req = input_ids.copy() + + input_ids_first_req += chunks_ids[-1] + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids_first_req, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ).json() + outputs_normal.append(response["text"]) + logprobs_normal.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + + print("outputs from chunked queries with session control:") + print(outputs_from_session) + print("outputs from normal queries:") + print(outputs_normal) + assert outputs_from_session == outputs_normal + print("logprobs from chunked queries with session control:") + print(logprobs_from_session) + print("logprobs from normal queries:") + print(logprobs_normal) + assert len(logprobs_from_session) == len( + logprobs_normal + ), "logprobs must have equal length" + for a, b in zip(logprobs_from_session, logprobs_normal): + assert abs(a - b) <= 0.1, f"logprobs {a} and {b} differ by more than 0.1" + + async def async_generate(self, payload): + url = self.base_url + "/generate" + async with aiohttp.ClientSession() as session: + async with session.post(url=url, json=payload) as response: + assert response.status == 200 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + if chunk == "[DONE]": + yield "", None, "" + else: + data = json.loads(chunk) + finish_reason = ( + data["meta_info"]["finish_reason"]["type"] + if data["meta_info"]["finish_reason"] + else "" + ) + yield data["text"], data["meta_info"]["id"], finish_reason + + async def run_session_control_backtrack_with_abort(self, replace): + chunks = [ + "Let me tell you something about France.", + "The capital of France is", + ] + tokenizer = get_tokenizer(self.model) + chunks_ids = [tokenizer.encode(x) for x in chunks] + for i in range(1, len(chunks_ids)): + if chunks_ids[i][0] == tokenizer.bos_token_id: + chunks_ids[i] = chunks_ids[i][1:] + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + rid = None + + payload = { + "input_ids": chunks_ids[0], + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 100, + "no_stop_trim": True, + "skip_special_tokens": False, + "ignore_eos": True, + }, + "stream": True, + } + gen_so_far = "" + finish_reason = "" + second_output = "" + async for chunk, rid, finish_reason_chunk in self.async_generate(payload): + gen_so_far += chunk + if finish_reason == "": + finish_reason = finish_reason_chunk + if len(gen_so_far) > 50 and second_output == "": + payload2 = { + "input_ids": chunks_ids[1], + "session_params": { + "id": session_id, + "rid": rid, + "offset": 50, + "replace": replace, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "stream": False, + "stream_output": True, + } + response = requests.post( + url=self.base_url + "/generate", json=payload2 + ).json() + second_output = response["text"] + if replace: + assert finish_reason == "abort" + print("first request output:") + print(gen_so_far) + print("second request output:") + print(second_output) + + # close the session + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + assert ret.status_code == 200 + + if not replace: + assert response["meta_info"]["finish_reason"]["type"] == "abort" + else: + # 2. not using session control + requests.post(self.base_url + "/flush_cache") + output_ids = tokenizer.encode(gen_so_far) + if output_ids[0] == tokenizer.bos_token_id: + output_ids = output_ids[1:] + input_ids = chunks_ids[0] + output_ids + input_ids = input_ids[:50] + chunks_ids[1] + payload = { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "stream": False, + "stream_output": True, + } + response = requests.post( + url=self.base_url + "/generate", json=payload + ).json() + output_no_session = response["text"] + print("second request output without session:") + print(output_no_session) + assert ( + second_output == output_no_session + ), f"second_output: {second_output}, output_no_session: {output_no_session}" + + def test_session_control_backtrack_with_abort(self): + asyncio.run(self.run_session_control_backtrack_with_abort(replace=True)) + asyncio.run(self.run_session_control_backtrack_with_abort(replace=False)) + + def run_session_control_with_branching( + self, root_prompt, chunks_per_step, gen_len=16 + ): + for x in chunks_per_step: + assert len(x) == len(chunks_per_step[0]) + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + + outputs_from_session = [] + # send the root prompt + response = requests.post( + self.base_url + "/generate", + json={ + "text": root_prompt, + "session_params": { + "id": session_id, + "rid": None, + "offset": 0, + "replace": False, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid_per_branch = [response["meta_info"]["id"]] * len(chunks_per_step[0]) + outputs_from_session.append(response["text"]) + + # send the prompts in branches + for chunks_for_branches in chunks_per_step: + for j, chunk in enumerate(chunks_for_branches): + response = requests.post( + self.base_url + "/generate", + json={ + "text": chunk, + "session_params": { + "id": session_id, + "rid": rid_per_branch[j], + "offset": 0, + "replace": False, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid = response["meta_info"]["id"] + rid_per_branch[j] = rid + outputs_from_session.append(response["text"]) + + # close the session + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + assert ret.status_code == 200 + + # 2. not use session control + requests.post(self.base_url + "/flush_cache") + + outputs_normal = [] + input_texts = [root_prompt] * len(chunks_per_step[0]) + # send the root prompt + response = requests.post( + self.base_url + "/generate", + json={ + "text": root_prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_normal.append(response["text"]) + input_texts = [x + response["text"] for x in input_texts] + + # send the prompts in branches + for chunks_for_branches in chunks_per_step: + for j, chunk in enumerate(chunks_for_branches): + input_texts[j] += chunk + response = requests.post( + self.base_url + "/generate", + json={ + "text": input_texts[j], + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_normal.append(response["text"]) + input_texts[j] += response["text"] + + print("====== outputs from chunked queries with session control: =======") + print(outputs_from_session) + print("====== outputs from normal queries: =======") + print(outputs_normal) + assert ( + outputs_from_session == outputs_normal + ), f"outputs_from_session: {outputs_from_session}, outputs_normal: {outputs_normal}" + + def test_session_control_with_branching(self): + root_prompt = "First, let me explain in one sentence about AI" + chunks_per_step = [ + [ + "Then, briefly, the positive side of AI is", + "But, briefly, AI could be harmful to human", + ], + ["For example", "For example"], + ] + self.run_session_control_with_branching( + root_prompt=root_prompt, chunks_per_step=chunks_per_step, gen_len=8 + ) + + root_prompt = "I have three apples." + chunks_per_step = [ + ["I then give one apple to my friend", "My friend give me another apple."], + ["I still have", "I now have"], + ] + self.run_session_control_with_branching( + root_prompt=root_prompt, chunks_per_step=chunks_per_step, gen_len=8 + ) + + +class TestSessionControlVision(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmms-lab/llava-onevision-qwen2-7b-ov" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + # other_args={"--disable-radix"}, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_session_control(self): + text_chunks = [ + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n", + "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\n\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\n\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\nDescribe this image in a very short sentence.<|im_end|>\nassistant:", + ] + image_chunks = [ + "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png", + "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png", + "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png", + ] + + assert ( + len(text_chunks) == len(image_chunks) + 2 + ) # the first and the last prompt does not contain images + tokenizer = get_tokenizer(self.model) + text_input_ids = [tokenizer.encode(x) for x in text_chunks] + for i in range(1, len(text_input_ids)): + if text_input_ids[i][0] == tokenizer.bos_token_id: + text_input_ids[i] = text_input_ids[i][1:] + gen_len = 32 + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + rid = None + + # open an existing session, should get session_id as None + response = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000, "session_id": session_id}, + ).json() + assert isinstance(response, dict) and "error" in response + + first_rid = None + outputs_from_session = [] + for i in range(len(text_input_ids[:-1])): + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[i], + "image_data": image_chunks[i - 1] if i > 0 else None, + "modalities": ["multi-images"], + "session_params": { + "id": session_id, + "rid": rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": ( + gen_len if i > 0 else 0 + ), # prefill only for the first chunk + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid = response["meta_info"]["id"] + if i == 0: + first_rid = rid + if i > 0: + outputs_from_session.append(response["text"]) + + # backtrack to the first request and regenerate + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_from_session.append(response["text"]) + + # query with a non-existing rid (the last one should be disappeared becuase of backtrack), should see abort + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[-1], + "session_params": { + "id": session_id, + "rid": rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + assert response["meta_info"]["finish_reason"]["type"] == "abort" + + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + assert ret.status_code == 200 + + # send a request to a closed session, should see abort + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + assert response["meta_info"]["finish_reason"]["type"] == "abort" + + # 2. not use session control + requests.post(self.base_url + "/flush_cache") + + input_ids_first_req = None + input_ids = [] + outputs_normal = [] + for i in range(len(text_input_ids[:-1])): + input_ids += text_input_ids[i] + image_data = image_chunks[:i] if i > 0 else None + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "image_data": image_data, + "modalities": ["multi-images"], + "sampling_params": { + "temperature": 0, + "max_new_tokens": ( + gen_len if i > 0 else 0 + ), # prefill only for the first chunk + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + if i > 0: + output_ids = tokenizer.encode(response["text"]) + if output_ids[0] == tokenizer.bos_token_id: + output_ids = output_ids[1:] + input_ids += output_ids + outputs_normal.append(response["text"]) + if i == 0: + input_ids_first_req = input_ids.copy() + + input_ids_first_req += text_input_ids[-1] + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids_first_req, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_normal.append(response["text"]) + + print("outputs from chunked queries with session control:") + print(outputs_from_session) + print("outputs from normal queries:") + print(outputs_normal) + assert ( + outputs_from_session == outputs_normal + ), f"outputs_from_session: {outputs_from_session}, outputs_normal: {outputs_normal}" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_skip_tokenizer_init.py b/test/srt/test_skip_tokenizer_init.py new file mode 100644 index 0000000..a9d2724 --- /dev/null +++ b/test/srt/test_skip_tokenizer_init.py @@ -0,0 +1,227 @@ +""" +python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.test_parallel_sample +python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.run_decode_stream +""" + +import json +import unittest +from io import BytesIO + +import requests +from PIL import Image +from transformers import AutoProcessor, AutoTokenizer + +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_IMAGE_URL, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_VLM_MODEL_NAME, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestSkipTokenizerInit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--skip-tokenizer-init", "--stream-output"], + ) + cls.eos_token_id = [119690] + cls.tokenizer = AutoTokenizer.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, use_fast=False + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + prompt_text="The capital of France is", + max_new_tokens=32, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + input_ids = self.get_input_ids(prompt_text) + + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": max_new_tokens, + "n": n, + "stop_token_ids": [self.tokenizer.eos_token_id], + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + ret = response.json() + print(json.dumps(ret, indent=2)) + + def assert_one_item(item): + if item["meta_info"]["finish_reason"]["type"] == "stop": + self.assertEqual( + item["meta_info"]["finish_reason"]["matched"], + self.tokenizer.eos_token_id, + ) + elif item["meta_info"]["finish_reason"]["type"] == "length": + self.assertEqual( + len(item["output_ids"]), item["meta_info"]["completion_tokens"] + ) + self.assertEqual(len(item["output_ids"]), max_new_tokens) + self.assertEqual(item["meta_info"]["prompt_tokens"], len(input_ids)) + + if return_logprob: + self.assertEqual( + len(item["meta_info"]["input_token_logprobs"]), + len(input_ids), + f'{len(item["meta_info"]["input_token_logprobs"])} mismatch with {len(input_ids)}', + ) + self.assertEqual( + len(item["meta_info"]["output_token_logprobs"]), + max_new_tokens, + ) + + # Determine whether to assert a single item or multiple items based on n + if n == 1: + assert_one_item(ret) + else: + self.assertEqual(len(ret), n) + for i in range(n): + assert_one_item(ret[i]) + + print("=" * 100) + + def run_decode_stream(self, return_logprob=False, top_logprobs_num=0, n=1): + max_new_tokens = 32 + input_ids = self.get_input_ids("The capital of France is") + requests.post(self.base_url + "/flush_cache") + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": max_new_tokens, + "n": n, + "stop_token_ids": self.eos_token_id, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + ret = response.json() + print(json.dumps(ret)) + output_ids = ret["output_ids"] + print("output from non-streaming request:") + print(output_ids) + print(self.tokenizer.decode(output_ids, skip_special_tokens=True)) + + requests.post(self.base_url + "/flush_cache") + response_stream = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": max_new_tokens, + "n": n, + "stop_token_ids": self.eos_token_id, + }, + "stream": True, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + + response_stream_json = [] + for line in response_stream.iter_lines(): + print(line) + if line.startswith(b"data: ") and line[6:] != b"[DONE]": + response_stream_json.append(json.loads(line[6:])) + out_stream_ids = [] + for x in response_stream_json: + out_stream_ids += x["output_ids"] + print("output from streaming request:") + print(out_stream_ids) + print(self.tokenizer.decode(out_stream_ids, skip_special_tokens=True)) + + assert output_ids == out_stream_ids + + def test_simple_decode(self): + self.run_decode() + + def test_parallel_sample(self): + self.run_decode(n=3) + + def test_logprob(self): + for top_logprobs_num in [0, 3]: + self.run_decode(return_logprob=True, top_logprobs_num=top_logprobs_num) + + def test_eos_behavior(self): + self.run_decode(max_new_tokens=256) + + def test_simple_decode_stream(self): + self.run_decode_stream() + + def get_input_ids(self, prompt_text) -> list[int]: + input_ids = self.tokenizer(prompt_text, return_tensors="pt")["input_ids"][ + 0 + ].tolist() + return input_ids + + +class TestSkipTokenizerInitVLM(TestSkipTokenizerInit): + @classmethod + def setUpClass(cls): + cls.image_url = DEFAULT_IMAGE_URL + response = requests.get(cls.image_url) + cls.image = Image.open(BytesIO(response.content)) + cls.model = DEFAULT_SMALL_VLM_MODEL_NAME + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model, use_fast=False) + cls.processor = AutoProcessor.from_pretrained(cls.model, trust_remote_code=True) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--skip-tokenizer-init"], + ) + cls.eos_token_id = [cls.tokenizer.eos_token_id] + + def get_input_ids(self, _prompt_text) -> list[int]: + chat_template = get_chat_template_by_model_path(self.model) + text = f"{chat_template.image_token}What is in this picture?" + inputs = self.processor( + text=[text], + images=[self.image], + return_tensors="pt", + ) + + return inputs.input_ids[0].tolist() + + def test_simple_decode_stream(self): + # TODO mick + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_srt_endpoint.py b/test/srt/test_srt_endpoint.py new file mode 100644 index 0000000..81671e4 --- /dev/null +++ b/test/srt/test_srt_endpoint.py @@ -0,0 +1,518 @@ +""" +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_simple_decode +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_logprob_with_chunked_prefill +""" + +import json +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from typing import Optional + +import numpy as np +import requests + +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_logprob_check, +) + + +class TestSRTEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--enable-custom-logit-processor", + "--mem-fraction-static", + "0.7", + "--cuda-graph-max-bs", + "8", + ), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + return_logprob=False, + top_logprobs_num=0, + return_text=False, + n=1, + stream=False, + batch=False, + ): + if batch: + text = ["The capital of France is"] + else: + text = "The capital of France is" + + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 16, + "n": n, + }, + "stream": stream, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + if not stream: + response_json = response.json() + else: + response_json = [] + for line in response.iter_lines(): + if line.startswith(b"data: ") and line[6:] != b"[DONE]": + response_json.append(json.loads(line[6:])) + + print(json.dumps(response_json, indent=2)) + print("=" * 100) + + def test_simple_decode(self): + self.run_decode() + + def test_simple_decode_batch(self): + self.run_decode(batch=True) + + def test_parallel_sample(self): + self.run_decode(n=3) + + def test_parallel_sample_stream(self): + self.run_decode(n=3, stream=True) + + def test_logprob(self): + self.run_decode( + return_logprob=True, + top_logprobs_num=5, + return_text=True, + ) + + def test_logprob_start_len(self): + logprob_start_len = 4 + new_tokens = 4 + prompts = [ + "I have a very good idea on", + "Today is a sunndy day and", + ] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + response_json = response.json() + print(json.dumps(response_json, indent=2)) + + for i, res in enumerate(response_json): + self.assertEqual( + res["meta_info"]["prompt_tokens"], + logprob_start_len + len(res["meta_info"]["input_token_logprobs"]), + ) + assert prompts[i].endswith( + "".join([x[-1] for x in res["meta_info"]["input_token_logprobs"]]) + ) + + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + self.assertEqual( + res["text"], + "".join([x[-1] for x in res["meta_info"]["output_token_logprobs"]]), + ) + + def test_logprob_with_chunked_prefill(self): + """Test a long prompt that requests output logprobs will not hit OOM.""" + new_tokens = 4 + prompts = "I have a very good idea on this. " * 8000 + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "logprob_start_len": -1, + "top_logprobs_num": 5, + }, + ) + response_json = response.json() + # print(json.dumps(response_json, indent=2)) + + res = response_json + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + + # Test the number of tokens are correct + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), new_tokens) + + # Test the top-1 tokens are the same as output tokens (because temp = 0.0) + for i in range(new_tokens): + self.assertListEqual( + res["meta_info"]["output_token_logprobs"][i], + res["meta_info"]["output_top_logprobs"][i][0], + ) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"][i]), 5) + + def test_logprob_match(self): + """Test the output logprobs are close to the input logprobs if we run a prefill again.""" + + def run_generate( + prompt, return_logprob=False, max_new_tokens=512, logprob_start_len=-1 + ): + + if isinstance(prompt, str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + response = requests.post( + self.base_url + "/generate", + json={ + **prompt_kwargs, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + return response.json() + + prompt = "I have a very good idea on how to" + + gen = run_generate(prompt, return_logprob=True, logprob_start_len=0) + output_logprobs = np.array( + [x[0] for x in gen["meta_info"]["output_token_logprobs"]] + ) + num_prompts_tokens = gen["meta_info"]["prompt_tokens"] + + input_tokens = [x[1] for x in gen["meta_info"]["input_token_logprobs"]] + output_tokens = [x[1] for x in gen["meta_info"]["output_token_logprobs"]] + + new_prompt = input_tokens + output_tokens + score = run_generate( + new_prompt, return_logprob=True, logprob_start_len=0, max_new_tokens=0 + ) + output_logprobs_score = np.array( + [ + x[0] + for x in score["meta_info"]["input_token_logprobs"][num_prompts_tokens:] + ] + ) + + print(f"{output_logprobs[-10:]=}") + print(f"{output_logprobs_score[-10:]=}") + + diff = np.abs(output_logprobs - output_logprobs_score) + max_diff = np.max(diff) + self.assertLess(max_diff, 0.35) + + def test_logprob_mixed(self): + args = [] + temperature = 0 + # input_len, output_len, temperature, logprob_start_len, return_logprob, top_logprobs_num + for input_len in [1000, 5000, 10000, 50000]: + for output_len in [4, 8]: + for logprob_start_len in [0, 500, 2500, 5000, 25000]: + for return_logprob in [True, False]: + for top_logprobs_num in [0, 5]: + + if logprob_start_len >= input_len: + continue + + args.append( + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) + ) + + random.shuffle(args) + + func = partial(run_logprob_check, self) + with ThreadPoolExecutor(8) as executor: + list(executor.map(func, args)) + + def test_logprob_grammar(self): + prompts = "Question: Is Paris the Capital of France? Answer:" + allowed_tokens = [" Yes", " No"] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 1, + "regex": "( Yes| No)", + }, + "return_logprob": True, + "top_logprobs_num": 5, # The grammar constraint allows all prefix tokens so we need to use a larger top_k. + "return_text_in_logprobs": True, + }, + ) + response_json = response.json() + output_top_logprobs = response_json["meta_info"]["output_top_logprobs"][0] + print(f"{output_top_logprobs=}") + + # Parse results + # This is becaues the grammar constraint allows all prefix tokens + logprobs = [None] * 2 + for i in range(len(output_top_logprobs)): + try: + idx = allowed_tokens.index(output_top_logprobs[i][2]) + except ValueError: + # Not found + continue + logprobs[idx] = output_top_logprobs[i][0] + + self.assertTrue(all(x is not None for x in logprobs)) + + def run_custom_logit_processor(self, target_token_id: Optional[int] = None): + """Test custom logit processor with custom params. + + If target_token_id is None, the custom logit processor won't be passed in. + """ + + custom_params = {"token_id": target_token_id} + + class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + key = "token_id" + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[key]] = 0.0 + return logits + + prompts = "Question: Is Paris the Capital of France? Answer:" + + # Base case json data to be posted to the server. + base_json = { + "text": prompts, + "sampling_params": {"temperature": 0.0}, + "return_logprob": True, + } + + # Custom json data with custom logit processor and params. + custom_json = base_json.copy() + # Only set the custom logit processor if target_token_id is not None. + if target_token_id is not None: + custom_json["custom_logit_processor"] = ( + DeterministicLogitProcessor().to_str() + ) + custom_json["sampling_params"]["custom_params"] = custom_params + + custom_response = requests.post( + self.base_url + "/generate", + json=custom_json, + ).json() + + output_token_logprobs = custom_response["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # The logit processor should always sample the given token as the logits is deterministic. + if target_token_id is not None: + self.assertTrue( + all(x == custom_params["token_id"] for x in sampled_tokens), + # Print the detailed test case info if the test fails. + f"{target_token_id=}\n{sampled_tokens=}\n{custom_response=}", + ) + + def run_stateful_custom_logit_processor( + self, first_token_id: int | None, delay: int = 2 + ): + """Test custom logit processor with custom params and state. + + Should sample the first `delay` tokens normally, then output first_token_id and consecutive tokens after that. + If first_token_id is None, the custom logit processor won't be passed in. + """ + + custom_params = {"token_id": first_token_id, "delay": 2} + + class DeterministicStatefulLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + + for i, param_dict in enumerate(custom_param_list): + if param_dict["delay"] > 0: + param_dict["delay"] -= 1 + continue + if param_dict["delay"] == 0: + param_dict["delay"] -= 1 + force_token = param_dict["token_id"] + else: + output_ids = param_dict["__req__"].output_ids + force_token = output_ids[-1] + 1 + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, force_token] = 0.0 + return logits + + prompts = "Question: Is Paris the Capital of France? Answer:" + + # Base case json data to be posted to the server. + base_json = { + "text": prompts, + "sampling_params": {"temperature": 0.0}, + "return_logprob": True, + } + + # Custom json data with custom logit processor and params. + custom_json = base_json.copy() + # Only set the custom logit processor if target_token_id is not None. + if first_token_id is not None: + custom_json["custom_logit_processor"] = ( + DeterministicStatefulLogitProcessor().to_str() + ) + custom_json["sampling_params"]["custom_params"] = custom_params + + custom_response = requests.post( + self.base_url + "/generate", + json=custom_json, + ).json() + + output_token_logprobs = custom_response["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + # The logit processor should always sample the given token as the logits is deterministic. + if first_token_id is not None: + self.assertTrue( + all( + x == custom_params["token_id"] + k + for k, x in enumerate(sampled_tokens[custom_params["delay"] :]) + ), + # Print the detailed test case info if the test fails. + f"{first_token_id=}\n{sampled_tokens=}\n{custom_response=}", + ) + + def test_custom_logit_processor(self): + """Test custom logit processor with a single request.""" + self.run_custom_logit_processor(target_token_id=5) + + def test_custom_logit_processor_batch_mixed(self): + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list(executor.map(self.run_custom_logit_processor, target_token_ids)) + + def test_stateful_custom_logit_processor(self): + """Test custom logit processor with a single request.""" + self.run_stateful_custom_logit_processor(first_token_id=5) + + def test_stateful_custom_logit_processor_batch_mixed(self): + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list( + executor.map(self.run_stateful_custom_logit_processor, target_token_ids) + ) + + def test_cache_tokens(self): + for _ in range(2): + time.sleep(1) + response = requests.post(self.base_url + "/flush_cache") + assert response.status_code == 200 + + def send_and_check_cached_tokens(input_ids): + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": list(input_ids), + "sampling_params": { + "max_new_tokens": 1, + }, + }, + ) + response_json = response.json() + return response_json["meta_info"]["cached_tokens"] + + self.assertEqual(send_and_check_cached_tokens(range(0, 100)), 0) + self.assertEqual(send_and_check_cached_tokens(range(0, 10000)), 100) + self.assertEqual(send_and_check_cached_tokens(range(0, 10000)), 9999) + self.assertEqual(send_and_check_cached_tokens(range(0, 1000)), 999) + self.assertEqual(send_and_check_cached_tokens(range(0, 11000)), 10000) + + def test_get_server_info(self): + response = requests.get(self.base_url + "/get_server_info") + response_json = response.json() + + max_total_num_tokens = response_json["max_total_num_tokens"] + self.assertIsInstance(max_total_num_tokens, int) + + attention_backend = response_json["attention_backend"] + self.assertIsInstance(attention_backend, str) + + version = response_json["version"] + self.assertIsInstance(version, str) + + def test_get_server_info_concurrent(self): + """Make sure the concurrent get_server_info doesn't crash the server.""" + tp = ThreadPoolExecutor(max_workers=30) + + def s(): + server_info = requests.get(self.base_url + "/get_server_info") + server_info.json() + + futures = [] + for _ in range(4): + futures.append(tp.submit(s)) + + for f in futures: + f.result() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_srt_engine.py b/test/srt/test_srt_engine.py new file mode 100644 index 0000000..672344c --- /dev/null +++ b/test/srt/test_srt_engine.py @@ -0,0 +1,190 @@ +""" +Usage: +python3 -m unittest test_srt_engine.TestSRTEngine.test_4_sync_async_stream_combination +""" + +import asyncio +import json +import unittest +from types import SimpleNamespace + +import torch + +import sglang as sgl +from sglang.bench_offline_throughput import BenchArgs, throughput_test +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.server_args import ServerArgs +from sglang.test.few_shot_gsm8k_engine import run_eval +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, +) + + +class TestSRTEngine(CustomTestCase): + + def test_1_engine_runtime_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine(model_path=model_path, random_seed=42) + out1 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + runtime = sgl.Runtime(model_path=model_path, random_seed=42) + out2 = json.loads(runtime.generate(prompt, sampling_params))["text"] + runtime.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_2_engine_runtime_encode_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + + engine = sgl.Engine(model_path=model_path, is_embedding=True, random_seed=42) + out1 = torch.tensor(engine.encode(prompt)["embedding"]) + engine.shutdown() + + runtime = sgl.Runtime(model_path=model_path, is_embedding=True, random_seed=42) + out2 = torch.tensor(json.loads(runtime.encode(prompt))["embedding"]) + runtime.shutdown() + + self.assertTrue(torch.allclose(out1, out2, atol=1e-5, rtol=1e-3)) + + def test_3_engine_token_ids_consistency(self): + # just to ensure there is no issue running multiple generate calls + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine( + model_path=model_path, random_seed=42, disable_radix_cache=True + ) + out1 = engine.generate(prompt, sampling_params)["text"] + + tokenizer = get_tokenizer(model_path) + token_ids = tokenizer.encode(prompt) + out2 = engine.generate(input_ids=token_ids, sampling_params=sampling_params)[ + "text" + ] + + engine.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_4_sync_async_stream_combination(self): + prompt = "AI safety is" + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Create an LLM. + llm = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + ) + + if True: + # 1. sync + non streaming + print("\n\n==== 1. sync + non streaming ====") + output = llm.generate(prompt, sampling_params) + print(output["text"]) + + # 2. sync + streaming + print("\n\n==== 2. sync + streaming ====") + output_generator = llm.generate(prompt, sampling_params, stream=True) + offset = 0 + for output in output_generator: + print(output["text"][offset:], end="", flush=True) + offset = len(output["text"]) + print() + + if True: + loop = asyncio.get_event_loop() + # 3. async + non_streaming + print("\n\n==== 3. async + non streaming ====") + output = loop.run_until_complete( + llm.async_generate(prompt, sampling_params) + ) + print(output["text"]) + + # 4. async + streaming + async def async_streaming(engine): + generator = await engine.async_generate( + prompt, sampling_params, stream=True + ) + + offset = 0 + async for output in generator: + print(output["text"][offset:], end="", flush=True) + offset = len(output["text"]) + print() + + print("\n\n==== 4. async + streaming ====") + loop.run_until_complete(async_streaming(llm)) + + llm.shutdown() + + def test_5_gsm8k(self): + + args = SimpleNamespace( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + local_data_path=None, + num_shots=5, + num_questions=200, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["accuracy"], 0.3) + + def test_6_engine_cpu_offload(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + max_total_tokens=128, + ) + out1 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + max_total_tokens=128, + cpu_offload_gb=3, + ) + out2 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_7_engine_offline_throughput(self): + server_args = ServerArgs( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + ) + bench_args = BenchArgs(num_prompts=10) + result = throughput_test(server_args=server_args, bench_args=bench_args) + self.assertGreater(result["total_throughput"], 3000) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_srt_engine_with_quant_args.py b/test/srt/test_srt_engine_with_quant_args.py new file mode 100644 index 0000000..e3b30ea --- /dev/null +++ b/test/srt/test_srt_engine_with_quant_args.py @@ -0,0 +1,60 @@ +import unittest + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestSRTEngineWithQuantArgs(CustomTestCase): + + def test_1_quantization_args(self): + + # we only test fp8 because other methods are currenly depend on vllm. We can add other methods back to test after vllm depency is resolved. + quantization_args_list = [ + # "awq", + "fp8", + # "gptq", + # "marlin", + # "gptq_marlin", + # "awq_marlin", + # "bitsandbytes", + # "gguf", + ] + + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + for quantization_args in quantization_args_list: + engine = sgl.Engine( + model_path=model_path, random_seed=42, quantization=quantization_args + ) + engine.generate(prompt, sampling_params) + engine.shutdown() + + def test_2_torchao_args(self): + + # we don't test int8dq because currently there is conflict between int8dq and capture cuda graph + torchao_args_list = [ + # "int8dq", + "int8wo", + "fp8wo", + "fp8dq-per_tensor", + "fp8dq-per_row", + ] + [f"int4wo-{group_size}" for group_size in [32, 64, 128, 256]] + + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + for torchao_config in torchao_args_list: + engine = sgl.Engine( + model_path=model_path, random_seed=42, torchao_config=torchao_config + ) + engine.generate(prompt, sampling_params) + engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_compile.py b/test/srt/test_torch_compile.py new file mode 100644 index 0000000..760cec8 --- /dev/null +++ b/test/srt/test_torch_compile.py @@ -0,0 +1,75 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchCompile(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-torch-compile", "--cuda-graph-max-bs", "4"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_throughput(self): + # Warmup + res = self.run_decode(16) + + max_tokens = 256 + tic = time.time() + res = self.run_decode(max_tokens) + tok = time.time() + print(f"{res=}") + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + self.assertGreaterEqual(throughput, 152) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_compile_moe.py b/test/srt/test_torch_compile_moe.py new file mode 100644 index 0000000..34c80d4 --- /dev/null +++ b/test/srt/test_torch_compile_moe.py @@ -0,0 +1,74 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchCompileMoe(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-torch-compile", "--torch-compile-max-bs", "8"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.50) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_throughput(self): + # Warmup + res = self.run_decode(16) + + max_tokens = 256 + tic = time.time() + res = self.run_decode(max_tokens) + tok = time.time() + print(f"{res=}") + throughput = max_tokens / (tok - tic) + self.assertGreaterEqual(throughput, 285) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_native_attention_backend.py b/test/srt/test_torch_native_attention_backend.py new file mode 100644 index 0000000..3af0557 --- /dev/null +++ b/test/srt/test_torch_native_attention_backend.py @@ -0,0 +1,59 @@ +""" +Usage: +python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_one_batch, +) + + +class TestTorchNativeAttnBackend(CustomTestCase): + def test_latency(self): + output_throughput = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, + ["--attention-backend", "torch_native"], + ) + + if is_in_ci(): + # Torch native backend is expected to be slower + assert output_throughput > 50, f"{output_throughput=}" + + def test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "torch_native"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_tp.py b/test/srt/test_torch_tp.py new file mode 100644 index 0000000..8349cdf --- /dev/null +++ b/test/srt/test_torch_tp.py @@ -0,0 +1,24 @@ +import unittest + +from sglang.test.test_utils import CustomTestCase, is_in_ci, run_bench_one_batch + + +class TestTorchTP(CustomTestCase): + def test_torch_native_llama(self): + output_throughput = run_bench_one_batch( + "meta-llama/Meta-Llama-3-8B", + [ + "--tp", + "2", + "--json-model-override-args", + '{"architectures": ["TorchNativeLlamaForCausalLM"]}', + "--disable-cuda-graph", + ], + ) + + if is_in_ci(): + assert output_throughput > 0, f"{output_throughput=}" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torchao.py b/test/srt/test_torchao.py new file mode 100644 index 0000000..77ec0a5 --- /dev/null +++ b/test/srt/test_torchao.py @@ -0,0 +1,74 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchAO(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--torchao-config", "int4wo-128"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + assert metrics["score"] >= 0.60 + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + import time + + max_tokens = 256 + + tic = time.time() + res = self.run_decode(max_tokens) + tok = time.time() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 210 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_attention_backend.py b/test/srt/test_triton_attention_backend.py new file mode 100644 index 0000000..829901d --- /dev/null +++ b/test/srt/test_triton_attention_backend.py @@ -0,0 +1,64 @@ +""" +Usage: +python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_one_batch, +) + + +class TestTritonAttnBackend(CustomTestCase): + def test_latency(self): + output_throughput = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, + [ + "--attention-backend", + "triton", + "--enable-torch-compile", + "--cuda-graph-max-bs", + 4, + ], + ) + + if is_in_ci(): + self.assertGreater(output_throughput, 153) + + def test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "triton"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_attention_kernels.py b/test/srt/test_triton_attention_kernels.py new file mode 100644 index 0000000..184733e --- /dev/null +++ b/test/srt/test_triton_attention_kernels.py @@ -0,0 +1,390 @@ +import random +import unittest + +import torch + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd, + decode_attention_fwd_grouped, + decode_attention_fwd_normal, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + redundant_attention, +) +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.test.test_utils import CustomTestCase + + +class TestTritonAttention(CustomTestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D): + dtype = torch.bfloat16 + + b_seq_len_prefix = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device="cuda" + ) + b_seq_len_extend = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device="cuda" + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32, device="cuda") + b_start_loc = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device="cuda" + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + o_extend_mask = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + o_redundant = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + custom_mask = None + mask_indptr = None + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + mask_indptr, + max_len_extend, + ) + + b_seq_mask_len = b_seq_len_extend * b_seq_len + custom_mask = torch.ones( + (b_seq_mask_len.sum().item(),), dtype=torch.bool, device="cuda" + ) + mask_indptr = torch.zeros((B + 1,), dtype=torch.int64, device="cuda") + mask_indptr[1 : B + 1] = torch.cumsum(b_seq_mask_len[:B], dim=0) + for i in range(B): + causal_mask = ( + torch.tril( + torch.ones(b_seq_len_extend[i], b_seq_len_extend[i]), diagonal=0 + ) + == 1 + ) + prefix_mask = torch.ones( + b_seq_len_extend[i], b_seq_len_prefix[i], dtype=torch.bool + ) + mask_flatten = torch.cat([prefix_mask, causal_mask], dim=1).flatten() + custom_mask[mask_indptr[i] : mask_indptr[i + 1]] = mask_flatten + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend_mask, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + mask_indptr, + max_len_extend, + ) + + redundant_attention( + q_extend, + o_redundant, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + + self.assertTrue(torch.allclose(o_extend, o_redundant, rtol=1e-2)) + self.assertTrue(torch.allclose(o_extend_mask, o_redundant, rtol=1e-2)) + + def test_extend_attention(self): + + # Define the varying parameter values + attention_values = [128, 96, 80, 13] + + # Loop through the values and call the method + for value in attention_values: + self._test_extend_attention_once(19, 12331, 12, 4, value) + + def _test_context_attention_once(self, head_dim, is_causal): + # Set up a simple test case + num_heads = 4 + seq_lens = [8, 12] + max_seq_len = max(seq_lens) + + # Create random input tensors + q = torch.randn(sum(seq_lens), num_heads, head_dim, device="cuda") + k = torch.randn(sum(seq_lens), num_heads, head_dim, device="cuda") + v = torch.randn(sum(seq_lens), num_heads, head_dim, device="cuda") + o = torch.zeros(sum(seq_lens), num_heads, head_dim, device="cuda") + + # Create b_start_loc and b_seq_len tensors + b_start_loc = torch.tensor([0, seq_lens[0]], device="cuda") + b_seq_len = torch.tensor(seq_lens, device="cuda") + + context_attention_fwd( + q, k, v, o, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + + cu_seq_lens = [0] * (len(seq_lens) + 1) + for i, seq_len in enumerate(seq_lens): + cu_seq_lens[i + 1] = cu_seq_lens[i] + seq_len + + for i in range(len(seq_lens)): + start, end = cu_seq_lens[i], cu_seq_lens[i + 1] + o_torch = torch.nn.functional.scaled_dot_product_attention( + q[start:end].permute(1, 0, 2), + k[start:end].permute(1, 0, 2), + v[start:end].permute(1, 0, 2), + is_causal=is_causal, + ).permute(1, 0, 2) + + cos_sim = torch.nn.functional.cosine_similarity( + o[start:end].flatten(), o_torch.flatten(), dim=0 + ) + self.assertTrue(cos_sim.item() > 1 - (1e-5)) + self.assertTrue(torch.allclose(o[start:end], o_torch, atol=1e-2)) + + def test_context_attention(self): + head_dim = [128, 96, 80, 13] + + for dim in head_dim: + for is_causal in [True, False]: + self._test_context_attention_once(dim, is_causal) + + def _test_decode_attention_once(self, B, H_Q, H_KV, D): + dtype = torch.bfloat16 + seq_len = 10 # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D, dtype=dtype, device="cuda") + + b_seq_len = torch.full((B,), seq_len, device="cuda") + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len[:B], dim=0) + kv_indices = torch.arange(total_tokens, device="cuda") + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D), + dtype=torch.float32, + device="cuda", + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device="cuda", + ) + + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + def test_decode_attention(self): + # Here we just to ensure there is no error + # TODO: correctnesss test + + # Test configurations + configs = [ + (2, 4, 4, 64), # MHA + (2, 4, 2, 64), # GQA + (2, 4, 4, 80), # Non-standard head dim + (2, 4, 4, 13), # Prime number head dim + ] + + for B, H_Q, H_KV, D in configs: + self._test_decode_attention_once(B, H_Q, H_KV, D) + + def _test_grouped_decode_attention_once(self, B, S, H_Q, H_KV, D, D_V): + dtype = torch.bfloat16 + seq_len = S # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device="cuda") + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + + b_seq_len = torch.full((B,), seq_len, device="cuda") + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len[:B], dim=0) + kv_indices = torch.arange(total_tokens, device="cuda") + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device="cuda", + ) + + decode_attention_fwd_normal( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + attn_logits1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + attn_lse1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o_grouped, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + print(cos_sim.item()) + self.assertTrue(cos_sim.item() > 0.99) + self.assertTrue(torch.allclose(o, o_grouped, atol=3e-2)) + + def test_grouped_decode_attention(self): + seq_lens = [5, 100, 128, 500] + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 64, 64), + (2, 64, 1, 13, 13), + (2, 128, 1, 80, 80), + (2, 128, 2, 512, 512), + (2, 128, 1, 576, 512), + ] + + for S in seq_lens: + for B, H_Q, H_KV, D, D_V in configs: + self._test_grouped_decode_attention_once(B, S, H_Q, H_KV, D, D_V) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_attention_rocm_mla.py b/test/srt/test_triton_attention_rocm_mla.py new file mode 100644 index 0000000..f3074ae --- /dev/null +++ b/test/srt/test_triton_attention_rocm_mla.py @@ -0,0 +1,259 @@ +import random +import unittest + +import torch + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd_grouped, +) +from sglang.srt.layers.attention.triton_ops.rocm_mla_decode_rope import ( + decode_attention_fwd_grouped_rope, +) +from sglang.srt.layers.rotary_embedding import DeepseekScalingRotaryEmbedding +from sglang.test.test_utils import CustomTestCase + + +class TestTritonAttentionMLA(CustomTestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def preprocess_kv_cache(self, kv_cache, kv_lora_rank): + latent_cache = kv_cache + v_input = latent_cache[..., :kv_lora_rank] + v_input = v_input.contiguous().unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., :kv_lora_rank] = v_input + + return k_input, v_input + + def input_helper( + self, + B, + H, + S, + kv_lora_rank, + rotary_dim, + qk_rope_head_dim, + num_kv_splits, + dtype, + device, + rope_base=10, + rope_max_seq_len=16384, + rope_scaling=1.0, + is_neox_style=False, + ): + q = torch.randn( + B, H, kv_lora_rank + qk_rope_head_dim, device=device, dtype=dtype + ) + kv_cache = torch.randn( + B * S, kv_lora_rank + qk_rope_head_dim, dtype=dtype, device=device + ) + kv_indptr = torch.arange(B + 1, device=device) * S + kv_indices = torch.arange(B * S, device=device) + attn_logits = torch.empty( + B, H, num_kv_splits, kv_lora_rank + 1, dtype=dtype, device=device + ) + rotary_emb = DeepseekScalingRotaryEmbedding( + qk_rope_head_dim, + rotary_dim, + rope_max_seq_len, + rope_base, + is_neox_style, + rope_scaling, + q.dtype, + device="cpu", + ).cuda() + positions = torch.tensor([S], device=device).unsqueeze(0).repeat(B, 1) + + return kv_indptr, kv_indices, q, kv_cache, attn_logits, rotary_emb, positions + + def ref_compute_full_fwd( + self, + q, + k_input, + v_input, + kv_lora_rank, + kv_indptr, + kv_indices, + num_kv_splits, + sm_scale, + logit_cap, + rotary_emb, + positions, + use_rope, + device="cuda", + ): + + B, H = q.shape[0], q.shape[1] + S = kv_indptr[1].item() + qk_rope_head_dim = k_input.shape[-1] - kv_lora_rank + + q_input = torch.empty(B, H, kv_lora_rank + qk_rope_head_dim, dtype=q.dtype).to( + device + ) + q_nope_out, q_pe = q.split([kv_lora_rank, qk_rope_head_dim], dim=-1) + k_pe_t = k_input.view(B, 1, S, -1)[:, :, -1:, kv_lora_rank:] + + if use_rope: + q_pe, k_pe_t = rotary_emb(positions, q_pe.unsqueeze(2), k_pe_t) + q_pe = q_pe.squeeze() + + k_input.view(B, 1, S, -1)[:, :, -1:, kv_lora_rank:] = k_pe_t + + q_input[..., :kv_lora_rank] = q_nope_out + q_input[..., kv_lora_rank:] = q_pe + + B, H = q_input.shape[0], q_input.shape[1] + kv_lora_rank = v_input.shape[-1] + device = q_input.device + + attn_logits = torch.empty( + B, H, num_kv_splits, kv_lora_rank + 1, dtype=q_input.dtype, device=device + ) + o = torch.empty(B, H, kv_lora_rank, dtype=q_input.dtype, device=device) + + decode_attention_fwd_grouped( + q_input, + k_input, + v_input, + o, + kv_indptr, + kv_indices, + attn_logits, + num_kv_splits, + sm_scale, + logit_cap, + ) + + return attn_logits, o, k_pe_t.squeeze() + + def _test_rocm_fused_mla_kernel( + self, + B, + H, + S, + kv_lora_rank, + qk_rope_head_dim, + rotary_dim, + dtype, + use_rope, + is_neox_style, + num_kv_splits=2, + sm_scale=1.0, + logit_cap=0.0, + device="cuda", + ): + kv_indptr, kv_indices, q, kv_cache, attn_logits, rotary_emb, positions = ( + self.input_helper( + B, + H, + S, + kv_lora_rank, + rotary_dim, + qk_rope_head_dim, + num_kv_splits, + dtype, + device=device, + is_neox_style=is_neox_style, + ) + ) + + k_input, v_input = self.preprocess_kv_cache(kv_cache, kv_lora_rank) + k_pe_tokens = torch.empty( + B, qk_rope_head_dim, dtype=kv_cache.dtype, device=device + ) + tri_o = torch.empty(B, H, kv_lora_rank, dtype=kv_cache.dtype, device=device) + + decode_attention_fwd_grouped_rope( + q, + k_input, + v_input, + tri_o, + kv_indptr, + kv_indices, + k_pe_tokens if use_rope else None, + kv_lora_rank, + rotary_dim if use_rope else None, + rotary_emb.cos_sin_cache if use_rope else None, + positions if use_rope else None, + attn_logits, + num_kv_splits, + sm_scale, + logit_cap, + use_rope, + is_neox_style, + ) + + tri_logits = attn_logits + + # reference + ref_logits, ref_o, ref_k_pe_tokens = self.ref_compute_full_fwd( + q, + k_input, + v_input, + kv_lora_rank, + kv_indptr, + kv_indices, + num_kv_splits, + sm_scale, + logit_cap, + rotary_emb, + positions, + use_rope, + device="cuda", + ) + + if use_rope: + torch.testing.assert_close( + ref_k_pe_tokens, k_pe_tokens.squeeze(), atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(ref_logits, tri_logits, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(ref_o, tri_o, atol=1e-2, rtol=1e-2) + + def test_grouped_rocm_fused_mla(self): + configs = [ + (1, 128, 2048, 512, 64, 64), + (1, 128, 2048, 512, 128, 64), + (1, 128, 2048, 512, 127, 64), + (1, 128, 2050, 512, 127, 64), + (1, 128, 2050, 512, 128, 64), + (8, 128, 2048, 512, 64, 64), + (8, 128, 2048, 512, 128, 64), + (8, 128, 2048, 512, 127, 64), + (8, 128, 2050, 512, 127, 64), + (8, 128, 2050, 512, 128, 64), + ] + dtypes = [torch.bfloat16, torch.float32] + use_rope_list = [True, False] + is_neox_style_list = [True, False] + + for B, H, S, kv_lora_rank, qk_rope_head_dim, rotary_dim in configs: + for dtype in dtypes: + for use_rope in use_rope_list: + for is_neox_style in is_neox_style_list: + self._test_rocm_fused_mla_kernel( + B, + H, + S, + kv_lora_rank, + qk_rope_head_dim, + rotary_dim, + dtype, + use_rope, + is_neox_style, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_update_weights_from_disk.py b/test/srt/test_update_weights_from_disk.py new file mode 100644 index 0000000..11b7e67 --- /dev/null +++ b/test/srt/test_update_weights_from_disk.py @@ -0,0 +1,293 @@ +import json +import random +import unittest + +import requests + +import sglang as sgl +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +############################################################################### +# Engine Mode Tests (Single-configuration) +############################################################################### +class TestEngineUpdateWeightsFromDisk(CustomTestCase): + def setUp(self): + self.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + # Initialize the engine in offline (direct) mode. + self.engine = sgl.Engine(model_path=self.model) + + def tearDown(self): + self.engine.shutdown() + + def run_decode(self): + prompts = ["The capital of France is"] + sampling_params = {"temperature": 0, "max_new_tokens": 32} + outputs = self.engine.generate(prompts, sampling_params) + print("=" * 100) + print( + f"[Engine Mode] Prompt: {prompts[0]}\nGenerated text: {outputs[0]['text']}" + ) + return outputs[0]["text"] + + def run_update_weights(self, model_path): + ret = self.engine.update_weights_from_disk(model_path) + print(json.dumps(ret)) + return ret + + def test_update_weights(self): + origin_response = self.run_decode() + # Update weights: use new model (remove "-Instruct") + new_model_path = self.model.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path) + self.assertTrue(ret[0]) # ret is a tuple; index 0 holds the success flag + + updated_response = self.run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + + # Revert back to original weights + ret = self.run_update_weights(self.model) + self.assertTrue(ret[0]) + reverted_response = self.run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + + def test_update_weights_unexist_model(self): + origin_response = self.run_decode() + new_model_path = self.model.replace("-Instruct", "wrong") + ret = self.run_update_weights(new_model_path) + self.assertFalse(ret[0]) + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + +############################################################################### +# HTTP Server Mode Tests (Single-configuration) +############################################################################### +class TestServerUpdateWeightsFromDisk(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + print("=" * 100) + print(f"[Server Mode] Generated text: {response.json()['text']}") + return response.json()["text"] + + def get_model_info(self): + response = requests.get(self.base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(self, model_path): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": model_path}, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + def test_update_weights(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + origin_response = self.run_decode() + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path) + self.assertTrue(ret["success"]) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + + ret = self.run_update_weights(origin_model_path) + self.assertTrue(ret["success"]) + updated_model_path = self.get_model_info() + self.assertEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + def test_update_weights_unexist_model(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + origin_response = self.run_decode() + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "wrong") + ret = self.run_update_weights(new_model_path) + self.assertFalse(ret["success"]) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + +############################################################################### +# Parameterized Tests for update_weights_from_disk +# Test coverage is determined based on the value of is_in_ci: +# - In a CI environment: randomly select one mode (Engine or Server) and test only with tp=1, dp=1. +# - In a non-CI environment: test both Engine and Server modes, and enumerate all combinations +# with tp and dp ranging from 1 to 2. +############################################################################### +class TestUpdateWeightsFromDiskParameterized(CustomTestCase): + def run_common_test(self, mode, tp, dp): + """ + Common test procedure for update_weights_from_disk. + For Engine mode, we instantiate the engine with tp_size=tp. + For Server mode, we launch the server with additional arguments for tp (dp is not used in server launch here). + """ + if mode == "Engine": + # Instantiate engine with additional parameter tp_size. + print(f"[Parameterized Engine] Testing with tp={tp}, dp={dp}") + engine = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + random_seed=42, + tp_size=tp, + # dp parameter is not explicitly used in this API. + ) + try: + origin_response = self._engine_update_weights_test(engine) + finally: + engine.shutdown() + elif mode == "Server": + print(f"[Parameterized Server] Testing with tp={tp}, dp={dp}") + # Pass additional arguments to launch the server. + base_args = ["--tp-size", str(tp)] + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=base_args, + ) + try: + origin_response = self._server_update_weights_test(DEFAULT_URL_FOR_TEST) + finally: + kill_process_tree(process.pid) + else: + raise ValueError(f"Unknown mode: {mode}") + + def _engine_update_weights_test(self, engine): + # Run the update weights test on the given engine instance. + def run_decode(): + prompts = ["The capital of France is"] + sampling_params = {"temperature": 0, "max_new_tokens": 32} + outputs = engine.generate(prompts, sampling_params) + print("=" * 100) + print( + f"[Parameterized Engine] Prompt: {prompts[0]}\nGenerated text: {outputs[0]['text']}" + ) + return outputs[0]["text"] + + def run_update_weights(model_path): + ret = engine.update_weights_from_disk(model_path) + print(json.dumps(ret)) + return ret + + origin_response = run_decode() + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = run_update_weights(new_model_path) + self.assertTrue(ret[0]) + updated_response = run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + ret = run_update_weights(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + self.assertTrue(ret[0]) + reverted_response = run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + return origin_response + + def _server_update_weights_test(self, base_url): + def run_decode(): + response = requests.post( + base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + print("=" * 100) + print(f"[Parameterized Server] Generated text: {response.json()['text']}") + return response.json()["text"] + + def get_model_info(): + response = requests.get(base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(model_path): + response = requests.post( + base_url + "/update_weights_from_disk", + json={"model_path": model_path}, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + origin_model_path = get_model_info() + origin_response = run_decode() + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = run_update_weights(new_model_path) + self.assertTrue(ret["success"]) + updated_model_path = get_model_info() + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + updated_response = run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + ret = run_update_weights(origin_model_path) + self.assertTrue(ret["success"]) + updated_model_path = get_model_info() + self.assertEqual(updated_model_path, origin_model_path) + reverted_response = run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + return origin_response + + def test_parameterized_update_weights(self): + if is_in_ci(): + # In CI, choose one random mode (Engine or Server) with tp=1, dp=1. + mode = random.choice(["Engine", "Server"]) + test_suits = [(1, 1, mode)] + else: + # Otherwise, test both modes and enumerate tp,dp combinations from 1 to 2. + test_suits = [] + for mode in ["Engine", "Server"]: + for tp in [1, 2]: + for dp in [1, 2]: + test_suits.append((tp, dp, mode)) + for tp, dp, mode in test_suits: + with self.subTest(mode=mode, tp=tp, dp=dp): + self.run_common_test(mode, tp, dp) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_update_weights_from_distributed.py b/test/srt/test_update_weights_from_distributed.py new file mode 100644 index 0000000..7352e75 --- /dev/null +++ b/test/srt/test_update_weights_from_distributed.py @@ -0,0 +1,605 @@ +"""Test distributed weight updates. + +This test suite simulates a distributed training environment to ensure +correct weight synchronization. On rank 0, the instruct model represents +pre-training weights, and the base model represents post-training weights. +The base model's weights are broadcasted to other ranks using the online +weight update API. + +On other ranks, an engine is initialized with the instruct model, and its +parameters are verified against the Hugging Face model. After updating +weights from the distributed system, post-training weights are loaded +and verified again to ensure consistency and accuracy across the +distributed setup. +""" + +import gc +import os +import random +import time +import unittest + +import numpy as np +import requests +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.srt.utils import init_custom_process_group +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) +from sglang.utils import terminate_process + +mp.set_start_method("spawn", force=True) + + +def verify_params_close(params1, params2, error_msg): + """Verify if two parameter arrays are close enough.""" + try: + assert np.allclose(np.array(params1), np.array(params2)), error_msg + except Exception as e: + print(f"Parameters not close for {error_msg}") + print("Params1:", np.array(params1)) + print("Params2:", np.array(params2)) + raise e + + +def verify_params_not_close(params1, params2, error_msg): + """Verify if two parameter arrays are different enough.""" + assert not np.allclose(np.array(params1), np.array(params2)), error_msg + + +def init_process( + rank, + world_size, + param_queue, + truncate_size, + state_dict_key_to_shape, + tp_size, + model_name, + backend, + checking_parameters, + tie_word_embeddings, +): + torch.cuda.set_device(rank) + + if rank == 0: + init_process_hf( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + ) + elif rank in [1, 2]: + init_process_sgl( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + backend, + tp_size, + ) + + +def init_process_hf( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, +): + # These two environment variables are very important + # to avoid unexpected behaviors of CUDA and NCCL. + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = "0" + + # Load model and get parameters + hf_instruct_model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype="bfloat16", + tie_word_embeddings=tie_word_embeddings, + ).to("cuda:0") + base_model_name = model_name.replace("-Instruct", "") + hf_base_model = AutoModelForCausalLM.from_pretrained( + base_model_name, + torch_dtype="bfloat16", + tie_word_embeddings=tie_word_embeddings, + ).to("cuda:0") + + hf_instruct_params = [] + hf_base_params = [] + + print("get parameter in hf instruct model and base model") + for parameter_name in checking_parameters: + hf_instruct_params.append( + hf_instruct_model.get_parameter(parameter_name)[:truncate_size] + .cpu() + .detach() + .float() + .numpy() + .tolist() + ) + hf_base_params.append( + hf_base_model.get_parameter(parameter_name)[:truncate_size] + .cpu() + .detach() + .float() + .numpy() + .tolist() + ) + + param_queue.put(("hf_instruct_params", hf_instruct_params)) + param_queue.put(("hf_base_params", hf_base_params)) + + # Init weight update group for rank 0 (the training engine in RLHF). + print(f"rank {rank} world_size: {world_size} init custom process group") + group = init_custom_process_group( + backend="nccl", + init_method="tcp://localhost:65500", + world_size=world_size, + rank=rank, + group_name="test_parameter_update_group", + ) + dist.barrier(group=group, device_ids=[rank]) + torch.cuda.synchronize() + time_begin_broadcast = time.time() + + # The last parameter is lm_head.weight, which is tied + # with embed_tokens.weight. Actually, we only need + # to broadcast embed_tokens.weight once. + broadcast_parameters = list(state_dict_key_to_shape.keys()) + if tie_word_embeddings: + broadcast_parameters.remove("lm_head.weight") + + # Broadcast all the weights from the training + # engine to other ranks (inference engine). + for parameter_name in broadcast_parameters: + torch.distributed.broadcast( + hf_base_model.get_parameter(parameter_name), + src=0, + group=group, + ) + torch.cuda.synchronize() + time_end_broadcast = time.time() + + # Measure the latency of broadcasting/weights update. + broadcast_time = time_end_broadcast - time_begin_broadcast + print(f"rank {rank} broadcast parameter time: {broadcast_time:.3f}s") + param_queue.put(("broadcast_time", broadcast_time)) + + # Delete the huggingface models to free up memory. + del hf_instruct_model + del hf_base_model + gc.collect() + torch.cuda.empty_cache() + + +def init_process_sgl( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + backend, + tp_size, +): + torch.cuda.set_device(rank) + torch.cuda.synchronize() + base_gpu_id = 1 if rank == 1 else 1 + tp_size + if backend == "Engine": + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + base_gpu_id=base_gpu_id, + tp_size=tp_size, + ) + else: + if rank == 1: + url = DEFAULT_URL_FOR_TEST + else: + url = DEFAULT_URL_FOR_TEST.replace("2157", "2159") + process = popen_launch_server( + model_name, + url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--base-gpu-id", + str(base_gpu_id), + "--tp-size", + str(tp_size), + ), + ) + torch.cuda.synchronize() + if backend == "Engine": + print(f"rank {rank} init engine") + else: + print(f"rank {rank} init server on url: {url}") + + # Get weights of instruct model, i.e. pre-training weights. + instruct_params = [] + for parameter_name in checking_parameters: + instruct_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + if backend == "Engine" + else requests.get( + f"{url}/get_weights_by_name", + json={"name": parameter_name, "truncate_size": truncate_size}, + ).json() + ) + + param_queue.put((f"sgl_dp_{rank}_instruct_params", instruct_params)) + + # Init weight update group with the training engine. + if backend == "Engine": + engine.init_weights_update_group( + master_address="localhost", + master_port="65500", + rank_offset=base_gpu_id, + world_size=world_size, + group_name="test_parameter_update_group", + backend="nccl", + ) + else: + requests.post( + f"{url}/init_weights_update_group", + json={ + "master_address": "localhost", + "master_port": "65500", + "rank_offset": base_gpu_id, + "world_size": world_size, + "group_name": "test_parameter_update_group", + "backend": "nccl", + }, + ) + + torch.cuda.synchronize() + time_begin_update = time.time() + + # The last parameter is lm_head.weight, which is tied + # with embed_tokens.weight. Actually, we only need + # to update embed_tokens.weight once. + tie_word_embeddings = ( + True if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else False + ) + update_parameters = list(state_dict_key_to_shape.keys()) + if tie_word_embeddings: + update_parameters.remove("lm_head.weight") + + # Get weights from the training engine and update the inference engine. + for parameter_name in update_parameters: + if backend == "Engine": + engine.update_weights_from_distributed( + parameter_name, + dtype=torch.bfloat16, + shape=state_dict_key_to_shape[parameter_name], + ) + else: + requests.post( + f"{url}/update_weights_from_distributed", + json={ + "name": parameter_name, + "dtype": "bfloat16", + "shape": state_dict_key_to_shape[parameter_name], + }, + ) + torch.cuda.synchronize() + time_end_update = time.time() + + # Measure the latency of broadcast/weights update. + update_time = time_end_update - time_begin_update + print( + f"fully update model_name {model_name} rank {rank} parameter from distributed time: {update_time:.3f}s" + ) + param_queue.put((f"update_sgl_dp_{rank}_time", update_time)) + + # Get the weights of post-training model after weights update for correctness check. + base_params = [] + for parameter_name in checking_parameters: + if backend == "Engine": + base_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + ) + else: + base_params.append( + requests.get( + f"{url}/get_weights_by_name", + json={ + "name": parameter_name, + "truncate_size": truncate_size, + }, + ).json() + ) + param_queue.put((f"sgl_dp_{rank}_base_params", base_params)) + + # Shutdown the engine or terminate the server process. + if backend == "Engine": + engine.shutdown() + else: + terminate_process(process) + + +def assert_tied_weights(params_list, message, should_be_tied): + for params in params_list: + if should_be_tied: + assert np.allclose(params[0], params[-1]), message + else: + assert not np.allclose(params[0], params[-1]), message + + +def test_update_weights_from_distributed( + tp_size, + dp_size, + model_name, + backend, + state_dict_key_to_shape, + truncate_size, + checking_parameters, +): + tie_word_embeddings = ( + True if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else False + ) + + print( + f"Testing model: {model_name} tp_size: {tp_size}, dp_size: {dp_size} backend: {backend}" + ) + param_queue = mp.Queue() + results = {} + + context = mp.spawn( + init_process, + args=( + 1 + tp_size * dp_size, + param_queue, + truncate_size, + state_dict_key_to_shape, + tp_size, + model_name, + backend, + checking_parameters, + tie_word_embeddings, + ), + nprocs=1 + dp_size, + join=False, + ) + + while len(results) < 3 * (1 + dp_size): + try: + key, value = param_queue.get(timeout=5) + results[key] = value + except Exception as e: + if all(not p.is_alive() for p in context.processes): + break + + context.join() + + if len(results) != 3 * (1 + dp_size): + raise RuntimeError( + f"Expected {3 * (1 + dp_size)} parameters but got {len(results)}" + ) + + params = { + "hf_instruct": results.get("hf_instruct_params"), + "hf_base": results.get("hf_base_params"), + "sgl_dp_1_instruct": results.get("sgl_dp_1_instruct_params"), + "sgl_dp_1_base": results.get("sgl_dp_1_base_params"), + "broadcast_time": results.get("broadcast_time"), + "update_sgl_dp_1_time": results.get("update_sgl_dp_1_time"), + } + + if dp_size == 2: + dp2_params = { + "sgl_dp_2_instruct": results.get("sgl_dp_2_instruct_params"), + "sgl_dp_2_base": results.get("sgl_dp_2_base_params"), + "update_sgl_dp_2_time": results.get("update_sgl_dp_2_time"), + } + assert all(v is not None for v in dp2_params.values()) + params.update(dp2_params) + + # Check the correctness of weights update by verifying + # the weights of instruct model and base model. + for i in range(len(params["hf_instruct"])): + verify_params_close( + params["hf_instruct"][i], + params["sgl_dp_1_instruct"][i], + f"sgl_dp_1_instruct_params rank {i}", + ) + + verify_params_close( + params["hf_base"][i], + params["sgl_dp_1_base"][i], + f"sgl_dp_1_base_params rank {i}", + ) + + verify_params_not_close( + params["hf_instruct"][i], + params["hf_base"][i], + f"hf_instruct_params rank {i}", + ) + + if dp_size == 2: + verify_params_close( + params["hf_base"][i], + params["sgl_dp_2_base"][i], + f"sgl_dp_2_base_params rank {i}", + ) + verify_params_close( + params["hf_instruct"][i], + params["sgl_dp_2_instruct"][i], + f"sgl_dp_2_instruct_params rank {i}", + ) + + assert len(params["hf_instruct"]) == len( + params["hf_base"] + ), "hf_instruct_params and hf_base_params have different lengths" + + # Check if the weights of lm_head are tied with embed_tokens. + params_to_check = [ + ( + params["hf_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["hf_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_1_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_1_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ] + + if dp_size == 2: + params_to_check.extend( + [ + ( + params["sgl_dp_2_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_2_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ] + ) + + assert_tied_weights( + [params for params, _ in params_to_check], + ( + "lm_head.weight is not tied with embed_tokens.weight" + if tie_word_embeddings + else "lm_head.weight is tied with embed_tokens.weight" + ), + tie_word_embeddings, + ) + + # Time limit for broadcast and update on CI is 3 / 6 + # On local H100, it's 1 / 2 + time_limit = 3 if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else 6 + + assert ( + params["broadcast_time"] < time_limit + ), f"broadcast_time exceeds time limit {time_limit}s" + + assert ( + params["update_sgl_dp_1_time"] < time_limit + ), f"update_sgl_dp_one_time exceeds time limit {time_limit}s" + + if dp_size == 2: + assert ( + params["update_sgl_dp_2_time"] < time_limit + ), f"update_sgl_dp_two_time exceeds time limit {time_limit}s" + + # Delete the context and close the parameter queue. + del context + param_queue.close() + param_queue.join_thread() + gc.collect() + torch.cuda.empty_cache() + + +class TestUpdateWeightsFromDistributed(CustomTestCase): + + def test_update_weights_from_distributed(self): + + assert torch.cuda.device_count() >= 2, "At least 2 GPUs are required" + # test_suits : tp, dp, model_name, backend + if is_in_ci(): + mode = random.choice(["Engine", "Server"]) + test_suits = [ + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, mode), + ] + else: + test_suits = [ + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, "Engine"), + (1, 1, DEFAULT_MODEL_NAME_FOR_TEST, "Sever"), + ] + + if torch.cuda.device_count() >= 4: + test_suits.extend( + [ + (2, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, "Engine"), + (1, 2, DEFAULT_MODEL_NAME_FOR_TEST, "Server"), + ] + ) + + if torch.cuda.device_count() >= 5: + test_suits.extend( + [ + (2, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, "Engine"), + (2, 2, DEFAULT_MODEL_NAME_FOR_TEST, "Server"), + ] + ) + + model_state_dict_shapes = {} + test_models = [test_suit[2] for test_suit in test_suits] + + for model_name in test_models: + model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="bfloat16" + ).to("cuda:0") + state_dict = model.state_dict() + state_dict_keys = list(state_dict.keys()) + model_state_dict_shapes[model_name] = { + key: state_dict[key].shape for key in state_dict_keys + } + del model + gc.collect() + torch.cuda.empty_cache() + + truncate_size = 10 + checking_parameters = [ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "model.layers.1.self_attn.q_proj.weight", + "model.layers.2.self_attn.k_proj.weight", + "model.layers.3.self_attn.v_proj.weight", + "model.layers.4.self_attn.o_proj.weight", + "model.layers.5.mlp.gate_proj.weight", + "model.layers.6.mlp.up_proj.weight", + "model.layers.7.mlp.down_proj.weight", + "model.layers.8.post_attention_layernorm.weight", + "model.norm.weight", + "lm_head.weight", + ] + + for tp_size, dp_size, model_name, backend in test_suits: + test_update_weights_from_distributed( + tp_size, + dp_size, + model_name, + backend, + model_state_dict_shapes[model_name], + truncate_size, + checking_parameters, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_update_weights_from_tensor.py b/test/srt/test_update_weights_from_tensor.py new file mode 100644 index 0000000..1f35924 --- /dev/null +++ b/test/srt/test_update_weights_from_tensor.py @@ -0,0 +1,90 @@ +import gc +import time +import unittest + +import torch + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +def test_update_weights_from_tensor(tp_size): + assert torch.cuda.device_count() >= tp_size, f"At least {tp_size} GPUs are required" + torch.cuda.empty_cache() + + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, tp_size=tp_size) + + param_names = [f"model.layers.{i}.mlp.up_proj.weight" for i in range(6, 16)] + + _check_param(engine, param_names[0], [0.0087, -0.0214, -0.0004, 0.0039, 0.0110]) + + memory_before = torch.cuda.memory_allocated() + new_tensor = torch.full((16384, 2048), 1.5, device="cuda") + + time_start = time.time() + engine.update_weights_from_tensor([(x, new_tensor) for x in param_names]) + print(f"Time delta: {time.time() - time_start:.03f}") + + for param_name in param_names[:3]: + _check_param(engine, param_name, [1.5] * 5) + + engine.shutdown() + + del new_tensor + gc.collect() + torch.cuda.ipc_collect() + torch.cuda.empty_cache() + memory_after = torch.cuda.memory_allocated() + assert ( + memory_after <= memory_before + 1024 + ), f"Memory leak detected: {memory_after - memory_before} bytes" + + +class TestUpdateWeightsFromTensor(CustomTestCase): + def test_update_weights_from_tensor(self): + tp_sizes = [1, 2] + for tp_size in tp_sizes: + if torch.cuda.device_count() < tp_size: + continue + + with self.subTest(tp_size=tp_size): + test_update_weights_from_tensor(tp_size) + + def test_update_weights_from_tensor_load_format_direct(self): + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + write_param_names = [ + f"model.layers.{i}.self_attn.qkv_proj.weight" for i in range(6, 16) + ] + read_param_names = [ + f"model.layers.{i}.self_attn.k_proj.weight" for i in range(6, 16) + ] + + _check_param( + engine, read_param_names[0], [-0.0198, 0.0227, 0.0168, 0.0232, -0.0178] + ) + + new_tensor = torch.full((3072, 2048), 1.5) + engine.update_weights_from_tensor( + [ + (write_param_name, new_tensor.clone()) + for write_param_name in write_param_names + ], + load_format="direct", + ) + + for read_param_name in read_param_names[:3]: + _check_param(engine, read_param_name, [1.5] * 5) + + engine.shutdown() + + +def _check_param(engine, param_name, expect_values): + actual_values = torch.tensor(engine.get_weights_by_name(param_name))[0, :5] + assert torch.allclose( + actual_values, torch.tensor(expect_values), atol=0.002 + ), f"{actual_values=}" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_verl_engine.py b/test/srt/test_verl_engine.py new file mode 100644 index 0000000..72c0d52 --- /dev/null +++ b/test/srt/test_verl_engine.py @@ -0,0 +1,298 @@ +import multiprocessing +import multiprocessing as mp +import os +import random +import traceback +import unittest +from multiprocessing import Process + +import torch +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.fsdp import CPUOffload +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import MixedPrecision +from torch.distributed.fsdp.api import ( + ShardedStateDictConfig, + ShardingStrategy, + StateDictType, +) +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.verl_engine import VerlEngine +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import is_port_available +from sglang.test.runners import ( + HFRunner, + SRTRunner, + check_close_model_outputs, + get_dtype_str, +) +from sglang.test.test_utils import CustomTestCase, is_in_ci + +_MAX_NEW_TOKENS = 8 +_PROMPTS = ["1+1=2, 1+2=3, 1+3=4, 1+4=5, 1+5=", "1*1=1, 1*2=2, 1*3=3, 1*4=4, 1*5="] +_TORCH_DTYPE = torch.float16 + +# Set to false to temporarily debug issues unrelated to weight update +_ENABLE_UPDATE_WEIGHTS = True +# _ENABLE_UPDATE_WEIGHTS = False + +# TODO maybe we should add more other models? should we keep it in sync with test_generation_models.py? +CI_MODELS = [ + dict(model_path="meta-llama/Llama-3.1-8B-Instruct"), + # Fail to run gemma-2-2b after transformers==4.48.3 -> 4.50.0 + # dict(model_path="google/gemma-2-2b"), +] +ALL_OTHER_MODELS = [ + dict(model_path="meta-llama/Llama-3.2-1B-Instruct"), + dict(model_path="Qwen/Qwen2-1.5B"), + dict( + model_path="Qwen/Qwen2.5-14B-Instruct", + mem_fraction_static=0.4, + tp_size=8, + tight_memory=True, + decode_tolerance=1.3, + ), # test_generation_models.py same config (qwen + tp=8) gives 1.22 decode error + dict(model_path="HuggingFaceTB/SmolLM-135M-Instruct", tp_size=3), + dict(model_path="allenai/OLMo-1B-0724-hf"), + dict( + model_path="THUDM/glm-4-9b-chat", + mem_fraction_static=0.1, + tp_size=8, + tight_memory=True, + ), + dict(model_path="allenai/OLMo-2-1124-7B-Instruct"), + dict( + model_path="ibm-granite/granite-3.0-2b-instruct", + prefill_tolerance=0.22, + decode_tolerance=0.22, + ), + # Fail to run these models in test_generation_models.py, need to fix that first + # dict(model_path="openai-community/gpt2"), + # dict(model_path="microsoft/Phi-3-small-8k-instruct"), +] + + +class TestVerlEngine(CustomTestCase): + @classmethod + def setUpClass(cls): + multiprocessing.set_start_method("spawn") + + def assert_fragment_e2e_execution( + self, + index: int, + model_path: str, + mem_fraction_static: float = 0.4, + tp_size: int = 2, + tight_memory: bool = False, + prefill_tolerance: float = 0.1, + decode_tolerance: float = 0.1, + ): + master_port = find_available_port(23456) + + print(f"assert_fragment_e2e_execution START {index=} {model_path=}") + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + for tp_rank in range(tp_size): + p = Process( + target=_run_subprocess, + kwargs=dict( + tp_rank=tp_rank, + tp_size=tp_size, + master_port=master_port, + output_writer=output_writer, + model_path=model_path, + mem_fraction_static=mem_fraction_static, + tight_memory=tight_memory, + prefill_tolerance=prefill_tolerance, + decode_tolerance=decode_tolerance, + ), + ) + p.start() + processes.append(p) + + for _ in range(tp_size): + self.assertTrue( + output_reader.recv(), + f"Subprocess has error, please see logs above. ({index=} {model_path=})", + ) + + for p in processes: + p.join() + + def test_ci_models(self): + for index, model_info in enumerate(CI_MODELS): + self.assert_fragment_e2e_execution(index=index, **model_info) + + def test_others(self): + if is_in_ci(): + return + + for index, model_info in enumerate(ALL_OTHER_MODELS): + self.assert_fragment_e2e_execution(index=index, **model_info) + + # def test_adhoc(self): + # self.assert_fragment_e2e_execution(index=0, model_path="meta-llama/Llama-3.2-1B-Instruct") + + +def _run_subprocess( + tp_rank: int, + tp_size: int, + master_port: int, + output_writer, + model_path: str, + mem_fraction_static: float, + tight_memory: bool, + prefill_tolerance: float, + decode_tolerance: float, +): + try: + print(f"subprocess[{tp_rank=}] Start {os.environ.get('CUDA_VISIBLE_DEVICES')=}") + + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + torch.distributed.init_process_group(rank=tp_rank, world_size=tp_size) + torch.cuda.set_device(tp_rank) + + mesh_kwargs = dict(mesh_shape=(tp_size, 1), mesh_dim_names=["tp", "pp"]) + inference_device_mesh_device = init_device_mesh("cuda", **mesh_kwargs) + inference_device_mesh_cpu = init_device_mesh("cpu", **mesh_kwargs) + print( + f"subprocess[{tp_rank=}] {inference_device_mesh_device=} {inference_device_mesh_cpu=}" + ) + + # hf model is used for comparison + hf_model = AutoModelForCausalLM.from_pretrained( + model_path, torch_dtype=_TORCH_DTYPE, trust_remote_code=True + ).cuda() + hf_tokenizer = get_tokenizer(model_path, trust_remote_code=True) + + hf_outputs = HFRunner.forward_generation_raw( + base_model=hf_model, + prompts=_PROMPTS, + max_new_tokens=_MAX_NEW_TOKENS, + tokenizer=hf_tokenizer, + lora_paths=None, + torch_dtype=_TORCH_DTYPE, + output_str_only=False, + ) + print( + f"subprocess[{tp_rank=}] call hf.forward {hf_outputs=}", + flush=True, + ) + + if _ENABLE_UPDATE_WEIGHTS: + if tight_memory: + hf_model.cpu() + torch.cuda.empty_cache() + + # test update weights + print(f"subprocess[{tp_rank=}] get_fsdp_state_dict", flush=True) + fsdp_state_dict = _get_fsdp_state_dict(hf_model=hf_model, tp_size=tp_size) + + engine = VerlEngine( + model_path=model_path, + load_format="dummy" if _ENABLE_UPDATE_WEIGHTS else "auto", + mem_fraction_static=mem_fraction_static, + random_seed=42, + trust_remote_code=True, + dtype=get_dtype_str(_TORCH_DTYPE), + device_mesh_cpu=inference_device_mesh_cpu["tp"], + ) + print(f"subprocess[{tp_rank=}] {engine=}", flush=True) + + if _ENABLE_UPDATE_WEIGHTS: + print(f"subprocess[{tp_rank=}] call update_weights_from_tensor", flush=True) + engine.update_weights_from_tensor( + [(k, v) for k, v in fsdp_state_dict.items()] + ) + + for enable_batch in [False, True]: + if enable_batch: + fn = SRTRunner.batch_forward_generation_raw + else: + fn = SRTRunner.forward_generation_raw + + srt_outputs = fn( + prompts=_PROMPTS, + max_new_tokens=_MAX_NEW_TOKENS, + lora_paths=None, + engine=engine, + ) + print( + f"subprocess[{tp_rank=}] call srt.forward {enable_batch=} {srt_outputs=}", + flush=True, + ) + + check_close_model_outputs( + hf_outputs=hf_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=prefill_tolerance, + decode_tolerance=decode_tolerance, + rouge_l_tolerance=1, + check_logprobs=not enable_batch, + debug_text=f"{enable_batch=} {tp_rank=}", + ) + + execution_ok = True + + except Exception as e: + print(f"subprocess[{tp_rank=}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + engine.shutdown() + print(f"subprocess[{tp_rank=}] end", flush=True) + + +# Adapted from https://github.com/volcengine/verl/blob/main/tests/rollout/run_fsdp_vllm.py +def _get_fsdp_state_dict(hf_model, tp_size: int): + device_mesh = init_device_mesh( + "cuda", mesh_shape=(tp_size,), mesh_dim_names=["fsdp"] + ) + + mixed_precision = MixedPrecision( + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + buffer_dtype=torch.float32, + ) + fsdp_model = FSDP( + hf_model, + use_orig_params=True, + auto_wrap_policy=None, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + cpu_offload=CPUOffload(offload_params=False), + sync_module_states=False, + device_mesh=device_mesh, + ) + print(f"{fsdp_model=}") + + FSDP.set_state_dict_type( + fsdp_model, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + + return fsdp_model.state_dict() + + +# TODO Ask: this is extracted from PortArgs.init_new, is it allowed to extract it, i.e. touch that old code +def find_available_port(base_port: int): + port = base_port + random.randint(100, 1000) + while True: + if is_port_available(port): + return port + if port < 60000: + port += 42 + else: + port -= 43 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vertex_endpoint.py b/test/srt/test_vertex_endpoint.py new file mode 100644 index 0000000..a899d62 --- /dev/null +++ b/test/srt/test_vertex_endpoint.py @@ -0,0 +1,54 @@ +""" +python3 -m unittest test_vertex_endpoint.TestVertexEndpoint.test_vertex_generate +""" + +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestVertexEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--cuda-graph-max-bs", 2], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_generate(self, parameters): + data = { + "instances": [ + {"text": "The capital of France is"}, + {"text": "The capital of China is"}, + ], + "parameters": parameters, + } + response = requests.post(self.base_url + "/vertex_generate", json=data) + response_json = response.json() + assert len(response_json["predictions"]) == len(data["instances"]) + return response_json + + def test_vertex_generate(self): + for parameters in [None, {"sampling_params": {"max_new_tokens": 4}}]: + self.run_generate(parameters) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vision_chunked_prefill.py b/test/srt/test_vision_chunked_prefill.py new file mode 100644 index 0000000..cb5c132 --- /dev/null +++ b/test/srt/test_vision_chunked_prefill.py @@ -0,0 +1,179 @@ +""" +Usage: +python3 -m unittest test_vision_chunked_prefill.TestVisionChunkedPrefill.test_chunked_prefill +""" + +import base64 +import io +import os +import unittest +from concurrent.futures import ThreadPoolExecutor +from typing import Union + +import numpy as np +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestVisionChunkedPrefill(CustomTestCase): + def prepare_video_messages(self, video_path, max_frames_num=8): + # We import decord here to avoid a strange Segmentation fault (core dumped) issue. + # The following import order will cause Segmentation fault. + # import decord + # from transformers import AutoTokenizer + from decord import VideoReader, cpu + + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = base64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + frame_format = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,{}"}, + "modalities": "video", + } + + for base64_frame in base64_frames: + frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format( + base64_frame + ) + messages[0]["content"].append(frame_format.copy()) + + prompt = {"type": "text", "text": "Please describe the video briefly."} + messages[0]["content"].append(prompt) + + return messages + + def get_prompt_from_messages(self, messages): + text = ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + ) + image_data = [] + for content in messages[0]["content"]: + if content["type"] == "image_url": + text += "\n" + image_data.append(content["image_url"]["url"]) + text += "Please describe the video briefly.<|im_end|>\n<|im_start|>assistant\n" + return text, image_data + + def generate(self, text, image_data): + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "image_data": image_data, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "modalities": ["multi-images"], + }, + ).json() + return response["text"] + + def generate_for_video(self, batch, num_frame) -> Union[str, list[str]]: + # prepare the video input about Steven introducing ipod nano + url = "https://raw.githubusercontent.com/evolvinglmms-lab/sglang/dev/onevision_local/assets/jobs.mp4" + cache_dir = os.path.expanduser("~/.cache") + file_path = os.path.join(cache_dir, "jobs.mp4") + os.makedirs(cache_dir, exist_ok=True) + if not os.path.exists(file_path): + response = requests.get(url) + response.raise_for_status() + with open(file_path, "wb") as f: + f.write(response.content) + + if not batch: + assert isinstance(num_frame, int) + messages = self.prepare_video_messages(file_path, max_frames_num=num_frame) + text, image_data = self.get_prompt_from_messages(messages) + return self.generate(text, image_data) + else: + assert isinstance(num_frame, list) + func_args = [] + for max_frames_num in num_frame: + messages = self.prepare_video_messages( + file_path, + max_frames_num=max_frames_num, + ) + text, image_data = self.get_prompt_from_messages(messages) + func_args.append((text, image_data)) + + with ThreadPoolExecutor(max_workers=10) as executor: + responses = list(executor.map(lambda p: self.generate(*p), func_args)) + + return responses + + def run_generate(self, chunked_prefill_size, batch, num_frame): + # launch server + model = "lmms-lab/llava-onevision-qwen2-7b-ov" + # model = "meta-llama/Llama-3.2-11B-Vision-Instruct" + self.base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + f"{chunked_prefill_size}", + ], + ) + try: + return self.generate_for_video(batch, num_frame) + finally: + kill_process_tree(process.pid) + + def test_chunked_prefill(self): + output_chunked = self.run_generate( + chunked_prefill_size=1024, batch=False, num_frame=1 + ) + output_no_chunked = self.run_generate( + chunked_prefill_size=-1, batch=False, num_frame=1 + ) + + print("output with chunked prefill:") + print(output_chunked) + print("output without chunked prefill:") + print(output_no_chunked) + assert output_chunked == output_no_chunked + + output_chunked = self.run_generate( + chunked_prefill_size=1024, batch=True, num_frame=[2, 6, 8, 10] + ) + output_no_chunked = self.run_generate( + chunked_prefill_size=-1, batch=True, num_frame=[2, 6, 8, 10] + ) + + print("output with chunked prefill:") + print(output_chunked) + print("output without chunked prefill:") + print(output_no_chunked) + assert output_chunked == output_no_chunked + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vision_openai_server.py b/test/srt/test_vision_openai_server.py new file mode 100644 index 0000000..5575645 --- /dev/null +++ b/test/srt/test_vision_openai_server.py @@ -0,0 +1,713 @@ +""" +Usage: +python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_mixed_batch +python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_multi_images_chat_completion +""" + +import base64 +import io +import json +import os +import unittest +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import openai +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +# image +IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png" +IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png" + +# video +VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4" + +# audio +AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3" +AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3" + + +class TestOpenAIVisionServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmms-lab/llava-onevision-qwen2-0.5b-ov" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--chat-template", + "chatml-llava", + # "--log-requests", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a very short sentence.", + }, + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + # `driver` is for gemma-3-it + assert "man" in text or "person" or "driver" in text, text + assert "cab" in text or "taxi" in text or "SUV" in text, text + # MiniCPMO fails to recognize `iron`, but `hanging` + assert "iron" in text or "hang" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_multi_turn_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a very short sentence.", + }, + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "There is a man at the back of a yellow cab ironing his clothes.", + } + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Repeat your previous answer."} + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + assert "man" in text or "cab" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_multi_images_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png" + }, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have two very different images. They are not related at all. " + "Please describe the first image in one sentence, and then describe the second image in another sentence.", + }, + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + print("-" * 30) + print(f"Multi images response:\n{text}") + print("-" * 30) + assert "man" in text or "cab" in text or "SUV" in text or "taxi" in text, text + assert "logo" in text or '"S"' in text or "SG" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def prepare_video_messages(self, video_path): + # the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa + # the size of the video embeds differs from the `modality` argument when preprocessed + + # We import decord here to avoid a strange Segmentation fault (core dumped) issue. + # The following import order will cause Segmentation fault. + # import decord + # from transformers import AutoTokenizer + from decord import VideoReader, cpu + + max_frames_num = 20 + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = base64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + frame_format = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,{}"}, + "modalities": "video", + } + + for base64_frame in base64_frames: + frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format( + base64_frame + ) + messages[0]["content"].append(frame_format.copy()) + + prompt = {"type": "text", "text": "Please describe the video in detail."} + messages[0]["content"].append(prompt) + + return messages + + def prepare_video_messages_video_direct(self, video_path): + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": f"video:{video_path}"}, + "modalities": "video", + }, + {"type": "text", "text": "Please describe the video in detail."}, + ], + }, + ] + return messages + + def get_or_download_file(self, url: str) -> str: + cache_dir = os.path.expanduser("~/.cache") + if url is None: + raise ValueError() + file_name = url.split("/")[-1] + file_path = os.path.join(cache_dir, file_name) + os.makedirs(cache_dir, exist_ok=True) + + if not os.path.exists(file_path): + response = requests.get(url) + response.raise_for_status() + + with open(file_path, "wb") as f: + f.write(response.content) + return file_path + + def test_video_chat_completion(self): + url = VIDEO_JOBS_URL + file_path = self.get_or_download_file(url) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # messages = self.prepare_video_messages_video_direct(file_path) + messages = self.prepare_video_messages(file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=False, + ) + + video_response = response.choices[0].message.content + + print("-" * 30) + print(f"Video response:\n{video_response}") + print("-" * 30) + + # Add assertions to validate the video response + assert "iPod" in video_response or "device" in video_response, video_response + assert ( + "man" in video_response + or "person" in video_response + or "individual" in video_response + or "speaker" in video_response + ), video_response + assert ( + "present" in video_response + or "examine" in video_response + or "display" in video_response + or "hold" in video_response + ) + assert "black" in video_response or "dark" in video_response + self.assertIsNotNone(video_response) + self.assertGreater(len(video_response), 0) + + def test_regex(self): + return + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + regex = ( + r"""\{\n""" + + r""" "color": "[\w]+",\n""" + + r""" "number_of_cars": [\d]+\n""" + + r"""\}""" + ) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in the JSON format.", + }, + ], + }, + ], + temperature=0, + extra_body={"regex": regex}, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + assert isinstance(js_obj["color"], str) + assert isinstance(js_obj["number_of_cars"], int) + + def run_decode_with_image(self, image_id): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + content = [] + if image_id == 0: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + } + ) + elif image_id == 1: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + } + ) + else: + pass + + content.append( + { + "type": "text", + "text": "Describe this image in a very short sentence.", + } + ) + + response = client.chat.completions.create( + model="default", + messages=[ + {"role": "user", "content": content}, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + + def test_mixed_batch(self): + image_ids = [0, 1, 2] * 4 + with ThreadPoolExecutor(4) as executor: + list(executor.map(self.run_decode_with_image, image_ids)) + + def prepare_audio_messages(self, prompt, audio_file_name): + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt, + }, + { + "type": "audio_url", + "audio_url": {"url": f"{audio_file_name}"}, + }, + ], + } + ] + + return messages + + def get_audio_response(self, url: str, prompt, category): + audio_file_path = self.get_or_download_file(url) + client = openai.Client(api_key="sk-123456", base_url=self.base_url) + + messages = self.prepare_audio_messages(prompt, audio_file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=128, + stream=False, + ) + + audio_response = response.choices[0].message.content + + print("-" * 30) + print(f"audio {category} response:\n{audio_response}") + print("-" * 30) + + audio_response = audio_response.lower() + + self.assertIsNotNone(audio_response) + self.assertGreater(len(audio_response), 0) + + return audio_response + + def _test_audio_speech_completion(self): + # a fragment of Trump's speech + audio_response = self.get_audio_response( + AUDIO_TRUMP_SPEECH_URL, + "I have an audio sample. Please repeat the person's words", + category="speech", + ) + assert "thank you" in audio_response + assert "it's a privilege to be here" in audio_response + assert "leader" in audio_response + assert "science" in audio_response + assert "art" in audio_response + + def _test_audio_ambient_completion(self): + # bird song + audio_response = self.get_audio_response( + AUDIO_BIRD_SONG_URL, + "Please listen to the audio snippet carefully and transcribe the content.", + "ambient", + ) + assert "bird" in audio_response + + def test_audio_chat_completion(self): + pass + + +class TestQwen2VLServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--chat-template", + "qwen2-vl", + "--mem-fraction-static", + "0.4", + ], + ) + cls.base_url += "/v1" + + +class TestQwen2_5_VLServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2.5-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--chat-template", + "qwen2-vl", + "--mem-fraction-static", + "0.4", + ], + ) + cls.base_url += "/v1" + + +class TestVLMContextLengthIssue(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--chat-template", + "qwen2-vl", + "--context-length", + "300", + "--mem-fraction-static=0.80", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Give a lengthy description of this picture", + }, + ], + }, + ], + temperature=0, + ) + + # context length is checked first, then max_req_input_len, which is calculated from the former + assert ( + "Multimodal prompt is too long after expanding multimodal tokens." + in str(cm.exception) + or "is longer than the model's context length" in str(cm.exception) + ) + + +class TestMllamaServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "meta-llama/Llama-3.2-11B-Vision-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--chat-template", + "llama_3_vision", + ], + ) + cls.base_url += "/v1" + + def test_video_chat_completion(self): + pass + + +class TestMinicpmvServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "openbmb/MiniCPM-V-2_6" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--chat-template", + "minicpmv", + "--mem-fraction-static", + "0.4", + ], + ) + cls.base_url += "/v1" + + +class TestMinicpmoServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "openbmb/MiniCPM-o-2_6" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--chat-template", + "minicpmo", + "--mem-fraction-static", + "0.7", + ], + ) + cls.base_url += "/v1" + + def test_audio_chat_completion(self): + self._test_audio_speech_completion() + self._test_audio_ambient_completion() + + +class TestDeepseekVL2Server(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "deepseek-ai/deepseek-vl2-small" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--chat-template", + "deepseek-vl2", + "--context-length", + "4096", + ], + ) + cls.base_url += "/v1" + + def test_video_chat_completion(self): + pass + + +class TestJanusProServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "deepseek-ai/Janus-Pro-7B" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--chat-template", + "janus-pro", + "--mem-fraction-static", + "0.4", + ], + ) + cls.base_url += "/v1" + + def test_video_chat_completion(self): + pass + + def test_single_image_chat_completion(self): + # Skip this test because it is flaky + pass + + +class TestGemma3itServer(TestOpenAIVisionServer): + @classmethod + def setUpClass(cls): + cls.model = "google/gemma-3-4b-it" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--chat-template", + "gemma-it", + "--mem-fraction-static", + "0.75", + ], + ) + cls.base_url += "/v1" + + def test_video_chat_completion(self): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vllm_dependency.py b/test/srt/test_vllm_dependency.py new file mode 100644 index 0000000..168f681 --- /dev/null +++ b/test/srt/test_vllm_dependency.py @@ -0,0 +1,168 @@ +import json +import os +import unittest +import warnings +from datetime import datetime +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +MODEL_SCORE_THRESHOLDS = { + "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.84, + "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.83, + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.62, +} + + +def parse_models(model_string): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def popen_launch_server_wrapper(base_url, model, is_fp8, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_fp8: + if "Llama-3" in model or "gemma-2" in model: + other_args.extend(["--kv-cache-dtype", "fp8_e5m2"]) + elif "Qwen2-72B-Instruct-FP8" in model: + other_args.extend(["--quantization", "fp8"]) + elif "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" in model: + other_args.extend([]) + else: + other_args.extend(["--quantization", "fp8", "--kv-cache-dtype", "fp8_e5m2"]) + if is_tp2: + other_args.extend(["--tp", "2"]) + if "DeepSeek" in model: + other_args.extend(["--mem-frac", "0.85"]) + if "AWQ" in model: + other_args.extend(["--quantization", "awq"]) + elif "GPTQ" in model: + other_args.extend(["--quantization", "gptq"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def write_results_to_json(model, metrics, mode="a"): + result = { + "timestamp": datetime.now().isoformat(), + "model": model, + "metrics": metrics, + "score": metrics["score"], + } + + existing_results = [] + if mode == "a" and os.path.exists("results.json"): + try: + with open("results.json", "r") as f: + existing_results = json.load(f) + except json.JSONDecodeError: + existing_results = [] + + if isinstance(existing_results, list): + existing_results.append(result) + else: + existing_results = [result] + + with open("results.json", "w") as f: + json.dump(existing_results, f, indent=2) + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary( + f"### TestNightlyGsm8KEval for vLLM awq, gptq, gguf\n{summary}" + ) + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1), False, False), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + process = popen_launch_server_wrapper( + self.base_url, model, is_fp8, is_tp2 + ) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vlm_accuracy.py b/test/srt/test_vlm_accuracy.py new file mode 100644 index 0000000..75c434e --- /dev/null +++ b/test/srt/test_vlm_accuracy.py @@ -0,0 +1,215 @@ +""" +""" + +import unittest +from io import BytesIO + +import numpy as np +import requests +import torch +import torch.nn.functional as F +from PIL import Image +from transformers import AutoModel, AutoProcessor, AutoTokenizer + +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.conversation import generate_chat_conv +from sglang.srt.managers.mm_utils import embed_mm_inputs +from sglang.srt.managers.schedule_batch import MultimodalInputs +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.openai_api.protocol import ChatCompletionRequest +from sglang.srt.server_args import ServerArgs + + +# Test the logits output between HF and SGLang +class VisionLLMLogitsBase(unittest.IsolatedAsyncioTestCase): + @classmethod + def setUpClass(cls): + cls.image_url = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.model_path = "" + cls.chat_template = "" + cls.processor = "" + response = requests.get(cls.image_url) + cls.main_image = Image.open(BytesIO(response.content)) + + def compare_outputs(self, sglang_output: torch.Tensor, hf_output: torch.Tensor): + # Convert to float32 for numerical stability if needed + hf = hf_output.float() + sg = sglang_output.float() + + # Basic shape and dtype comparison + print("\n=== Basic Properties ===") + print(f"Shapes match: {hf.shape == sg.shape}") + print(f"HF shape: {hf.shape}, SGLang shape: {sg.shape}") + print(f"HF dtype: {hf.dtype}, SGLang dtype: {sg.dtype}") + + # Move tensors to CPU for numpy operations + hf_np = hf.cpu().numpy() + sg_np = sg.cpu().numpy() + + # Statistical metrics + print("\n=== Statistical Metrics ===") + print(f"Mean absolute difference: {torch.mean(torch.abs(hf - sg)).item():.6f}") + print(f"Max absolute difference: {torch.max(torch.abs(hf - sg)).item():.6f}") + print(f"Mean squared error: {torch.mean((hf - sg) ** 2).item():.6f}") + print( + f"Root mean squared error: {torch.sqrt(torch.mean((hf - sg) ** 2)).item():.6f}" + ) + + # Cosine similarity (across feature dimension) + cos_sim = F.cosine_similarity(hf, sg) + print(f"Mean cosine similarity: {torch.mean(cos_sim).item():.6f}") + print(f"Min cosine similarity: {torch.min(cos_sim).item():.6f}") + + # Find largest absolute differences + print("\n=== Largest Absolute Differences ===") + diffs = torch.abs(hf - sg) + flat_diffs = diffs.flatten() + + # Get indices of top 10 differences + top_k = 10 + top_values, top_flat_indices = torch.topk(flat_diffs, top_k) + + # Convert flat indices to multidimensional indices + top_indices = np.unravel_index(top_flat_indices.cpu().numpy(), diffs.shape) + + print(f"\nTop {top_k} largest absolute differences:") + print( + "Index".ljust(30) + + "Difference".ljust(15) + + "HF Value".ljust(15) + + "SGLang Value" + ) + print("-" * 75) + + for i in range(top_k): + # Get the index tuple for this difference + idx = tuple(dim[i] for dim in top_indices) + diff_val = top_values[i].item() + hf_val = hf[idx].item() + sg_val = sg[idx].item() + + # Format the index tuple and values + idx_str = str(idx) + print(f"{idx_str:<30}{diff_val:<15.6f}{hf_val:<15.6f}{sg_val:.6f}") + + np.testing.assert_allclose(hf_np, sg_np) + + def get_processor_output(self): + json_str = f""" + {{ + "model": "{self.model_path}", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "image_url", + "image_url": {{ + "url": "{self.image_url}" + }} + }}, + {{ + "type": "text", + "text": "Whats in this picture?" + }} + ] + }} + ] +}} + """ + + req = ChatCompletionRequest.model_validate_json(json_str) + + conv = generate_chat_conv(req, template_name=self.chat_template) + + text = conv.get_prompt() + + # Process inputs using processor + # FIXME: the formal arguments may differ + inputs = self.processor( + text=[text], + images=[self.main_image], + return_tensors="pt", + ).to(self.device) + + return inputs + + def get_sglang_model(self): + self.model_runner = ModelRunner( + model_config=ModelConfig(self.model_path, model_override_args="{}"), + mem_fraction_static=0.8, + gpu_id=0, + tp_rank=0, + tp_size=1, + nccl_port=12435, + server_args=ServerArgs( + model_path=self.model_path, + disable_cuda_graph=True, + ), + ) + return self.model_runner.model + + +class TestMiniCPMVLogits(VisionLLMLogitsBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model_path = "openbmb/MiniCPM-V-2_6" + cls.tokenizer = AutoTokenizer.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls.processor = AutoProcessor.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls.chat_template = "minicpmv" + + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.hf_model = ( + AutoModel.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16, trust_remote_code=True + ) + .eval() + .to(cls.device) + ) + + async def test_vlm_embedding_output(self): + """ + Compares the embedding output of vlm + """ + inputs = self.get_processor_output() + + with torch.no_grad(): + # hf + model_inputs = { + "input_ids": inputs.input_ids, + "image_bound": inputs.image_bound, + "pixel_values": inputs.pixel_values, + "tgt_sizes": inputs.tgt_sizes, + } + (hf_output, _) = self.hf_model.get_vllm_embedding( + model_inputs, + ) + hf_output = hf_output.squeeze(0) + + # sglang + model = self.get_sglang_model() + input_ids = inputs["input_ids"].to(self.device).flatten() + sglang_output = embed_mm_inputs( + mm_input=MultimodalInputs( + pixel_values=inputs["pixel_values"][0], + tgt_sizes=inputs["tgt_sizes"][0], + ), + input_ids=input_ids, + input_embedding=model.get_input_embeddings(), + mm_data_embedding_func=model.get_image_features, + placeholder_token_ids=[ + self.processor.tokenizer.unk_token_id, + ], + ) + + self.compare_outputs(sglang_output, hf_output) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_w8a8_quantization.py b/test/srt/test_w8a8_quantization.py new file mode 100644 index 0000000..2cb2fa0 --- /dev/null +++ b/test/srt/test_w8a8_quantization.py @@ -0,0 +1,75 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestW8A8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "neuralmagic/Meta-Llama-3-8B-Instruct-quantized.w8a8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--quantization", "w8a8_int8"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.7) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.time() + res = self.run_decode(max_tokens) + tok = time.time() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 140 + + +if __name__ == "__main__": + unittest.main()

YJ(;}>v?L6g5;1`FYy3Q%3#qG zlL;K|teG}b%F4Q*#A9>IHYBE#q{DvS_noD&q3h0BBT0R`qZV2Vvd-|nEK?D+Hpho9 zJm;xA}bbU$mwr1CsP$T7vRVQ~w{;087nI9UoZ(A+xMs z{Ut!e|E;I~dzBd_6ssW%%_YF2Id>skn-Wqt;d>%DxmJVytJeQoF zBoJUjPF?}hq>`sU$^Q;AQIVtszHeL&W? ze>e!CuZm_DUXlOxPN0zx{0$&Hb`BDN z%a2}QQ!NhsGCifI&Qy44UfU8^`gvs0y0cJdYVhMZJ$0bjvin8G0{Bj;H(SYKJphe!2yyyy9*g$ zbbhgFJ4qf23H4n#xqb*vU`)|{Ux-goSW7e6AH34^8<6`eJ@roufP{G2zov9?&A=irOAWAg8)kr^mH?r zhY_r9@-!V_FWAxa;Ju4Jn`3%)6Kar)43NUa)H)1qx6Etbsd+-5;LiN-rT0q6L3`&7 zU`w+G-6Sv`?k2J6d~&-G>KS4*K6f%p5WHd`kBwd5lG(; zN&O;cFeVG348r!8mEwvBZvc0ZQ79^gXUsb-D)@6%YljU@VtdJa9_2j?6}l#!fa+q| zAXc9nX%Q9H*LLp)Lod#}lkNI}RAY+aW((q!8M4*-7QU$Q8WAEl&WzxhlKdx-JVIur zA5+aGL#)hGiwFmu@((~ro2P(is?;#8@%DPy^UOpjs&f_EaDtUA zNAKniOnrV`$d-$?mP5o@Dvaa2hr1K;1Sd_;F^rnQ_B<>gukYWzp4UI(Q|k|1E^1)P zP}64M?-MK*9l>9kGD?WrLV;bj6q^$h|3GYRI@ZVBSN&STT2E4=KHSfHGn!)?#TVKP z^g|Q2F=UAqCwzUGkDYYq4oNeyeEAA~%S)*kePK67PcjTWw@aj;%nq7ic*eNBGQNIr zY#7v+TYT2luTsA1R%d|YxH(N`^0nz2eQi6$zgPZ#^ef%EssC)@>~8?#^PNcjq-oo_ zl$k~JNS&Y&|MH~4JFG&Dv7`!V4KenwJ_{KALk&DmS%}R%hNNuePBhRRG8}-4`s*1C&$gdW~NF-+ZLywjyUs{_6#7G9e zhe$mf+SbSdfXQl{OYA^XAR#IJk5&x~z+}}*OBtV73KiI8pL%0Ty0*HiS;Hj@ zn2yb8tJw1_M zKTlQ1$n4XfY`t$6PW(b=g&;({i>sL4G#D87rdhBGYI@3XZsunyeVt0p!2wP@_h^i` zA!A&_jlAkA`B(%4RbyACcKH9GQ!AP}j*U(e>wPl7mn-?Ob#Q>;uz051#>dhSH@P473XQds| zi5h2nxEOprn05D>^JhYk|Go=*<1xhf2jb;ZoTm{iLR2Q@o3FS6&Fd|TMkCsen`opK_P_V;{BUeQ#In9cErgI2@E{ek}fyv0pai#62AEOd0cS$Opvr<1KO0cOg0g zLB=?cbbq4k$>RBDqFrx8pubb*L;Kxr{FZ;(%u}a*F9s}oT@GvFlWz^1Z>^=jD9Nbc z1gayK3*k5JWY#&yYNxIx;$ngIoPBo!CI%W*kFwOPz16?e-sxqG33kk%Dund3T~@!m zv|n|-n$8h%loy-Z|GW)|+HSL-Ia23RD=vhCAfuVbZ%!g3f1L z(}cd6ehBlRR9T9^+Ak?9C>tz2*=Vb%5<~)TjB`{S9|9Js%5*J}GFp0YI*VFMiGfLm zmKnJ7Dao)p2XE|$ANdMok^1>c;4h87T3=SQkpT*CQhx)eUk-$U0_;Wx`(`}8nrJ-M zH+^qAK=)H-QXW9@Z%O;i%jRQO&JT z0<}^8z*#TQNJkL?-!*BY2J0U0YcaD6Ld)`BS(Y!F1iX@O@;E0KkQ5ix^`6vF8=-!{ z7$r9iqq{t){O#J>=89W_pLSIs_IsIj z1y9~nB`y8--t5I_Wt9ZQwlCkND?q6uW_Sh*pGcojnPobvHywy!2(CR9PTI5ln(gjE zIO;GJRiI}X!TJsG8}Lq~A-8uOG(?4Z{R+_knqzZid;5a4(Gdt&PRn)%B1Ic{jMf4`CxH>?ZIex0P20cSxUF_+Qbf(4%x3BwAL(`2% zy-cIXo;(LgPJC#Eelok5NtsM-8jCte;R?laxUQBowyeS_6_FQ+p)*%L>((hY!4&n} zdPd29>KluzBuPI^*0WqTE`y=bg;R?}l{i!iQ>}T8gt$WWc19*k5DV;0T(xUk$->Vp z0n-pwvN4_%-v32Vl130r87jeA0OMe3igocU-$Q5KFDZ$YbK%lLP$?;ANyOL8x(ZqO zHdpXRh;w2H7PIo@yC_b@b0nQ5ReJ-HoWjXH57>SKj!l{xxC|=CLHt97mp!7D&pxZ_ z8g-7j@jHi%%Ejg zG+Xt_A}IZ8iN|zJ)n>Xv+=j|<)~vx$VLYJyIYlsuhtQY8x@Q796R?`^Lln0qepNJ7}<=Y*`A$Ea#~LC`3rKXg)MCpUh7 z$*k?GxTVK&bUVYPkMNXbGpq5}t(HbQlvW<-=tNwpQiISYKR^9b8b@1Yck7P}D$;`m zrQRMgFGW@)4ir{*k360&Bj$q~_8;zztEPc%S(O6bmfRc~ja)K(qc*ytQrDg=8p@?Q z*yBwYRakT`>{#GVVwUD*?q{!236&Efzh)<0q8<0}iGw#^O=$gEqiAphLq1xh-|Lbh zO(wC=Z{Vpo(^-}SZVqJIpBX)P-mkao-%x!Yk$ z$UkALVG1;5O|hH&h)APR9DaB#SGc#&nq+GhUrd8E!&OXI%;%&HfT>7Q9}3?*NOMK4 z>FAcPO%%bax@n|R&^&$vLg}-JYJ`V)igJtt8gpS5)fPT--}n+vCZi)1nq}Kot$1n- zeaeaVhGp_~Ay}dG`9DyQB!NNYnwRlY@Y%MA>(4I7M+FVeiDJU$)SJ1w5--aO9v*hf ziB={Kx6}NKO-g-TggA#S=#4jLio%OZkol-t>}3_;JAjL894XWvl1_iMwuIYX6FIY+ zqFznM8#usLJ?b=IUvN2Kl(fJN6I6>;z6k|{$Ln_?2?n)V%ja=|F}@a+-;A4NeXt*O z_rqy_Kz3~~F5-JxKJWLiHQ>YOfqLkKDMW6VH!+$3NB}`=&Dt9?O3t>x{TSyOeo~;W zr(cizY3TRz*;H1zoG|3c7AFlzv#Y=HXifj+Y3Iey5X1&A&TN*N%KG?pz1AY9p^{34 znAui%mg!8pYzq=6HM&*Y>Btej!e7FmF)Sh2*{b64^+lHvv<_6VC*Eg z5&L>TMn=y_Rj3yRlHp#qR%}()!mPDBOy%m@H%8c&99W8N-iTw(|6!C77W^_V}5?wr>u6E?oxx^(~_mEu%I5Y7ctxcN~k=K*w_sy({2;0eEsQ z={kQQOrn80y=hLWk%qQYUgBc4C%}ru| z6vHSVe+GA^+iE2uho4XMgQD_;uUcOPigmw14+Wf^v`BoSy@{|^3>RY<4n_GhXhn4n z89fPoQgcRk9`m<`MbFEibKS>dM17m2_Uktw&Hz;{;`Ucm z=O*pkp5A=V#DY7Dl42fG0!0@HkueCrP&`TJ(HwbeszYxJB?pCRK8nTbHfT{kUI}p-5pzt3o$*V{42(;AV85; zALz|;9;7vZ=;~X9zKy^aqH_P)d~f+j!3^mXBPhN1fc!JD~4LqlLOwO>=N#=>3EZ4ZqgE}Y1NxZ@#y{Qmy{IEhHv8I$e|n1 zu2q{$Rsq-mIR6q@$WvX-pK9Xp{hfkajYP$`p8*C_{tm44&lA$6YI_eC&cdtHVhrnHeeGTeMyBcBm2!H~u}<^#@`xW7 zGUTlY+|)ouU7H1;cgYoW-lu`nzY-Wk_NsyWlx3B@abk`eHl*XXu>Y5CgkxQ^si?16D$~OGE#KTECt&K8KxnZD!^~Lfw2Mclq{7WXPoJT(@msp1! zd%W{}Kl4{*31E3dSH3vh&wibzpJvsWPOuk^7i||JGN*UW8o9_(-5Y+1ky5Xpv{9nH ztHTT32IrRb$JLX0=h$#9sg(aBGwcvYmJ9=QS@-NiUEoBGC#>AK9ynbBH)Y*h`v-`* z+^)6`wePlpiBp07nJWf{wbEgf*t&u1Cj5vm)8bda zcLLGcmLubM0t9rQOvYJ0^iDrgqg&0*`96&ds`ZjwH#Nnk;y2%^q0CY&6&0x0dC7|` zUuyT~o2z$nb31isI_>8{`&}tQ)hTq%8i>uqh?EDYKbc%XPjTIZ^B9ccgRqXNy>i)0 zy|R4nC)_RjYfu7Etiii)9ULIzw7bt+V!NLl%q%OOhpnVq#NqKog{Oqy5=#B5+l z3JvhtGfnVB4Wc0H88X+hdt}_|Vm02!r(gvkS{sgy-Ke2A@iij5d|x!RpwRu-hO7@A z?)Z^=Yt1T>5D%Z4z=$G}@M8!pxzskpKwY9eo$X4GdILqbLyTVK4;^GKYD_ z$45EWrl^ql@XtHUQ@zMVQ$mWFql$Hq;=m<=O%X-CpJ7LeX=fLHlI?5nN1gI}tjEAa zrAqXbNSuAcG9k5}HB+1X01tf6e`KtMck-ucnDJQTr(2~LUY4`_5PvUs%MX4E*)gzrhM6zM9=qFMaUJrK%?-U#r;Fxde1_Ll^Qcs_z}co$e=x$9j<$I1 zbw1Y*D31H5{Hs~%ujNihvTYCLOXn=JrXi=sUbq|+rVjhG)DJe7g!4>f83)qL^{WNfsCpyYu0+ zd|orW7?ZR8)G>0i2x#7C5GNQ5b_iZ-qcbgj6;rF`a2l3+BW2`e0^>y+HI(_X)Zet? zC)8jMcf~Vc3KxSSFYb^Awzrk0FUuC$T+>R0}Iu z)3$bj+bb-r? zX#_rAQGR%bK9sW*)1T3)lsMEf_k+66%_0Z;)TAMTd`ZFrwZ)3vPirK6FbIJagukM# zq(!Ea*-&hND>Dx^O){o*p>)@1D}eIAgw%BP*tx9|$;s>*Y#RCB9?#xZbY{`ZToihq zFHa_H+JHim8yVp0N}#tUesfF4oKQmnD2Z zAp{%Nsza8jc!d%j4a zVrZvnB%un4uBeh!=jHo!^906WoB1kIF(dLeFHH!$TmT(Vn+cT5RFu zvS{EyCTP5$rkVdKTqd=x5K|K+GgD_z)@QlgSj9nY#Pc`cvUjqGP%<)qob~v4Pec94 zn5j7@s#8;9rfw>3oB?ULbErYC9qBKd;lyT@Y2%v>O>Dpb#gXD6YuIRxDug(x^kZjv zxaCJDBuYO22lu1Z{e3CMW1>zjNb1;6CnE$E&u;yzLmjea4u*r}>EoyBTwVMJk4sIp z?sY_Fgv?8GXSKhiLT%;C^BX%Wn&Md|ZCG4sSZQ>$JaxO_v&n$6*KRv-VNBNH^`=uaU<3*ky~n zZ;DUOq3_?&^Y1`7_*QkS z__Z$bkxF58)IcJRya^I@dOys z$BRGkaODp!7|5#D)8=l*V9aNEsdM$18i+)a7}Pqf(r$n92Kl%~@7$8e@fX|A>y2mF zUJy;0t>zb8)SVbV* z0ZUz_lGVMVw6Pqz&fy}cQd=g+QesPSv?~MGcg83Lx?!Y}8tTGG8QkN8z<+560F(fq zT>>LlE+n)}ljvWNz~J^30jKPL^&T@ytl$3?0Z0BV%KS7>xh15?FETb;;&Sl;-H7#1 ziO8S-NsVI|gmOpZ_=M3PCEay0#kn@#ftUrVrmR)Y~CGN zC`qBHaD8Gp(GsnC(?-e|ocDw$KIA>87tdf5@r zcjG%tX}TAGEXjY}!~jC+M}q|4D@+8F2$ILb|D_M$DDuA-CQEvR+{^dKDixW3KoU;7 zvz7qN`rFT7%xLIt_d6w8|M`pD7TN8<*{NdWr{lRn=5{*qAv*|vwBFK`>Z=`tySuK& z7TL3dU;aFe6&I}V8=!Iix6uF7>!AM_mla?0=Qj#uCjkBLCMagC`aTVo)U_0LYv=1X zc6M}`7yhgK15727wpyZr^d|8&pb7=eP(4*&#U3$aps3OxTl9aYjp?tFoe<Y?F`%XLHBDs$Ig7y+WNma>a1?yr^Gw zNV&L9(=P|}5k!d2PKMxpN(JxP{4%28DD^1k^uP!$o%ixNo3sTUcvB5BvY3qhtQ@os zHgM-k&4lm34wXWGrW*R~z#6V>`i%*;M$<1A)!1w4`1}WX4>Bitr>KJUf7D3!PbJqI zS6^3;#ODmc!2aF%>vR{~Ecy za9LtcXoeH24EeTUj0VffXatjhv;df0U^d#*(Of!ONp*r40lBhyQ_>s>RJIu$^Ba!! z7#LP0{=wkvmR?RFNj9DgeNH@a3oV?a$d3g104!JNS$q;!iQZIU?5lFuNzWgGu^0fI z$iC%{AK?TH1rZ*N%@B})6;wHlWrpSY?DLM<6yp8DjABTLXUJ9n9*iC~9!4VHRi3WWoJaJx&vm-Wn&7@&=%p)SlR^R#LZ0JKg>=DdU0cvaEjp}cD zM2^LDtXNYujlDOBxcQw&E2ZDD2mEL{b`N*C7L%>Qa6{F8RQoyxXaZVSM7|htI_V(n zGMp_+Ra3Qga};*L3RQ)_D}Ao5UuD{>J_M4He?*J^=qlYPANdGC!~-NJ&yu^&NjiP_ zyG_}xkBX5=&Tp75uY1Qoc0Va%)MzR*E&RpU)AxN(zoj2Ge}N1Lya!@Vqq5{`gX5{I zigHDw{-6Lans$DF`(c|9sx@hT#bbSOczVcKRH$Ui(^*=y(+}3QLFj$La!beV(r!Ec zlvcvE4gfJrI~TmhBUCNvaj);OQEEeGh*iJwjrwj>d@{p2!LT3T@f?!VXQr*?l@-bm z9gG(=Wy&cOwH1TW#el}E6y+R{4HZ-^P#%3+Y()Yq8iI@6rJJ5sGrFvruHA6!zp-oW zG+p#W4}$bJN~DBBShP(T(Ge2NFZ5)+-&CY{e$7!mO@=h$h-zWk@06)B9%-966cMB_ z0(Wy>Fta9>UA-=tP-w6f_K-8cYXX$x@33{~6#9{NF-psZ$4e>$nIK(yY#`qRQ$)WX zV3(_|r(_VMfsX*`XfR=g&J8UKr<>A_w+dBlT)uoNQV?GnWdNIC`MMrrYx2>V(5xAk zg6o4^5kQ7TF|1Y$Is>%yl)_L}R{E}PQw=rWXIUDXq#myJ93fBX8QyqVIBeI?O$+W& z&3oT%76#@P#8h2`^!|GDKPPUt&33ifH z%Qvx5%91Fm?tPp|q8S>+R9llev2dFEi83eN9}fPkvQ8y4S%e>Sgmq;5PqB@_LOq~T zl;^0F!$3%6*UCwNl)qT#A=a^CRGbkk+9=FJ(%*70V+nl%i!3d8rPiljU0!|`lGj_A z3(Q#_X*As)-lerssLH zZy7ONE4sSBM9#yC9}J}^ixqIF!w16?*XPivOp_4q9E2s5A-rn?pmu2>FdF|^d`?(A zm{tmv33I{Rjq$@z1@`r&pdYn2OgS+Ou*TXCC*;h^xl-Eefhg$nIa?gkVE2R%7t*wql?4%Y0Q4Gl~%RKW(4~=$^I06Lwz^)i@;+hw;r7kG%j=g z+eGf)T(QJ*3FE%eAnknR4UWKjbkrt%*tSNow>G?I%yJ5(U2OMlJeZXAwE5O_{CS#T zUK>GoukRQelQN#yo-9kTs3^KZkz^E zBw*0S0M*O7*Rk;L)d9}{@@scILt)PQTeAnr8C7N9MR>d+%H*kGUM>DKn~y0n`I1nR zkU%C9_0tYe1r>&!j?6ozZh7`vL~0%3{I+1h6UC!V{CR{Ms79`@V&!O~UsKEaxncJd zGt$L;dbTC-g<6<;*PMnf9BsOUy^|?g)cA+1tOG4S?AcweO`)Y3vp56ssTv$B0p4X& zT788G&+K>58M~4?g`D3|+v~rqvmt!3(6`c6zW|8q3?t2F&8l4^L zP@D|84@xwUD`zlcHxLnup<>tfk+cY%(An}G={*(oFUzMXKv4K&3!;{2oL%re5%ET> z&+G@gw8mh_FJ^ac`X%fd&?i3v-m9?Ol?Rudx;?R=WfGYN4C_GBgR)^D;H%a7!ygo$ z_R@r*8bYYCWWWqZ@B7{}gRq&JnVN5D$y}#lFs&L!`E2d34QU6#n-bC`CQ*OCdUW1T zz>_!ky&ibc75>MY)C?|;E@R45H*1mVNn7XVYx8(eXXsc)YZA2G{AKQR>8_ZHf~w)q z^k=|Nwx6`Ar|?FJP7qqmtr!^SqFv_ruO|0~zOk1$nx9l!T5})#Zhdh5)))J@z!4TT z#GXI*P`stQZ@>i7d6?RdIsu2A%Vs}LOMWyB27=OwP`mJl7-rtLH%qrp_)lFSc7ve(W&YR_`^I;GX?i}=4)plGK>$5 z1#tQ1I&hu33nwldp4F|5nWW@P3?fsS!tU_=4qx6Ei_4s&sKI5N{P(an84JtGY8LS-y;YHD{`2r_wYu~UH- z$MWvnR6vMCpS()7Czko#)oYEqZ}cdg^%-tSd-QUx8%}|jI#A4)@#nsMO~u=~gEI*| zK&FWc*=KQ66H`6~&L%qZr{wNqO-Ra4r(oy2H`^s<;vj#T^fLK3_QxvCQ<(@URH=#n zU*R~>Z7Z3a;fCJ-wOw^lKwU$vj%tnZ{df|C?f23&LiCRMOgFyeIw+IGy5}IhxedRb;n<5d1(!l z?@?mi>Svw-O95}c#;fB+NND3EJv==d5+s9V7f1ewiJh|TWy_;Wc!+Z8$H~F8XTGT03 ze?j}9trfYPr>FmT_?Y%oR7x@;+yD8X(d1>Sm!jF9&|Ypgwp9MxiO5&Kl$H?+x%Dpl z*N6VvWPf~$zb5+WFRu&Pbk0iGShNW#QZr=ek^q%pNWKG0S>2a^`QK}YmX+~ddqj&E zlwW`QTPu?xSMyah94kCi@z5G0Um&7!{WV=u!cQ+L3LvJRFM4gmY5hXfe#qosQRA6< z!-1`h-SNV}IXl&c4J;&=!u8b9uwgMOQ6$&!;`TF*ZY}O{BE}g)6^tUV81BupeNunX zE}r0AV4Bft?wj9>^u3}SpSEt-d!p?ZP&`M1?ps119M8KnerFv0U4Pq8Row0R{cXDu z{QbYwGURvKJ1Q-e%tLoc=H2R#`M0l+J}qdb(Vn~Tg#V4VaC)0T&_4(~W`A#keDKJY z%>4=OhN4*q^$~8iu)a@4fi>taNnULJ@AE-RWggl~k4Yh?^dA;%e=)wOd-N%{r!6iu zee1_CV7kHAfw66Mb9P%h4E*;csdC4UvCE*5KV~!l)N&aV;HF#hJ@)ZXg1Hj%lf80ebz6?rx=?5()0Ff*E0`Y4)?f z#;xaRAZj2x6;|i_YfS%i=%ud3xv@FR$jDp!V|lqvd12;uP3`Xyfmg15uo3v_X_bI` zWYkY^lJou^7AW;QJ|F*q-B8hAWZpd3UA04{)ai?VBmEoJFTQf+U)5iB{UgNzv^10O z{V(f!>!<}w-RA8p%kWuvc+co}!}^3HzSO&B6$XlD`w^bVHz56F5z-1gd$%F&Pq%%s zzr137hD$&RAEgK&D#O*p|R$?DOHM9IN^!M=kjb_FhAttr$ zx3l8Nz-yh`8Z`G{#^k~r$t#>GxtGOFYxd!T=;0-4`mfFS2V<{+s=@fq{r2K~Uw>UE zj89hU*t_!Qm0l;jtIH2jDW9~rv|iBolw8X&xcc_Xv`gCXf}EPCv)NQAKwP{#6gRF( zI_pdI&*T_f82N^BfUR`BP}ZrO`D%Ab4z)pi(rR7x>6~cxC!P1Rsj4M>q3B|EvWe^5 zteAA@zkf|EMykTZU9($nvI@Ma>j-Ssja+19n&U2l|^lYKd&QSl7>F_f}ZIue`dkw#I)!iCq)e>TV?p9+7$W*gb zenW7kgU<2KDLS1ppC`Lfbt~Wb&qGg(M)6Jws7pTdu@^N8a2n!S>&R&NK888FiUuv-8n1%(uNC9>;Fq}-yUmv(=fthC9-NZHQzVw3 zx4bJ9^sNg45Jm^Ub4F-tJfqDs@t`^*PI;T#8s?-yv_?HIVJu@&()qO6t);oLf|dFs znx?0I9!NL$9~9V#g0*3CPE$##;VZh{iW8|EWXs4$=`dRxFp`K6Z2vrzGJrO9^>s947s?yrLPE=Od{b zO!rd1y7pl-BsGS=2_llPi`$Q83Kxu(j*@kbV0ItRIlQqfH-k(S9ZNfR^KS&J*Ag&F zZh~l0a~td0pEf*=u{j`ao;FYzy0%(&(eb^*Sc_f}FeaANac?(X+?(aP$}6XRtbqx{^C0l_bJ+}(Yp)F zZQKHctE>dMDAJPea*Yk|68^#R9Zt(rhGD;ff35Juv^xzS4}oh%)qxj}k{XMl*#Ock z$&Xq%JYJy8st1z*xQS?*T}4zv8mWk}19HSTbV1gz@12Ib_6_Rsy!OY@(-`mI?TEZR z>>`xxe)pzge?jY8`wrYWEuh5|w+2SeraGnhPgP7iq8?v}O^#SvjPIr;<&lbwb3NO`dD`#TB28 zc8db&3a#^k=(~dMHIyqERU&s?+qL_I-$&hbkW~=dKp^r3_*z9NM!g_sySwsuik19q zxA9Oo37gCvE6g!ZRU56Xv)Sxr_8N-(kmN57-76p&B8HtB#|(N37K zK#vN`h6`HOvfA)6$rIB!2EPlGE(mZy2j>K+NS%uGDE7OlUEMSqtOr&^DF7UET@C}g z)8_pSdJ>}Xstz%XTVA(%9HbRIV|o!}y>dFjw1%uqp`c=R?dbf3fTE5(Ne2&WRoPCW zsBcnaJkmG8Zb3ok-P~v5a*COxIBK27xBLhB#o@t12wql>Bo~9`=y6KyhJnfcCBE;l zfS#h(35TRv5@!dB(3kHN6M~okaljVrLmf{{zMm4x04aWl2#-lE0{L(^;2;sk|*G&0z9;h6mWo|4CsCE^dp zy^%T9;5cSCF?(pF!!7}#zWB(+L6MT7@>;){jYB-aNFd2PPEYB$wnH~!4=PG&Gt~J7 zUJiN!l;-UNr*Q6O2s`%vR8c0;HoG#!k?(X^>XWvj-wHWTAmH~R5aU*|4lmm|zmYe? z{n5gg7qGoM+2r80N4D*-$B13$8c%*P zkS7Evc!F+^mpV`z3ej1uaQNy!RO5uicRu3*fr5+l)SVBT?G0K_)>O32o0~gZPnMi` zbB?#PH@?pSIg4Y1_POg&8_o30padZMgz@y~`1Xn}rBCBoXAo;f7KXJ~Dr&k*-u zEc5E|xo_o6%4-QZywy^Sxh7afk{(2XAS52##Xx5=KnA3I9~w_55bC24i60Og6-G$# z%#)jqp$o|^RtmGSooO04s9xQ^pkctUO*x;YurCxfX^+3Rf=#@-maB#_RwtUO&X5o_ zR;rK1lgMjX15QuR@l%$tzHhl*VM6ruV>c^?=>r>HpATd&56$iKPUI>%1gVy99gLs9 zusVgfTSg*V@R@XcvSD&=>~ro@l4iMJxnKjexv)<575ASzzMZ@EW^*-WwdCoJht2d$ z6vpiG&62Erf@gXU^VjQ6&t--wD__jVRp8AHF4^rxfm#x7^%UaPw}?0;^oToo*~LO; zb+xR=cT@~F3ePw3HX3a-po!9jlCV>4sDKfv80pg&VouH@;EJPRIYWx-3v&E0Gc)8V zxCs_UZ!d=WHVCQ!1~V6aPtS!6X6I$#!gq*#P!bEi+E1;b4>DZ27EU zqOutk`VuOZri2cFt3x<_gYYp?rywiqsL~DO8Bd4?998rOku?x+low#Z7SSwL= zlm$YD;WRY3X2i_QpDT5wmeL}bG8T4Mm5)2J(Za6C+3!r}@LB7<&9JkonWWfMdD0!n z-TP50nLj9FcC*KsoG+aS@#`uIkn^sMtmIt4v8R) z!XW}kwsP$@ZhT5xePB3~Xj;YCRn46gz*J!mfx2FLh-H{F_{fk&*xl4y`}cW%FWC0c zg7$eP<}Bx$tHGVLdO=i~3_|StWZZCfkxLT~!t>3F0IDJ{slLfjn}RQbW>AlVIc?uL z(s=Z<4XnjyrLm=YIsW}-5qorDeTj@996L6^Bf;2gbC_3yc|_R*8F{OSMaFDV{>&B! zs(&sr+2Z>{7B7^+ijyYF=8B2lLnR+CIsPE{K!BRx7t0sgo2nlB0j#>P+;15*-~QCL zXWd#E5~pQU(dhK^@-{LM5cQa?n4T24dAxJ;W^mkl{^bdfh!)fQt?UCG=lZt;##E?| zMiF07Q}Eq8A=qtJXDGFr0kxTrx!?~~8>VMWI!{Z<`F6Q}UpIyP-cz1&;1VQ6t+Gwa z80a+yWXPB}UMzv*_&E{F7WIXC)>M$6f$M^GmlT(AVwng#3k0d%od*rMla3Gm#m7wv zhQHmTob3;~g1;_(FPR=9ZbAv39=`yKkZGa;U{04Bn@lJG0TaZe@%S3Y3T+eTdI>%fyHZGBC&Bt>$J`xl z`>6Y<%j{@(^;sl>qP|#dAF!+VFq5D#i4Ak{EE$;e_`bFG2RVa;EeYDo*Gr#7`YjKQ zWB5-*O*zcRq0W;eftciK(_LB*%eb0< z$yw^9GrrYbNske7+*F|%doA!jp5G|A|7E7d5V6e1ahS3<5y&~bWHNV3KR2hva7sU; z!1w<0PTb?|o%kK~jE}R2+<6Hv(yGiW?V{tPt)`z^!76QrIX*GLacT3lFM0BNAlNbu+CvQkHt>Zko=+v8~kjxk6vV zk=Z3s&d{c4bfC=!Ue*~C>TorJ1(RnI|5=soc=3d?kG-I6uWl6^ZbHrqBanP|>M;^J zYVg*ZNpCTIeRRdl$Kq<)zVKB43W#`y2n*m?yXW|t1se<_EHh)etR984^bomBtACxS zFrG!&NPc%Bxl8yRUw^xobVoXOq%W{(90RX1V+^KDaqhh;Fj&%W>-bI4yOFm2hD*hG zlDa91NEB-*FEZgHz=EV4oNN3$#>l3JXUit`ch_`eYQJanKK3zCk+aUz1?glE*@W(; z>OfS&=4764%y+b>t{6J7owp^&#;L!Fkkn$~3}tD)S^Bb|5Ua6!!fC6Ys%#U=uFPJy zEcdk|$*#A?M))Lp;2`8chPj2N3nH!RvIue#SZxS*4PvxT&@iJ4+P`7x6u%6NJLHI> zGgy3kzm8oiF?Xi064=l*c46&|^QD^LIXMZf=K=@+S_^SrohJBs@_0YqVH~+lY-PK| zvY2HoYycw1QFm1U9AcE!o$_K^iJLk(Tb&E;!nYy20SOb39Q5Pf6t;)XP21L4vvi-P zFuty_I-^MBzA|A%X89b;IB36yg#taV%bD@<0vTn2&K)xLwM|A%CB47Lq1-Z793!}5 zs_R|#$(DXNg$zV4tiz?F2U-ct{B>clLB1{A5%~8K~5n-AeG9;>X6^CoVEsO zX|k81^r4qYzbDNhp${&P4Zq#qqWm8~@8kdYdPcyV9$wL*&&bjq<)h%QC!hHU+n?eU zmhvSgJqP~hK*9aTkPE3;6I$x}515Dkj{IlqU#SimUe}}C+OwRLLZtCMi%ysF`+LwwiQrT)cq4;+|vZnr6rm(?xDyzP@IXsO4wJRnpO6C zeVX{LVgk)S06}S~N8raNJ=E05O+%6x@cnEWkpuUFGQg6`uGr#cwsSI~BaKC} z-7!Oi%H|g}DeDv8;-G2Yh5I|WlKw%_E35l;dHNP5n$^zO;}42q<>{<8@>Bl{Iz*vI zl9NVZ@BV+-z$Zv)tG(e?_fhb(Q00b;J|hFE_WGm1V=!O*UjRjL?O$|19?b9T><6d} zLlgy05``Qn3<*Il30+vdfw%46^?!Tv|Hy>@pReVgz2zC;-F*A2tHZkMzoRFi07-of zDcl~gXB4GK1u2eLTL$o-q9hNDtfV=9jX@e2m3eu3vnFaL?h;1Ou(mzYaOec4e77}vL z&b%e1FV@mzhGVO|o`gx`&YQ9ha+-35nh~qktt-P8 z3HNvgk^d3{HwOLvgfVu(Ta_WB7Pp4@HiVD$2`%#eB@@$Od&c=Szq!qiC zWddwYQh9+r;~_A?@OVQN$RLZ1HA}HIA!M?!L}~m@g}9lX+rc`RQodxqp> z!dy;HSdN)XTAM<(ZLnis6LEitJqX-4`g0mW4IBx8Wko765XZhw3)<6f^I;qtu{qa^zJ;Y3cpM^l4sVSP!!95tJ_BjT~f#C)q2{~E7g2)iH9Kq%g%|5=ZjI!#n`z`HL=@T#8cGgpUhJ4Cc`2L_s$lj|=l%Ek- zOVzYh*-7cjc~>}xn1e1@!MUJ;N&rJb^y*}a(|v86_Mp~$Qn)80CnuU_dl?h@=O=mH zA2g)Nq1?UiyT2m}D(+_<2|XkSFFxI2{G4`AEAB_Cz;C??ruJnzer!;{Q)8xna)*CyukJ21R~IqL2|c8 z=9|PO=c(c$(3L&&z8rSpb6Va&cK9}7c}eKoDEZ5zOGtqQa?}@2Zc(q_Td})c)};J# z_#`IhaV+gYI-*NB#mT&Ym^fdW!Av(S(o#S>Fet&|m5CGDf)#*qR>vl!w0BN6Tp@Lc z5yU$+rDtaU$kh9!bUYIt_0-%yBl0BbMRUhs`!sIbB4%D|HQ78|;v% zMGhn2gclMmlJAjuL*^Fj&W&>|U=R(+EaqX_-NCaB*4uS)!+@kF_# z5dQXKs|QWTNs*A;<>yP{VOBxt;Q`fDrpawgP@VE*MyzSSvt#}_!!95tV4BuLc(N&R zVz5LSS$%}fs5j}sQp8!9lDgCv#?$qqb5|Jdj~9s9Kzy?XmP#Vy->L!wFAcOGFC_LW zoj}BA^rxYj_o0`GF^F}$+ZW7j8{~pM9e+^Z&-XvJ2#G(c`wr08mtc!_<~$A<9a0VF z+yO2s3LZ!GILmpJ^>|2YEoI+q)UP#kLG?KKM(L)iZ7ovLKNY7HjdPIkgoCwTx2#^F z^fXtibdp_=O7oAGYT_=O8LUFf;~S?10!g&&R7tnz8kIkeCo{<`@15^moMxstjWw=vWD{p#}k|TO#8ih=qsrv)^wtOtd<&noLlCWH1-+z+J zZ8z7Pbeo#Skem=Vxg2%Ax%YB5$}M@Fr_UAurm+U%=SM%Pq%o>72*`xAs2dC|lQ*RfAxm-m2+_jWeKecs=@} zu{+e`PMDwIQ^cv`O1*(p(KswA48M?_fm`I+>s@XOjE^P5d)#u4-9ao|ljvC4In8!FT|r$6;b1Z{cX21+ zKw+0Z1@vBYw=Gc%q$7js5k`S`H$P1A%)<)t3x5+S4LasEt+Ndo@xNIUTzA|cd>vl~ zMAf5^c-A6#f(wV0NPPD6Ji<-#x(9Tff z&Ar{}UP*sYOpbhvC~y7wFV<8o-9(M<5_I;eULn@4e$P$3$^CX+%E(QL`?8P))BYFO zIlB!z2fSxKI$pO^0j}xA2&Ma$A1QLCgR@oyVwvWA1QPYSb=%#MJofJq&}B|tLX)Rb z=%r2reiHk$o9UjY=Z`vUSvbT_7Q)Dy<+IEbQox*(w_(6nfSwdaaccZ-NzVcMs|R&5 zvOQ$dTt;+x|8k^EWgFBYH?+IEN8@Y}*&fj4RyAiDGNxmbKvReM)-+F(s_=kDmYK`1 zn#2YJ^yc$3->1O2sqA-`tIb{Zl2JIXNbt_`mq~IfGWNSKNV16;6?+1FQBY7)b-b<{ zZ(iM0)|(R=J0%d%Hblg%MKM{nIWDhfQuC~MSj#QxFeQ!H zJT&DKEFe7%fi+5l^0U5GD-HF6*gY$p9!Km6uDJ_()t=}#otow(bbV{e!HHg%Sk$>m z;H>O5x~+#hcU=4Zhm!hZQU<~YwSoU)#o&>c&@6_~WtQvi_-=?N5(HLB=|l654eREQeZGt3q9Q+~^*2u)$E zHN#&CNr)auA(!IsXHL)F4NCWpavphquTr4~b$+y&DYLAmH<9xGdH=n?G|_c%^n73f zO8%Xz_<0-+oPnrB*fJct*EYm|(Y9O~Rnpq&Q@5GyUhH>pe28@H58#muzM)n=t=Fek zp%L$&+ARSqqq4eX_gy7$Y}%|S6NyVp>nXbj+f@j0tqlG#x)B z`RF}GY3&()Cjf=S-1aJ7O7^Ea?&>=h=dvjbYgyW=!`bM!Frqh9Evoh>lZh#c4RXyi zar3;l2EXhD0WIzI^QMed^5h2g%Vt}E(WL_i$JK+S{$nkQ9z`JfVFC+!-30;gc3^{W3=^Pjj>ZDH>13(UdLy(X23Muam9 z;rsE3Wo|19^vAK6RTBp>hoO@c`R6Va;9Xb6l2en?N#B1P)imbFj8EgWivZMg3{C7~88V#nE+0aXejHLCAa(Le+4gt)En&r*mU| za}fggj8$Nym_&X&{P2DFVTURA?GhGucu-NPneLp~$r0rzXfxXfr|EdqT*f}D9K;f) zhpa)^y@nCuho8nbGJrnU^`~rR{)^&?T-AFCMCHU-S-e`9JY|)g`H4XaOM#vBi{}NW zEJFhf+T-{h%$@6w?r?Jbh>deasp*~i61gQr&Ui>j)`8j1>4DNs>XC8yWWI(eW367n zXbyNy{-ESFYAgc342wZFv?*3vM#Zhl&7!SIZ~LuW^9fB4J;mZg94!!sf!fGMwp1V) zzud3mbdG~*Fu3R09~7Lfg}1A5*WA6%=?&jh^PXH^J|-USdCbr7VH%a9Nb?-p-ySdQ z`XYnv5HK|Ua{CfU9qyaWwUa3@abWM%P=aqWR8Eq z@A?I&zhIkGcX2^6O8RF@dJ+oYu6zBQ`d_LbQWy%QSz~$Vc@ozCS0xH#`kNPt@|RnI z)K{La1wUK(a*s}&-HlVBv>QBG(tE{)WNyR<8 zIbsI>!+z3*%hftAGjs(igB25tM5rz(GY4YdzgCvKN3Ywj58$2^i`HI%Mho##db?8J zWk##;b*>x^Q;)8TAR8K^+Q;|BrUGUnq971``n|61_>s zi!SYGE9WkMa~hvvYQ(I^Sw*q!&r*&MyiOf)%}g9cgKNyDW{QZk8v*6@f>1%RuR_=N zuOln}setH{nIbdix2wvU_H6G7{_fiFDE7?}A7&Iu#LeT2yZ;2mCra;5{_;cYyRS~$ zQP)RMUdrZb-07SgRH8(Gc;|WY`t_^BJ+^Z}gF3*!H+u8M?cLhHh>rK7c0$YE1|M-| zR{mvQFix%17d>@&LYaxo(7Mbv@)Y!yzuLZ!JNtFu(wqEuEKQ|&nsjS=e}X{2v7cIq(%~qm56Po=(W9LBecTBn(?#35&CF2qxGD?T)wF@uI&FzL5*CZvCKEzcC ziqbN^+r}mHce+ z#lnP!gW1Q+!Uxx~CvSC~kvPxSc@0sHfsFadT9~4nlisKWj7NFuHpc`iUd_!%mDz#m zcjrz&@B1lFhjwA4f<6+XJd)Z)4%!<_Gl}1qB5lwi87?b%iF40SH(mC8Wbi(a;W3+DqJc{ur-jh~Q#G zt<~o@wThNH6o#nMXx6*>^Yo z{p=bdiQT<$XMv+}L`fS{ySbqM9k)=HBYpx=vT|HC65>45)}O5VaK^NBFAl%b^iMES zD>@#Mm5>#~p4yFB`GO|I%BK+;p<-R|MQ)Y@Y8Sj}Xl*L&64?}fNeUYx;n1T+sz?o0 z_sYmKm!bjkgsuU8ykPOS1-@=cXz|My_u{gj(r<1dXv9Y-UMVh*saxw5Q(@X>EBwR{ zHs!Q%M(Zu!a_tIzDITv8)cwi+-P*dmsc1=^Rn}NlMpkP+qJ7A35{OPpj$F)fE~AP;UpCXm92Q-D_X8^q2e2dL=wf#HRWDqNVK#Lw2ksw z`HVvQPIn*zp`r>_9rl;flNj?-56|3c!@r2qfkKbo+7ggtfTL$$mYD2li@KrCze2=R zE+{?yjQ5HC$pys@oEz7X9~6GR zMoa3p0Ht!zM|-pI%|=Bk8`mpm%#qe|a1nXqfwOCrSa-nMb#s0Rq^o9quSA%a35WW^ zt3~WwI)x1jy{j2&Qe|!O(DJaf`w16cm3h@w9(y=O)R@t?^&G0Z40n=kM9MUgO6ylG zf7=mFNa!P^zHFfDN)2a-wy|0~+_W~*5h&5RTyVnb^ehty;F6>2yX^Fx`F?jW5M6SU z;-kv%4z6(GEIXBU*VEsn`CX{ibdVoAZ(Kh)SV2d&=s#@<(t*5=z%DY47~)A)bW!S$ z_pDHJ5YlH2RoqY`l|zinhc=8GoWqHx^s}+_v(FTeP;UrpHzayLndmSM$f33rk3CmX zuDs!CbQRO63=6=KpsnnbTT)UoB^0|k<59R zC2Q{fK~d+pxAZcM*7H^kQTQ3fANXXKb3zwB32)<4`>ZIuPFz%#keKF`Y)@tr8VL%T z)T&aUbeMzH>zFt@xMBst){0!1Gs!bDQnzo$^lQ&72IoH9ni9qnY_bzX_vpfTSr{QX z#b8Kr9G)X@V;qBAFH)a1c2>G=8O zS-Dt1nG7qgyxhLi3`=iK5}7@&tihBN--B_fftI#OXof=a$9lQGWQih}kfccwpr3ll({x{^A>PgTdU z$u;bgE4k;#tbJ|@V05UN30pBEd}ChB@Vt>ncMf8fsS-pE3B@kzXHIJMEXVP6|{Meec(3x|dljTB(vnM|%X5z|J?bdi=ZY;HU(=*~IVXeK)`#5?o zP-!^CP~$_2xFH8x9F&9+cl~2HPKk*EXzT{qlk3kybvjc}kh8C-1EANxbGbh}w3?Ur zaLB${0+K12h{Qcb=%MbNYcrLUvl*m7q4z2?S9-WJ#Zf-|Hh1dZpW^X2_2Ev>zJl*X z;(f_+~8-A@<~7MYtwejCx(AWOc2+uL^SlKN)6!$NfpVK|GI!E%C=1} zx#39hYaOnPwq|Ds8E##lL&N=e{j`PGeuK+?!xb1AG@nV{s%E*jUJr3v?44(RnOZPh z@0@8?g62Tz32?tRwMf1hgHFL&4NW9h#%nHka>?-+rxnB)oa|&XkYm%wP0yrTHTrXkgebN zOV_hIF}bNb{`6b}KJEC5Cz&`4-gMqqS~&rcZUj%S7lI^b0P3U$9-5Qpl10dD>Onoz z9)~FbclJGfywN?RTZ66tjI=BXRm~EfS5_ zq~V&asRoWc40X_)CGVSu8U66-<~mr8V~6qk(El=gX=%l01-DbL3`TprX10%nz*Mz`H>Y+RSAt)1>x2w&{!Hu% zy&6#{#i{9R1=pEz_i}pAG~Erg9Jy1FVmy&x^3FQ_NyG!Jd6izz`~32SUn@feqvm5t z`3)`-%}lvN`a#2ICFe@?+LD6F*?1lrlj&${HuzjXDQ|{JjXy7lhTyIS417ZodrJ&{ zWJBlQ4{VNgN?nZPo6g9@`#o1;lajlz&5qz7Ya*?oW1in03gAAYIU^}ab9}}5zJ{t1 z!%SD^leD+M4;(5ZWNMbLQz3AkFs29-r1}*fPm*z!(IO>$W3zB`ke1^mcr3J9Td32F z=&V95W?n>svg&GH2XJTTRKx!`KxwV^2W5{=W~h-rmnd%vdf#1MtK45XIhcF{=| z<4kdQS+{ETJy_o)ERC-+C6S;@rvLHMJ@>51BDhCdj5Yt^wC2XVqJQqXqN5XcmQ;kR zzn{E}r|kilb#74agRB9Ho0=wOAqo{j>ahE1*xuPqcFGRr{kx2jGP6cCU`4HzmL;*c zs&1lOUi$zsYj9G|Cp^LfhVdGj(k)Eu`2ZIz=@{j3*sisVJ63cz2da9=p_)YkNX7#h z!*k}@KNf5ZlUsuInf=nb#1%=sSG(Q3SF4Qd?2THex&u_tNO4X?H`z@N7n^&ICEV&? zx`*&bUy2nLH{wz6pOqk_gQ?H2-sLd&YUK^2OM8oJb*}u#??X4Tq@TEfLK1c3`UzEO zh%_Ye1ujEO&*;wW6b-`s4g=nq2v^$om1bSu9Q42j3VPRV;W@8P00J09hVOB zatdV8#sr5LaOeEkBhgru+SZvXDaP9wUmxA*(*{rYC0tY_^iiRZAAos?NPYP#dlnyi zITi%J1_;vlkWb?WwpFE7Ly}D;1Z%GbX|NPEN`qVUUhYv_e#xC!@nQl;XQCGw8=+d` zblIZw9I>Jh(HeY^{BE8!$!M^|xpjk6nyy2Yqg6&BxKd*HYHFa?^!>UIQNqWZlQ+ei1-rr!qmOU%t$!~|jiCQ8DiZHMDCU2(4FaT-g^03Z#=Eo4`ae_) zQGNN$!o=O>f9W7-v$_l03wO5*C&&LKh&(x4nCM?TzMzyp`4`kdw?1O)o#DCqkKKVr zceb>>d5+Tk@jpTd{u%Td>ie&{Mv*2vsb8}R@eRK@-1M+R#zwB?3S7DQI`%xYwrYaf z#&NejKYqM9yA;6YJ-`3`*Ls7>ZD4qHN?ylvh6D_y(=7Gl{L&K@sVQR?VI zH68!kC>eo;Q&Rs0kA>rfm_~$p>VQvX=HxT9%oyG~Jc6M=C?EfA-ZKduo1u2JD+*7$ zmLK&axc|z0jK+oOtw%8y4A43seJ z`riKjKZQWh(N4$U_wi=v-%9++r@DJx0|)FsC@hEf3$6>BEuXj~dH)}--ZHAKE?V0T zUZ7}kiWT<k|bX4P|$r~+hu?Hzx71j7Y}&|?*8ZH z;*R3sNX_L6n>oM#Yd{Jx3X=Q{06*Ph`~8por{EdY&}WRm8&O|hc)(v9QRgA?T@YdvGHvsj?*t$wz*VL5MV*?>veARB|6g2B_Wo z*LDZ+JN%2}K|)_i8iM@WTgWwP^I|J!LGS+=%_u?R+W+l%p&B1FLFSnt;m!RhKskMO zXyEO+Ky%;2(|bVKJ1)T8^~I&=bvJ-^WmQ1T7akw!0}~qI^)oNbyg% zul}Ax*KoT@AK%%6%_i5`*%&jkxlSM=EGh==%6azVXb;RV_bf??QFLMyqhjgff7VA~ z7eUqz^}J$!YVxCOfwy>VTT$*G&ithD7wL&m@$Dxu}fMi~JW&2FCD z)g=0`YN3}3JDI%24RCgKzHj4KUqi1NIjx5icA{*&wQ9EG7s zOlqUD>B*Mbt5N~hO>Sv&O+0xQS^nWTuI_oKk{Pd_^?}^`GZXuMPL*%d_If@Lt)>^1 zHYTcf_)nLlwI?0iP73@!qpieNlT95Kalu~Z>Q?KQCfrp|#0v)vQ^CZL>2RRPo?qgO zp;?Pw3s>M6-&m{69uCOb?7!I!P}!^@y6*6)S%zj=_otH|{l-pRQPD>6)tm(mpS3a3 z7CwNq!S&jz&-b$Sy{!= zrQI9NDExieO50Xu{+A&mSRBajoHeFdCG6#$Fg?5|if&*%Gt?Lm^UfRLd~;O;k`%G4 zsZ}?zyC@?;s8VRQOoFlYA2^=u-gAp|g2wqP;?2iW9LPUF4by+#H5)qpC_mpuaD&`- zR^2lE0TD1Kx)MWWC*EZv=cb|t9-HxO0Tr@_YYrjBgFI1;FHPaJkBZ{_!2y1g-GYA8 zxd>3MBT&q3;Rlz9gM9%)>kr!^4xsl*Ji5o1Pv!NebmDegzIt^FHoMY2-?T359|v-Q zTV%2y(cgqp=bQnM=5<T#HXo1JfIT;1EE4ZHLjruOVr1|B9x8%;Ec0t7sbOSfJ)Xt1nYa7 zoO0p}JvHE1g^Re1t3ngE5B1}`*2fc7z(1aEH=oqD-Q7Bk_wV zL*&yK5-a2SsHz7SOG^1xSzKO|_#%*FXw|bEtto!xDUO-+;#2wCDz%@@{lTCNp^k`E zAeD=@lIE7Ix7eMRT84y}VKqsuN(OGVY~`0w{bJQ{?T@%dWPYGfPl)lFF^!35r2JLK zm(?ySGu4kwXN6hABKBWd+NB#nOLCU4# ztBA>>0By+o2-2-(_w-uYcXH|>*hnn)!1a-}xD`&Fb@2Tln@s+=nVy}7yZuc&qI|bz z*q+WXv6Rx@X-FEB7`P+-axwgce3F`2+P!()M&zOJQ`zy4IV<&%C7Vq0V=T=e%9!LS z=>>HeZeWctU&~cfm^gAfPr8vIDf#e$oVkOM3t^Qi!&3&SZmnbscUtTB=*N#iSHa1y z)+D4R%r=ke=5BD2GJ<)YRA&n9xYdR&(_H*wAjEEdM`ey?h9rFUY1t> z2818^`H3l$YDQjdWsIFA=UAs)(3{<|C+AIh8da8iA+qsWQ>M9xT$MM5+}EZ}9)5mg zqpxgj5$ti1S#2>B8S^cca&{mH!V*>%AUNw%{YFV7_?F0RYIT`07UB>i8j>rC#@6bcr7_-a-s)i;kqb+=d#O^y3CL_ zF+O>nbKVF~$eBEaFK8ygj$(_GujHJALiqW6wF$^fmO}yk`ZT2N0y5$ZIl@Gs_4W1o z)AARyx2*Y@R5~cwpNmxY$fOUaeI**Aw-a!#tWKF3*n)_A8CbG2a`QlzhHA-M`9|D> zY{qr`#==>Frw*6)9@u_SY{2J3E0{`}0zhf>M5h-X0rX!tzfMcHwb?Fruy2n!P?C zM|U#hyAgl*Zj(XQRkb*_i`$dqLyA+blKU+~MJIOMG>7FqLFkvbD?uh0p*Q?5qJz~&S83&T*v ze4eB2d}=>XaAH<$j;P^mS?fURbBcCkH6wfQ0eJMXk+kcKa zqCH~#N#_{BTZY=t?I7fZu9PGy(`YhU@py-swktpPfg(=`0m{c}mVHmpQoYg|0-E`m zrQ?bV>NZcuQYxKM_vLK4NniQM&+8Ijl3QlyKwt`!WRR0IHU#~C4~-!ir}>WaT&>8& zRB=*lb)lMci@M6s`$=bCTcr&&;zy-TB?#1PpK<9M-ovd`5a;~9;Zmz9IWF>RQe%Og zR#Gspr8|nmwmDcC7MLm&y`z#{RNcH7sWOu5Q!228V(s8A$NTTLwcC=5T`Dfo+zFy;k50G(bP&T z&d1NNAoUVRL$RQ;!k-zYU9w|j$PZ_)Zf~&#YF4F>JzjhoVR$E`awPOJLHG1*TSvon zZTPWRT!;@$34w}hy8b65u9>BF#$0N#Hz~)i_Kr8QN@4=6t~EZ*5QVz>ykL|vwG1`p zg9HP(rG5j3a*+4Rd%QUC2)Qgf@hls3+=v}y?c=J5 zTb7kuG?mMmmo(PJf$Xt6JbWS#`PcN}G`1X8x+;s5M%!GjUwU_F1IhDgj zINIXPynhZQM(TSc%SReXU+a8&iLY8#S^Q}=n2?B)S6QA@qo+M><@A91LaSt`LhHxe ztAsm;VzQJ)@zEESt&Ft@^!wB$^vE?Xu+X8pQk1)u%(5(!U#PUOb}@$|OeMbjnrgw| z?Y)ya3ekXUS^Sbrt`DwujEiXeeNfji^pS^RcPRAs85@DHx?Vy&8TbQV#Wl2c94Z9s zW)QFXERFtdkaEX+efSfi^{ugAi@?L*Xgi>%=b~TZe`3aszt~rVOUJ_xWTCoxQGM{Y z$3XP8&wwP4Z>^>KCl3<<(u)T~dE-L-8leh6&h}_^b6Iii9)jLJwf3UkKA~JVLt8fB z0RTYj`i*mNSqlLD5uq3|4Q78lK>}o(4!I1axRwT|1K4^|jgRXQ%87a*K!{TFK-ay| zk?5W2BA^n0(szu2#*i*wNpk%L#MA#xP!Qz?r$2SNor^#6&=Mm8S)6+#$E)^sNpNJ) zI{@Ver#JBXLw}Ep_MKP&=iq{<>lyvdp4znpG=#$E;vAk0ucz^V9!>zzMBQCmHgf^+ z8OIy|sK632G8#9r+PY}u!r2WV*>MSKJrHcJQm41P_<@|y`6z#NN%9jud)`L*1DVr& zr~4jh{NdpUiD`SG7ft57?+Iu3!uCS+B_$Kf_Q=@Y|;_#wsUa~1K`UI zFRyjFI#-)tV`QHrWF^Ho6JO2iHeF?00)5-y*&)wZnS*)XV65deOWOhfvy}2TrcdL+J~>s01#84-%%{JzL!X`oFN+O~n5i_0Ku6_t0Auc)KHqj0@5kA7%W)z)Y z;(&R!wx4%y9aUa{U(P*qL?6GaDqUUbUOzC^xB^}I4TB|j<6PN1^O(5e<|S?Otd9!x zmd6rJCs`_YuqLHC+_d(H<&-BHJg_!PT=f`imq+aDbSN^FGWfp{Ex!BlCJLPkH#Nv< z=`{Iuk4GfqkSR*`J$s?CDX-V#?qT0j=MHSPvT{CkA7XFFj(9njyPsOEvVQq-Ml3ST z!MQcnT~}iy_tK5Q+PG11v@9?kDN&QILrhV%3x0w;VvOpbxPs3U9$;5Bs#A0SUX5&U zO~s3e&%csnEO~oxuNOfUL;OXIlkNu$?cgo{22`nC)-I!nY=1gazI90ASpg?!;W@bXu>!OKdOfGNLogwe#hEDoN#~@qqU0hGaTS1Zf4LDeQfJLceq(L3Qq89$u0Y*ALIJP&sX?$03#vP ziiZJ!(NFlpNeF%L8WVmyei-$}Y#tGu(sBIhQe2PIl)6c-W)xsI#0NFGxxTkeEzS-m zwr6Q>YC{XH%8hjd3Ygh^+xSC+M%Nty>*b{zIlQjV%D^O@Sr3{1o`GpN(PYAoCI>`~ zVL2kUw%E;{&MYkrI$An6UzE=J3@Poynfh{M-f|c=3UwIOk%GA>|3|I)q{P5TkwD+C zL?6trf9{`S`ue?1@#C>LgoekZiBkYb0a+zH2>-}G^}7GlR%TO0)^sr|yMwIY0q6|{ z(8ONqgMgrj;?Bf1M;2>Vl%4Q-zfm=2Fr`gpLII7h)5toB1&FALZt5pIuUELzbMINf|D(9mw%-Fbs9UnH8_9Ip9 zv<|h>)}OZ2HP%ZaXFIS3HAWtl3+19`s=BkQFg@t_1sJqhC0_m9x=W3cyyWU`{Ia@> z*Ob}pw3Kdlv6XM+kQq7Aw;go|j-`kYNkMrOZPeckTjhqrF zvt5Rz(NQ370?~9;Ks+xDf+u00SwBeEBKD@-W#R081Q>tB{lsD`oyeMoaj;}rU06Er_!i^U(zbUdGX8N>$4SPzC; zh~}zKV$E|m~%c)nUT856L(Brs%WiRRRF6$nI^kZ_*Tq&-fBU`P5*e+(XzZ$)yGirJN- zqjRBTU~EG6q*omYLEW9N*92DAh5uC%9uz~^3}2pn0cgHYP7O0oF2=7cmqn?5o9F|4 zrr7=J`SI$8#Il}NQmIIP>NU-R71(xeyI!2P7n8T5MLSy}HG>ktM_6(|#=@ z5@(6dqA1+?;HK<>aMsUmOPmFAT8xS;@!A4hbje2gE?0uaVfL$Rss}m7thY)^_cd@) zf3X2%AE2|Nco5+!nKGZh#)Q6G8^03aJR(Wa*(LPOkoG7=(@?uP>_mrVEtJ7H+!*oY zVV|X=N$GqTcb4SFqm!MbBYGqxwc}Y&h74VJIb`5wdY(xYKONQi-ySxG z8D6;Qi?}A$xgRkbs}PQoD3L>N%bFm9p&+Hz)q#YA#)rEJvKY*<1=ZAME|pu{wgzC1 zQKeBd(>0rc0n^WGhI43roYp4PwL9%$0`@b=z{d6Y=>la<+SklX&s|h5MwOOduLhJ^ zHRwY~%h(<}&$rs?iilt0&y%^k<$lB7X1AERDuvsBnz}boD&ymX1+TRm7nb2w8*^?8pP{mD`|-X3 zE3pK-e-(!TW>mxOMN|MNCfw*I#QY}Q;&M{k(fCW(qv3&yc6(r9^I9tf26vAQ{fuS} z-sx#XB{%3u^sv#pl!%nd74Hbo&_qz{^@dACit(`;bAA^@ANeUUl{0&*s_F9Wn*7>? zWQ@9V%d@sWaQu#XTB-06eE-JRY^4oqua^^CIJ(P)asoZf{;oK)Ol7qDv&m-dvrRh) zvZVq;=e6wUZxR?AUQ!kWQ6`D66y+ic4*#)dT39XB{cI}nz}2X=<%rT5&75dY%r{#r zQ;1NT3@71??U;f;d-=TQ`T4TmK4n64Kw4p7tyQO?ONjNOxSU8D%cUHmS!SLl+RKmo z5<+4nZ)h65kCzUiVrgw))~pGdAfJ{K2tHDBVQsbOS-)}AoFxPxFIRcf79TrDY~Q;i zq-jj0jMB8b#tJTpcMLthGEo!bv6+tyxW-G>nvZsAjD7a0Wyno!Ke9V_!ngT#ArHZh z^mG&IF56r~O-k#ziHzg1e;VEvgDuO|(oYIf+mUfwSf)B{iOe-HMCv1-|dk$FS2S(w!d751tpU{RtPJ6Wh z)udB4H9H%{HR)+FF}g!D?xazp?A2y=;J+vvbAXISbbyh;?|dlnygv-V4aNA${6>=7 zBw=|6@Yg;hoZ;h6u-cP8CgykV+as~aJkcNYe;$QSE#+qrd;y8JekXL>)d|&)X6Xls z0JP;Xs5T@KpiWbpi!4q7Uxr6oe*h|8-ugZDyhQGB`w=7!=-Ii0Z-xA0fNHr)W-y$# zwYug}`~UbGkl`>pC)8~G2E_^oj&t!?W@WUN=9^FpJU-~MAH)Tw)Kgv~)$3^cyAgpsqe9Hjkzd?Dw6aXhD31cfuLWmySer+@`WYNz-7zy(-aTrj1apQyHV70QX2 zy;Bu({lJ@6^@oZ5Cj1AVB1tR$seg(mh7*fm68>8v{cb{ahXo#x=EDYbs%}9*B z>kz{jZ#Pllyh|yI=-vwd-Wk}s`tFhMg66+U5&swWc-v`0%2OS$!G2y6Bd7?{h^lyj z$XAuB{@yEeqSQ=QhyvZbm*6u^k`SvJO1D zaMi@0uT`&L{<2iKbH*k7vbKyTSKEHc5^`H`C!4zXUO2j%ssi1F_AHI7R`*lb;z$3m zL%n03MAUi2wjLI*Tb=x)o(%D_K{Xr3}{GTX>akn{QtAiCJ+>{Qrd z!?vc2m`$x_1xlbDe`{Y^Tygkmyu8q|LsbAq)O{KxG~#k)&XZbIbx&Z(2fU=IEaf1O zfnk=(5rx^{a`;GybShzW)TRMSbFGt(mH?*&a@XZLrWuk$)^HtkD1MD6mUDHHOS%x* z%=sYF24-Or)pzhpy5Kvc2_r7 z(k_y|At(2q1W(PfLzY;xCco;ObPwa2&vrca;%OPFBXKd#h4|}pPUlHr1aqDegVcd{ zNj`p0eWo|i9pvPmFX{B(K>Ve8I{VtBa61@^Q6%r~fUQT27Dr4qr-Tzwoc znk(QT&%55eBg>D)s$rR&`e|)b^rl6pX+>JGRmnm85FD`rs|t0My$`*8+JwHOIAVAiYOYgfZ zm3csW+$-<*cChigCqll;^X_Rl{GCn13p^m8n>)Z#dI za2bN8y(}4y75UVA7w{;3Bi0f3)%$Pr_3GP6a@S?2h>u=3YZNe`G(-VVV5Cpn1VNrM zj7zMxF02IGEs^!8vW#^L$eN_=czR)b371W{AMdLr?ur$w+yHMpA~>;}uyU(Sn4H0l(L zDFR*PS(bijXD=9g?l9Z=@g4GPb15*f73&LuO@-2~rl=Q*=;pG;DHkH`i0*?Kd$@nNP5-wNp#gStDd%61fYNi3P>sH=^q&PB;Yl0aMB|^wP5T5NZbo$ybG6 z=3T-=N9{idij_DtsUMDT&i(ogI0m9d&A&i&mahd2&G>Su*dJRuE4j^x=WNNxK%{BT z%s9Y7dS(~rh|CSN2;;*UTin;qxTcFR6)g7Y5UV%k0|rhwvck5DM$vRGOv6BaPV-iF zy~n`$<5=H8dTAY?$Ct|v0j%?)Tm4AMo7zX$| z$i7_-GWKr5-w#nQ^UkOuWa-kpV*WS0);h2G(nIN9$#Es2u^4{R)Z|tiKTih^GxlOd z=@!x_9b7~^WCkM;MsrS&dpE&2y96`PMt)V&gPo-yb)Nh5qaIgrqjSU8wGTQhmvlvC zGIq{&gyHt>b`17fllWdj+pv~J4V;R7O*F>Nj2D#^uSLom))enCMpfD1*v#uaaaoZQ zja5R+I0GZaELDtHdiPqzgoGh+jI!*&56P)}`kGoBzZ8t=ZF*>`c0DBwfrsJs`S)E z&?f$_QX2Aaz>~b3A$ye0m&9;JDQ)E#xd3P1K?wx{$Ey^ghv(7Etm+6C$08lSm}H%# z^A?K{RUI62KJv966`wwtW8%_97{9;IPkG(PDYhS*dOzBqeo4%%8)j)L4zsDUrr``n zO7YTli{dst|C-xp;~&%SfJJ-g_~Pc{Im~r8CSZ?e!_?xEL%T2`lIMqJf8aYhWp`u! zL5rd&yr#)`h;Tt3L4T3q7q!B$G!t3M5S$2h(Oln3CZ>h|RBt@iyL4^FAvTJ!r;)r1 zOTNz!)7?#-V?O`*?x1GM@Fky-*=ScoifKHYx&QPfD-y+YD%DX@$&i&Q$;}?@KiHAC z$x_|KX^!2|t$dMK<0MaAdm~B&WyBe%Vb+hION^-LQbO@1C$ee|of*O`Psx{Og5)_ostXXUf)1;fx_?PsAaHz@##! z-8woJG{I@U*7Z^kFMPB;dV!SthK6;Q;%d#oQKP1y67SCAV>rYFC`5 za#lKq;1v&-pH%fOOU}e6yb+|9yTh`3Jhob!NngHZNMtJpP4wKTzjj(&vRvL|RQ8dj zP;=Pzwu%1^l94vGAH5yP7JrLIRG{q2s3jeKY;y#?qJYwlbJ#N}BRt>bego3{#C7}j z@Lw(Rj#QneAhv;YuHW3bT1{sU_TIbD)w~BK;$^E}ZXXFPJJ&r2w!3t7NyKzJA~S%= z+Q`|UEXRK7%vH-ZBShJ|IL&RP#`i6Zt&D-LCb$CXk?=jR{7$%|OZcAts-ksBq2#%x z9Mr4SKt}Gia;N52D(J4fUQ42V~S=E7DmapA67O=4YT)2~|(UadwGzE4ja^A_s% zJ@+mp^0inRm?CfH6O*gIjD{+`S`b(gR1)y@v6t_f{7<3CA@ds~mb(t2G#N_fvqnQS zQ3Su~i;@)Y6q$~%DV;NIYK*C70`;%I=44lz6*sxEn}Bf7q*7Cmkc)-uC+EUYlewF_ z4g*iyr>cq4n+5LnTK4huR~mS3xp5ELbMAv$Rs6Pqi408H8umVf6wYt-uo$;J(LTCH zY^D7ND&$ZyL74CTq$VtGnvi*R_^QL`W>JK)lJ~l-Ey3%q`G@%Um2`E`&99%jx^=Cr zHBz&6ksG;3hxoa&R2j#Fag6 zYY|dGct9x=gA^iNBfe&8_J-CNhNPjhp2ris5PbSCQ2Hnh{qL_w!!t~V6RBeu;BTZ; zD|IdzdDZ(j)?t};b@bV3{R7_r-@%QY;0)Iiq`%?Fpj_9Z5d1*>^550WO1FV{7Z}Hn z1ATvo=}3=6sd`%AwY`5M2%Ta)E$7fzLo>%cD0#Wa{C|jaFY*l+j^3Za%_rbNPw(*@ zbJ2#zu;h-rzsw)Ha!YW-k+%MHK!+s--bmI2Jao(cctt{Oa^$2a8G!ni?N8D@6uhv1 z_x2S)-|#nJa|Vg+kD~%6E#*^B-@S(=;w@FZCrvDp5?gzm%tiXc^P{8SaKQ;VMi_L^ zxh{%#4FGk5-{en0~J-!)++4{I>k$@xpl!;rJ~w*g`U zhQ82<)wV@=oTfSanPql|2Yo}9nPhX+aG^?tlK4mlUDOvsOu6!qPG%4qNXV-$(` zJx~In-zBcNFefv!Zd~}3#z^1(1GaPZcN7bhte#O_BiVxV$CiR9o2n+*thW1qKbhNv zq5HT0d?Vf{y>!7jbbJ!^2uTbPq#C?W7^<(^ObWqjLL9*J*pFS=vvHtEh_3b~oCCF; zmU+?@pPrlliRT)B14z$%HSynXJmuvFQ{ow;PM0ZOMGC%JIdJtqy>a5g$oOZTLCdtT z=+Vmj@q8y&K#~L11M&C(#g;2cWm8yO8)7nq>xHyuXlXAd9MqfBvacw{EL8y17kX+w5Ubp&*g$KkB(`dE28=aN*Y~@1?K?N_Yscq#sCl@Ms00i>%;Xu&IuBo*pceh z3LNml<%+Zy6(3IspdZw_8SjsfEBxJo01}4V$KL>jJn#wDpB;$-`+k0Y_HaT?5vS3p zjcOEJD(aiE@(wSf1lgnfGFEsMAi%7I5qNA4TR7P~@0Ft=a6RI=x;MRidx5e@*z{JB zr;%HiL1k1sp9ue{>f7dNsfx|KZu(iZP?bocG_{eylJva}bR_8EQQg$em$k?%~c#~k^DS;AVGZc@hCSo2K*FB4<0whpug zc%);EtaK)kjzQ>WuLG7A9_{Vqi0cm4YE&xU6mHrX@4|?A^qb7xr*KHTKFxJg8I+>?YGAjN_?F}U-k2|-yX1x3)Oh;jG3-*$n6JwFY63;( z=g9O-d|PK3qZ+H#k+u>j)u&xk=h*+6K1UPi_)>VAe!F#Wsnu`aG1m8WY;(el!oi_; zf&OEbFTrx9asvA+f9=bx45W+^Jpa)luO3k-P%#AKs_5R$R?vd5PhcEyp? z(q1ftauX0E|LcePJ;N7DNt4+eZJPF|Wg7uYKLlo$8kd#}7Shitt881{KaWn^*RNP< zQJ1$fXlG|_x*}&a^Zl1ThdFqGzw23+4Pm9zt0H4;@N+i5IE%|I8W5;-61t5`n8@ur zKT9{Nu)*+~Quc~gA%6i8{>)o6yNJU>4(ZrWm@CrT1L@l_yzuNz+=V7D$1j#m4a6DV z{!$njKBBhAVORJKh>YZ=4)`*0{%cwck9=XHX>S@fl^e(+U;xaGI_9ir4C5NsjWlvr zD;Za^wWhV4$+6L}`F?dKz0R3h3=Z_pW$eF{eofxWwiDryMHIy;cKp~-iOTs#%E7`a zu#2C?UFvz8!l(I=;&5v!GoqRN<+B_wQu~|Svgq>iyTrP6C*e;c8cb~?iRV~a&K!V} zO9NE~mN-DDSIhJ;h)`?eKbO>URaWnAg)9Cux6`>YRZdl18Xu%!x$?FtDNx#a8X(E> zJszQtf@EJn$TPjiR-q;7Ez`y&nr9wO#&}x6HZ8X9X<(5UN`vRKzleUhEA)fR=PkCK zDzy+GjuF$+Z6(z3>iYK zl&6$)r0WSv)S2EtnBJEB1wD;@8f8U@NLx-Jc6h9eLqYub zVdzutAF4;_1?1+|=|S^%2iW0kdiPZ#7OBxK&SEa9PboBfbi|#}C_A6`Ue(jN3dv|@ z8BLpNB8gHaR^sxT3?LU+Bqj>d6Wy_@2A!{BcScsTWyqlX(1cSyDC7ic>@hsRjZ!~N zimbk3-*H`Nd@U zttDSqLzWR}M!kl?g;=(%oM49oLi;{~YPqZljurHR-U2y%lq75%;%WMBb24BcdsJ+b z*LH%$F^_Vn(YLjhN^DeHOQJM$mnZokm_#(_YWgRT5h^8bk9|tR22I4*#w{*eVn|)v znNCg0rIEQK8>GwiNSGHrp;qeHQEwD*tg#tGWRfjs!d)6cRqb-4vwPt#)inj;yV6*t zOr%G4K`)eXnpp|i;&5m=>m=xEn=DSI2P# z>p2ChW?6jGM&kZ?BXH}aSCVwG)DYw)w@xEqJSsUuTRh(Vd~U+FuBoU<7ZdSwnqLR~)pW_44}tH>(8fAbju4?tSzOVCQOG z6Dtg_i9UFwih{aUh0N{UWo+1w-iv^o6sY^7F#~=4QY} z>zt3s0}j~Lq8Z2w%Pz_?S;K@VzIut^4)zL%M~&?fjqMu6)9_H z%QG)K>fQjM!;z^m0Hw}(p6}dsoE_OKD-6cjHqpwmvtMn=H|w-*q9InImqU^H*S1%} zUM;N$j|U6)(jMZN3*~7KU!`}03`Vt7K18`yOuP3OeP<(}vUtM@46%`Si%dM+9Q8)9 zALjD6v^G9>X}U}d9Oo78qli+OUV!{W-5PAEB19AFt3G>`q@{We@QsSg{PZF_PeVVX zpDy3yoQ;q5^kWIAgw5vmgr|jBN_X@n-VGRs^UC>^s%NO=9uub(~nr-L;*W0Gs1Jj6##Kswc;*!D{ z>!!K+`zVid3vymja3_Tn@Y`<@d9Mr^QSV0-qK*>D+%P$l>tJy_BH9^6%^LxjIEwZe zM3~Bs3=GzULxMHGM!|Y!pSC^IY?kJ#8qY5e7h2Y@vei2zT*Ui3i z-3?~eMa@s`Fr`QLNff^i-Wx7OrrDA;7;?khyx}zYcjogo&WQ6N3~p<#-sG=!z-ho< zY^&1NT&eBDlJ=?EMXSJ~fY2kV;nS0wn$jy6$uKGO!03&Ner@uFD6U9ETn1NrOFTF4 z^|Ds$tyX%&zC!2Q3qY~qmxc4z{T$1|PiZ=_MFbQ$R{|t}<&CEx%gbxbKL+>22|fEg zHOTxaTDglS;jNC(QDy#5HSkX%@Q(-m|L+CcWB)gzl$ia{qx%GRe~0v^s%U=q82E=J z^*`4lf4TmnKKwKCRtSKIm;MBMB!@#Iuz>LH-+)Tg)Z$FbFO-wB9NfWEw@;lpsE&Ue z`IwOiM&w%BR#Mt}m==B*u0mCz?~n%HFl4*nO)OkqZaSYJk+fbv`kGC40{~{`h!7{Q z_tyKeXMfZQp!nvv3cExl7(Y4qM zz&=3q9H#jO>HjToh9D~)1bw|r`fHaMBBjKfkXO1t`ZtmXl;tCep@FSul;{6O5nh9L zc*jwI_dz`o^fOPB_yqaKgF~0sB=_en49I+yh(~)6+yvk-?>k=I8Lz>PuQmaM=tH}~ z5m)da%ncYk>6{FpZ*1&mc-1PV?| za(>4J9l%-qIiyVbpN@D7QU^wt%6v)TGivQ`KH7BxQHhq(hb z?tQP%&=&#ucrpI*aL((e4A}7?K-gl)+xXk5WrJxFlZ@uF9ZLC-DG0Qz<^MsqnU2=* zXRh0!4y($!E#a$Jw+{S>cAzZh+8Vh?M^{&iJ-UiX>X}D7tk{;YTQ-O^FT1B1UjC`- zQlS)zS**;Z=_j5>TTwFOsQ1C~vh3aJ6W12Dw>YjitKla$XAdi8kT#^0(zk%sVR(tA zt`$4rVX*Z;H_A?%N<1AeSTN1AXlm$M&$hH?qT%AwNxfo~;P&%X)`TfxZpdiyD-jYf zYxmW{c+&^fW&%s+Pp@wJr&TKj928qqTF`Y*)7KUw3`N0SG;uq11yq|~gQ~Y99ep!z z)@Ocb%@&9HhfA!_HXHs-PLIaNd$r&F6HYPCoKgt(isf6z&DMk2 zq~O)2DmkGq`k1?!F98*)4+!#~278uPlL950A&1-phSqBmIU{4`g=S~nF(VYLeYIGqwN%Eb)fg)-3dJx{D|)v>7@ z4#oKWxKc3<_lwz1LL-8o{Yh%dm*_kF8RVHfROa)=d^1%dVb0{#Qz~bn*(5`^l|cNx zt{aCPiKXOSWq)o6%?icN%0&g|-gIsZ(5^A^R5Rp+E+X6aOL0=ABR1R}t&hS5D+-gO zZh8Ak;9v}t#>TrE*lqmz24`LMMWt0cd!hf1^{Q&v22p4MlmU%V$>XbGmDCb$J}tQS z$tu*tf6)}k&nOpB^vmaCmwZ)1d3wM55J8$K>ovjFGHe>Ic$<71_^ z7)ie|OinWs6#(jJdRumA&UK@`FMO@nB-nJV?pG;o%RJL)ZNX&vf(&d(H1d3hNq0<# zST$wTgV<7{f1MG5)&I6E8C_6l+&m zejowYvYX*Q<6-Pj1HXE?*eqqoL{ppGmB`{os<<~ndnVo!-QSUy)A6fyjr@pGcMH>p z455jySIIn9kqlIbQjZgGsh%h^rblv|HkG1Jr!x}z7}4}NyChKCQi8)G0nQs7q&9GR z*BA>gJ5|DQyUf&lf7{hwU2nrI4=3fX6PCh?qpnO@AR{#AU`%UPR(7c+HWG^3!ajzi zhjQ9vT&`3TGIN{bH-4WXa&9*sE8>`sV7{R={XI_ei;C~G*6*+qS$`2D&W4tlVLicJLb{wdSH`soyDq(CDYq3x zzGSC|^dk7RUi;A4(wnhT@^#|YWY~YUueGZ=mq1oR=`(EFLWr)ZE`v1vSlW)5;ERIY z)k-G5)#sOTws$qDf%daKS?h5hIwh^S_=+m^aw10AusJ+QO;~qlFDlql;c`&Nu#RdN zMrsf|)8wjb39ELUNHAmn%>5jWGoq8p;-vhxD%1)>vr9~8WUpM)6gS%hfBTIzVVC*Z z_3Eg@v^G2GHy|p>H#!Jj*39>9C9XIswXC$ZVXen&8f@a6OJscV={!Ci9#7I3jDx1y z+=ijYt|8vMLMMm{!4$=jf3zn~^ZrI~bwI=Ty7F|1Yw@IWh#=IOo({1j7pO{e)fc;Kj(?Y?y>zeHUBkZlC+G_i)-9QU%p+NCcoB|~{6xS9f6btU| z1efBp#l5&wB!u9_ofdcZAjOM&aC*|`dC&g7z4tid{F%W`)?y`Nv3~cQ^P2Uh^So^h z6zm8mvBqhuZK}h;+!`G$)%W#}ApI+L2aZZl&(`fN!5Xm|&9yiN?(xNOOlM+dzgSK* z%Vwa=KWS{4>4PF4$Cv#e*LDI_BJ4lDKD;hvhDa6RG*TFi9yq4GOuKuA*bJ-o=H{Z3WWnYy3_0#m(GO93cYKG3e)Cl+}}J z@k>PZL&;+XAg1+{Dv)P@$U6`aMfxN?BPrE^S6hBiy|uOtKApC&rwn;Z4SzTaod3`; zmy^B2+N~8yMKl*xRhdxGk_g;%A z_<+82m|&)T6H<+Py+6YjV{D6PfR;(FLMqj3ski31<+UFN@-o@*V_$t@{56q~L$LKI z=N}|2o6d_gd-!Z>?5jIbUODGI{h~PB#608`m=bhZrQ+G;YCnHNp+F`X7T=^&BB1Bq zY!X;+E;jJ+bf_;eo^73y$$WVQIi};?IjW$zh!<93fyG7yx=-|Qea;aj6(5sWQLQ!E zfNHIC#nKZSp5LsSNp;v3E2Mi30;iiK_2#R&oUw_(%&UKKbofg<2-|pt608@i31Jt! zp(tS@fR)zd&E+5JTeq^@O)=chwyjfZ-tx-Mi*=taC+TtcDNV%G=#_IQp}@r^4=htg z?*8es#Zx>6nAmIiv>JS*$?)%H>J(AqQb4?F$_4Bf6RS+9-FXvU#w=eri-cxvL(8&d zg@RyjC^>s{X9l0hD#Cx!e%ts=t%8yvuK_Ve9i6V37aR)Y3Acf%#-BL9R|rQhq0%RK z6wVCR#=>bKb^95|hFtya%9?wlN+pVWCRJ^7TyO)DW>mQ<@~@Jv~{q{dt7-PqGdpIk(Od$*je z^CO^@jSGTd`S@rwh7X^w0EvN0+na${=ZwgeNaVvDRR5S=%$ONft8EhNZB)Ifw;db^ z9OGDCDLFg8bEcokg3NpbSW`A$>-+8ptWOf4z2c|r_3zE8fCY>K05P}MpDJ%4JqV9S z0K&qJQ^A!Hmx!@~T*cp~=+-ZnvjYh^L5{@+Rr&+h5e{d(NT9#c~JKk7k0J zQUo1Elkp`0Z{9U5KD7Fn3Qn)1WTE%6NxdRRr|X|kSmR)QpO%)kKr|}P+ZfMtv22fv z?sr5x*EiQUS>=H1?A}2Du?=b69{^X~{VnACV{EU;gFwtLh{?wDMiOdHk;c;>fLO`; zZ?!SEMv;W3A0HhAKAXIMhCcfkt+u=I25qzL-*NB-_(>Y$9ondP?n)i)KNlZ3;W>A3 zdUtefLzLqnZ5y2pgkI$e&XMAV&XIw{sF(i;K4D{Q($t{u5CK z;BWb_OyS3}{FPP#`F$u=O8g|)^+&^XU^M1({hhepLw(j?%V(hKKf2hn31t#-RaEA3 zNc#FN^x`4;2YP3p*U8H0=)Md9IN3|??cn$HoCNjxETWga9y~^0p+lj!DgJ3?577UY zS@rnQ2r*$O;Tf7z(6#@Yrz|%sck}=-o+IsE-Rr+#s>eGdwJi@Rr)>;cP(3YjVcM?8 zj{%Nt%N{7d?6b;6ZO0J+4a4htR2Wnv5c5AV&KiOJnQPbH6#yVkeB+`1P(T3i0;d8o7T|#TqaM$B z8KXaKzq*Ts6#^Jn;;8NZ0DJ+`Ah;zqwaeIJbaM||g-h~5dc!li>2XKi{VI26genjbw*Au_3ld_p&MZk2ER30Tj z+dqK11>o15%z>GYPcQ4t3U32MFV{)W^?2$7EbYR#OfT!DGv#8W>AJS*D)S3A&sLkv zTbU9iMQ)$)>zg9YcTctNzPB2gKiQI8&=VWj#ZwpRoCb0L&(h93jEmlZr%O5un?;du zu5lMR2R5#TuVWc=w23g+hT$4WhJxnGKqhjjiBHGrAdTZ)VPyXTW{@PXVcelf>jcH3 z@98aLiFy6Q(S~AZ-Sj(GFHXjX6e7^iN;FTv+;JSlWD($8+RCFQ6ZO^VsRDM4$1XAV z_th+FY1GAuY%Z+OQmrytEyE|dr0)t+$phvYOm|Ne+pyc97J}(F7W5q z%ne#QS4l>Pvol)7_+q@8$6G0UxMs0K4H+NTR3-+C+CRYMd=DYGAZT; zRJH-@tjHV|{jMl8pOicH8z;v{dTc)s&mh>tiAs0#4+<`fcs5Pc3oS~g)2!WgJK(!z zvU!?TYI)OI^2u)|pZA#bnGh;++Y<6+H=^0Za?xI`vY>q|K?;-XnARt=S74c(E-2Kv zxxS6EkB_1xG5@~N%IfpWB{9KorsL_I@XldoUUu*w{&D5yHL}Mcr=2uMi z$Nm8H{1R_Y#sxWARf=W4ZE3lzS=9YjPjNM1EV$s8NRI61xjVvdJk7RICh9M)TaRR> zyJD8knXg$Qm8ZCwEEf+3?(a7D8uh)B*HPWSWUbPGxE`R627@qDpXI{`cA6$)npF3^ zX*bQJXtz?D-fvh_(b>nd5;T+V^3h7Y;CrGm8B;k@tV?xUasCNOme||t_zdJ)RM)xm zT$r`$Rrjq-a$pU8pYe-=d2ZfXQ!#uiqF@>$)Aty3v!r~Cn|V4j<^K9h!LvVf={ga= zP_@$$1`7Vn%H4Kwn4VuKyPEmZ_x1dD%!`#}q@ z=NtFVcL>=~Di(tL3K-cyS;Vpmwvsl~7cY`o598J6B^`+8g+6zN)3UtmYV!%*QY005 zC{=Qqp5uE%So>6T@iq4^AE%dV%}FY@(|HT*iPObc)X%P7`du8T;BaD<)sr_+#T-JAKIUoQq<^K4xb& z@*u3rK4GQZ%KmPhHliTY)uuNa5y%@u%|M=uewrQ8gUYtG+vrQWb%)M2E=NeUMRkJk5&MCeX}jrEjL94k&gJp+)GR23YCj$B?goibQvfo#3OW$saMRN3H#&%*RK2c0`GfS ze-asrIkT~*M3RNVZ*UqR-N&V$EOzQPL2}GZtQ?K>*oXWhJc?5D=E%O=U+gF-4CNod zUH*8xsc%tc{Y1o;MclZFS6w5!9t%(tsT!bGk85C5(k@lS5G*)?<;qP8Z+8A}@P)>bbkBLi!%0IQX%>0dX_~6Pu zOL|#yU1t_zVW}}ertyInYV@$ZX<(qRX@lv>(nViPo;ONVV@wH6swl8wGA4#ph4Qe>a8*jY)>Qew_DJrz&`m*_e?Prr{E*gUcex3j8=_Bg2S1D)^pZ zNzKc0&HghpSC=89$6^bW@0Ymo%q{SiaBa@3P&;p>fmKVdqP3Suh2i|0{EajY@eosiQNdO> zTh_3Fkr7Ya0FWx=B~N*Xp5vV16P9mmGed0qDtZ4)Xw11!wju_VYliD!=Di8(b|$3r zH5`#zw_+%ImPhX1Z&QA|7lBr4RZ z=r@`tb>-n^ZHSEf52oT&odd$yX;T>!IJwqg1o~nfiC9IWXW76 z{`SMIkm@F#pDf$d>)AIwR6H|lO^ZTZ1ZzaQff4%n_h$->7Sg6!IU#niVwMKT88@KCC1mjao;-dZ55=DE5tyt zF(?=|7gw$|+zK0|<)e6!l=X5kW*_JvvZCDjZ*fwk_Xv_gq{%wozOC_(2(WZ;PSD8u zYHRH-En!a26ig}qQ6OYddcO=|Sa?296R$@fjA2F;Vr#kpjjz(b)D>55jRxmHHX^_5 zviiG=*4fV5b5d8f+q}f*=GVL-&dK9h)J`gGPheKloB+=hPTj^8)0#X}{9N=iIjJe; zGn-GQ1u(?M@~+>L-<_^bG;^s%`2Ncly%ORC@l6C{Zx%*|SIf%6k_p00IIG zk;{6Y!HMN%lroS&DOt;!2jd;`)5-4M=AVo0Y!9HzB!0vXQY>Z2&z;X z?NS9?=uh0{DgTjqo*S$*KxZKN;yh9ye!<9>Z$JYwF1!sa<5X* zMWn@HBLm5iIih+5fNslvYnY~D*`C2tOJ?t!(sKhnFj0P$n0xp1EC}^=w#d828#$`2 zk0glHOnEl^oEZ`2j*6T@sqfA?_DPwimg(?o1Cq&D-QD_8 zoUCQN5v9beeK77i>2o zLX&*QdEe$l4tnBmLc3l)j^x8qC?;WkiX~Cp!lSFEL2}ix}Z6in#SEiLat_KDfxjl(V*V! z%@1vaMdCneJiAJK4QByVb77Y5B$5ZtJ2|}>J$Y2`F>Vk+ppfacjk`vzrCt@(R7=31 zORqsTrP9b+*ZFCqFR0?V;s*-`EnRe72T5xMML2Ga-cyN{D#*?o{j>IaSDfQ|hX&ai z%Ozc4hKz!dmqMD}FD_nm7**blUOJc}e#&~l*#y{c9I{ znd4C1fO1@;^cmDE!Y3DrqV`KgDIxX-$IB^w4G^tATU)=_ zjH>->c@HUNetJ3a!8V15HaO`-cgPP9>+rqS($!& zsQ11gUG1^Cd|E|Zf=YUFgowoTuLAx5Sw!GV)S4=TgLNq8_&ZmOO<0-^fi`u#igBpC%)3#R?EGcq6D44WwBVZMR^#l&KP|iIkT?56;vv&pWIk^?83>LVl zlC9@Q_hEhHo5WAF{r5J-(aa78=_!9VInOc|c^Hies_!B5%OkH)U{SC!mxviCCqK0JAWI0KK^nGinY3qLaAp9(oV?V0|dp+;&XF!;=aY>9xY zTVIDoVCq6*@4DT2jo=uXjll|bdSto+M9!i>AO+$6j6*tsMADyVF17&5f%0aZ zYy>R$r4W=I3nup&c!;)KdM@x;MlI>RfyGj)FnKzJ-C$Y`5p6K@%Ca?efk#7X->G^C zXu|MOChFn$NOQFeJwq%5$WE=~S|~9NSN=z?mz^ytw$|e%=I=v2`FcM#naS~(j^xeV zmioP6()TGNakpu>;4Q6I%8o3PW>SA<2Yz3FCJ zTRdv-Gfr1MWFs}=ah(_AoVPx*xh7v6V=eZzyg*;WKf)6ecvV^lo-AIQO)+at1T5r^ zv!(kEjt@RNnP1~DnE!5^AW0(og63o?jWUf)XHC&CE>r6H&LNw!ih!vshzw z{hR-4Oa*0Y-dz@JWe$GFlqTZQRcymk8-Gp66yL%Xt>TbVQRGMCInxvH)qO9O=NhZ( zaq`0%iT&;;cLlBtI4heGVX0azeOBo4hMX-HR!lzJu~p;+8j!z@X=yB!Va1}kOZ`7y zFTK%8Ql_Lx_Q$4#B8PsbL?JE6Q zxW8u~VD8Fn7GCO4qsCGLE)T8a!^KW(EEW{XM1i41X-aVM)&h27d@P{6#I6ZS1Abnb zuQsrt!y}4&K2I&Mj2lyzqU*hMWt5%-YjDf4@M4@~hECSNK zck3Ql(2gD-#+NEYm2K`Z!05NRWxRwP=4_ zGZ|guTr|esgdNGFAhImFuv&BLaWQH$b?Wkk=~Zqdlc1_<-jc($QTo=$U%W>G44Ey9EVf;>c+IE$R=a+4=W^t|G#A&GN=8nBpZs}&{ThcWnULXiGeH-}Uqf|Pw;eh{6 zVzG{qfc%-Q$V5RX6G1drwZ62a(vHf~E<`$oEQs5+LFrG7iQ z$Z#f_D~$&8G2%coiTGCS zFA3J}qyy(Lq(&ot%pOCSnaz<)0t8Y$Di53^)+u=? z{(b2XieEfYst#dhEB9C16H>>eZVgzgr26UKH^3Tx;&L%oPcULv^idw9nO8$6yq)zDUTvP4UGX%5oBzis?4l+9$9nXqXMuolYGx!on@YD+<9k*K6175~1 zq#Qhfrb~w+-}sm@<|O*Tb{8kXU#02CGkHNLG2@rdFdy|LGc3Fh zJmjDg#{&P2bMF#&Y9h=%}tM-G$@M|gu$x%2@jkWCwU z&R>2tuB@Zc*~QrpdGTkCM`HHWvZ}m2NOfd(_Vk@v9L47(`W@OvF6L;FI7bM=kLHul zoHq8Cs3nrb-$!TFp9c1jyHjQo1$d$d{PSO+p~%B{k2z&^a}M!As>YF@8%8y_62`aB z&m)>Tt=WYQO5q&@CBf=+%`Woo-NmU3>`FhJ_!#f=OeC9noGLF6Yho$=p!fNXw##GI z(9s*rNN?_xRSQUaMA%)@t(~fRH$iD-*5?c@3e3%#+yJ%0J95IEX#EF^ln3}_>dbwk zAYA7yC;oc zG4WD2THCLC$k~cN3aYAzs@y(43y)Dy;6ahnAa|KzwOt-w=JGDOWTC&RZbR3z=#v;( zY^&sLNQCrhP<|-q`_PcWF$w`b5Qq5R6i(BAagT}W->I-ap3 z=R^7)p_x!u@*b|OeRq5k9f-olq+S7!K1igKYtGxQv(t+W$OJx#{%D=-Co;=5P2o$+ z)vD^iTxw@I<8U@nE@HL5$sv1R!R_j&xx**1xA`n_xNI}EZ1e7%t}Z7gmf1ATHV<7P zP&RKX#ZZB*pv)4}hI=ECSJ}%tKVI3$YC>84b-z#O+C-Lx+4LQxelrw|td)YXREdQQ zrilArOS{`@T^Ze#Os^z)Q@N@opW!kXvRb}S{?UlP0w(^x-r>Kgr}Lgkx~>W=-!28$ zP&E`mFs+8Ho;D@ePz&{0+%>D}l{|0g0?v{!J`O6>{H1+GAVR?R`j>;VwlI|RWj~j7 zQD=QXBA;i(&;pw%#wexi=*zXY5Zuh$@hLh%+T=wNeXJ;pzK~xOPq6#(WHq-RU|k8{ z=51W%svL-tw8vmIck2Q8zCFeMcuqbI`#GE>xVFC~-9594HalO1tNIGme}hjC<_$SZ zk&;aA^X=|z+A=eP(&9Q2RSW4gQYZzq1}GyPGOG1E?YP?dU8{bky9ekh6!sxTGcL~1 z!fbWKi+SjQ;f=O%F%~wUQdz?;ymC@bNM2DX6E0`2ZGh>LxF&r_c=tua_nV&9PE!ES zo*=TLBM5h2rt3&S!%8)BN*tApyDm5N)aJr4oruUtQ1A;)@H4?kO3Zp722NL2$y_!O zxBgs&gc?xIM?73+n3&At;h0uiU3wPWZmDIup}74m?uZzHB&5}iW+F^|cc&lwjqd02 zPc9CixL{zk)S-ieysA)mw^X%v3HWRUF(31Y6bX4X4)r6Y;8if>{}}l>L(zc?E!`(D z5+U(QTmG71$rz94wdQhNmxfF$A6uz@>uTQe-tiN#ZVmEu42ozQQkjVP@ z${uPL_nYqueIi;;X=}9(;j$4x>caJLJz+;8jmYm>5?#vc)vQ`(&{=n z(FN+kL21~pBqLrCH4+wQA-EwHnd{YolR(*^3G4_m+trbhZLU4XYR311x}Q|`#LNTF z&KP732x(LiCiBEtk=2G$W{eb;X?}kIRMlLH6Tiv|TcqCKuXEOI3b`u5GB@3sCejPY z%+Ep>E4}w*M>p*sSSs6MkO~P7*W?L_u$l<3l(umC{wBIzHwvFchj* zAgVI$J=B&eJv~V35oBY!x&VobinsY(&TgYRam&=gWfSJ{4@dNK&0E^#F0C0eL&?I+ zp4a8zfXr#ihYI-OtM7ON6KxqFaG#tr^;yhoETswvk%kWSWQVKp_u$NG z#?2kY+v+v)QWVrZ5y~L15H=CFOYH34fc*|=raM4xqr?QoBPp1!6K3D)UoTSFpIt#d zTip`rLXILScY2LZYc7lIC4!AS_}kQqxgNWRKI#L{d9l|>f-DrTB9uctR?LAA@&chu zJI&|Q$2y*rDXz5MYcQBcaJIz?>RI;4nDe6Ny zES%Bfi~>yVvnJvqITRu1QLA*bVh#S}>;NsL*5S-az5o-`6&)=e;>XU{$5ME>;akx4Hv zu-VY6r5hyyq`m*U`!Hu zO&TWxTbVTyiPNY}g4%teXV&p3WD(>*6_YaDdYQa#XKZG?ybLVvLpG?4 z#Ck7)KIRZR>4hU?k0FCEplV3ZF_y{<+r7>@FJb|yT%XB{yQ#?h3;lEalM`NIdK+1w zR}$aAyVi0z2mzs=BV$JSg$`tXD)U!_7VRyge7)ak>t^1A)20Rph~BSUv9;tE#Sm$t z(SLR)S1Zc9deiH*XtCX*PBJwB0ZB~lDg89jgLS)&_)RQ#eQOD|8A;rKDjQLF=s=FX zO*D~Q+q^Gued-#RSK{pSxowf<@#Pdkb(3=ubj0M%kg<7!zt$ue}J4g zyU_WqMnbBd#vyrX+8y$}ccGPMVAt#b=ohm-ZC$TCtna9OQNe^)nPp&bAGr~5BN7{s zk|whURU0)r*Mr~X!vnDAp*G4!NmTKJJDY?Y6U&xqnZaUtogi8wW=b);L)OBdhRln_ zgYkQ|SMdmkK`NU0SH#hgu>11Yad4=GVrf}D;a5qIvZ|NNxhkWaP_a)_R7IM`ziEGa zeW8&s&HN=y)TyDs$|MwwUJyzF?|{%01fhH6PRL*yEcLE(F#vP8531;vl#B;2EVEz( z;R&rbZ2%E^C*h|#C;Cs$U$j~z>vP}`L3|Cv!JGAwBVdoyYqAj5cdHtjvHjAS4$IZWmw_(Zwo?0~ zUmHvkk$5xAu8s(TvQE6kj%DMci|gXPYE3eMzxhE$Y)z@=~qu3+R+VDQ5}+0Z;IcOOokQl*C;7$gWqr4)V-( z^SMQM>OFeDrY+LvMySfm>k=<^3VU#}{zM&_c)nAyX=vblObv})0PSWE>zL$Wrt(a4 z2C^9&sCvo_&{%^H(F4K3F+XY2y`;uL)_cVwPCY^o)%Rouwj15tr$%AkY0FNZWcHsz zTXY}#uc5s5U*56_9X5Jt7Vg+!NR?b_m?(WPE+EgK`<5Q(8aL-2Epv$dYnso#_A2TR zpoX84D&71cwh?LoX$HBAC||0%({Vf^2gisNUgh!d_%gJ&bZbBRU*Zn7)BhVv`@hB- zF{AK@W|T#(fdJ)~Gnz~YEU*r~6EQ=_4=nzhS^IBF)c<+K%+nkYhzSh*|6K6DWvqXG z84JR`@{1bf5j?xbezbI^tqdnL{pHRXL9IhY|Wfb}ylLX2Q#nQqw! z#dfTxwtpP-&~D@l;KT>pL1;mxPmkJ`S{g+HIZ$TKf2fQ4*CQ`e_Ja?Y19=5p%Jj1$ zV#21}<&?A6uOdiS93U>4_Z0DChEZX;=+)*IdR?wU8g;P;j`MljXNjJ#G$)P@GJ7+; zLYUvi?yuzmzpn`~Tu4G`G4wH;;0 z%v|xj86=F05c^_{?3abtDNH1ZGhGA?kUReJd~qz+ke58`Z_5bx&{|?5UWUjMD&NcK^ig|@ zpytvgu8bSUBr{om*2eLrB-MCw7Yw#U+UuBH*;UUUt`FI7X@2WMm9#=(V1zZ4M5B1Y ziWt3$VZqW>j63;za=tcn1*wDVc5`Ze+H;Q(CEig_x8RT@u2&N(tf6)vRA2IO37(1xZjIB6%-mlLOufK(tt@u zbNR&5pRed`{ammmy}!A9ie}yIw>YGJDGPG%q%cEN_L3mK?q8hI*;tmkN-c-_@2Ts} z9)7(xS~Gym&xCO0Yvd9088}N@DbApFkn$W&?07m7bA12~xpw4P2Q>`Uyz~(V2B%fd z6w|CzzP)vauLUiJI=Xlm?6TBQlsOd~-cK)eZ$H}i#DMuQbgAUg5k)xVFOCy^aeOQg z0UpARg!k^rMZJ|VZ-ydOM${>K7PsPZygo6=zQ+|o9P5)@feAh_=S`^NvBMJ+R+Q!% z%9F_0C|gd}I9`;*dZwg|83G-?#pht87~1dRwwJT^OrsvkgU(7!O=COQR%Lwoz-4T3 z%`Wv~*i)~>dkj+rrRHMWC`~KPjmk*ukO+uauu1eZpqtSON}P-fYIJonm??Sj{)say zLL8~DE~;SJe-B4GZG?JHbBALRR_<^ObJ{fNe8FUi<^+Fpr(Wag*c^|GHef~UR-!Ej+UPZ@S2dCci_fKo9>U!vt zrQwTz?1`BuiXDVBB-*M*105I!N|SAx*j^p>|Ky#SUE#lOx#@?S4$1w-z_6&`DP2305RYD^jhwD%YuRQAlzoUTfd&d>RrTM(N}`EjzA5| z+}m26CJW`7bj{%7>L@-VNWO*Hertq=z0Z|X8{ZOMlVjauc`h?FA+_Sz^>4M9*k^B2 zl24QH2lU1_3nhpKapiqhzE#r#)tGF!)`a<9Msw7}7v=LfWs1eIcg{xbkyaDIjokL; z^Y()pQ-l+L?RD z=$8L9#k64g97Y^!CWzUnQ2bGmdn#;52~2z>#tEsLw-)?RoU4Il`&cUt)uPpNUp^^r z@WOcRh~=<;bjrFSdvt}{0H$MsDnnP6y)Jd%?>Si;(HpJ4Q5NPre^PN?!E!*`G#_?5C|RSE9-gFWf# zdCNphx@9H;w6o~{^_@k27Hj6ttfWcAo^BMUX+lSRxnx7mgNs`=qIQIgQ zfc`AkTChBKZ|_%zO<5W+eTh^(rEjQsAl0cthTS6xBe#a{u&Ow>Xg)M%#Y@l*7&6a< z^fYL2;p&@ebf5$q*u=dDC$5oyTxpa$X4R-9ohmu_Ubow)QD=7kFXoSOr#9>_;>>MuhM0%~nvNb1g7OH>!Y7Uo;yjiKNa-S^AXK6mwN<$+##+_>-HNXsCgL&m}7b zoY-*T$d#-VUK^rz$TQV^IFybYL)uJn+ys3ZsWMTpXv zj_^M3in9N>OYB0v=U}d;T*plq zp7V6hrza(tNm;bM7sCM-Z%VX-b+0`nYl4JnEo_g~OFm_Ky7RHl)5bOoIZpSOC4;9D zHkq!@A0DPMP$jW@%6h);+HC2|AER}nHkd7q3Z~NJz};M-wgk2WDmg*5RJm!Vn{+Rn z3O+w_w~lxPSp#-{Tiil0yB>-jzw830o5c#8^$0vDD|-vm&Oo>Hzb4TDH&G$9!Xm74H5qkMdEtsyu;hSzZPGU_1QkLRl5~FHRe-8`drph|0VKbX@|P@;lri8A zmLHBYZ#}%(5SbJ-#Ev1Z`oyNdM^KuWRE5}HH?9Zk)M#Wh92@|Taw0|FAsu#M4=|M~ zd#e1cM$Qsq0>s{~G&&T$nK>@y#iqTAh348+HL(UhoVEY!5wWd@4peZJA^35_uC~Fe z9jggF){jT^S!R}fnW6rJ&x>=5NEs``ftdDjT{a`-7DL-K;@7D$Z_4E&2t(L3BX+8+1<>PxGS%43?F*F?sP)PT+Wcc>n{y z#KLgIym)y`IN(6dQuR2Ij_XzBl?)qua zPaY=aJ`qM1$o`2@;T@6sR~%b|=(Ni~kP6i3X*c~i8-BGfNR2$(NRV)v zWICYUb5FHRxYBJ^08dTzCzb5QHHBUbU2IVOWoiW)&EIhig9k2sGU$Aqj22k*&p>tx zV$+wqyRPJ<`Ufrl@@bhC$sYiRc#$zMc{|~A2 zUzX&5R5!p>d%C8JDuz=gH_$@vT$j~4k->O^evpI3yj zkN@c_c!}ly7vK3D1N4`QiHXkgZ@tOOr?Z+O*nOP;Fb^@&=^R_H#U;-+s8E*7zh~(2 z(wIJpJ`$H}+3?dp7a9GT2(B`DEF5r#Adv0SyN!~je&*^G75wYr6zPJD$qx#KFUG35 z(Jho43|@@MDQzsNPGo#?{$Brb&bljo$tcds%JRjk7p`n_8Np79;YiNF1XM2e+Eyuo zJDKvefEoT)eKqzQQ6I{SjMwCi?BR@l^2xG2H9I!-i@K_b6)4p8V3%g-`o^Y3{#Pf{_(#~wuEl_D>SC& z--DrjD%dd-p9kW`p5o-cci>Tnb1!}e7Z2l$DeVnf&r{4n^SSj#RZ&MJx@g+B4<#gb zkGG4@?XhUf*;KzRfrEoVZZ2!K7-;h9K|OSmwIs5kE!#S*eBd>FA+>NOo&}(WH^fJ| z>H0?IB-0%-4wq@VO=!Ey9(~TiO^Rb1I>S1lo59_7!ARz_I8>w*7H7()d7K~76W2wv zMwEEPp$_Dn9mp>L3c=s1`WN8t9f`m99nYAw(6GN4Zo0_cx8y?=0c*S}C=FKoU2k~W zRi1xdm$xX>x|4dg>)u2Lt3=)DqkhVktL2Tr7)w(0Gk^;?rWxuPLPYYlAKJwz1DLeV zFm*D5z1=^0df$SgFY$81-R@YXJ|1V|Q-T5qKw@#Ze*j}ct?oYl{iv;27Er)Q;`4&b z+sTr0?(qQcYm?%8S0ux*0h@!S|IS|juCiPT_2u`W#2HFyE2j6CA!|)N0fWbyw^MRX z+j5~;VootBE`EBKwa-GCc|@kwQ{t_gvz{mkO@tSPar&B^DOK-6-IU1rUv6Cc?H6YE zB@gAi=-biHKn=DAB71)$CyoOC#J}5Fnlz*El2^OEt;9Uv-FoVC56Ljafk<*w%mhBU!<%^U8Ie;wePJst zkwsaD&t4~(*<}t`q&3%54SH2o^TF?3*R9Mk`U#81;qPQ&<<&IW7{W?lG%|XDSa3mh zRO>GZThF91PaIl}Ym((y!&2Rwa*5xGS?Q(cq|moz&gK^K!W@)=u$pt5iEyod;vTlpaF!{`&L%@W>)Hxlp z3H+$QMsCv8#g@Tn+OAx5GS;{#IkoEGBsM$uvap`h=3Rn<`}e!>SSk<+PU%;-Mk4LV z0!nEhfN?#FMdNn|XWO7v#S6+Hk6ps$XuJ=N+^8sR=O?P?3uQ9q!3+*>sfaN0)WaRi z*vhOe-$dQqz5PWWM6TN^F4Xwl_Vjdm`U}a+8|91;P-NZ?v&l(9@T*_Qhntn-nDQpI z8LF2ZCsp$t^VdaV^F#M~-fVQS)SY^Qx>)h)Ss;l=@R*3jK3uU=LfXPiZ!x3XX66bm zTv?7ms31YER6zPfn(?&)<CR!W6?yea^sx8V+#JqaDhvHFr-MU*5jMUsHdn`CMCm z=F=U=ndW`b#UFr0(~eT*T|beG*GTK5-4&G^-)#6y)zE&#@e-n+qep=qDi6^ypTA>cRW2qPr`kIu(!`1x;-alN`y3lYJMrIf_saTBdVoo%Z z1%pCJH!zx7RTo|bXG!Zx5hQRIBU5iK2%$jXMWhvc{rd0Y0`o1X4zi_>OD z3Ni&`(5=uWRRIjTe6LiXG0FCVT7p)ni)hL~#a`N_)U;maZ7myHYMxzHP^BqJQ~}#p z-{cBn*FJYqJWt_od==+=^aeI7=O(+^?zd)_Wh~MYqsLA17hvxPyGx_=xQkKdZDtFZ zpl*NR?~5UIj}u$6Sg7#jG+QMG>Jo}CvFe30Oi$9|3ub$`L_CRFJflxor^a(%^B!GZ zI@XccQ6{~GpEnhO*Tn|Es=&aox^c_4Ct@sfXr@osI=aXK^xbvtda+t~Fnpc0Xf+c1 zlv``pl88I_An#;x4ZOCrYas983a0vN2;L}b>LceO67fA~GOCgiGE;!% znx+%+QTm$D6~L< z0>!mZ+`SZcNbm%AcP~zn7T4hJ5Q00wiWDgh#fujx?!}>SC%yMSGi%nGZ?Tn}WSyLI z-u>HqKi6SNz7#6Fh5IcxrI&pHBp{EXRgY@Iyl#P_wAz%r$|gghhJOHhyx{>6iU-b+ zYPrxpDc^fEZUCk zhSau9!l-h?(pthaZh7Ncka)5}@V!OF#_vhQ3%F>aokFX1<8k)a!=3Jz zyKw^^Lwa|Uiy})lKh|Ja;ueM49~d=?M^$9lcB-lci&2|Xt?Ev{43=3Lj;bn_X;{`B zpbee=ETGlp2RrM@y@q~j;g${H+`tgKZ%Xs@?&GsFzFN%WAa z-&;PR%#0uT?m>&x#)(a)_CtQ2o34Cugcky2nIbgWp0Q@nK6<)?z)4D3Af7qw^9OOi zFtxOYPL9bohj)-Ef%%yiB1$n+qKhC{U3jm9-n6xI)zDtJ{>=g=x9#ji=T7_dB1a~- zuBTaI-TMc*qHN|Lovq+_4h!b?ea`QRiTRsFKKPlLDctdKz$Yk~`&YE35TAku*fhpe zd=!i- zMx=kiYpCYA)VX0quKOcvOA4#oaci>Ti z#E@R_j>o+8cByJ7MXpFvriz?IB7Sx*lN8vwg1&4lN9o}ainFJD>G5wAnpPe++#EBn_<23OP{?T)7?XNfwTRJ{YsAp#bS$r*cyc>7Zj6a zTU8&5+A(K}P>hbQ#6MT}`jq(%pDt>GJJf8-Hpuj8o~75z{xEU2vN%SD4%xf=MWcy< zz&Siwk43Xt@IroyMJh=Zhxj^0zQRFUAM9_J)NOMDDjL#xD!!5q2$-jCi%X!avC1hI zes-?&b41?tK#mjFqf{kCz_EifhOgklLM=YRqB+yFy=fH5L`?CBnbIs=`M}=JsHT}| z3w!o9ilYAOQEWVB$h1-9j-?f9{8`_RH95<~*Dkh;e*3^L553yT{vvvi$>xObN#j$S z+0M$mzKhwKNl4Y#;!3cEfSy5>5muGylCHMpFMb)HQ|1x6?PqTcF}pXS8e};u=}M>4 zpVF$)v6*eGv9*g>4b*&QSjiIY4W*+O+6%F@o1c^1hsxe5i;012^E-*74-Rcc+B5aK z&g&4@1r^kB4#`jBzugvS_q=xJ^KzArmCIlfry$?`xdr68N9(=t!Fb$*vW9MaiTZ^E zu$;Xig?BU+6lV?@6O5B$PG<`-Y*uJ>Cub`?&adE#F{{Bjwjh z>v3`c0~MAIuWPTHi{L@BkY)KWDJxL!><_&pUsqVPGkAoX{@XkXvqqVg?Rab2^d{lx zz99+=Vs5LhTuIVwom`YpIZ?=Ha$3KiV7ypuk%Y>dWWlvB^_8PBVt2b!d6*xx!u(1| zJ!(ovsZ{8YXHY_w!8*;%+r3^QOs5NJ1{didnm9FY( z)xdUAyUp>d++U4++;bn;5_6?L`nVlnl=E(|0x`jS{C64!$yLEDv!QjqRt*Vp{F>w1 zH4GE2nTtI}49XkBBoUD-l=;bx*3!{r)-l3R0dLu;YE5MSX}hK~ENnZrP$13Hu^DjY zl&hd?tN$a-HTnDa2d^?UkK&4~q_pIT{^sQ_!)xr9^TxH`g*~au4p$*ja$}H;064Q2 zsE;Um>7#7$QEgx)a5Qcp(S6gbPm2MYwx^}^9SiW5nR9w^)BJpOt%ZTKFN0`VKJ`J+ zvVB+kLrh!wrD9%ApSlqnC#P(*W5d;Z0V1q0vt5=ZeFM&sG8L=VF4om@w0h;FU!cz$ zbI^Rqn}gS$2wFW_2_^U(;EX6k~hOS(3#h}VQW_=rsqsCSb$BH}oj_!wuRzkbpawWLtOoKiD zE#I_g=ukDVPE^+iaq;|T9*_NEuAG2uG3N}8Xu{FC>|c(+W$eh;jZC5J2s!Suf>n*UJ=P*b%DVr5V$|9yl1;D0EDf8~m8+@Q2%F|09tP&2iU%)J!8||6k?gKkx@B#{bn| z{`cy6HpR0aYwN22w!@+pB-NGzSbETc)kp!KIsWo2kxS_1|NanY|Fh}nl1Q6yzA})d zx-QfKPJQP<27%)|T@-kla5DQuRbL*oS^8| zg)R?4!pk;V`N6-pmbZ!_4cb1FzSLu9RB7TYnAfK>0U?}Lo%FKKN` zI~1!15_4WwP)Y{hj=$h?V0~|9&l=l@-iH>Pv;ATjw$ffqCi((B*V_p&C`1E{q8pywo6~^V42$sWSx|4w;Q~YlJ+bkPWsT&QCg=I7Imd)EuzZi zQ>QNC<1daZe%1qdR*Q+;PmWbBH!XSAyyOmZAf?DNYs)NUWR|{}Q-tVlu4e>2(^jdx zH1u+T04u~VJ15DbKuA%&pRhF64};6tpBrDvE3_s=$0)ulVinu&VLn#-d3HvhD)YtE~n zdT|Q=jeYCRl2i%eF2Pi0Ez~RxLfIq~^?@wrK8LF&;laUCJffJ`B%-$4@L+u&mlS{E zW4=-LJ|7Z&&x2a^*{d4Y%e<>3@$c?2^x(Piua(7DxNGEE&p^lu1s6!$+~I{u`(78_ z;n>~V#m35`t`89^y8Am0ZY{a;Hv4j_nW;Yl-7~FPovkT536e7#LMs^FBQ(!Tw_~GH z8^g`>S{QezOOz66ol8|O^RzVT>BWSntDL8Eqvm$V*K)lMHc+xkayp~8ld`^aPkG!l zW|?(wmXRQ%GM}cqPe`iq_orR&^wn35qQx}BNpg>jFKxDm_jPpk+S>EU--4{4JCs^u z=Vq}sfFxoESVNMnJ?YYBErm3ftae0~(@Hww{qFhrmWZ4R_Xjz(%j#z!?#KOw>h?^U z84+#MNc>vCy*8#l_yQPFXk9(pqQ)`jDBh=Gc6oicD&#E&VUUFM11D) zeaf-y9=tAGywfRL{zL!HfXaN~A3zVZ$+9e=Drbza8pho9$mVgh!YgllZE`T14s>CZ zAE`z=l&!e(XM9KDpzHsntZg>NQPgilKwA~e4nvxmn4q?c!h!NbgWWyeUFL{pH#bn#a7#5eql*_x`5EVboO13OmnokM;n*W-k<>!F2hCbo2 zs%=fEd;sVEo5A!{(u|W%{2dY{uU^EczF*P=-KWB+5Sno@;#8~rR8NU>FhxvlgNcnV zAMxUcw7cu|OCyrGpmWXah2KkDmd{N`U4Q4Pr)0eX@g|h(Fw)Cw$qilP2gceVH6lqb9W*;5v%%VQ`~Mw$O`h%O&DN8K!UUZ!ZkHS<#| zTiFldHgDjAp|1X2_QKi%@!R5?btdgRXj{3KrT5SvB%8@kY@K)K;Hb~;A3#>`0~5-H zxQ^8oPfG2v!L;P47l4f1z0GCeqaT+n?a+I?k(ot%9yqPKTh<}tMBRWFk)AAo09 zzexd#MUb$lHYFtU=_lRW@DcOvC4Kh|ty5pp&`@rSJrF#3e|CI99+S&DH-4AAMI5e@ zp{t|j(RvLX+-+JoLb*olYq{`_?{QjT>{_r57$XYjz20y zuFd=ds5e;JWwQJ_OZhnC2upd>Eb%ot$8G}T#NMfqB%{2;Xs1!q#sP+dG@SisBkW(7@@lY1G&w-y|% z1aNY&fSZb91O<%9YXzt+CJXNELy*k2_V1jpp!fb;F_o2-GgDW{3fH;W`-Ka2wr$hH z&WyZDE%z{IB6CpR8LNP;ns74C(dfm$!fBeipDZucX{i`3R!Y$BQ4{4)^13A`P^Q- zYMHVlmc-oox! z6VrHqX=cmBjyB|`Y-TyZ88n$@jWQt>Y5@pR<*Zz=N zl<52J1|2A^7_8e;O_lG!dcj?pnD@~(!dS61v}yFPVHQP7(#~G=6(0ZQKY;YjH_0l+ za$27k)kckpZ#M-MD2p9Oixx1399>}7tk2II&Y^*~alfNR6iNf$y|{rBuXl*q!k>u66W9hm zsg9)VPa6f+%0)h^N+(!q9$ZEXT2-u~&O2+nlnGq9IQBtBSEn@#r0Ck2-|;UU9Ip9K#&yV2cPqZKhXTio+3gM(LV|iVT+~!y zI9b}MsNv)4Ed^BKFmOt=zvrKEq90{?<|(s=Q(8EoC6#^V1ccM16zm0amm zx*4LV+mP}H{yV3ysNeah4MUecf?Yb!-1=aGnEJtOXSI)DYGVUlAHI9@Mn11fKF;76-tN7c$?W{!UnBWYPEWN88BL_@;~8t^-}o8b)Z*! zk_?KA)bU=d@C2G)zuPYf`_gi+yO-#14`mWi1WTWq&$_Ih>%DYeW|#ktCCT~^U?eMP zY1*p&gcz%Tik0gNXYbpe=Hu`O_Fu8J0xf3SBUkJUe&3;2x!@!UFh*xVZ*&^~o-G{a z*4fgXU<;EHYv{F!%ysX%^^F}=31Zuha$DB?a%5x%Y&V^(*5ZFdfNEp}4y8%JRF}jN z$~FCgL$@_@Oi*cDAn!gmL3fM^D$OOi8oEw$DB_YGmxBoGk(FLe;)%;}kN-ci=+S>svxt}&tAe>QyEfAqksO{;Jl zG(hZI%HUl&Aww2*U+IA$?Qkm?Q!XtTEgKHOXMfHxd^XP7-Z;r+ zZ&YuzX@8@H07xuQYieFiL+Bp>>!C!Jkg?duYvTQnLa9r#bA6aGs?O|+9LyX&q1@q% zmM$(`?~m3VPC50>ZK<{&xB0F>`JjKH(NhiHDw? zmf|mpWZI^im4mwG8_^hpI+2^fcxH_ztNleN)HPgNl!3F_U(}J~=KcYe&>IdKk@BA9 z9YCYF2XC2A?c@76lpo-KBR-2L`&qT1TMZ3tJ-Z@tX8#IX{)4tJ;xtKf37Ls?{fQEc3CNSsSq2b*CEMO04XFs^oQy6-f5LI|Cr}pv2N0l$ znxb-}yfJb3bkT!usS9v%R*CFH;u7JHEevtV3ZC1Y8_0&s=>2ym|Tf{_z76 zQ-T%@tm{GBfBT`uMSLv-Q1X{rLa)DGdm=#zkO!~=j->AL*@B=K!C0>Uc_0+g)>gxw zG5~~$lRoxkDuAKMY{ zb!*ozKVM%%fE2KMe)%{k z6ilQzh>UFUkaO&*h2mnxAw9K!0Z(d*Cn*ac>&z~)Ki#R!pJaVJZ zlANyBcP-!JJOn8NK;b=PaOi*+VESdjgBvj?yIa-S<0M` zefO$z2fS$lzgH&e!L>xM{a1D+1;rOgP56Pfj9ko2-x%j7{Pr!9Y(@zoPrtFJSbr5A z#sSXyw;VPVA<<6noC05Kk$m(@;N&EZvjrlP`Umxb^~fH@4r!C)3^B*t_WCV1Xf-d# z-CEx8!gDV1$BERU;km;*xbf&U&irG+Di{1JAJ%42tEUydjRXPkI2@RxD)KqZ z=uavu@|l9C`w0z6-0Hx}mxF?8d!g5|Z40K~9>`&HNljN`N9`GpAv1GKQbuz@goj>m zxRIYG2TjQ2lG0QrnS{990X8W}jz*}-?uwzX<(b+)XA*zGtX&)eL6E;|#B2A< z<+Gb@ep}sxD)Gs^K@jaDO9O{0e@KmXTLW^^IJ`Du4oZbl87ay6J=rb%sh*dG^&hm%a0G1KaLkgYg;ko;5txU&nsSNg5b^=(j1M={uuwZ?=NY18JMhjMu zd`7Cty~5D*W+&9&6BgU8Q_$K%eZ&{QG}zlnEiezxh2RDG-@I!VKCnHf_|*C!$qJG)q7Nj=(4-xP)`shibpuMy;x@^Q>EpWroH zzyD1R1bvQAFQ2*$1Ek=*!DOD!HjuteFN167x^eN?BMWx*GJZ<$YF31`fHL8I%GMiU zAyzE-b}4ckDarLbyYi|(!{D15NkC<2;l$ox_efT%&KBZvS*;P@W9nYuIf@d0g`O4L ztWh-Gh!B) zr}B~6<-9_&Wti&mYH?}FFUPsrtV?|(tb9R-87gOb`B4Yjrm}sIzLOH0v`g8o12#i_ zS6G=IN$ijEhA-EX?RSRU$K-tGpTXDH7d1Ywrl3c+j~Qe^tXR$g2La*&;A&WASqP`F zap-6^M~Tq%w@j{rN{w^1js^mPWfpBXkob-SqpXNM z@?4cLV0b^X#$p^bpB>Vq?!-~ZA_>rA0lcI~F-5O7YYwL$&jKf<_`B_9drpo0`L?8* zfSBC?%`Q8*F~3k`0*|LOA#gsg;8eY#_;DT?ylp7L6zs_Ioos`@H5j6+nkI6HB{ z1ivd5vu~22W0ewG@UeXPatT@jgDA%AG@Ch0`&{JhB4D(k_U6~@ciNT%(3Luo2N6N# z$oOX$4@)Yd7is*(bKFmJa~)e3GxOwZ#g(-q?d^$I)kIVCul=W#sF?jep~A7)INlS& z!lm`yXvrbi{VYm$uKX8evTwjk;AwI{QN}ds70s|hM-c(GVnQ-+eTB)$O`IR)pt4Sd zS1&ge#;3a8jB&-`zT2pRu>z&P$Qy!cVJfWzsc5yJ&!W~pI0|ppHJyS6W7|7cOJ{3tv{10=37z{6bt_XdhbP z(lOc!Z^SGI5-J`h=Lk=!Db@YHvy8&i-}8 zee{I4G(SSQS{6}HT(1<{B(#Cqo}V!KoLS+GEsSL}LGxUK>@rF|E9MBPUE`A?yLDv3 z-peqE9!8g0X0R(thj4VFz9a~WDmgG@_3#)hz;jl9nx33fm&#a&Ex0Aq@yFjFbKdq;kyX zZC$h{aI!9rs<+LwW$yo$^Dd@=Q@&aNXWq1!V@#?cnOG$CWBdS=Em@--n+mxDFf4TT*=qUc;b}vk`FH~v#SC1 z1zB%uiaOLhzqSKsf64qQWS%P_x1VHkIhXq;T`U-T7;xO(o)Y6Nd?KbdwwPCl&xLNi zvLO|m)WngqJF_J3Y_`B*^@251iGMaHAtli=120z!3n)me%ADt^-Rx)fIREh=^qe{7 z7mVo)c|ICme7#k(Nj{S$oSbE#pPB~qco1MyKl??Qo?AAKXd*(5z}G7vFh=AS-^PZ$ z-l$dt8ncwEynZlz5v4_`^7!X1IKIr|T((_Z^2IWlSh;T)&cYC)0_C(N`yxr7 z#o6@xi45xLQDXGVD$kTJkC0&vrb9zSD$_%;czuF*2CdwBqg=7bQ$5{R=ND>ATvGSzC=aHJEKIt}klbvM-`D^|aA zkiR-vGUCla@9fRTB9lNuKgkgO`s+g&lrAlK+#E!D#ad=!h=cj}{(KeLSXP(Iw97C$ z?oU_IKY&?_fpU>o zlx7uAXM}q%rXM3*M9kly06uhYJh}q;@3YMZb!~)Q!0(2`-y)H6we);bvld;{v3n*>kr}*32#BR zh&u#hVj^?4RJD@}*^lCeN3LM|UN_{Xi%xDl*5eTomSiuV03YuD0T=?na>1RCPR$Xn zjzWJP3;;Oqh{ZdD>&1hwTdGg50dSSmAnf-K*yj$KtlFKi<=qZ;U21%K=~83 zn(o1PFfjnp9FB}+bOBfgdQhIa({2Hu-!J>)U)*2ZZmyvHL}eH4R>=9wuEAJP6@4|t zjaJcQh4CC!Di}Q&sgwK-_8WAkU;RyT{PzT_MiRm;?#lr-x@3>++%vlNbM!YCyFup% zJp0Tu6axLiL5rJA)EGLVFQFNxtU$TSD4Kl1=g$UMYi97xZr771P+7pS=Nq`*VG#)r z*yvJ180A`C8K(cD$xJb|Y)A00(Lw;&Xaq3w*w{|dO;XJNV{>@%#I*|rGL|jYykYqn z11&@Q*ve$C_M+>y=9!;ZvYjGXkl_4CP!wzJ^DJU8CwXb#YYgD}UAp#4h?T9}378|N z*gV}UCStRCC18ezB`|^i6K76=ZE+e@PE70{!0Ee2FEQ;YFP~SVCdYUsIjP5Xvxw=X z?_(?D;`&(ADfTz%f6S%TU!NW5F$-dWF$BA+Xfo%ud?w_&3`J^s0+f!lv!)vaKh-|G z&{0s4tIS&*br6dZUXU5rGcNz`__W1argaPLheau=L|gHfuEwUPHIRDuPq(VQv4_(0 zU&wLtPf<^)p(Q!~Ts|oY^ee_7vuBo`>U;nWFAVNz1{(Z2Z5t8IVX`S^0%y-umZ#v# zy{;SHi^liV3KxxfEM4nyr-JXYhq0tnmTGTxXpfM^;m?zE@ivR8lr?d{csD^a)YUk+ zNTwtsnSQZ~-R}8hd^M$UWpOfe`my+f<5I}=;?eQr*Bxvcnp_WkjoByr7?=^=<-zrV zN$RxUa`UNbr}j-UO*HmQYU2x)##9@&9Y4j84EIO5AADx2rlQO4q`Gbd1T*_*46+Hq z5kDZuVpx{%)DM>2(s@!D7%GK_L`}VvhHAvLWdk+0Rn?~3nA9Y`d-T;NmYc*C0s6ye`Iz{7tnh_O;8t$^5X}?CopIQj(Ic3JHKXI?ZI*F7=4Xl*x z4bE)puHLt5DGLpL6HVxW8L@Avb!}Bp{@@!_b$A~wbHoYDWaYQ5RAPk^vrT+U*RX~} z%P>;Phcs9!B+@@mu$b8uH1Y^9zO@r(F3* zf>R*)@a3JhrcYD!uwy2turh`RB%DVMS1qUmK~jly!T7PIs7b@(=& zl$}P1AWJmv$8X1h#64C?bJ;|i<_kQ~o!YYwB#&9TU2BfIcEPk4AMcSkiecUeZjC+0 zcDbZ^;{jWI%^PQr&6#Pn^Zmh#k;=JM%F~##WwtwZX1qzi@^R1-CT2Jy%Zo9|!fn7l zc&;rkXh!x*y5S{nBeu0Y?iFr`VS<-(yOR#fJVV4lNYz+~VzKpbwdnNRMj5hBmY6V= zu>B{CkaaRm=lcGJb-jRgg+_SyD_xT^E6;k>=wX?e{EES8>0h<|fq66ciW>IcbDY>f zHR-y6Vm$r9ns8KAWQ0a<)N|MIjBuy;%Vb{fPvPBd(F+W`3g$R6)9^5|a1)gNcjqnT z>8wsZhuTG0#mVk?^6M|Ew(g$<54%WBTGcf!i^H-zDjMfbJtVP3K;M3JTHKP)8 zUbyyJHmkbQqm?7$G(T7MXR&ISf}}-E9dj9Vif{5T^@ zwVAe36y+f96otMHtTt3|wBf6aXbaxUi((?tMRlvzrkj#hWW(4m)tho&R4-wQ8(8&h z@kXl?E<^bEC2uf65=ftQal=vQ7=br03bA&)-u&@% zPy>mhRkD)6WMEdulG;*90!d3Uyv^A{<6u8a5AsaACY{e;;~S@KLdYAA_WLWKI{3O{ zV>$AUx%Sm3^9?w0w3n)UUJj>k!Ws&caz@px8foL^!{Zj47O5^YP9Pwr zyIS;m{{|Q=_6x`Ys?OcEar$#iqP%kX;V3NIb+!E&(N256WSFAwfn{o;YH!B2bisvs zhP}Ge3^^s!rtsMCR+IC&1^4_Snh%|*&vhGh#iFA$Hc?> zs&;}e6UzA5Etc(m+C(i|CYu)icCKLH^?oW5?=rW0n-enrNYPlt(e zvpzu&?|F$0hgKEG!krvvcZSQ9N*8i@Ho?Tq=PPD((2aT9%(xj^Ehd($cSVrW?ee2Z zWulR?G}9SI<-zuP^HiOQqb;Gd2|g=bU8veM!K;8adjp)vaF`UTHLH8jfxb!kR#)EBV9iw8a;Pnj93c#-k;Hi(ULWH)z^3B9MLJX~)ISCt z>5I%*Jg~PEAuysYh*(r@YEy|lnr(;T!12O*$V+Lb({+`G&r*q64B6=ra9uaj#Kf%K zEexlQcDk$;(w)P4hsV7Mo)qC(rn*T?N>!4#da2v=!WOCAz>X;ccRfrX0Jg^hW|B|f ztXAfq-TVQ>En{XNg=v($GYCiwk0VW&M|xl#oyRURziimhie=cTPpPhI<0m{UfTOpt zQp)MMJ4(`!$#7b_k@J+4Z-!PO@P4HUTBnL3K@Aph8JMx~NvPWo+AO=XK0~JMF4U3n z6ra>GEN9U+x4!B!+P(V1Ib^L8K}rX;p?++{q!W8W^*f;PzG9ja$LsIinqw`h{+{2Y zH$JDF)MV4e2EE1x$&J~`z4D`Z6`3q%XB8loL&WZWn>{2@ufN~w*uxKqhgaPwpr zYok>J2eM5;I!4SO>nEN^h=+Gh6k#-q8pmX_28M$dko8Xr87WPE#c6QJTGI4>;%E(# zJ-G86ojhtajXY^qyH`>eoU#4(bFxM0y3k_&8ZT5-w17`}_GF4%M%PCnkDHRmZX7kA znqUb)kDicPva~Ts_)GUbNo62gPVDLjpk?Mv+Z#oXebeWRZa0Z)bB`ZR zh80LYM`b7JIVWiH`Eyg4ZJ@z(eY}&Tq|!Bg;?IPsg;-dMrB$oQwxnYptWcQKoJ-k* z={EB_GmW$%v(FxN+8b8&LpN4Z+4zwobE7pawBr*c<=T!j<1iDC1q7v8w`r2rboAIm zHScb}p;C6F*c{~JIMMr7Hh2fYuHFe$z>e}zq5aqlvAeYiZhZGy!cLKdUZS+ zB51##wq}8`##$&a*CqDljCkcBl*IvqEOBRn1NM%XzEvqsV4P*PMf)f`Gi%6^&A6yE z+dxY7HcBP3v~+CYLH8*CV~6+4*cAyYoGpX)T=mg5<4nd5K|i+oKD9C`4((q>J?me! zym>K(z>>EHkzv)N1q`CkoGYSl$?p~`_%=||-OOPu$pRA}KDisWPZ-;Yk9${j%7m$n zs#)ulPnTARVt0!ttCd7Srtb|@RQA-jN=q{|03LEsa1NrRt62R;8`*8~Fz-a#*_3JZaqNPGkK*G^8W^9QqI7EG(s}ku1Vw56L(UI#tov_*3=jMePW5N}<7}kyv{# z4H%^bOcDM zOM4pk0qN}kKR$Xr!_v58pLSHAANRS|dns^0aoRv*9Y;=q#4=mDHl4;^AuHLGePFZf z4a>kC&Io1>X#I`EkB=DOByjP?87l-(>v1)6!XwblXqEk9{co|6*hR4qDGqS*!0mj?S@B1mFgq zc_>Kl`G13#DP5{IvT^$F-O`HXW*5W*M=SR$Bp|kj0HiX-PA3)YKVct^DuGoWY|8c= z6~oB+cW?|ctWL9pM4~1~{Y{%+BQ5l%a$iyVTt=a#Vt+wkQ5XQ+o)3ey-vi%?x1QUp z_6?HScYi_Gjf{>R=}n^zPSEu=*ZP}m$N!fZYmWSs=YTgyI1!UAV|TAF9^%db%h22^ zQRuS6<4q6RQ!Vmo*6CWV)9YQ5|EQsaKZ$0b&wgV={x)2ryg3e9SlE5UWeGdmP3pG8 ztiQ*7jDT+i|7Q4d^5yCbKllO}_*^|l@_d4Uk3rd{Z>OZFMb$jM08mxJZ~qPFhXvAP zK6)d4>V5zE?}JMW;@`DNoCo!xZ19VW_Km576R&%0=juTeR%HJF zlN9QM+s7BFqt7;WSbF@wKTV9L+Qx-ZH% z^6%uEkRhHJm|7+!!582`VD=F9@^ALpl8b?7~lLR*(=l^R@sO%HBzy~E|Te;~idW)ca zKrSg`OcySn)2vf}CCwKJt%Os*`1J`HYVQD?ci@O zo;iI}?w=F3vp(RVEAnkMv&O1{7rE(s$rbB=O>I zG!*^;)+JVzMYHZc1yp8FhpWu`RlK1iQ+Gceo5?UWIa`YP8+x;y+i=-aIartsx^~If zm^P+0VYCATsqWKVRL9Zdnb`_{k4b%lzN*}DAvF)^r70CJWeoI0R{u<_Swg5)FW1FY zOEk@t!Z?PK-*hckeVNIkwELFEpTbNL26xA-r;lgH<+i_3Vf$tkkme{B(G!rFteHiv zOCcC4FR#w~J~3rzgp|^TjZSQCzeSywsTr;eRTU1m;LFGurtgeO7byI4#pSK9V~YI6tX9TKS~o+Z^t^>YmCS|k{kB9U+LuWL1Kj#c zyA$dsLI#JQV#1xTh)%P1k9zmgsqv|sExWP>+tI}czw{-!r=n)!GzvOa{XFrs%NSO& z@)XXgs!hL-US)a31WH%e(H!TmI!J^7(XS5}Yaacp+LyVA2xAuOXFpsm`uz0jKDv-k zXc{{Ra}b-SE8Mk|-bct)Xu}A;szqNqUT9U>7#1DAF}y9W>XA{b_YHSeXVvz$m@%gK zBXW{bL^mEY&CIV>oK<|Tbprm(}8N>mtVMC$+ls)pr)P1JtV#K z#pjnM-IcyeWt&)<3ANsV)OJO33Dv6f`{r2ll6p6mHGM=M!k&gYl*W(@DlY-VkBeSB zvV1mm9n=rZCG1S54l(o`h?Cl8$24GD-&v*PvFP$WlR<>pJKvL3C!l?(wDaCtA zd23TjywK~giH-etl6IM?H(Osu-S*{VeN6U8}DYkpkCytNktBJ>260f!E8+dHDQspb}E>uBauW!V{>92lXJ>USLSa`oo2Z zm_00f#kGlPHH@E@THVYMh1ID0rBY|><~GxDox&@xL;l@5tDRNG&g--LZ;$@~boxpL z&FRTCQzkV)+GFrUIwciCn7sf51($y|d>O;dBg?&1#gveSSX8@8{=v|j+`vxm zy=_t`2W~mYp8XWxZP|&4rQv(J)b-yZdzd_z8YVLj@5$U)giHpX1-7ILYvht{ypa=r zB30Ut^B*4!%8?HJKi2&o42V^kJzsXA8DNr8LK1RP)d3RWcfA(cG~9X}S#@&yJacFL zqbtXXDUXW3z|4kJAZ;vpoC^E}j>bX?974N@+xW9R(!ta1M^e(N*4YeO3`041r$x=} zrKfFd^2wBtp2zzLCj1O<%f+Hao$j-`ozYiueIvDY&h0xG9$yY+i+6r#VRur;K3q_K zxG~qjRfza0Sv*uZlri>R3zLB@8ATfH8@()~`I$Osb1e2a#@Yo_L(|^B&}n78&J%<_ z@U5|8extZ$N1xV#FY?cx|2xl(oZ>nCN1oUFh5a^mg!kEwMek$(fG5Mq1>`BEBS$n( zsrGhwMd{^3lv8nIFI_yG{YlfD2*x*ywtkvGxbVBJq``{|yaQDR+&j7^FTD&UQP zD$naV=@%;Jc&Ur}BGlngd>7~(4yA$+>XQDQwfCQ1mR94aS#ml7dCa3~tqzn~x7}d| zF@z zhigpjonD}=%>k!hL%VnOSP!NKxkbva>k z)$=i4)N#Z1E5Z}nW$}vC?LOv7ATSpqA zme156D(O;m4temL16|SfA05F}is~MVB_7+!^_|DblP>qL#y{+Gd>0mO!x7?2mr;Ng zJ5C9z*_N{KEP^_Pegz5{ z=H3S{5&#_UoU^H~HC|kibNDc`eZRwY;s-%Yrx(7#d=4=(w;;aM5qj3lc~@LSW}oYMQ}L)y>2!YtN3$L9>O-BZ9D@YsYzTo zn`zhec(r0TGuQmG$0SlN#kyi{My&S=v>wy&`*XmI{tRHWrbd)T*$@sY;JJ<&FTux4 zt6np=o=humtGDrVR@_zT&Rt_C-?JxXB;t-8nmC$og_yNd8B4CR6wLGNRzr>FR|Z;r zP4(EmbS!G#j}qwG)R5T)3sl1yK84>^OE@~M%9&=xOPerkyX~k}I@!L#W!f(FJu5Bs zj?gRAbD!l1&wBms+-r5D>^Xt5MN$DW zy}dG#`gx^3&k*dCRX|3p%ENhTx=d);dO9C?K3PogCA|ylsHB=5r5((gnTkD9b#mrQ z@4My5bbVHCt3=hLdtdsnn>}8EM6&@UmUU1r$f7{eTAk(dJoXEKI&U~u5jZfIrD>js z5kJX)(aDAbN4Ew5ED5==_S8Q&T}`%-@M0TT%BWirk!^%zr2+~#q_1n(S~1xeB#Q)- ziMN0ecgLKt-c;Ey^3aHuj$|0{UC9=2UA;qca%_NZjcG}(P^qs!>q^c2Gxeo>e=iP- z4$}DU-}dSh90>mal;0*lKUvkvEpwv-kRdg4O*XI(Tumi zk3LBPe2gjJTEbg(2MJ}0^G(tB>8GZWs<0JwxqxCFKd=43WC9GQgh*8Tr*{h!tWol! zD8%c2ZP%*~b7{9XnS6vU1&o@a20`>D#?snS6avIg5#Z6@6Y|9c@8y}dG{3X2j%|H6 zl$^GaLuOR(h&n`b@WmD|%Tkls|0FFksy%y%6<);im+0FzMjI4ro<6G^@m?jB%Gspk z%v3b;uDJ=i=oosoa;~1d>A}cV_rygH0eU6aVFzNMODR!qNWeS@rKY(QE-V-RD$5Oy z0QR6D-KWQf8C5Z7K@PuL@4a1--mF%q>3MWRv{2xf)m6K^5L_WAw1c_Pe(C zByrD}Va!~;LHutKSXHOuc8YlqIscowQC9Z$)fGI{?1xtttOpU7!emsgZ+#FS!8u-9 z69q)8N6}#^hy{+`8J9F)IM(gkw|K$ubhwY^S%2}>J9wiw=_EzW)0(0k+0XssX_m9w z+VwS+b*5Gv!%|$d-hKMzE9M)z|M+>QRG%;I->_*ap;u5bs%K_#9bQk>>{s4Kt1jRq z&PiH$ah00N7o$w`JxVmXEl z?EERHt}1barbbo0xMU6kGfuJ4z&ia;6W(Kg5S5Y#8WY2Mup@0nG4hgJgUTtR{O>R5bb1%tZSrj1En+dbgU`eJ zt9vxH`g->*kNKQyV_FSsbpODPW&{5o5vw-*UjF>f2#-ouoprU>T*}tbU-iQbN$UQ8 zXsv4mQF)bA_C!IsoWAAf@2Tv2r~gB<^YX`Vy9v{3qHtPd{r=!{e6xlqLU9o@QQ2*j zRpR^k?8=6JyYHh1TI5STZ?Ei|c7b{{8q~LA_Q!QJG&KHq6Bj7E2b#dmZ*-Flu~{^< z=TFW*qYI#+q^xIXPmRn1uwCJUw~y|N{om75ZO_+rXiuM1^%e`kONQqI zoCxFG8ZYYa*Bbso4;)@s8(K&MDj);ZlmHn351{2uKH7&4ok&6EXPW?M@tdiluB0A~ z%coJpge3zuJ$_#0>@Fe>0XNO(L3xCKRwXQ+Fvz@RglBN>JvCxG)m1E>1@4VLRl6S! z9DTQ7kdRTnIn=0QkHw`MRwVH%IoU%+2mrQY!>6RP0uIz+()^BEXiT(pLl`W4uFrzF zM_bOeGQqV%$~@MGR(Hgi^Y!O>)qf;4x(YzG-`$tjbu4D;W4Jexo8itaSJxgl?K&OK zT~a<*;ep!c$T2d%l^ zq%o;)nXOHfd6sMY&HU@Ln$0}u{{PzFgrQGQC@IiVdVo7}44{Te9rWO3pGMI@d}>bvikNIXszc{GsD4fT z?kkq#JL+>Spn7ra1@+a->@O^M!YGI0{2EJJU{DF%x%&+^u?zVsmSu|kOWmjugrn`u>L|dx76N5rJgqim=tWVw{LEiB+|3a-G^zYDB*;DRWu) zI#;;x@eToS=FVbJWyDzg%o?1dwlN}iXQVjRf+CB_`R09h9g~ldfh8w0O7Ul~@b82I zgpyFb3-=|^P=1x2X_wJPSuF_`u-C9(!CUSz7jb>B$V{QUNgKNH+sN8^P3!(A1xxSu z>>1$f={Wr^BIkO%vVMBpyHnss4jx35+4yjzv;^^pK{M<{hqM^~Kvd!YTQn@=*ZEn6 z3G$U(eyh-<_1ON2N>-~?rKYomzm>R^*ZpnhIc~m|kR>_)m`-5&j_AugwWjAuXXR8F z*mNH_H#maFn_N=3#lprGHDpVip>Cg;oeh-O4Y{uqW$hk?FBZKnFF#cV{1ApUTY#kj zBbFk9)NMJ#U;#m@Z>jIc5h~k^rh?4<)r4j1*tH6X;t!unhP)Drp;1-u1?gnG`8M`i z0JSvvow!q0VPB5>luMGv*jZ@R z+MEZJk7cWHuJ^oCuV4DOGc2a3&JxzItY*w2ECJugPvoeXG|88MLP^Coxe^Y{Mc2cl zAlMW1LEhc;B?Z{fq$O_C%4jEJwV7FWy;#@aX1T)B#ckI}5qF}jA9hDZMVlen&goJn zSwC60Bb@UH*sPPWh1+Nyyq8SBt5e0|8+$A6Rxt2s7q_f|YdF=N&Y5x6z$H<-+y9{j z?^PgU*Xn+Mj!uh&a@&HZ+nt~}1l#Jb0%5L?<@DI#9&Px?OE{%rsBTnUQcYzS7bA5^ zvNh+*3#e)pZgtQ*!h9}##iDBMVg-?_+*kUa)2i&ocwxA+p6mUeAZ)d9)?+%+>KbAi zrd+wNO7aUM7(fpBW7m4AY34*Sda8AE^e^KRVYIr5YZ$ntAxdZF8l;M$Le1SJNwo%D zjU0E+hBk+ODu%!5Dc;icYQ`Qr@ zVfx9LF;9Q=7x*;WBP~PXoU;`j_dGaoW zfX!WRnf+XUul4HD#d~$3m8YJrS2=1Jm-1DTR?d9SZ-gC(hF1M+WHrI> zo|r)ez}{yn$uZF#n1XplZoA#$(KeW)X;n{Zgn`K-&BbKhZs1MIbsjUzWD&@iT@#Uvz0;$H$tR3yr<{E#x|Y^dB(@Vr zQ67-2$J;;5TF$2TsPqeqSeE?|>qG24W2zsT!O0AO#HXrC5aLFOnOx60qC6)Xd@QF< za43!NkOGfY&^fVpZ1mbqcXvTYxa}3iey1Uyp`XJ=?!2D{H80bYP>`+x4UYB&>A7}n zT9TJ}aZHH}C$Du?QmmIJuJ@GsmI(t6GogMf$hY3SKNHt3llT}5(00|?Hl$d++NNHk z$IESzZt96bnMd5KzjMPseV6Ok;w@__R;Veb>>O2$Zv$i*E%HiNFzby6$yQ-Vt~Od5 zHIeC6W%(&M)s8Adz?!oTE>#O%HC`Sdp{S49RbnI1Wuc4b($gUNg5w&U$NjtM<$amZ zEPu;UkdSRc0o0Zrys3pGKO~)7dot(_4oRx*OE}kfy(*LXcA4^pR0_D2RZ*hSLsCno zAQEEdvf@wP?90}WZ^m;W(b&#hLYb3)b2q67XdfSQ^%lJ|nc@mUobL^ulKUlcbG{G?mj7M=9$b%^%ng04bffgJFwIr8{L_OU%UGQL!(mB%tecL>RM?ZWYMa2G(& zo6%IBVp3O0vvFZ?-2UkSwv)b0%eheI-Bn`mV6J`fECm65!66p9p|*#W`X8E7X#S)% zPH!$JevTJOIMtAegBJ1&&crjb2}&W*f-C#F5m_!I-)@8%i2k-T8y+>>zo%ca2}ys zoIa;<1!IxDj!YkE*W{0zn)u)PWnnTyc5Ik}T+8LAI|d^NszcVW;&e;0Ass34`%1{@ zKr2+MvSU^ScHUJEW-5{TTtver^R#V~%4G~@0bf|M2f*8Mos{2tixNxo7bo@D8H7@~ zCN*d&D_9K+i#J{Bm@IK%)XVt4Yq#q?F=7P>p}JXbDo>=xgeY*d_kT7>Ce3XTsQ&}V3uG?7}) z-Yn(!JibdNhUsyNbbWcwqZUrjmPild$OKz|g$X2AF{e<3(>o6YqC9olFwGn%SSXoK zFumzif$WgiYqplYAK}2lq~rfQYh!n3KXf7zmChJ z@1#t_<~7VYW0p{5-b_=rtZVgEQU@S-lB~hdOuWd^4q1|9$3nhKx5y0=t5dAG#~vu- zrVJe~$>zeCRJ#Se7fDm_)kN=qQ8JMmGOXh@38vuQV=p$5Z>q`C07)0~F4*qt#$J^k z{=m=Zzk20vv@%4Ed^d&Q=~YVwF%$rxc1+(RE(Du`bYk!e+$`%NKz~ehEcN0GCUeHo zsi*8pY$4sUimFt#qkIuzR&#V#TDYa7e7Gel`j7$@w@7Wmy)5ZGt5BA#v#ExY|Ik7V z+@$4I?kX$`r6Pb-Zcs@IY)RsA{U8E%b|uwfMZTdk)~f@5s8`H_&=4y1i8n@ z^xvOg7bvKeN}0j{Ad?v&LhY+Qv%gj3!*_g(n|<&f+WXz88hEx9 zT-C{Tis}6P5GK?t)LAa-)+>W*orvl^a2YR|h9($PYMj{})C_bc4$YrW2WxcAcj&%P z<6MSnPVPtdM$OT8Kl?>g2w~FVEH7` z{nCIX6#W%B8Y_D=x;XUbg54!(K%KMKS&NozlXW^*K0RK3orIK>l!6+(q|8F|HUE%= zG#<}j!yRo}W_8_N}KhF50|f%b_Y`aVFWM!tHC;G8pp znhlipjwgA)4Q0r@Kor=sCAUZGMx&QE|x4Qwn32Sb>R>iVCBE zE|bgI1_r3+d4N^5;Ew_A4DWGWKWWPaXgXf5A4Ld-WTux0qM|~ih|2yGJAAz*x#mm} zy+>a%(}_tJ9IwN12rP-TND~%yce5p_FY#O|z&XpD_KiJi=*=#4k6Ga8*QAb*k>7O+5!jxlN2;p5DX0u!W}7c}xy>P=5|}sV{|F<@YLp5J zEL!i;k^lZu5`)*s#AO|UZ|C;p0gmk}SnvK!Sop!?m@E`2)~skje)}I96nZe;a`5Aj zde;aE^MqEUsweE-%UPY45JH(&)D=}3>|7U|U=7GL3)D=AzTit)ZBtsO8|j7&$!i4; zDMaq;`QBI?G_70X5dwyIE%nt&R)HYzTVIj8*+=ca78Dc%L64oYKTjM*i7B{VW4(@b zcluv*0fC7ldl>Pp(S8pN4dcbte`o^n^m1a1L!UnW%OgCaL9dV7$a&6~_nl0T>c{WY5+syOmJN!T;` zzscT@uPe77z9Y&6Qqvrc&TypvH@0^rWqi*vMGp&rOZv1EJe;un-E_v%`rDzv6S=&y zaB*5E^7kqmVwuXHW279Aerl;OuBpP3Bz{Xivtl3zou$kw)9QikfPDe#A z!(}rE!J8ZRr>gb7b(i5N(Sh@`2vO2PjQjiL=_f;ro{ynuc<|BIn66jR_m^#p#%O52 z_4XaaLT#z+WJ2x&176f!3j4Lnce8|sJDsA5WjT1;d%HNoXY4#D9%t}YZR>pm_G5Nz zFxWS*o}d00&(Fo*q8&rs9W;vH#Mm{ggD6`DI@!N6iC{-3+m~k#-~YB%gfY{`|F674 z7UQoKhYrKz?f*J0BqbI2)<-9t#fz3bQc*r5xgzH%oGV=xltTo^TkV8HOlliM zC=XxsGg4>yGy4y`EpI>Z%9)IRPMf|;jBSsEFS9xB?Wqe{|6AQMYIR|MMxDjQ4`W$a zap|{2BK~l=hqK3YJ-9#4{`2b4w7kq-!xpJSF>(Y_9F1P9(?oAm{z8l8zU=&47kdr_ z%3XK=Yg&r%_yPF!1Ji_GxmV0cA8hG&j7PS_Qg1MaC;#+7q)i(^$AOb=Aiu6AxQ=E?}QNFpR+ zl&LZ$0wK)$_=IVqASVx z2^MlmSVb@c!qqC&r_5HLkFQQdpG0Q(8!d7=Mn%vY->}lSme8M+Lf9hMXji_h+*gd< zB<$HgVYQHhQ&c7DRLFDMyvZut+I7kO1EH~uRURUwX#A8c7p#!gY5i(AZ>tFo+~I<< z*Xu&7IhCu^npp}m1IyY8FP_6o{hJO0%i}3eLyK~9re;>o82p3tZ_Cg2jd+X(M1^J{ zycMOB2Ul9Sz|twWy6nTAt@Xfp8qYyx!rHWyA^u*{Vtk6pxN|Vew{`KS;jT?|{2N5< zj?5~|Hkw4c$W9CIt$L!gE8O11%8`n;lc~fqlQXq_M=dD4U|$t@KZ*H}h@3c~vgm=8 z*XvEd6r3a^B=oc~Da+U`w09eX?IYz2o@{&Uaqtwm@E$iTArrDTElVFqsOFsVWNc|K zsMBi8YDL?foOlWEhM;m#>e}jkXB2}=ypt+PCs*h#G=RzW~0f|Y~y;9?7B8G{EY8O0V& zK|x$qpn+sf;Q0Wh21l>Chst%*e1V;?N*v4i92DCw!;#R}g1}OnGDTlg#fd$86TO`M zp(D##B;hhurkS6w_qncG-+4KUfnCms$|!>z6~gkx&%?>=h$p9SQ2|}rL-d1wqcY_| zaN`{EeLaU+bxH$GZW`AGG|T`G{Lee_i`7<&zt@(M3TW9B$Mtb*9ojEYuYFyl7&7uEbowIn4ze zjsq_Jj^MTXGBLfxweJFvwE~qA-l!9OfO(cw-{P&&(6x+y8rBdTNr^iMR}OjWFs3w} zKk9wja`Q#5$(h2QZ8B-np#BVkzbLwVY19wo%El0tOikUX;bT4?$k(v_z@c7TcmI}d z$T#Bw^8lH}Y9KW9QJTL5nv=aPU(2M@s>17&oV0tyEzH%Cpi3{xb@klm<3_>DZW+3g zNh$WMmDRh+-h3V$q8u7m0r-IUrV zWvgH|*m1~>wsOPe9g-!%2cu#+DZk)5=V3R(96unS&$+Q2kC}IYcZh5m)tIB-FM~pC3fe4T<6`}GV8%b96{?EIyYa#_C8{pZ!%iiov-Y#v}6mMkE zXu>+D)g(8i`^N#VJXBAM?xI1;{VuLRe#$hd9|Zpw0|khZ&M-5gfl>Ju9c(Lfgw@bpnIvh&Qe^gq`XX$pB}x5g+Rel8L67 zM~Ywb6YV6@3Sw01L6t=@OzG5T3@y?LS3UZd*rp6lDtfh@)q0NJBgo7y1Z+_7V;oljny0DMw<9bJ? zM$PlsB_akt_ULUcgkVhRq_CLBw%aec4e^=^ZE6s2q`dD&G_B#^RmlFyQg@m`cM2B9 zbfktCwCGt&EOvkFE@ejV+5J_0A=dAZuVKK8eeVr1KrYfH(F;yhwj2J3R>AcaM}Jgh z(DhtCswUwy-8%J6QcGhPy5SK5?^0@vH>rg47n#3Si-gH51sQc`L;JC8=)o$-ZgTcl z1;7IDtik2_t2-wmE$$6BgmU;u?b88IAl%l2{$=&FZ&8ek;Q#$u5K9*~V&AdAUIjkAkY~PHmv5f^bEhIVn;#TSL%2y7AtrC&O)D-SG z)lph=>fr1CnV`W2H z4-o0T!|UbSJ6T}bl?79qj-RBielY=g6s!YxVolPw*pJ?>Td-yPPZq6tW0&2^?(@d6 z?AA{FD1WbJfR}d=w5hA%)f!By{5l$#l%xpf7W};RH6yo)jw5Sr9ji!2baGe1O9($T zu3Mq;NIt2Xd8cZMG^5{Ihxfp=SF_nOx#1jkHOu>B_Go%r~RH#vjVjBj;R|fLVgtk%>Z1x)oC%ml798pnO3%$ zJvpXXpcS?!Ip2Izx=u-!X*NRw)sn01wvF9tZ1BAKG)(DmX;VoL?bDF$rkt%ZZ;2%= zan<$N=#S_yYXMT?kvp5$yk~3KHRXz%NZNTeMcN9P=b~R2#aduhS~c!$Q$JWHtsSa` zI`j-P&tUCkC(_J>T_E$tmL?;RpED|zqr!L@SF3ct9u)c=TN8Y{>m&jwBJ6FY{uHqR zx9b{T*!6`o?;QD^Nfz>FL1qgm^#%5-g+s;t@}^iqL%-NhtKo9$7nUlv&IvG zk<_?_!}3fFLYuHuG~Z{!5=>554Ek;xGEUYwoMaYfd||2*9uCN~ea?zHm$E4D?@`B5 zY(FdoC=CqEuE*fi{c3Lp`-`Q)W-PQNS^*)YI6Nt1c^gfWAa#q7Z~aM?&R(Kib|q38 z4Rt&j%2rpG1KO+ZAMxCwRWB{m9;Ws;#glVw%Pdix+^c~H-6%YoR|>BMC_$rlNLD3{ z=Rm1yMm6&?kuw2BZ1>%4`nrV2>bkSp7%guh#)g=ZO}UGNRK4wDD8P{JShtvF z=*`v2lP^@sb(k;Zaa~)A-3r_Gxk;{%V0&a6O5;Pyq(s^YuWxV&xw#^|Gc>TS8^3-H zC9IW5N*?^$v1Bl14DQ;)X^b@AbEBPXoXB+Sw>H6>go7pZN@PQZ zH0#`|?*fQq*kgRgK+qz#N_i2byMkPYj`!H{B{_PuX{i6CN_R1E<4c7@M_Lg&MIh2By^m@4hQ4*!z{oqL*o@+ zypNA6_4mR*fkCCxr7!Pq1WqXN_AEGnolqxfg!N?6Qc#^UuSLF{VxMD&CClVy-7i*G zmC#qlNiepyv#2~7KGQJ+LAe{P@+PpGQ5R@zo?BI-Snh73Ja^gh<%66?noE#SL1au< zvBstV@xGAlpjI-MBmd%yy4GhTmGHQgfBj2(AOA^0|}I+>Zsa5O6)6p?|h1Fc}W6 zw^AkEV`DL^kR3W_88(fX(%2Nnqu=_yr7fFHaJ6Lr@$7EsM_7wZ`m~|{IwxlnCMrsNj6yyqA5%?@FAW+z?^7CIV=t`W(R=-ZuAx5A4ERGHkW@PRhVb@R zW-=xn20zX^|6AS6`#QbShLC`IAvt#Da`qZmVIN#4j-}3_9OQ*qX;w+~MmxJQicK7r z5*Yb{BLFZ+-SM&-Bc%wa^NAWJGBKJjjIGyc;OPw-?X6(&Fd>Qgv9{F{X;vW26==P` z-SY&*l1!Iej#F%nDZ8nO%jRU7^p)(|H5Z533lIaRyw(WgHnpio*4Nk6M>vkoB~9vq z=IY#3yedODiK_<;=-3un&Aj6zL$01)HGOHMSlEnLL=nomi3ewug7LTBj38 zU^y2ZVJ3NDon2B-59fg}?HtxspLHx)F%y!$B?OAx-m3el%5ZJ?E(WSr2D2g+3n3* zj*7xmHL%EVKW)J$%*J(-T#-3hnR)6a_E;vr)!@dkhsBjEJi zoele`nCZNChly|2b#zCLxX)g*S4H!~>v1V35d2|f=GLzrM>pDk*#+!k%p8uz7(2sw zaYlkx1~=R7UD$6#*-A_^COjR_&TeCIIhpLl#*dgTBP5{&_Mx^9etz22{5i+sAHJOz zU@drLW1ruxeZ#JK%^fzWLZLc@N&oEbWz3=}3joDlc-6ay_Qw10_Gb03EP(d@itqlP zg9JTK@3HcK^oozE->hi+7V2~1`+vGhrgmjNHVs--;;m+3GtlvG+5er#QIdm*q{myw zZ(jplOjhi>%Woe2mnI%Z-(>$mM&C8pN1O+aZmDiRh+W5>J0+Dy_qDB5%%UM^M;%qq z{AkE7PePsp0%JTlP*-zj!|MQxk@%wK_67BGP$3aZZ zP+7d0PB;Egk+WSP{>8+bM@zhtT0Mt%l6Vo{Y37}dKD_w{(fIz?4nrw^aPmgY8?S$R zu-?JvWK*iE;pRDN${4Ef_=&F&Jq;qH z>gbG-XgJ&cvp~gEO5E(ek4`3Ed)fHp__UiN_p%|D7I6?Ax{ek0*MsHxgQOo5+-v+If1A3vjuIC_)oFWBbBuYcyR zd(``lUd6Xy=49rleC)4CS^t(6jY!+?xnu6VpGUOo`j7j+-679681eCOBpZ!=M7mg0 zo_7vOymz+u`D@0)NXN@ERu9m}eK@=jF`NB6_vcFghXy|2UesJ>{9lvH-(HoFzdo}O zoAn)L7V^lW=j)rOd{iFOCl82C5ccRhT$R;l{VVdz3%mT5RlluK%F@AD{>RTyI?Qk^&eW4w0t~%l>Jntm2D&17e~1>t1lGm z&6CdhRIg6x{${npU>asjU7WzbMKzMJzqcaruV#1izfxJMt}2f4e+L{wsG>pU#n%{2 z$}W<>&r{-G70v&9#v}-qzoZqvb|W7Y)mQS0c>ecFj!gR~<()A@q=(zXbJDu$)0OQ2 zgnE`!%Fn)RA$OXq-wU}rY=SGk&1>@4k^W9$KlXceK8L0yed+ss^5E^|nOnq)6+uzk zo4bFqy&F!u#{STQRo;Hb`Q5D2hJiq9H<$)q)?hY)N0N|K22+d3Yk+Ik90BI&Fo54s z5ns8kHX#kF1!38yI$6<-4{oLQ0XO+MP8tL$gaxl>>%AD8U!NG*So9FDiM$@UNTWJo zz@JP_Ew+A-4x^U+jmfH}_EaCi-j!X0nBV$|G_TQ6))$Pr95I&?>YmE?J@L@jGZPa= zSr%@kJ_xLzjN|7!{?W7bKSRU3+L&F*Tp@myHY1kn_%YNeInX1|njz}WWJpAoKUytO z*&7i4j;PZ4*~W@(!Uy|uJJWe?Iz(&^LZP>Mh@AxN?e> z!n|l{p36SKtyKkcxDs#tKX!lxkS+~N^jLusAaF>&xZzhy-*{J7 zVBXtsMaJcXCEv-~q*X!U{l2R=W5n}WeLX>kpC;%5jUp7-0bcy@RHQp=Hjml3(r?*P zLj(E9a=f9zpfWZGDe=G$3^OsN-XrgS>>zIw+qy{LRGH$r(+=BVtxp%%9F zZmO%e8{@^o)9anWv7~~j6F~UzLRnWiqgL?`kJ?OBsTXrH5~^{(_<*B!3gWaHNA3Gg z-;Xc01gi#x7EM}>Z1_|XrPw$O*`y3x%q|h7tmtw}&%2TZwNy&$kb3N|u=RM*lx!!11uKKeM(R{EgKQtKcg9?yZEF(4c$C<}6LO%$J zsyM_Goi~33lMl3V50LVvCzbH&7imElic{d=(ppls{`|2sQQXS$`RS~z;U-Gb8ud!v z33b2GV$qj zLx~u#Ff7!qKv9S=i8G}ia>|EP&YGos_R{flM`klmNtpYY-NKlWH!86&V8d2IB z`xN=%IY<@3a`o3@q6l^&;O;p0K@Thou_Py@lA*!Gk+IPs@&LQPgBt$O)vT2ym+h;>h2t~N)l4u5%E_3Q^2wwp znm%6C4AwMYTbpGmLv!tLMULKitDCCPG-h}O7{$`BkZx69wTm8)GZ6A~_Sh1=m2!}j zTvLf;DGBK$Do}cnDwdgQ3SNcCmU04gN}6ri>OfoCCKz}iULKYx)puCeqiq(^mKQ>m zxi*KN;M9O3PP(2i*l8WJMO|*d1u34EcBL@U(|R35Q&CRZx^m+&*P?Vr=1unQjZP})8}A+>Hz7iYb6Dy2ettg9EJdHqU# zg+H)xY8K^kf;Byp!AYC4=mkJiO-1SEqI}vMffaKRvywlGO&P8;L03kV5CRIFP+mc) zqOM_$V!&0&psM(%U{0CphJNU!eo4bZ%u^)hoU&ppWxos{j8&xi z@a%+@cb?9Agq14{qL?mccO3HsPj%17N{v9bw9i?q8P@-Kt?@neGadR--#B^d`ZsW? zq58aziI|UrPLYGaDk63TOCy~{gEYkf)3tI74JGc6$c_ zht62lutAlt=S*_l-?ow9Uq_)w>w<9&3mh{9H`&h;&VW|IjReAQ!(-v1IBmhhF#awXm93w{0%dwUyJ zPv zICOAtC-nFQ7B#Arj=kSjYT1p(?2qm&ACJ2m`M|PeLBGp67qvek%$ z)V?AsCj;&gaGmaJL<|#-*rz>=R2<&(>Acn7w5P0J@gw{Xjb|U1=62A)C^5yzX?%g3 zTKU(KbJA^6hOc^|M=(4>m-_P_r-)@LZr+>cX3X0Qkds2?aIOMg_{HPafiTpe3+Pn< z{1Xut*$jEdN;^X}{AZSjM8LxkyD_dubHRHesCa9vBIyFIm`7n{tG=PVcw&nV`vruSqhhri)u8o>;hyF3d$2Z#mkv_v#sGy1XP!y%@;VrW)vl(%l4Z=Wrc~Q1;hk zEot(qUUfN?Ve49bd)f3dbA*z=R5X>FFd3xtfsTZC)?uo;X14mL7gs@rqP1I0dU4Y_ z*m^|L`UCS4lrLVzx#*HNKZ8E6%BW#wl%AC0nbUH#;vmZr>LaNhaPR?6K$=;kGs=76 zCwIItXvVcEYuz_^(_092FBIerJ1N=4D{Ztv(mN<4#uq(*aw51bs}HH{tb&Wj&s=JB z@1PXv#k&;=-4*fIS|#lJr^v+8rTSgaJAH5QOTX>J8pX;5hj$he!{?}?vq2X2UDVIi zHj3h)F&>e8osarc_j$x6rvthN1;nD6>q_n z4>L0)=T_-+pL1wTdAP%*SWj2|Vz7^TbBKP-6_c^QNOMp~W_PIWRNG+fire*7&1bWE zDT&`z571R)>$FHqr1I_P7RtD@Ygu=lZARJ^$@%r2AseWQEd=1)8MEj6?@rXBS0cXm zi2D8jw9zO`Vl~jk`F)7Eq9Z?aIi)atEQaT~S}alOJ_hP}c#RVniMkV{Ms0|R%3hM^ zY`k=APwuBY{#j44@H5PsNP5+iLf~~1)8se)Cxc%_j@Ur0;WxL&Ozhd))pXt#;?zl8 z6Ma#IjXl}Nj_f}%?gRe>U^YiCygks>5Zf0NR!tzW?18|*GO!{3xvdXLkc=ej3Q?7G z8nO*peNqPQ2ub=+zINreu9pdw*!FxTCh+Hq`TF2x{pQft?iE!luakJYRbN8LNIF}knHIkj#M&@J$1^ORLU912F2Hwu;a&m zf@+n2Az6L9bM9rp_k`2bA{?K>&5(1nA39WRGlw##CfgQwzU%0!3>sRpAn7u(mTqrq zsdk54!G#FDN+)UdW5O=gy@W?zOP@l_W!_3sHxF1V7dB)H^`US}LqyEYczy;!A+NC zgNUfYD`w$}0x+0=&Qc@lw&q)i?jCOQewTYCfVROm|LE_sfM?X<-P|W*guDz7*l%NpUp`N>!8^YK5LW<;=FmMldM%eSf zr>E1_zDgiMjAmugtZUJwrPk0lWu5EEaGCv&S#tFL-?_B&sQ8m65vDZ$iv@4J!CdiF z_~dUiJIr$19OZI9@0C*#H}c;2< z{bpX(;$!S{T=1{tu1sxDBhc-Eg0yZmJs3=Zn8Zd8NM@sx^^a?l=83q)VEPT15lc%6<+o{fnoSMtWU)MYu*L4i8`bv(B96 zt?B!GQD9iK^NyWa2n;J`$n-pUL8ihe8XAlL25iVuz@#-Xajbf$r@s>E?;R{-($&%%Y&agB5F6x|-C=L?{nkHA0juT8^TzjiVa9}x z?8nrPOAF}YZ2w}82ConP;9w~Teik!+TA0=F?jw#6^Au$fCE;Go1dhBsCs}xB_|5;6 z#gG411pcdha;h{{{Wg@${#W`-6l~n3^(W`(^nWjMPHA-t#o763a!~mUjwWzT;`-3b{p1B@6rnG5W+V3HXXB(1tlev%Qm6{F-qPNR)6sn=O zW)}dhX}_7s?Xs3&9^dgq*A39tEt40eSDO0O>DONA^fCjt)Y45}qYb$*Wa;qMxoNOo z>gz+u)k>h5t%KXO{36vtgtdpVZe%CYQUt=?Uoe@BKX+Ez>u+BGj_JyWRinMf7f+_O zzp5ubQbo(mak3dNl{x1<+kCOmayG(FMS1B${#%Z_ZfWg&kn%%WBjx;*a{)Q?uT^16 zZW|)!MQD$kJ+(%LtGiCxInx+q(rVxG<2X|yymRPgl${ftI{_f}LQK3X;K;@xwT)8hNFL6@NuBWN&Ce zJYA+G+rXrIXZ1Odl|9PDem+E<$39tkUd@w5WqFNbvPeTzC~=B#$C_uCBgsfUCK+as z?}pbT)RGW>c#so?C}dK!{2*sBscBa@4zSQ&?-5xz5BnC%C&;ok^>gF=mX8r83hDyq zwMyU4454mq$C!b_)U<)d)a06o+urCwC+u9d95kXM78cd^fE6Adn5c18F z@*-`$CKV}?IWTlcANR_ICv}*J_yR`xcDhYtcLo6U;@8d{cqBSL(?tc|?4UG(+#evZ zwq&(J(Mda_eP%GR;Tzeq;n#GMCjF&(0~Jax9{};sQd9SAU^1qSY3sw8lZJ>-h0SSQ zyDlYMqWMDBu|OA=iqpXrKXSmy`i@!fcsgSeA-Px0S>V+F;p;7c+UommZ7R?LMT->+@VOJcyZ_c(%#SWz2}@cbC@CQ*)t(K zdm?-Ne`{Tfq!%>iQH*YYsa!HcP2^9lL?2}T-pFwj%P}nBvbs+gRxS3mr~EPH-ooz3 zx2!fW+0@6a76tv$G-NBP1kg@*wJSt!=5LBEzA0ILC*TWv!e?WE9;KGROsPk9&wFO& z-{^G5fqq%9k8F2X`Xx`&=G+K_*X54dCtLM0e?(Okq(i2SeE~*@&f=#vZ!`v;^?;Kf zDhP=m)z+w%w72M7>+0rxHysgH9WMlI-<{Lkw|I_glL~D&-o*L`TH9IDhj2)m72BY( zZ!P=~YXS%G6lDTw-_ctEGZ>3|l=dqvVhI~vF{^%11Qw@lUn;lp?{)1d<{5HBI&9}d zG%B@nl7pFtqkFF{SAP2pD+in`82XO76Z`b%fL=&=O@(g8xUv5$iIHJm!yyRjgdz~B z#&rp0wX*S;P)Uzrv#~vks(>)#x}>n;`d5+d*c8S|?-&GUe7PxGy_ptctK)OBE7~*} zSf+6ylJRWg-r0>XR17a4v3rWE`}s_+x{WitN= z`{#}Tp~R%5l6s=ydoRN6+Ei`pGn{(iS88~@k&WOfzN|50fq7lxM|g6D-(owX!O2)m zY-0@H;@1{EIunSBp!I;WW-XAQ@n|GJ2%J%Tum@W=JHfq55SUISN(URoA8XU{I&>Mt z5P-hwhn8aN0+J+j8o~!+9K-s2^$3lfhJm(Yit+vlU%qxv2$jthw%-0uu$hf3JrDSm zqTBziq~O}&sbl$bM!DDQUaCD~7GhQ~5^kLWAtxKhMIoKy$w?)-IFX-&0-N^_rPUS> z$RODnf>T>k{%ex_gBr}UNBMPKevVk<3nPzK8v$L#topHqo(zn*O20pm2{mgp5HG|N zu0ienyXkJojPG6sFjg6!S_b)1`>yeY(OYe6kONrF2mtG>mnz4NDEpR0gQ^H&q9c+?7;`?;JB!lBX(Doxrs#4lV`vyMox2*( zaYPn0^-x9!yWhx&o?e~&EDi_#o%hmI;cMg+%TP=NK4?!_$EIaMr6u~{=1xh-+#v0k zBGn7Q&X|;uLubHS)Ds918;|co04&rS!dONXpVj^HXr$Q~dv{2x7o6- z?rh2RWWnbP`>Q49^eFjv1tNuat86X;QR;OMlZq#(#MVbaGR7c!Sy>!C;tt2OMjTMd z0s+1!glrq^Fe$m1YV(M4(;knZFm*Y>qa{R(k740izx@x`W%cPLht_lvA%d^ixhw)4 zz9e$NbJ-cTrNL{a&cCy5P62d&|_#< zcI~+N-d=hDc0E(kI#(tm9b?;zXM{d;QyCmLl{r9xh>Y>dL<*-97D_xs0 zro6q*EQv0iZnfv;=R;h3;kBXS`{D|pGf@Sx)q0sAKTB}vTUbw8N5U7KH^BJG0Z(-b zANp@Nr4Qu|f8TmA$Rsc)-PpSQA|aVmoys(qQdppH8szh2jquW7JBo0IM-*E=4VtLl zc4z5KDxyQEgWgU7`ZFy22Z7*0K8xXJ{Uq}(PfU2gJ9aUzUR|e57VCmkQE=KKl2-oK z%J<|sd{0m`T*?1nn8puowf9-XmwCRQ1wb@(boB}+Kl|T3di9uHqS<3gWEWdI7noR_ z`~#T4kf~i*VX{fDV*yIQK}ZoZOcrFYH1<~3+(RhutogOgXN9~EQWI#_=C{4~`H`os zP8$b^D7fslu9{P$X7pqs#gKr zEMrrV3!5G#yoHJ{#NBRi%NHW`7kr|5-E z(SEdPGF9BRR?jEm$OmuszZZ*1d)EWK%xzRJ9;(#y^MtdH*-c@4EM^)PZ${QPAl)V= zZckDA%!AN=?@ws!|cd>6mp=M39vFKt=p2oScG)f9o( z7{cK0n0(xp+WX+yQHmsbyBQWZmr|amUe-=QdB2z@)jI)+sIx~eUcgx0@z~QTNin)> zjQmq`Toz8y>lUQaxDB}B0uG9S!V)Sa9fLvoIt|osx$Ig=hA9?aspaa`+<}rxIJ?B` z1jE;CZd=xJOn{Tu_Zp=`*4y8UPu2m^VN!v7l}5QXeQtKm(J|m0S+BHaS|RTMW5o)J zvL4G#G%`P{Z;wvVpX1zV4NlGpC~UqdMsXZhs#hx9*RdL$73mkB=~qH;^^GFu?~_lh zmvcs|bp^d85lQH^KQ|rsahLd7J)5Q~Zt7iZB z`#08(O5X`|J_?6(nSAe+>TbKGuR1uPcvLE>bMfMS>vdA9l~WPomAU+j8VNGbLgM9T zDv8D^G^yA#2JvSC<#^ntkRPb`$rHUdR9x_-w1e>CSpPW6shpu4RDy2D3rV8evUZLq z24Am{cx072WruBgG%gf!%M{jC@A)h(>RQI)KP2EV0hoK{U=7vF*yNrH|5)qh z9w;YkO+Mzfv~96H9V?OgIM z?wbDFNr?^tv}v-cs-UZko}*YMe)S`pj@_qi)iGz{0*>5vIu*vaOsuxc%_7}*S#4WK z>aFb@OAOOu6odsG&^=Lz57@Gpxyv5X1-F%pYndpA)tnGh7||xshm}Xnx;Cb{LPWw+ zfy`R|dvVpUWxp^o$GL`gri4ddthYQ-(Tv-%A7FUh@1ML;xWR?Tm8nHM_Uh6pRlS;& zWjWMo*)YMJzz0HJK5|&tuiot3?%xw;eRg=>p^){o_X{v2uuVGb9x6ytDa&GMMmIVfxaW0NR7<(@4 zF!WrsO|Q?KT^vRxFh7-bZPcT_jQZ;Hq8M6S1eiAm%Iko1w4wVv1$i&nD|`DpJ)C~v zlpYA!j z(qWDJJ?wu^pyc1TiaHZRn2CQ&OWTJeo>>!kg0!}4L&u6ErVOLgG`V#60)}vF`xeQ( zY5P}g(}i2XzSoca4ECN-<8K6+y=wzRQuK^RtuKhm^?cK(X_n4&p?%u!bl1cFXCdcr zH!Jox5OBb)>b^Dv02Ec%qnmaiNhDxEM@WMU0g)t@9Ns>C%F5)8?!k>O9u$vE^y2Pg zaGfpQ9_^?jcWS*pZ_|M!I)=4FvV0gA77Q-7&MZcy`O(N`w4KB+xx}?j9p_JXG3{Rm z%qvy&e^bhoc~!o&N(*@=FV9K9NDPV-a$jQt@m28>5q`zEyARHf4xOOsF*%AXR^#O! z36LWKmpamuvNv>~@9^s1w&8#O0NG0^6d8bjVs!Yg%E!wq!{AkG5`wRX^?VISF{)bk zc#dEEp0p3k$i>Rn;he(gt^bp;qn-Ja$@|}|F0}fb1QBV+e=R?R1dOqN%|0{)`!2-6 zpYM!`i15&ac!Ks73?hfyhC4d(dxJ7G&sw(dkNri6R?bP1m!2trxxJ;Af$BNHFe$O% zv)U9}EtO*3uB>?i;(S`^1P*z!PB^>*NRoTmqz(D-(W${%S9&4KhnhPyzqV86JbEo6 z`U&W+_TxnD*jwEMCBFEY(8vncX& zOFp-F2t6ND0gs}6qLFJSQOQP9hJYn_gqHAn?h$2m%5k{G@Dw9X0;xK(VTxTR6@58? zYnu0Eb>VIJv}v6R{r9cYP}*G{E|J7bjzs^Z`vBR25HdDsGPY?f=bX`G~{ zHsMZT^te67H+5~W_5^L-EmLG063uQ*-{56wmKb`q@QF4ZuaIHFcG3V`!f$`5&A0zB ztxJs7D4!Am3Y00~!pc!cD5VN%SFhGtalTzfLGupIj`zeba3rUu6Kot!(`|U+XrPvG zJiI0s)L&x^NRpG9Aexg4tAa&rB}xjc8(c_~f3-H>cys(nYDHyg+z9 zh$2oK@+DkswCV1KdnkPQdH$=SO4Je|85}0{t7#lPlN9*Al+mhXD12E|2}1zZa&$d; zt}0^c>d@&kr&`Z3bpqK%@8Wu+VXzwbvnEW)V-C!snkr;`{Br;XIBPpicPrp>V+IAz zkZeQ|G_4n>+o(!!e~#ga|#+KTu|0#73*>ME=lR5P2SU1$+v;r0HrhyI z$1R#=;)1P(yU|rAab^pPwx#R!ZsOlR!t10|>c5+tXU81^wzZ?P;imCEy+A+Rcvv*% zGzgP5JE!!%UsSyOG^!e@D<&8u>BV3P8tlo$8>+`EB6;k!{(Z%m6QuLXSQ|HOGTowQ zq7Y;P2{#fLmg3<1Qq`evyvu>f(*QRyG0Os=sIDP{dUD&(cMQ`mA;a2m$j~T$89@gX!V#G3SRIC!_4&$d7pDoG}f^ zSj;vneokdkv=mr3FG&B=+@j;&gLNiF>-L*^^dF3GD%l2h9^IIE8|yi@Gq1-p^o<(J zC4&Gnpj6$D(=nIjq`&I)&2imF^-x=h92I?|Iy-O?6=-;zD}JpIOiK^euOY}@9;5M=Pit_8i?q0N-{*-bOEKW+8KOXo1;eX( zu!)nD!!jJXE(E(;(1<^4i<6B^#rG9Yq?5Aj^3x+SxJTl21q{9-u5)lHHh{>5IO@Eh|JB5U$I=eQ+YR2x3^x z$$a7UaPwQzGaV$B|APnJin!RuZixfqxhG;tBz2EVWpF?svv)?(y5Ne;q7h-z%uvHG zHv=)80)|IN*`MP|DVmW-D}lX9i^TxkBSSEBhmkXsMMt=v*|~e=vN{_)K&UJP#yP_D zNH(S3ks#=NQ{rhlYp${P!+rN*HS0~OobxB z7)OvGt1x~~$ks$&yVmL`8ZMzzQex|i)&UH1+U||oxr(0Wq=3SW2AvIkXo2A*Nm4`V zZ4Ra4Adp-f$&^_zhin8PKJDl5Z;D#+>Vmg=zqB!`qBC^{>vTlt1b}fVHTK-d&4hpi@@KMN9m}Yj4901ffORW z{t)B`1XAiBjN~2ulx6^slG1gr-aIYc;<2=tH4mRgQln9!wS4$?9XOM>^2UY=f8l1} zu8KUq^4a3@3CB&!4QeKv%Ja1YzSXX(QkHh?WD>P78vw2L@q_`*)K2B!)I!?M3sVF? z&2jrfI21Z3PEab@i;|A9@9e+ro(LLw%F)~4WD?nV0&&viOVktFJqNDJ9Tg+Hjls%p zT(tGHO!oLrIu@Q2&p|^3+NJf>38{b&t7u9Y=fM4@cFsH23Puft8AXF$Qs-fOK$gER zJ;L`yQqMxYxHn@hZyA+D#`=+jz za^wCJy0_1D=e_Kr7`a9)ID0FEZUq6gPO0JXffCzQn*rx>nasA-xdZmP$>C~ElG0K^ zyT=ucb{ZPy?7QXLzb+FdSd{|PZ3DgXwcHWj>>cl?ngm$Pimh~K90YIQ>msoWRmP56 zz4;J^6R*Uv7zHM^j+L}Zd{^f~e@*=wb04uTo#qQy7qyqnjAP1!PD!`9Jgo z`TE<8zDuw1^u5`nV?X0Al|Dw$H-go z@ac)zCv(2oovq92p@O4EHE9CH-ZO<-H?b=$_M@Nw|G1p_JR7cFpc-Gv~o{$*smq-=SI(&;U(go%l70 zENhwjZ3YIJnF-d~mwY6X?=Ys9!!>;m{93Ob1>+U(cTS|CSV_sdfpS>FHRI zvf-^u?l5@eAYxy0f8a_xdP6;?i(8}%7*vSe$uj8I`Uk_iS^gq3CUZ9E^ilO=gZVC8 zN^7&DDLPcw)h7Q}w|U71==uH0&2+5Rw;9ugo>jaMT7z$~U*9kwh#^~FdDTFCdLz7z za4kZ?Eag?}rQ4ln+Jw*iKmt9(uC$-h4J>+LUm#Ii`U%fZh%0;u-wL91>wniaP%FXy zvmr-B^z1L_Ag7i38cz zo&LelJP@Lr3}}}l<@l2WVb}j#B#Y+g(k1>sHqpcPXBfPHV-)}DUf#b2Hwg~en7u>1 z2Y*!JpPSccVx!(NVJ+PCaq+ORkL-x&A4^E&-zoYw1m#6?(lS zkZX!&8hx}PzoqH_l=EC+{TGdkRm&GjA89%N^Zjr1X)vH8c-1Xhe0D*eDio=vRN2(h zTnElm{NX6tep=ld0xkGS6KbxEjdg3j`)_hv$%Q3;sFcj-RT}P8uFiP^fEiG73@;>JioH zMo0bEGm`utpGv3m$ohQM2Ue`zA7nj3cf4eINUN4^=XZShb2GK9(yY=27%46P|vx}zT`n5mtR@mqh zUfQtY5Xt<3-oRDQGxmB0mV?vbIQ#wb2q92FibT*y$J1do(_(Z{aBx_S76W(v-R@OB zt=*Q?XWd;bEaT82)p{sHGv>5gJ$AcCn2@QEMC6f!Xo>-QW(G&G+h@a6T8QDy+wZz* z93DeCk2rrKZF76O8te-rG}CQfDc;RP z-E(Y)bK_n%Q(w_4w}vdQ4>|i^tdN!JcDE0X?(qLzF-)k2rs@zs(5J#*ckdSFm=#@U0KYc82~+b19N!~Ntpb^myr!4W z$NC1Hzz?r`PW8oDKUqxq1vhY2d2$a2U~%W0SbiKU+k_qu3ng&?ab zxk^WrBR=MEf4l!NP&~QyW)HhNQ<e^6Zu4v^b8R&PIa~sO~cU zGp5z-OMsuKOI4i9+hYLQxSQ;D%u6a3a=Biizo9W z&`%SUt8L^&LzYf4#llRPHIP!sSo73T^13hN2-SKVq)OiU%1XZZQm9X9we3E_R!Fk?N93+Ky))8F0zo?7yU ziS6hnW%?A0r|QYYRj{~t;&9p6uGXmx3>k9{am_3Z)kWheg=bKDSOuQA@0z1}X!I0{ zuhJ%sTlfc`6l$dBStL8r?X=$n?%qTF8m~D=@AP&ri|>&CU?}`*PG&)eb}J?_;goP<>dt7zeCy9&D>o$Xa#2HjF7Oz z3KVh);@@vthY;#Tq=6TXRJf4zviW4(`(hMJ<0ng}45U*UU>ze5d%u_)y$I*qNoIUs z-Zfz~rKfR+MQu>Pv3$Jvq^8F#f3##%)WgnoYw~#6cKEE)HTJZTpGqa!FbwCta#1zf zK4T&=MJW-E?pQz%=yp&gEmX-ZrytsSQc-cylU1v&OGT2deHQ`k6ky1nphx6%Ka^n^x))L-=UeIm$l>ztc5-Z0j_aSHeI zcK-(hL8Ai9JIsde5@#_^$4%|UJdH^$Suz5So&{75)dTaax|RA9KdR4VbPF>nBtUJC zvp~A5MHlNNRyOR#_TR2Uuj~yCp@@=ILD(VrX+n{ZQdWawN+9-1Z|(CVng-+D(#h|U zM>MZtR_yZK1zc%dABa=(Yr-BQ&+JqnOoSmp%*=kW^!s7#+SOzwoe>ab}*n7l<^}2a{^g*?D@*@!b=HN(wKpu_S^Tf z?h+bsZ;}V9yDW(G8`g(gW-nB&?P##T;&3XJ4d+Wlk((% zm@@Pa_>6X#t~w_&nNs}*?Ay<2`B$xvs{%;wi;Mls>=NoW`ZQtK{AAwH6hr*SC~VY% zZ#^4f!NFq0^Ea~2Bz5KA%|$F?3-w8OuwsMGY^z$Dzec*|msG7+uUFGQ^+hNveF5mp zA?dj#HGG_??d7bRzG*UcJ#}qm=V;L>DdxJnmz=?g^47JjJaAg~@OC)tOm8opyaqvc z-CNjr7?*#Dk-F1FOd2KQzs5S?OC?1BHQAllJh3M^A?{Hi6j6uv*yx$-QvjPa=`7_(g^w$J~btk1p-2Tcc&BlQ6$1m9%2!;4(Kh zn&z{0XZVlCimccac@C|vaZvqYfAh;mfXyE^Q3^Kuv}hR@DutJoviHP3q$gs5SCpy5y6ymy1r|qdI=62H|EZ z5s1P=DC00Ig0uw>5|F0D1)OHf#8Wx1%cg!hQ47X-l-fkBPtrkxCEqDCtqbZ0Lm%>sd{QHLgC6KNp;>=#P3j+?SOajpf(cRlJP z+NBdG>%lPeqVRaOSjo zz!LpIj*|2&_tT#AP%>QJMFex$-eLL+1dZJ0kC!|KYJ5jh2qnUVna>76Fm~{F*mLgB z^cQr+x`j-rhV?H}+5;9a77i8x#aX*{!?E&g(rm~9k0krA&c+e1gwMqnv9Nw}G94CG zt)Q*rI@3`-v=G*llvLTwj@sVsb=-l=Z}+M*R%9l>$!pZIrls6*r?Bpe6M;f^Y+g$J z3Jn0iix+&%%x6r;w6SOsn`RrbXS&ZpQ-$=_`+F-&SrF@^_?hP9zivPR_%A^rQ<>mWIHWPrL4G^g1Z8HsEd^ZK)@b`#tJ=9yr;qeRYv%SNw-%$jj54^b2MmAbbossWeO<0;FfO(v z7>Zkt(qBV@xSi^XhS84=+BUbhXhg-@Pq0hA+*x;X8;XX2Jo%ChOTZJ9}2C;J1^ZIkVV{Q=a0EslslPlB}Lg zbp9cWk8$Y3K;f&dXiq0ha8~no_Bx!dC#yQ2^=y;a1xoJC)3F*A{YohYbiyo`lg45c zD~8i;s9s6lc1|BtM0kyObkpvtYANM>U7bdip|w4_^YL`&3?cTbJ8Pm0I6iAoH?yR?+$m`ZXkl>Mr?~-ZzFrCD zI$vH7{I=fC>KZ-ur~mZ@%a9JtVK4Uc+{j2j0n_ZG>h-wzp1aD?+qXd7pCkD%Hlx7K z@9j6mVk2f3-m|Rxe+o7iU-8fMK5m;WAJc1s(98a3IPDcHqFkfvrWV$`@4gAKtUqFL z%qtC`A5E)sTo6jYy|dtNw@&Pz6Uu5ndm2kGtUdCXp=Wx!^trnW#39+T=e}8y`>LBt zd_O0hoHwP-9jem+iE&2csEyGdIxD!XT$>j*8)dBtw4D+xCpp*DPflAa;Zhm)-9{`PCtEBQHKQ7im;{9bu1 zww0h{=siN`qv$gT`lp@ z<*&j2>A%GD;UI>yReg0t;X89(kKSB1g&+o^x_~_#d!T1pz z+%AGXYEB-1+^dVfoiH*O$4|~~emoRl9Xt}ju+vEW(^zv%=ZFEsIGJ79{r9>40mc4T zhyIU8!r~`-H^89j%H&S`E}&fxWBsL**GkgUEoWq-tDEp_EQWRzvtR=u&cwIshP4ZE zKd&DLPyfYtU_6d%SiCxFjch<8>$d9E(FqldQpNhmMKHq5Ogp7}1wC8|y;VNf_wxb4RWZoC8k zoT@h7;ZkM;FAEN@7fYtPhB>bX=~M-ARfRlfX*I;(Z&Y4j;(ue=yqDb~Te}13zD*S# zx!k$7o~J7Fm`}wO8UX3SOCW-*%wQA8e=yFZu@{hSxq6AaK`#OwWVI51S*Un9766CO zCe(8$_=cs6XGZoWyIJOf1M5aJpVlwF?N{GHPzk!fyDzsmVV0`DDZkSTbhxS93}^RJ zNwSi-sSsJOY$}3#{zUeg`gW6|cH`N1TxawUB5VEZU}!mrWdn+4fjDBn4R@1Y(|m7F zjRe1L`|%*?7C$&;0UmjJ^>ztVRg4UjRT zR!qzfHRyYzrZY5l)03I%=l|rwRd&9#CwUTduYGr+G+wVd2$@j|&mHVn=#0S>u$`)r zwrF$?JZdrlqxlPByue9|V2?hTR)i!HTJ{Uo6UC%taBG|-J*JI_y;RT`8{l%DL#=Bl4 zR-|k@b>!Ig55`YZ$QX?^RXeG0QTstvOc|DZp8oR*F{XgMTzn^F^XlmlGd;rxBjc#l zoNVJLdR;N?@|!l$$7dA097sKvQnkF^TUrkyya z2y0BH>GziRMXo&CTcsXFl^tl6J%<2?xmUy}+yizMdOQZ|;d1Z0OI6-mHPzIY8%5WX zV}qk%ZhGnnfW9WC{5sj6Gu<%Q1&zsRE5~DyaM3_>xgn|#I7!E-sS*nZ%z!#s!j&iaFD=W_orqiU{%*{t8 zMirx=Z_L^0JS=vm5*uqW^-eMcHKvjMQ-h}!#*Zme_{rWu^QSg8w~~6eZA~8tZf&bF z)EmF{wyE4JRQDgI?3T*-km>NS#z9><WbqB3_a;yNq4|k|gf|xdz#Nv{f0zA}p9Y{#5Hz|SvuxX|q>p+8 zOdZy;&Lgq1GNMO8)vJ@2?g`F1XFKPW=29SX|6qVo!MgSz)dDW(=0m$sJ(?hu*s8&L zoR3=R=_z-cDtVk_{19oL;jA6KV%`E_#MGG8QZy`+PuOehO248+=^^8%d-it^S>G~> z7(O`h()A}B3`kXZcRIGE_Fett#_#<-)MM<8qRpvOqMs)Ne}u6IGxf9nO|sQ!A^RN6 z&G875LfOQfar6=q5>_Q+dSZ`Z(F=dwIENPTQc9qVd6wC&+cWvmSu7C5E%nH;rkRoa2BY?(SM>qi|Tz z$gKS}ZdYCT(O_(Tg}Aa4mhyR3^mV531USrpx27O(hu*1&a;E$Zq2{|G>(H3Y&{-#Z z4nVD-O-uP!hcIrTr;aIhD)SJN5I1X`HJ=U{mCiZbvmcxxtVz$|$17|6nq`}v103G* z^&|kuMS+wOqI=HW54(s7W4h=25-W)t&R7QD=g^2uS|9 zZfX^Rc{=;Sk=Lnk8`LP5fPaV+_3mM3RE|VqcXR%H9R2vq5wU`rWQy@TLb_O$Pb4+u zZw2hF7c!A`^PT=$K0X>fu`I@rYhP^U9K|b_h5k2!`l2_QGaQ67G#sAwReK>6THx^} zel(WfkxPH7grmFhg=RAVVwIThYF`*_I~rS+;Oz9x^|wR(uDB~@QHEH8X;}}^jy8>F z-!8X85`wfG3XqrjE{ubRX(aa=36vcQU6AXUJW1Lfp&E>DeO)+E`dwHYo-b);EZ z5w%ORHNwvn2dBo2{53n(E&fMqDrn}%!2WOfV&h6Sucn53ZEJ;7V9^Wb=1xzVEK%j^ zGII`Kux>efv(zJ0`8&+sO}#1RbY9MzGOWn`8D5v|xZpSBAb!2FC(k z98Mb;b3Y$KY(eTrCmyp?%PFD>DrZ-KoXKL71G-dtIq}7M%1SY84|gA3txgSpo7%oQ zY9BzM(CR=#`$~;sp1>q=x#Gk9Fn;A(xTlbplZ}(&;CRZ!5Lnu~Y#GWvoAYGL}wv%{xA-I;Tw$^bf{;^jOZR zaYgin26~Chh!frKC;4V)NA8qo@8=D-pZA!N3V(?X%1k+a!(Z0{mTH9DvR@#JZqyRX z+abK+-S~nRv6%nWK3&${h~(DK#5xsR@(+ehoxa-73X9lLSHsY&F7FajQS!u!auYuM z@X-cmv{fnE@17m%zpJ4{+;nQZgtgq7BgQy3tC%wp+cNCe5I#P?GoE3aE0(Q8UhtVw z6V!%LA7g(;K4e()wykXNs;G2i88Xjlo&la5FCDKm3}?O6H=G+{d>3Br`KYY9NVLy$ zrxgh40Z$hGP%z4f9ajr^{KzxYfKV~n;)#5ehxcU)MQv) zg!-vV%<{Eb8D@9=Lk(#upB}+l2bJE2z`Mj2ClZH}^kIMRD{tKKkLiq_uxKaX5ZK^d z@#Lk;FEnfXq(G>HUscReKtBJpU354QQh%tX5Zld&5FL@>`F<-x!#A}1DKV!XmYrag z*2v)otba=?ig~l3o5d0G^;;1o>{BO zi}8#_Oe1%=rlsxwNLj7m@%T!0bPOid2tJwd$W)+0K5+)j=ite9C+?=rZXXzwOsOV| zFxln;BBiJf^eu}p8fv&W3tsc~HMlv^TqRj&m^ zcbbbe??%F;)pt^rwAF))?78jm9o$m7yxkV(m|fRuA7tg=rO>48ZEI+kX1(qA%^!W0 zIq5j9G$O7D%p?_+uJ^j7x*F4VgR9T{qcS(`?Rw7d27hR__dSptaqOn zGNZt(@o-~{w|D|-ybwdVdjp@}=P_2wml|kgfx%Ch0K}G}H-!}PB{4rap2`aVn7f`} zka;Of?e!#=c6;K75DqLV0$2A19Aflgscp{3E%)1}b*{pj4hXta%bPUMQkP9;u8(um zi7DTq1&`5}KVWDG6!Fp!08;oMs}KTut&FbTd_;Zg_$Z~fZDApJ2E$1s1%p-NG5pKj zJ^B^a4*&8D<6*#|g~P0~`+~po#aB!PG@#-9hkJYaeh5wyoGIH?L$ov@7bo&bBEcx( zpWclJ326QI-=@vK6`Oy@z%QRih8Q^Sj?SPTaUEU5$@%1ex;PcW4lRNuFw`*sQ zHw{y88fAXsey<%j;6k&2su*Ss&OH-c5p#mzsC zx;;JIeDwenC2s7-<>cEKG`+mk(C&yXF_j1mYbI-Y5?GM&s4YjWU{L!Y|Fq`5BE?^TH&>mV(58wX zL8@t^bptaaZ$_Bs7K!mjHyzb_Q^~8(q@7+FRdC#6OiMBuQ@Xnv1ZTp-80Vwg8R z!Z_&}iQ#K^B^M9Va<*}-1`b1`phhjh zv`lp*rRs^h+=SrN{BodyTpx}mWnvu{Q^Oj`JT%$k;B>CP&^N~P+>e%3JdX=qxU-UG zdl(y~s$n9U#W~}f6<~H=axPe6ZKW|R(8N*l^OUsxrYE~!znS4DkW?9I=!F^isF&F# zMQ2`e8^5WLYIbfsKB0Hw0x#cAIo{;t0Yp&?@=u7InT z8f+{}fP31GC zA+_J7a&}W&`n~#O+Q(gR(^q?pfB6SGo0f|Z zOWXfucgg$Ez{dC$rCkcL3?v88m5LtFCg2>SLReGszdAlQ&DUO?MgCNy6f&B>+5FwH za$C9Tk~X46f-A-nXSJ8mcB^}5x4Vg2{0D=)$A=&ZQ?*E=%ij?=OSB;>?!x8%*3r95 zbL>?F>jM5m4g7ZXIgPIBN$wO5Tsd~Ve5=RssebnF$yK1>bbZ#q_w^TD=uYbEXF>0Z zD&sOOtZy-YM6Vp&qN#Qyxo2-bGt_egD*NGyHWr`0#R<{oQOqctpZ^|a=%}24d;-E} ze@8r=32?$CThhVo4IM9-Do|P+TFTnl*S-U+E-FMx`V% z3}Q!zD8Enn%C%_SNUri04=gRUePpt}FSHS(DwmcWn!@pq18@sis(IEm)ER_5#v}=2 zii2hXq#A{ghWtjtrjCpw>!fLjqZ}EZ(Yormw4=1y0oJ;Dr^aVszbN^^CHW#ksu@Fz zzWQmRT%jVsRZ`Z5eVHhS3|KWx?2?>b4ZsZ4L087U3?Hi919kdw($jniThA;ps2>$= zmDJM6$?)`x^=uW((e-cNY{UcZ^zN;lQ=CURDHyZW^5>=>*njMEjmR5g8)qRrzNHIB zXq)vrgf=c66%Chf7#JoMtjW?27aPGCX_@Cfpzte=Bt}bGzHLw2*jR_Eu0LY6*@L|C5#NL~E$=q0iI%v7y`4^7gJ7X1pPdGV9GCgZ}dz|&TNRMT&v~9wi;lA45 zo=N#cvGPujyn{}$VFV-Zp{Fnd0FOSxY>>!$$`p#knSAk%V~XFnIC>$OqR_Ef3s6RbkP?NcAeF#d;ZhGlm$c~kI+8T``ISGZPXDHGEYXdn+ue7RJi*(K2Uh()?edfi# zb*7-`qJvLP(HU}CAhc1H4hiE-Gk_SZ@QYHm*Y~VT%Agb`rMYQJdF(v)zpG@7HVT*q?M$HJPUpN4l-2 z)`HXtoeCcV_qO&P7tpRIsg!^e!6ra(yp31Uuc=!h1`2n#vfyS$9jRqzDisc;ec?k3 zoJ)Q1_-XmVsO}72(|Pb+#**7^7>Ko9i{+;gZi0|ekf-2QEy?A3^_LA%aup>8uMt6JUGs+Ps!(M(z@Nnbl6U@mv`!=#XL40DAxSd zi#k7%>I$bh39<-;kB4u_3<~@@?Pq>!5*^$ElX770ezsaD>0a{UJpL}ZyhfjC$%DWq zo#!ThS_d58YPV^PmTrReGbN~{yhR~C4KtMYL>jjiN+y}U7c84eI#*6W@jX#{4RC?; zlgEXGSdS2YTUNUYWmD4n#Ktg2!U^`}+g0Y-%d)dq_oufD$L(SJ$JY*6UOR1eBgC|y z%qu`pSY~9uFQi+czeY|OESJJw#WB^edD#RqEL!F{ET7tPj>~&CH>IYiTKmlLFjV!X zC=`pbSpR#Rc^CHV?|_EG;s1}bw~T7Lf4e=QEwn&^LU9TdcPQ>sAV_d`D6WAbf#RhF zife*9gy2w|pe@C%xCALqu;Na8a{uq=oSA24&8#`Yi>yVG-&%R`%gT3O*WRBU@#oW~ zduS8TUMK#~%_RvJsc~CrLAm69vsT|s>A%#%DN(-6Ly3pItq` znsIG+HO+c}*!D6T8qFl8n) zK=Q({71}7!Jh(+EKvDEyG&_QYkh~}S;h1i)BCBKFJ~r=W*k4O%ig1B4FIH23^^f6 zEzLjSeh!~>FGe8GB;UJ=+??zJFpAm-Qc&)F=6;?0tJx;#lG@9va|i0`+Z@QT$o%R- zQo>ePaYTiq3w_V+hnj+4O=w0erk`t-hqHT69kKR~3M`G~)9&74c8=(Z>?L z-MO?#)@2FVgqLAQ$5F#>R8kU$)C_Dd%Ya+zWa=Rtd*^7w`LcNfylR8WHpKa$W~LoQ z$6y;>MfQBb&c(Vb?<@j{a??~*Hh^9)q?A!6aP+vnWyVZrQ-xAiSbllL1XqkzQFCHu zX?1LO7`a?{-`kh^d9|_7cIkSNsc)Z=?q8ON z2vErvBO_IbY!wFEh|MrHGP5N{(C|Bnxy4ZuGnSHQgd0>Vv3n2-e?hW!Y{Vz7BUSvI zjeifl;@kQD<|6{$Q6xGAaWNFZ<=$WY!+v-;ySo9i@)0HUIVF%AmJ-hg47Xa7BYLdx zuTN>UJYBD{lx0(d&S5yFnJQZH%AklrukTKn_NV^-uJ==tXGV$7y1;0k!3_ph+16L;KQP^ znXx~7hvEDCtfSL%juyl8>sOkl2f|tk^mgw5(2)PuKK@Td`Ty`A!5fU))15rC|9PE9 z>65sKo&B5D^;5oZAJjw3DxYiQtqJjair>{Dlw>zw}@A|F+p?18RdKJaN_ zv8>43T$~B7_s!~m+rfB|1o!IOiC^jXeU9{4C7G>=2}#SF#`;k5zb)(jxqva~KJr{9 zytCvl1{a1$(ury4Da%gBm&N;FihrH0xS);%G(&%@X?AtoZFwDC$BBWFl|{&db$2*k z)s`D<7~P3>=*n_h>`gvB#h1YQr1YZ^6te9BJa6pqU@l)@-P*Z$VXvj;At3YUm%?j? z^{abn#rr><9}IT)>u2W|f*EYuhHva!GvI|;vL&kJV@6M!DXEaQ>?VOKyfj<@TaKQ- z#HPh3VNk5Z6zweG1l-agYL8h1bN+gZ$Cw&hLii~fB{%9aZK8Fd%G3U;gV3?DUh*#~ zvH8hjYU@0-4c>!NuTeRs5<5y8>N!-L+d>2)?P8eFYk=KQdqk|?OPe>4O5kU#TbKCc zPcv2}J@L<{I>>fMGlc!P?i>TZ?u!zAeMVdDYqpfJLL`~Lx zjlTa);a+uhSEe@_Nz8Vq;C)t#FB~mpM^ui`vnbFH6czF@c)8Bbzz6QzY6=T0^L#DE z!;cAk<;S~k`)*O#aU6a}32j%~3i)vRl=1Y8@w7s)cknL;pGw~S?8S^l5qi_u1)Z#| z?Ndrfr_}#sp_GAX8s;gx#WF{ZIC^>(!mrX~dv94f-MU7|Ee~gjd$;J9a`@dsJf_{^ z(B9ULja%6?uB0oyY^(&22eMyehdLDfk(m+;O&&GqqO|8`Q(KXGq!&G)Ez?>E&@#bU(wkf32H0G~pYvm`Z+bX)-`KCUE*kc9_WF z$7eET++dH9(=Br7V_pgK%AuED%qo4xNz6oMgGqHS*<(nfz_?W^Z%WVENW~w3Ihr}a z*+^k9mC(bU8FEQT1jZa=*gP^DRPS3w76ZPYjYNY{+kki4`&WkM?0ZyPiqd)0%ATm! z=w?g;*ih@ z;bV5Arg3P_V*8)|Et-(bh5PBo=dhRZyEPis zjPJ4BX#gTO!(9v}hPt}Cdv$yfewq?+q|xpFT!r^V+|uu3V%2C9(sB8U)k-;wU|V0wmt^6L;{9cZlqWY%#)Y&7>N^81N;U&08Yu1#jPs{>qV zwdMBpRLtl&V$2P*^X`bZbGUXwkykNW+_%MFx$AdktWDf!L)s>arq9EN%yspuwnvjL8o-l?x~*6Jk$JQV8JZ$g zuM~;lOe%?c$y81eS}^v0v-btu1v11@ILb`xC5e+l6c2b^bMXY3{zZ1LnVp+3$=;qBF(PeSKldigx$=LCFhx@No-O1a!wbmoeat&<%n0^tWJihOs(9t zr~IMx{2V1a5i*3P=+^w!sQ%jIkydmQ05>Na_Vau*`T#5&E2TIn6N+{>=twI3r=l8n zI=dylZriz@9+!%o41|VXR;-B&RwxRo^9Kb93l>fJ$Xw}55vK>Koe8=npV zQtETxZn}5f^Dd%`4gO-Z>BM)fnS&kc$ON-@RE8#3L#Hf?aJQH@TkLcM8tHg+Jh!Yg zfc6|#isg1Z1z=tl03j(f_-2fBC5I(nE0Tm|Z^qbz-F*QSyHIf}^nmI*Lf^Xxu8%IiAazPD)6VC z`0a!>eMU+gLkI8BieXR#RwNa&e~Hsb-OKONtdjGtEsKsk(8lkxV=jkrYmFFfV&B-YAJM{&cTfyoSVUy1_?sOQmoA#c1H*P8r@XH29s#dv2kJ){9l%mfum9=S5Wy zj_5KRKXCe~Puax5cp?XsiZmG4ID--}Yw1|Fn_MKqy_1=S{}%(Fe)g5C8CTOH(lUA^ zA|rWtlwrnM-SKtRYMtj!Z71bW<(HB=9*7G-_jxqmUX4bHKj&1;pGiLp!#L*y;%XgzR zieYu@ul>j2$JoMZ*PpJ~rVQ;QxnvUjfW`KY`qHP=3gKNWFbN zY@8;@eSeXY0pc)<`6a>G3p4{JBaox?nGzf)xmMG5*Of#5{13AwTUuD6ZDDOKFolVe z>O8vd2y*=$J*8P=fh3;RSN(tiXP%hB@liXd%I7;tSv!` zUlWiEj9S8P!(5pwEZ0`&O`E;rZ8v8LwT&WA^XqTMa44@H7ctHMZ#gXX$XkVfZ)*I-@Dx3C#owI?^F5A|HBhI8 z94zWCN`qy7JKWTG{aL~4CtsYyp!T`0T!Nv{4b?^;q90mpIv(_u6vt0UVrZQ7Dlqkc0(OTa;= zcnl_s)(&2jmtO{s*9WZOED8K}>=B4BiENNen_9-q-1nYe48CAqmHJV$fTMZLho@{O z>O!)4V>hS@`^V-Lg#+bnUuVm18iWNG)fvh%w7ci467r;;-OQsITJ z%JAJ>`aNFFGO=M%efQdw1<>xoqE)x zqxF89ZwJDi++H)P22Tlfm*j&APRSBoC(;o02CSngw7>t1k0I_gz_m;F)w zIVovBGZ+(x?ECqzhnEhyJL_k>`}d{_I_IaQ^C!ws<0L6iPL0~tcD>ghPEYT@OcSS( zPU9Vuc>FFE)^RC@c&$~ zK$JlOeBvKTW@a}kIUWAcB#35O*H2%qZK@MCs#n&At6|B9BBzF5 z=))WCy7_+Zg>?MFbnm=8@8rbr^SAzw=*F<-=2`uPdE1rC{{Lh5i23oEWL(q^s$Vo& z*F+!;L$3XbHLm92*_n)+;=ib@$8n4uhi;l<_pTv3#F%Wt^h(wujfYv0f4xKL~;PJKNe zBdp4$vE4BHIwxog^ImDiG`^(*okGvHo9xYE+$A}Q?Few67UjHPcmPTX1r}T6=H(*g zczHFkPv0}NjJ;s&@m9)`X-w5kd`LN4ig+c@PpaV~p;5+FNrGm&`4t*8us4xR()i8n zLZ+%3S|XmnfV{F7(Rvf=&NdN8b}JHewO0axL)qg;9Zg>Lt#Tp5yfxk&F$>9+`>?sx z%>^D&*maD_Y_lGW+u7)Pc6|r}Gb(eLErvd~wy2LX)XeBbx9w z(Ct)Zkj5|3Yx4v6(=D7Sv6!7@O~oXbsnpaCLs`;f{wuKjfNyk_4ITj^oHZLZKB63l zNm=cYX#sTk4fm5>{8YK*l=W#_*VToV`nA-Gx3Zf!RLuO_T*v*Hq8bGYt8n^GVdQUG zl@gbo%5B`W9ryN*%VX?*x7;kWnfHM6gQ@})2g$?*aH^!XV~ zN_>3Fe`O&iS)@tYu6kPaVyXRnu>pmp!I8zq&fAH4tnh(7`e*!4j%Nu((6SQUf|A74 zNp4eqwCyN>txv=T=|dHLqs zH`6Fw7()(mX5WXAmv-&vQDWuKwMmysh6*9{BOOz-Xbs#kWb?JcF9OSppQXlp_-Iqu zbHp?V$-?QZz9sgmX~=}PwAQdG+Vqh1R(~v_@U~wc?Cn398y^EsnVLZCB6v;w{H3_j zATNZ5cD;bRxfkLgyH4FepkWcDD+*RECvh(NS=f#zcpnE9mlytYASlCq`MFa@%S)A~ zgT9P|%^Mq!fZ0V4as(CQuesu0$*p~uGpO6P9)veNZ}N9MUm!fHq!s~BXad0A{1g5a zbNcs{wtNms;>)HbhG;;*4(@bWB8LVuUpetyseH=&kHg`Z;x@|H-j#4wp0LmM**Zya zAHb?|)qP?I?|c+3$l`hynj1Of;(}e*APK)U$J}S7Isp3s|2&t9S-!oAB`JpCn|R5V zgZ8NbKR7Xz8C1#5MEk2a#adMO=~NTfA5uL_sig(>gzDX)w=*}(DZ^P)6=!tIVhfys zo`M{Uf-zj~E=yT>mtAD@6z{O&*ME(B}4vVXrH%bm_GK^(zo{264^*D zBmwju`HA#M>0SKNPKa`QqqDF$(u%Mt%E(M|d782-#Dt-8y3(IzMEik!B{NguD9*o9)7&GE&RLS#&Tzb|J;v7L=sG6*Q8<@XdIj%e?Y5=UH= z`$N#fhl--PEpu;6v{WjTqr|1F72-}&Bfx=rNs6Wtx_`fyvdxB zS4yW(tEA%(%8f>g$73h=7oABit$tKYSO z!?sCd>O6(~%HYUbx2!{ax4Opuq7dcQC3H6%19Q^yU*;wWt~zr+sJeW!=b*&lXZ$DJ z`!-5zqDYm734Q>!dv5z3(^XHu#wpeuM`kde7KxBj784NMIBrQxP0LK}MoSTpX6 z)0*!XAuw3`rG#JxwG%`I`xWNVSkmO?diOnrksM0UdyfQ@-B$hNX%3*+Svmv^Eu7N& zSKFGc84lOg9i)}?mx(UZmINlL}F!Hy${)n;6+&GYAY*bA9t(?3n9qg;& zU$}ryF{MI$O#rrH^ET(c%*LIaw)^2CiWfWL`ZF`=TT~2z(q@HQvisoi5Cl~8kNj5S zuFm2Ch(|n_q|=&~!Dr@Z?xv!n2MJ^gsGhJ&Gm}4mfB%Ix{T{y5;`zaKBC4v*)Q%nL zBk40QVXxlK{w>SFR!QK-a;QTJTw?V7LHZEvl{DX6cYG1(c||Co%F3YNA<>KkUC<@1<; z9$iWCY5Jm8?{O~^(XVCalXl7>O{Z3G`%x;nXT|y&P z3#V=d!6_M58qz}VFTu&0Ca)Ttbs#tJ(qOjTDl#gr1N8JU#}_~3vK(c3a5_w~7fvS` zU;)OC(mt7-_MJ)h518gj*_n?5(Z8(w{6Ic9v7yV6v|wZw4fcSS!#EvPoe0YYAH^tU z2&?T0u8I`#vJVY2>AZp&n;#ajtrV%~Kuob;-dKaV#}zKm&(D2;1JI<8rx78t4{3Y2 zCgS>s3JBK*`u+{2F>MZR-SRM)8-IVOMaic9FBUmncPh*Z^(T=2aOUr+%X531Z5wLC zyxi6< zyy6%}Z+d4FZLWC7s%y+$wh>d+P-I15iIUr>XFg9c_cT%MYX~ys&xEH)WB8iJO2(>Rg))`V7yLHvC67b*^M(E0H&azi^2H(@|)$^j&Aq4shc~iA)cwLr09CADXKPN{2RkHo)H43gas*(a%OveM&^cP!9zZuJtdD~6E_jhR=> zluntNQT}ddivUqG*R%W_as1;Md;cy_X;g4a@B}FAys(8r-6pwt`rlVQ=sGC@x*BW= zD$kFird?Cg&|M?59E6*TPFcjE#BgrnePf<4+U51yBgjnrD6@LF#FX)g2W#jO!x9b%-$T;9-xdZMfNj zeprgq$24+|+RkWIy-D~6^d1oUwQ=0lS`Wqvr-5~~)gDUp%N0K42|xH4DCGpWHpx5k zYswPC8elyvw{EIIQLmtnc%*%$l}Pr=C2cln)xgn&3wt`L7W)=mqLy1eo13(uZ^l3A zJ~2xQXl|hf)qZ{b2zy5}%~iD!vSe$ABv|L6Rx&E^WR8)X@i9kKLnQk1b7*%Z2X)r# ziL+r`Wwm-#;yPIvZWX2}e}27e#r|P5<&f=I{Ux_`W9)ITZ^5#8%9LGt*iGU^&1-Ll zAlf*Jsu<256=R?>ImSzCU9T?Bf}q2l*h0tIh4%IB_j8VY?RG-6M@Yr7WnKx^EyoeT zC)-OAe=*{evN#x>O$_cq%kTlMTH|2q%nT0~5M~zzGe4CZ%4v4&I!4JL_L#r>cQe8F zuw(wH=U!)AxkL8GX_p#R5CvQHmrpub-UxhNAz~0vNrLGsLlgmz=d){OZb7ml}ZOd0Q&A$fVz zPu6ox>cV)NB3O=PEY^jTDmIM z0Gq0XedHr?9)i1Mav6$C$7x?I1?N-zvw>!UWLEC{cuXsMEZu|A|B(Xv&n%*wkE3gD z;jvVmOer=)@yz@?3UqMz-xRN6m6MzZk^qGEXA?7(?)lYn^}GaCAj%tvSQqkE$K{%s4BeKWehwukv| z!uT=n|M`WAi8*&zQr*G7Z+Vmw3~1SmyF1`M|BJyI^`F8mERI&1-nc&jhc14Pug+E` zfo?qDbi4%4&4Y>Tt#o7SIF9l?y|DH(WmPQ((qc%!^c7@UxmUR@u3+xxS((c7f?g+; z>d}QFFD@jTP1LxVhUn4AhT=Sc1354 zET~%f(japDvxJm>(1Snv$H+@NwdE)1>jXn_L!lWYi=>H%&OZ2V z)fIufeATif?LXx1cHVPp(q_8>pRWuNyi~XXTKWzl*0|ZCcaqZ=qsdGQw$fEXWnpiN zf2bdgu#J9Q=a`*6oqU|oQf;t5WEvw;=0J&UoXTDrp>>epOK|pz?>0H8c*UesN7G$% z9JB?Kl0+%d?9}akKY0%awBc8xlhhH+zxCTf@;)|cco_IyD}CGu+Rk*X>V-Yi*fWX5 zQN~EtD|&}P-pYizbTep97i|_$$VrcDocuytBJxXOULz;pl)zKDGjlSoZ6?8Gep4s8 zS4t-(qCN329L)$6*cqRzpnB90{TsAS@ounzaSaVBths6^en3VIEf7|8eB9hX zLYd*664bc8#LOhmqm>xKXQKI&vY$uxpEd0Zt0-h1Y8=d$t2Vc2iQsChU@V2Dc*^N@ zT$1;-pD(CWdE?dRd>F^P5aNDYA7GZ@HL@%KbKx*pHe1SqsN&wwiYY$%09> zOZb(p>UG5BiKjck_3ci)3&exQv%|K=S>qY*&F|{{o0H88{-97ue98I2z0^w^*L-}x zqf$mjzPepD4^V5GF?`B1pt4Xc+J9$j@FLT?M+k+5au9HKHuAAH(g=9U{b5jFT3|}R zVyoR74R{1|-n1BHsPc9sIbFEH*qe=W6nw8NCENR_^c@ za7*mLwvLSyG4Q!p6UN*GWmOJrNaU`NiDIZ}o1%uRalmhMYWSXO&RY>rU0C)#yQ2Y& z&6!zZDXHsm(;$Xxk~?DzxCvNPqkSd+!)6B1;Bdz#_a;m%{?45mD>u1GDhI(Z6De2! z7sIre=P$;QpWaxa(tD$j8@8|Em-9@At$o^SD(cjLf~Pm|niCT)u7_D86NA`*Ov4hL zDD<)Fe7~AXx1t7Ggvqa?9#`qJLYyWuH)-S97n&w-*$rbg5~Z7zcRz(L)=mE%*Y zlfIKqR&)+qg05|qi_OO{jd-EE9$CDnNGJDv%ql|H27@sH*JSx&POA~0y*i1@XxcnD zf6#{7;GQpuZ8~TFLu7&XnmYg`_PvW`60 zooGbEn;L{|?Vv)2@WSj5Jt$w>-u-0pucm{fm-SfeC236kp08#tbYgVhAxa>dhI+|q zME3@a?Z~PcQ+FlDQqi9taGngfNKMKai3NWW>O_c|zfW5R??9?&BKYH-tL?3D;DB*N z3x}AshB+GPQG1+13(z1Q6|Tp%DYLiZjv^)*O6*qgbfv)2(#DivC65P@^rB{}4rp@% z?}y*@OpfL_0m)Jdv-th3;M(pQErC8kyMwKp)eT?`&9cAeBJcVCETc&Nq~CdhiSra;&Dii9 z&rY?mPgN{$i74HQloU;q4@^w73tgcjLsP4SgFz*f>G>NctnfZNlNqjFd|G3kd7Eb& zBU_b054X~~qtpT7-ac>N*2l4{AhIUL=BhZTxHitTl6(jW+4PkU#r4XVcg#H8#n0kN z5qD#iEzM@ek9C$jR868IucL4#!;0~QotX;Zc2oTXsDym8m2?3)kl9h6dd! z1K?6xh26t1>IFPr504xU1#j_o5ywFddgV;Y4pI}>%XQE25G%=t7BDSmoAU+latSpl zK+yz=Cr)u(v=`4vJ(=Yr#J)%T$kV}A{kN0u(`R?v28|-yi@LKh_R(%#;S(Vo`y;ke z^A8E;ctaLQ@Ja#r=^|FtO_QR^2#>ZWRM-qo-(94jfUBl;K@(3#pl4{ z7S^iUPhRqsT+iO3ojI&Gy904%N%(%pqVIXAnIxQM1oA`|W~Rgx`SSw(CMx40=L(7; z(Fr2>Q4j4Q^@auDRqHE(F~Hyp{%t3ehV!ug2i;Pw`2=AwN4>!p=w3sVy9mt!|7+=G z5F&cx^LndR$<(rHnGQ*jowELjps6sr)AsWBcfPnY&OW^P%o#Pi`ZydFDXC^pnEF0& zUn@R(@AQ11I1J5t;Jtq@u9I=rN>#HRQ+*>qZ(x2grHQYyItao}I4C1^tH8Cj^Kf!lfwS0q#vCSBtvV1nh@J`-kSXhmDJ3R8xe@K`Yz0w#RoA$G z-}!y}o1vN)^uc$5hAHBUia&8UW%Juxjkt56!*FJ|KT!6MXUmys?P&S zm+$SvTn`Vz8VRU)4H1p$*(OPkl9Wq$|4b!YkNLlsr5n*(2Bd%cp7qiPi1o?4MJ5tq zLR}vG_^96ED^Xurpc@{=s?EH_h_)a_8xylUYX<==m(d4_MqVa z5gB{P5aS*6%X_@oaGBJBzc^?p#xSS}S3Kl+ca$tBsXN{y)8FMd)+Rh{ z!>->aH;r}Zt+~JzbAM0RbF&88Ol636S-8>VPrurZ3;eM=5UB6Y%rwi6_2^Dy;pHLq z%Xk%-aSfPLVuxTwB<|qSdr0eadMNHGpAuwSaco3HbM&jV(p84{&2#zk?);r-F8wn6 zwhT{emqstr+H-cHk$RIcN|G5TlBFHp%qY<&7G8WUh)l&_R0R=(pePyva zu((R^J`nt{o1>wDHXn6_gx%D9?{*F2by%RZXyuO)i>#xAcSigu1?pvbUspFRrIU)= z=1<9iX{$8+xeXGu?a%^(!&N-Yy5-cwq`XXA0z~P#^GtTFw5Ho9nkD=_ddY6NoDq<^ zBQq-AVS>g}c}R(OfDO^`X~kjN&cxTLT_|FFG=gI3k;X~@fv<$6tEm(_d#0$@GgUC0 zuT6Y{%ZO=@znS3uAvJu;EkY&p+*^pC8$wQ2oWritR7mn78NcmRA}Q!#VJC|%F~;Le z+Le5i_LI(m;&{ZE0Kmb{U860hqRoP;m!%`<`$0-tI$!z%1otm+ z1Ph?5B@(h=GTPrC^6wPwcY7;}>UAPISI7q%)6YGh0#Umkrw9}@k{30QPmrJVo{G#H z9dM)K26@!Hz+?k4DI0N;^iFKX@k4qk0R`%KruqZd zDJS^d*nQn@Iepwnt%JqD=P?a!+)WOh)Oe1CK?TA>w0@O#w4fw^if*yA;vw6XmR(Z~ zVvXdFCfck?Uhlay2SVTRcvMSkxu~3TMg0~#)%i{1{P{9Gdz8Lj?c%Jhb95F&ar=B^ zCdqPwv|pP<6vmOq%;}$t;+TfPI^McMi47W+bUCyazBfGjT-`1%Xoq`XZKyORUc{!@ zL!C8@`nNMTLgaWCvmfvDn_!Tc}cHYLTYlk@KfWS$oyb z%QWwE=!v!=1z67d$*C~M)>%0i#;Svdy9e>V8te24NsT5d3G zhW;a7{|kHdUy4{yQZ*qFTk5}EUm%kETZT6uEv^5F<33lcxe%63u#G{BU>F+r^nVe~ zI2>Xp($<*&DX{K-^B+bRwp1y)@8I=w+5hHf|B-Y;_GbeV`x2QqmoBPj@Je+Ai;kR( zWh1EVil$PO`a$wUx8NZcgOmKXdkNz|IIt({D(XgKe3&Bz&F%tSuTAfR_*8Q0N&zOSo-w&>g=80vC=A~mZ{b!JIiWTbf z0$b{TCAk0V(><9#Y?k!@#rPjaHU$n&ZOc~~_XG7~T7_R842Ne7ACo1JE|1ZzTSuUe z(fZUH0WHey08yNXe~4KaSeF!-*(}zs?(U|Rjp%7?Q@3KXqIk=~0+BGa`v5sc%-V2}YoRz! zADCV5exw>KMLKZoE5SEFkK(S9oWCDv@JFcyPNi8aE>@{KE+t-ZSr@lYl}AaIYw;JE zs}&w7N!Oj4s4+fi%d{5ZPg5+>`qfvW^IATXfYYBc94n%*-_fje*DkD}0a{w-_9Cmw zS!cRXPi2@H$Klvm^p%9)Ih~ZGRJ(MO^Xd7*)>^$yeB!D9+uZA1WVC4Fp?X9GwNcEc zr*40Wx`wkgYA$}SzD%7^n*@d<^2MWboHLZIG^ZUDBZd?U%LuW5ztjAlEL5}kQAL90 zFd9w6Z@i|Y%qZM$=jh z;^T?{z4&uhKv--V-^NvgOe2;<=?RD44e8UANAenVg^f(?+XfrX$1^Y&Q&&Vurs-D- zk95_bR9lK?9u2QtYP@57OY@ORu76SoDjDn67D|^gdtn4ejO zm5eHR!|D~Qy5<$nkR!_PPBTWi&6gc3Om%8Zm{?D1Cv(v(ZT&y&sA=ad-4ZuFmdKG_ z(=+A)Rj!Y?D@pWM#FIo-0WBNV+-w?+pBi&Us}-U$O}N$&lhC`WHTsH4JyUW6p)-rE z2!Uk+J9?{~dnhe%P1^~j+Mw1(GoWQKDJfj8YvDamvHkSLBXEa7I_$1~{I!O4I9a$7 zr$^~6>MZ{Wjh?#ac39kTXaYlk?SF@q=uO{fZ4 z=Z43;*<%+s9(Pb-iCCjyJ*r{Y`#LUYpJlqZQPFvSr`FtWpFKnBIVzB}`CeH{e?MBz z2kXt|mUO5DMUs0skBYvMvM^$@4eyahqQ8?#!g~`LLYR2Q-ef}s+Ak{6gpak#GCh`A zVxBs@DgrKC@d=!&x#vn4aB+=IU@k0-myCfQSh3z&f@8Mz~^09N54Z4pP zrag}i*XS4iVxW91E6N1;M~&$>Ie&oI{6Iri?}(GbnYo9oR2Z#wK6|_`u(2+^@|iHr zvX*%F!ByhUCuU?}c0ax~Ub7#6@lW_quLl}>cnVcBB?di1$v3}ah()hJVYy&437}De zZ_t_PWLSpN?@*!|nd9YnoR{j~zul;mPFv9PI~Hc*yx{y5d#P33pIaSQ zs}mNng{uO3Y@KNSA?^rv7Hxlg)D@R9h3~UG?&L}-?4B?ZEMoQ-;~4HL{9%>5ql3ZC zL@zaY*=~S`ZK27zVen)Y{|L&xJF)d$8JBVW3dZE4{i=Um&1gw0HV7opU`H{@Q#L9H zg1u=z?iB969;QS)JwvRr2fV)2rOA2m=c%Qbpdx@3?3*+y@Adh6Ir3kD4Cz%YJ4l%W z*{J6&gIZC_y1@IR{z-nitQ*;HJPpflb`yGZ65LKW#p?%?AUg~T-i&4{YZdx zO_nOox^hOoZ@gJv=}YVAe39{V7&+LL%T`Q$1$0sWs27r$PmDqFeGLKQC5$zs`VT}Fye$76V26VnzDs5v25n_M*bp+JM?I6 zJPNFd?mgyoPh4KgSe$bDVADqxA&aU~bagAzUY9>1L9E+rcRor8lT6`uIbJf1XoS7F zR}jw6uZuaw=rzu4H))H?^wKwnEqW;GHJU2)}Z| zkF16DzyTG)$))3R8P5PAy+Ln-1syCS%9Ck-wl(ajz77*^w|&}!vIH6$*X%F7acF#a zODal3@6TpJ)^rM>u_z3f* zKF-`(=f0I+MIeTl(J&{8rX?Zl^2QQ~R-vp8=$)Ism3Q*9$LlLsWRKm=jOhAua%~P+ zn1iQ|^N3=n_#*KXQolKyf!SSwPR`D%b}Bk5c9)PeDjl?4sALxU0%J8)VIY3Od~11m zE!PP(Gt>6W*sb$}wTw~7y^!%O*t2X7_eoti53@#`{%oqLod!!4P>-qVHx53mU!@=s zu9vu9jP|$rn#?r)EU!}RYBNpuUo;!?N%)LLC?RP2VqvFNTcdMI)(-$F37gXzKEymq zcdcS+d~pXtF-BM4wV0d7T->=t05KF0bIj)wZO{8 zg&^hKl?Bi&bs(PRAR@^@=?*Z+8jIKp}MFRnSbRnKP&KWB6p zGCm#mKn7MPWMg%Si0PMv3#8R$p?FwEyNtolbX{C z(1mH_CFS=Q^*?Q>#M;Znc1mv#kRx9dJy~@GtfTi*ieXv(;|@|q+XhMlH|$i-g!R(-^D=iq zVr+@mKAUAcB#?&58|{;2CuedUU}z_0ny+PCnaDO#_SzUS2)}prXxSm*ku>cWFx05R zkP}nO3002!S1gg++0MjM|n8n8O2_RS`oa?j2Fnb+Znn=5vnc1%0B&z=gr5 z%RC3SlN|njWnhedQL>u|PBTYicQZV|h$wQ#aDP0(Y2loDlrFh3#MUD9#<;~bUI)eb zu1RY8u}1*>v&}^GHOS7vF1xyBR-!NQz6|f=CU-fsJHb z|BK<+ku{IU#B}N^jJ-+iU8b^Y3BLM1_L9r@+@I%*?Y;{yBeJV=0e@<6S=!t=PW=mm zs1OJ7D`!C{%U+JWX$-{a2qinikwlNudEI_%aEeMUo?3-0nQ)Wgr9DSF;n;>hoZ&dL zTBz>78(^NvAwHkkqGP#tE4g}nV^{;7?8^LHR55p219&u<8-esj7C8mEmI8#ySO(^a zI-*uVgBVA?JEX}+&GDfpU!>Z8amwh#nk~#(w64ZQ1g=PtpbV8v-o__*itCuN|DpKW zlru0X*~DlAwwOnjX3}JOshSe^=Oo8{{=8LU!JHyoYJGu6U+xP5pM@0pJ!R@G`eA>N z&&Dh+t4t=Db09&?JHOZ;)U4#guIVaiCGF58L%(I9EnG}%GVQE&3Y{M}SWBmHN)%*g zf6dG+VS;>=)b-jdAq;A5%X9jZs^Ste~X$?`W?i#;I_m=CguKG3GxNQj)$Y~ z7n7M8b>0>^*=s*!Qqu`X`E`cxiS#h`gyNQs1DOA+-3DiYS5qo*jD`i|=%>hiKqAWx zY85$sG)qQJ`)ifrs+>FK2l?2`<+i3E7go=kLXy5B^E3!{`F3a8$fstr$nTSo6rc0b z;*aV>pn2*Syr+yQrCnMe%Jcbk9|+uCc(28`J!kmJp`p(9bh;=OSD1GB!|%kVQnAG@ zNVXb|lV^dt>vixJvjI4aycoZjcWaAJcpkz8%)V4b`-^ANBYXXI75mkgI%FcU3Z?VD}&uH z`88BMF^w2Q_3PX*FuLQL{HQq)SWN<{SOdJ#S>te z)Nyz!>5jxawCR7v+eG0;;)MJhRC6j)d$D>ViZhFHM#r#CgCFH*{q|^VSxv~h>GrcQ zunCrkRFs!wB7A#$b}xBMgb@}$ck`#}Bh=1jiRq&# zZI+R%-)XmkV6hLK`CkmlCviZD@wOHnE#4*n;I1*uf2y}$|4i9w{vUL`Wl&p-7q=U% z*a3=Lp*RF>Deevl#oeVi1b2txP9RW8{UY7>mbi)bHNxpNagp^L`4#&{$ zFFwE|u*fz(fsxhJ2hFrG3mqMPQS#^!Mcy(PeqH^f;2(7SVM*E{-v=T+$&uC{>?NSE z;hqPpQcDw60_DocNo!mtg{IafiBW9|*P5WO5r9a));rzXWwp##J zXn1M9XkqHYOyShGCW$7*B)Ub|ZraV>q+yy=Zfx4ga6Xd8;m(!rv6`7exg8 zf)Sd2uQ1E6Psd$8!r81@jCD4VuBc`F&%C13dEhd;VNK&DTm(dGe48lI?rl+M(JO_G}lJM*~?$N zU#or^p%m5JT2Vk$Zj0Gpdk?barLJASUZxzdavK`Z;q$5YT-;0y|EyFe=2AN0(&yJb zoLq>sut$hIAqu5Jr-j(f81#-2K~}dwrJGqDGQ&*e?uo{H>%+#^H{*a4fOG4 zyLST)+FAgnYYt$XaJkeeoNVxBMQ>RT66we1k#?!)PW`w6uDmMa4Kc zsBlqB<5#W!u-O_tYlOm9=(U;vl_qAJeD0;6Z2Icv(Ta4oY z*w@l*07vcNh=fa1Nb_ryh4bv*75)2!A_Um?8`y^n|oweqfK$wYpykTRU%G)xS$j3xC=FdbsmSs$&qv8RAu+^ zT^gw{p?!{1dqLWG1(Di1;^U60xS_n{6AyRPe{^iiz}p{qWY1OX_-Q<{iH!^Ih`suq zTERb6P?J(T36(vOh5jPh&|eL2WQqpsweAl4(!PgWENoUzX_IyZX(_P7+%Fn-xw;^2-xC@a-J37 zn~#QHRd3TtXh?6;7LNo!kl?WA<$63XGa+)z|5Y*)H7Z9^kk&&1ar0`E6oKmQD=d!1)>=iq8 zwsC!t5lek##qwR*@CxTDQ13(c$fOQcq<$pr&_C}CSfw_b#Q(K@nl>UvA`=k{wWKMrylK`etXA&wuNJeK#GBpQc)X zVsr!*5gfo6>*%gqQasulS(LO?2+wxWF;mZ zmlb4Xo0ZYCP!xWB_4vpE+9e&`EFVLTCGQis5!R#E~}s)vp7y7@w}l)D|j@@Lm3EyFf?W;75=fT+Eb;8rxn(E zHBmYhYyDW|q~A@pqVbmU1v)V?lm7}kwEaKS8e*DPExag6Ievq=;`5Yf1W8g?FG$`h;RlhZHuJZ<%3~Hd8zLR3eILe~n9Yq@=)_*@ z;`tY#JZWevZy-Ey$>Q}rTcKR6UZFzUyt+ucfbO+blG@8MqV&`W%fedDBg37{wfqz9@xhHs4YwmVZm6VUIIIhg zMjSX#>@w6AI)}R4pi-wfeVWsjn0eDBv(~CbmF;$1(R;irmS5vm)XwpZX=22)^b*t* zj#G{A(hag~U_f@n&=_iX;gkdJ+IR)UR|Z*;tz)WGhc63>v%D2UH|*==k>w)CO@Nrg zR6zEemJ#GMEa<;cVV~Pmzl95IO$0?>QC?Q}@lGip`QkceM8nRyT}3QePJ9ffejjimcH0Nn!Dv~`$fUC zt5)PqLB76R$$zTDMsQVd>A%HKlbX!XP>$)(-m3}n zjW;v~yl>NO-=|_CRay{sGLOxSms<8Z;eV4Pv2>kL^U_gr*YhKB)QZa01DMNyyk{Y2 z^3A=3g+=VRrFxUUanDtDAcO9fPjzKQ^Ut@_4_iik+wW@ycg~q39z}$QUC5U?-N#tY zj=EMDV74f(N)dxCOkE*xew5f{qM5Rf_cPKCE_vz`PF0ZGe;(6XT~kx_!p_~3Vh_Xt zE1?bvTpcNH>kbAbtfj7dJ(%%%t9_Y*-NM$?)}7?>@xuWE8ZoxW?}6gOuUAiE@$g5m zp<&snHBYE0CG*LUAp{qrA;1z_rM_#f5Wcu3M#4Qy>t;eD(pJX7jP!OPa3(!NP)+6#)6ui@L?Zu#bEN}bD>V-Mw#wndm>8{5D_<>p2w*zeKOv1l2@kTX;!9L0B-M6K%spQYWSH(=w%WTc=15 zj<)C1x{AhFF3~K8eJkVYqQ^Fn?|?oQnE)+3{3r6|0$F*YN_r9DUtYMSq&gze~d=NTnfcXBWQwo0i|l5`+_yVN^_u@s`YKa*0d3rg1H~T#H6(M#Wy0N{84+nE?4;GPm7CMzvkX->ED0p)k`TnQj#vUEN~=I z=Xqk4Izla!N8b7dz81v(}*uyVwm)@th?pKS=>mot^ zmFdXUQVsar8N}(gv5&=(X#rE|I9gu}6Tr)>6<;EhVq%2L8J(pI#H*HQ)V%_g(s_v9 z>UxAHMNXQ-8kJ^dA_jjQDo$I{+Ngstb-L>t3--AjFGJ4#3#iEXw*-M6~HACaciV`Riad zA6Ih}3KZ$)WG-avKn&3ea`FlQY$pW4A_lCatNfngD@>apPG*aKidz`XRzWxmmE>Ql znwW(kPa^Hr@;nmOK=kcx)Kv2Fx%fKLu_e_L_8^wz@iiF|%kXnQ5)HQ7uKyXYy|PlGRNwxe%Sg+ELAcM?9IQO|3OU=J!MH}+gK z*Pue2PJcU~!(h?3M}qwmcYWCr~xG*~K<31Q4CI=I985E&w!`HTcaHOy_gUPjul7mjM0 z7?esW>J)hOuQ?lu;Y@sDKCz_>j8J5o%2xRqdY(RazCkZ=38P2pI2A*FM;%T7>-jVY z7W{a79DJ6rm~KJa$P|!n%=Qc2+z%aq-y?V*jv#)Uks1-l;R_G#DYH{y#|h|35yE zXjR7LQ^4u(J43wpV>PV*|QLkuVh(7 z@Uu)UF=Vb#ip3^9N}*XywFEGh)B^S&;YiLN1ci}NNc_mX2Ow;weZ(UKYoW_%L;cYQ zwS%|>>+)enO8I#`;1M(?U-j#i4}LO8Mkb{ zSAuEELTCX4Hf2Mw)V)-F`d?Rj@D;4klaZDh{Vu2*z;$F7{pe};foa9)V|;7|KJ1L1 zJ3gG!vWl?Fp6}D>)E=;>MEOvhVtCaU&Q8o|L5c!_t@VMi)wCK)dyL=3c-vIwcx8GJ zPYQqR@9vw$RLOhG9Wkh2_VC?eaCVEO#ICh-MFKSI$IcQTe(eO8Wd_l*HBkJkoX5%t zXJi}#uQvBKC1c|{$P#lf@&QhrWxHO~othC};*kQvvz*;2k^?130Y?kZhHQ-g&~gg` z=p$P$_~w>G09%V&36jj>woQgx@}QstymNq5)6zUu4Xivnuv%T8T@%fXUoH}e*5)Sd zvtFg}zNb{@sY9lqm9?RgOzWu{@y<7u2y`fM)8w*f2mV5q9Eedbdvo8aGhi~sc@nn@ zoteBHdzcf=UaltLS*Z+9DXI3^aZtY5+QDNx{=ExiZ8|uuL*TAfnbc~9@2J9m=xfv( zI2xwO2WqSs0vwfhZ`ZbW`1fbr za}Df|%C;7#+38jB4n24$gump_tdMBfJ5|4E^lwn=8S987@hyS1AYmNJpFcj&;Ne;c z4fwjpuCGpa%ulaS#`?s%`soj3Zt=DXb#QoM*?pVJAOu1LU4rjm)9#<0yPEhgc08FL z7tT!;D3OK)u*A*&0%i-G1*Z!_{sQK7W!;57-RhWdqHl;X*RrbPowXbn z9FHZU1Vh|@SBsb#^KO4Q!#T3uLGkn%jBImn;5L}tX zcc;-u*hfpBM+3iDDij3_v}!Cye;RE52cUHSY6Cz1=Uev8C$^^h4{wv#X1@G+nY=iT z@4P6?ZVZa^Ij819GysS-38$|c-V>RS$?5&DMF(%h;tyf;twRWgq; zmTRkxUJQ}*2+(N6?vym72jKlESzZ3k!&b`*8HxSWy>U8 zW1A6?!u?r2gE*MLuvp{UOo4J?lgnQ~^FYjZOeuwdwh|zMgr>eyul{16)7>0U15)L9 zYtvtD)Zr2lb9G?SE+m35;UO^Ra3RS9*;#Y`vPCvV%coy!UVj>_SLtV0JOiY;nv+(( zoXnWZYuX<5d;d{9|2+^({fU4==&cnt*bw}F_)_fyj^HI|;yL;2ZRK>Pj!dz{V-+;V6pJkF!Z8Y!HDb(klv`l%GUAuFBOTodlb2Qd@8wD~9hEi4 zApzqt*KpAwSktAxrb;WSDAJ*mbgSMpdpaT8JY03#`sv&-uuj5{YZu&MR@9Js5sziD z(q-sz)Fx~964F34?6z>&8X6Gq$aDe*qwiF+QfC~8R_y@Ke)PDJcs_8 zU#Y$?K8PNzZoNP2uMYTf@fV=`Aej2WNr8>$&3cv%Q7f^Vh^jrT$#o1I47ao`O#W%B(|!TQ~-DAB>Ed(j9QS|Or0 zuS>`Bz%Cnvrz-jqTwn;@hXj@gaJ@jPmCbKW^V=uRwjW$8VxVWb?5XjPp^*>kcA|vy zw>rjD(E7lP0Qh#Hb=|FQUI{HIsIX=g<(O+09B=Br)>ls<}B*E?r3@lJ2U}GjWF%Vln zSSq|+sq80^H4K}~Q)T#3{(!KpHFXL;)rMCUvcYpH*bmc4Jp^HEU^=Hd<`*BB^vb1P zIs(wM^nxPhr(@aBU)p8a)e#BU+%jjLFT^P9xLmt>v*_GvdUZw+E5EgVT+PQ{)9MuP z7%ba{9rppo*ztPXlPq?ZhazavcigB`;bZu-;{JuY2BHrGIBv|jnL$zzs%SoeEW1RN z)O(t^c*o^aP-Tks%zxO+lvBw^1TlE3GsBVE%)6FWLXknun4~jK=8jNw*uOj&?tfPF zYQub*Fml>xO-EAMlaf?XKzCp#_tM@~rmlTjTB=}*G)YV>x`hI6)IMPHu<&Eil5@Sa ze_2v&0(Y(_7ON^I@}Yix>N;53@h%1vH7pNSf%(GznaG7FPw_fectgb7Tbju3bgzCQ zVbLNXi~W^+@G=Anv(pOJB|(~MSK$ED0G1zi0N)K{<>*3e1^K<`)!P{TZ~F}gZHf9- z8P|n1%K7T3+>R2Rv?$Z3PqJ!rk<7)aqFHf&#IvBDy!j{g$pA`H`}l)x?G37dYyO`( zTP6Rw=c%UplQ{tFTfT({gt_vXz7^j@?2NWN8kkU~PNL$RUTvq;WnN#`DchZvb9=G|xU4Z`m~{ z4y{;~|6F#c3gW@%mR(u z+k^S|j~$F?w#I7NR=0;7g%twulUlqdIml@tp0Fo+jI76qFzx&qS0$g4tJ`!E8UEBp zV*AhTC9IHM95H$>T6^36iSEXjpDP6k?l*nP6@t~nuXOquoEOQK#2WJT1ueA*W@oR{ zl76;)EEh;lCCbQ}vLxV+!$l_$jQ=teAd_#^>JGBiEgEy_A^ziv+yg2OBd#Q}h0}=? zQ8}<`dhdI!>rg5lCJ;n|Z5g3oYkBL0rao{#n}>}K@sZ=>r>a{|vSctZ2pFqdw9`sV zadEQI`&s;421~6L&N{{jl{kH&D~G-NAYC`z!y+u)phw%9v5%Y2;Lk>?6m<;g@QeMt zJiczI2V?*zR8Gw=;wsqxCkQ8A_w#Bv-}ql4C~o5LFAN?rLSCCYMihqtY?BQelV5?+adp^nEcA>L$x;V|x1Blp9(BX(${X7^M6TxLf6zS^-G1(yMO31_0 z84`uuJm6sgZ~nf^kFfXgBS(-@W9O%3$GIoJ^I)0t##NVIt>81AqEBWElxvSikj4Q= z@xd;QD(>t{I<_K|0lniH^Efn#|64u$ub{gNb>aSPBg$YJIeWc=tBGxqGw`m zR6xQUtc)_v&Gs?YCY$wQ@45q-7hz{7%rpKHqY19d1ytrJMsm;5(B!J*U}@=s-%}?i z_mg@@v(V23I9(tj22~X>#PJvSl{y218s^@pFIBfq4hmof?F`a)NAX+knS?tI@`wAu z@vaHwK3|HE-d|5|D3+>3DuoUgjODV52sqIAt0+Pb^vu2~56v}%9jGDEK;cNPthBtT zLzoK|x@Jqp(v%8Qxb+6H(~rwirhm4MF0QVuQ9jqz$cz&Rvxi|ZsfEZ<=eh6EY4`@E znNMyrHZg`_@`AbmCD)`}SsMT~bdOqx#E7H-UZfAL9 z@WWkLpLI=$G?=J4H6M1PKO{PBK({U?t}j|jMgBN};_c3_{$_D;87&d>J|j8nW6bL?8ht#I1H(nm=8r`LDaA%A zurf6@J?(G7P^YKwyQ`^4{PRoVJUY_CYt}B$%;Xt~@D&>!d3VWwG=Ng5vM%;)G_okk z;BKPK61LHF9JrMC>oZ51q_h7$rLttxXD%L3MV-kv!2I)DCuri`X+Lz!`$ z8%0;rNIph$Ta*WpOO!ATOF=t#8EA+Z6?9oKX7gCEgp1LrV2*I@uskI8B@m;AXS?Ah zlC-AvZ=iCkA>F1dSRa~S5=5?`o=G7daiBPSrruCc|Hjtt9G`*da&_t7=v*;Ezy z{V=qkbYlSx-x;T#if#wx8~{gX=7C7eV1~q~rdzTmqwuD#zOPRbM6wlw7tJj-e9OB& zrwuh~vXc=cj#nho8-{-I;Vr@t$XNUk7HOxwiyx(F>gz@{pBRtB}z8Sun zrnHSB{IaQ8CwNlh5o(IIN0Y&Z(EL8gXgrq=RI~o+(;bqz)6b=aLQedD7lf!j)oT{N ze$BP?7x1B@_vRAR5SfF#&r59P^5LAy_2Pgu=yT@M(9#Xv9L?#ASC_UT0&mJ-fQ1^6 zzv0DN@FZCiIH%&~=jzQgkUfzDtZxenZXPZAa-YyLS^(wR^vO(3@T4{x-$)P;(e3}2 z24a?>^8WHIWridwqH3hf4yGCs^A9q5YK%%qi1U7ZpPn-P7qAlC%=94fHtliY)b9b` zoajokpi;z){^Bj`g`L;SYrvVr#d>G4cwV!ta?SkMT&@6a%To@29pBQ<{^JpV*|FF6 zJh-sPyZgnU3PyB>86F+2c-+DdLFk>2b2N2mm&pe>iA400bwM%6IPtaeeJ|UO=3;nfO zE1a>F)Afm#y%q+IfufHNp3Fh^V!y~XRocmoE|n{*goE^55TZEVzks_in{KzYQ9>-V z6t+^Am^P!qdrVt&Q~om$z@-Hd!W8J4O7XI&Yr{Qa597lt`U{W>t$aL2n!g1j?LBER z>B2gg!{lSA*!R(-`DtVeKyklSy3ff9n>VB{el1C5;Uh;8P)oEvFQO?|)R)-U+S*RV z`DMOJ1Q$nC&uCQFEH+J?mP($-zd9H?|DF6ND8~1xWZ>MSbxxWB@+jRZ=y0i4%8gl9HuIc(WkAD~zZB=5w+VR@Vk`lzHnob=E zMsuQ;mkwJVn8dq+%So_0KKR!qwiU2)os%pEF&6vxJbYAh(0gR|9j##>iJc|=et0Nz z$f9_zV>cuE>jr*fJ^pNjSQt|I@FF#uIwL&oE9#o{g_|`cZ?A>}<%0SO0@7R6j*ROdF?IS+9?=Iqd*!|eu z>5GjR_b0T`HCTvPjTArZF1A$urgIayjEAsbbgKU6LNn{NqC~6` zL8VzsMcXn@pt|O8$8(UPSm=4#Hp#GMM*4@4&Gp_ut>nzy=e>EupTuC+2~mv$)LUcy zv?jw?rcOQ$>t06vp*ybarQwY)jF!s-DTrzwJ z2Md?^*d|~RL<7R%4ZAEWONRQ5K94!x6aC8`xi)xCSi{Wv)bW%(JzkJ!)BxcFE$2G*V?$aSp)gSE)h4IDw1K-;Mdjn^|7 zj3bo{RIh~W)v88KsNXKsSooyjR4_=dCuVnoMA8em{peiH8d$G${R4|~CS;$= zsc=2Bf8QCGe!9Y+uZJ5E9(*N*uWwYY{}8)ev*>ecS0709@NFb{cPOrlqp4yB0Eb&X>qkZ@|CR=|l4S0; z@+23~3h1kqX4IFk33Kil{d)iK7m!k~NLqM3r?-E|_GwfYF`T^E%5?#&JA&E=g;$`M z!pM$(3vu>#Y7~u&n^e`uR9cxOjO@vHik^;}-n;g!p=ismHe53eMmlNSF80P#(<)?) z;Mbz+q(`fS+S|($LH_aqge(lpchWW0qzx%96KPv(o(m?#vLpPOH>7;J-re$|^E@B3 zQi|@R{eqf6HGDgsDAmjXKhFOX++rQAPQi#`@HriKNk=Oe$VEw2&b+6b3RhwcAcG9p zPyrMNjX!0Ed`YFLjC`a{qFa|mzh8qTEEyJ(E!K510HHhu2Xfz~KqRvTEK!dg_E`7r z{-Sbn`M-$;)HWN=@rz5uU;jey-|axb`383s3$0$#DtOWc76TFMxwUdl6jEPU8JHD} zLJ1aH<|t@E8epR*-sZ4gR(kCus+3OMY}0ihzE5lSs#y^a`}P%24_7RD{>DO-HzhJ9f3f z*lT4Kd(*GV5}^RmN^zyiHcj<64Tl4%X;BxbbZS-MG)bF47e|KB5?QH(C8||bpVFhK zBLwoyW^3j~Q%hQtw(_zQXEM&2QZKD`n&-wqJqd|KRi)R1fS;}XR(l8L)IOwjUMpM; zRv*#fmnPU|`LcP{_il0@E+Y}EH}Yl45AE!zx6dI|g0EFd!$TMQb2E)0>H<=YhzTi{ zmT@RY=Y+?X{t@QNyqV?tI68GUXYPdMqi6>bOZC>Y?wB;b_ThQBFi-@!>NkY~MX)4m z4OZW)c<5LbHdW-NWt(-Ah3oe*YgBv~6eO#$puKFF6s_>-z6HjrC*h$l6^I=2%|WtN zUD7x<6WH|)Ekt3;W;Y0?#`i>X@vlaSEc5WdP4!{nV&#cxZcY+=TXfNnJYGyifH^dc7r3>apUU1 z@$&Cnt-Ll49(fW4$dXtT&3sBP^wnslsJf0Y)PjFLlZ5N&P>=mu^_~T?@;f+wXhd~o z$+ac=RGvW3HE9=fM;xGoNm*V@3FkxKM9mJa!Q2VuDk9gmE6>heIck;8p5kP%vx}05 z;e$nD$ZPFdTb`4XpsA>V+Cz}%gpmTd%0M)y4Jp6v*I)1d1=vT;n7lDfnRPIGi?h%@ zP3ubMt0_pS8^o0kJK_|}6YQ-#jAM18CMnNlQLzHen@5ba1Nu$^D>y*g{n(ni_)a#?<>>EtWtF9HL>yipt~; zPIaST+px1NjjM)AAAq`lsR%ZWk=Hz?UFHVvcOS})9G`!_%#q|BE~Pq@EUFdUpUe5a zrh)^UorAgi{+!*PMF{ZRpR{8R$>I}|h4Q8yzbo$>JEU5c;FbJnFbt44+?AXsc{9rn zV^xeR!HTCHz)obxnBZ{L{291-2ThMCsov0FNp%Lwp+^t%W%LfbbMbN+`IzJ};)|4E zRv7;*Dl7a3G`6#Qb#+V*1<9jBfehcyv{a4WB-eJSig-~zmwR39kr4apzfft%yMj0j zKR(@8>+J16EF=B-e=FMm^MV$&V*rSzihTWlD>7fJ;|Tzg01N-$DJa`&Q-=!cW_qaau62bgmI2%Rp z3mnw4aC3V1aNH#3za>3WN+8UMn7W_d2XH+HetG4*?fI^Q+8~?)voo*p4)Z@rR!bROr8Q@|~{ePR(Anp;+*-6g?mbm3=#C&(=s9*&i#V!#jj1#{@<_U0qumyTvU zP~nF;!^_}e8j~QEPQSl^q5l>OMAWzG*2|TX{caHp9xq>#;y;Dav!N#8R4uGu??)lX zd3C)Qn^czAWLAO5wivohl{?-I`@ zB47OPSCQy${B$O=q_g)F(LK@ki6A*`GIK%1y+h^_q3Oe@ehy?lTqlt4Hh3JBl_wPQ z>8$TM!MpB_45#Np>XmJ~<$s_=a9}`6kT5p<{=*o`VP@g}(MKMO)3uB=M?2_vbBj+h zH9jAUk^c8cN;VKpo*c3Gz`1G}5Lc7-2NX-)1?pn0r#0`gjxEr89*v(?)79Sen;_d} z{!Omw%;m(cijxj?%z@?i?Mp5-0ht%gVV&PWHjeeog;#@D#oVaA6|xnIu+rfWD!a2^ zh25c03n79(5?Iva7`mjoja@%eK3G90oCV%|!X6y<^NeR(*sGJZta}7Yo~H#24bQv9 zQ5!47joS*Ov>2d3V;E5~(z8}Q4)z{Q%82~8agKR9>aOJAX^g3qbuR@&DXZlSt?*@V zib`2KE>Jx-MgtBcrInvg8#OXs)egPoX&||0YJQCh`v)Z)v~xXbHzPJ}ocpwMAE3 zdZJ=59#-y+#qZ~3E`+7cRMm?395n}&$hW~)V;PkBA!5p6<`Xha1*Hcr)-5}+h^H8; z{_Z*DThgtCfw(M1f@9j7+L6knq|5^vq%)+a{o2X~B3N-Y4a=gueQXf`3f|)MQ;BM0 zxsrGcZu8*Vf)#LZ@O%CQ8+zR>sW(RHdpuG65@WdAhkys7h59PT9wOs&Sg-J#tlyp1 zz9W~4P7C<`54_Pa364gR;U+OvJvGB9@J1K16Y*<%EbS|Xi_Ps0p8=R_p{8QIO?gx- z-^d?M{(gIrOZ`c0(DVAAuT!5m{;7>LREBf^N=W}|5XGjLfW~Cd!B*9XQvCjRi}^-U zgGLgJXpQna=p8VZQ1tO!G#o$X)5lD?UnVR3$+=r#wKy{j#fv4SRV5Q+!q39h<13T#@3{AjEeUVHfx^Fp|~CemALbDjWN$Iie=7>wJl}aW|M2#RCpNo=GxDYxqiR!hbw`Y}_QCt?gl_{`@`=k?h%O=OMYGX##`8MB#L0VKSz{nM(TV1n91|NWRER|&S(G7WM;N*ADDp|nprfT0M6 z-yU8OwtXGxA(qlwn!DT9Fow9TISi|r@~YY zk4NIGg`8bRV@LAY+MEp$-Y&IdB=%RSa5tTDrU~oogXbBwd@4ET=brATjLwBK!#Jw> zg(N@#s4_1vXB|leoS^oPqGy}KoVeW|5}^61w9v+(?B>0W!Ta5xZe`qa2K&1v`L+q-@P|4fTwBD$qel1A{-n=UOFPDEoMFE z{Cezik^{^(w_tNvk(=IwIm~y`I0vDh@Zk19{5+LaX-%zl@?4ku+CP%hmUP<0E#0Gr zr0Uu@D4%Fb&3G}b-=I@MZpJMR0=6F}5IQumHvYsEaV5?K(9YL)aG>%S@yzkE==PQU zF$dSmb8hPZ>u=Uig)Ku}PWD8BL-K*=_qWNT>esLg zp@HXi#PgriquP8}L+bC~CuS`;xLk}1x}8W>t_?m)DPB5ghaznlG%pIuIHI7G=BA#Gy=F?o0OiNqpFj}xo5N}&CQM;Q$8(SA+H*anuY7z*sVT0Rmv9ln zz$ZvFYt_MNQ?GVqmiODF9~-|dSwhpA0IvZ;(M^v8so6S|e*yozMw@{%{3C;ZmiBT< z8Xyj2Q#%7`0KzlV49)ztYft$8s7+WlFJ3~|4|KHWN(!C+*m^7)gL{CuG zmip8c7F^FLRKDe2L6e6nW+)Yu(UcwjO<+kM$&r?8Mw((M@_p7Zhj;9&iBY|xqQ%jg zCyuiYJ!>utH+MEEMLQQvJO*0O3{Fxu7}B5m9ov$?PRQaef$z$q!7BB-hg5$lX3d(~ zf$e(!z>hmX*trgxJN#7AW$j-BO$rT-Ai~N5X*X&oEr;c+5BZa3`7lIesmGos@zUN9 zG<>rlYbmeHA(J?3K1*H?!NE-%CMchoTds8#?R+^DPy6AJpgII+OiklKC8_{IC^%fz z^Un$#^Hxza^(cmwAL+9x9PY2;+s{+w-weMg1lKiC3k zOH+}t@W{#@;+gtFVt%P{zq>eYvXbwuYAgpCm+Q74b%_toELqX|mlxi~g`%BW8nTor zh~onRYyoV|OS5(T4uvvaZYb-O-SrT6BZ9)l4-;lAiWuZl+&|!9ugEK{-oZ$XlC*hI z;fg>7tmO@93w!mIl|Qv_Fx4?^L7lPT4%Zsn^z;c+(F%hFa;k#Mbj|L*+|gS4af^iH z3oO`mSlr9y(_;8XG?56Ft9L?X7}A;p+srqwm2OeJOA#Sk0`YRyz!e_7m`gN@xzn{V5uS zV(w*AP-JGGn%{WI;{3z)tL-ez0k`pzi=8hnnZTzNqvq5Vgd}0Ou(Ce2^>l*S$mJGY zVSJ}o);cushiC4jop8}2U(hH#ZQY8W-b1oL&o2RD%Hg))xomd5vppHE5pU3(;Vm5% zX^)eKuC{HjyK3P7=o1>)yOnLhH1hu8OQsO7oo1dx5#4#*NE|K8_R(YpCBCtgAtni} zOp?MvidhY%Bd0v%wEFvOgXTC@_H()~^`fL|sYVa)L+Y;XqFD}MEy{Spk6q99C4JRD z42&ixDm^tjD*!)RlJ>D7x^3yA9DW$U7+C@{D%mFKJVv{5DGUf}YJoTz?z3!(`m33Ph$}(Ep1~Q>|L-3eJtT; z<*Ypi8X5ft5|V#7byY#%YzK-MWLB0*QuhfPW)CdXgjYY-Yh+|$#LIJ&GZV?RvMRK! zAFn)hRwA=%Vrl0hjhO9-^&o+nqi?>11_$sR2j3uez=^_e_l}>BI~Pnv$Ei{2B|D4O z)`PqJveG)J?yy490}z*4RQob&v>j%tAd}ofr;qctEX22&WTA&SA19gyCy9@v zOskdRhlSa!JsLn-?q8a^{0uj-&*MZ;OZch3L7x8|*oYqnpTSTB74A95a$bL<+wDD{ zm@y1&^W9*wXDf;n@elh$y6_jEala)hesoT)^9Mm>rXT8mx5Yr?g{X0(@o=o&5$#Cy zj3@C%B*QaKF>?{o`{na1#G4gi9$U-Xe0U*4qj z>ws~p9iuh#SV%xi)8RCdz&K&8$RU34l~Uqf|C3uk%3!)jZR3cnB>ywyIsQ@iJ_}XJ znLF%!cWR!1*oL|@==|CFb!E9XX;Kx^|GFYdi4!^e4L|+@w7USv-eiC{00`AB7y~hS zKR)aS&?Ca?90OGdJ|40Ja~wbpLCP!6lQ3Qvy?gdGMP`TiLG}>Gq5z0dQ<@tA4Bze% zFgzbq81yCv#PK5wh$jWU3 zk$_kTu$;r|)DrL}O^&_xpSjua^9Vlqh?+yWxP{sYUfT?d-l)-SIyQQN5gp2r>9rJ# zODMM}UD~5r>F4;bFhgCF+pG=Pk405h#hk5QR$rz|rrwxS+M{gscbyFIoeWRM(OX7% zEt4mPaWCa+YgjS@LDDM?z6g5%oxDv-9=<|8#*ZIaU_7}?+AXi9A}vzk5-pqezl(V> zujL__MwMS|Z!NgY`6!6;y40glPh7yMHigP3rdMv0lt) z!iHil_m!YB67X8m(T-R%ilL2FMfcfvEArFXz9zt&?ZW+WT909Dr0k!}CQ zldUJSb92lhF#&Sds81M(>Xw!p)42^xrp3~Z9CX2AI#ky#yRAVbwiSZ8n61Z2wW^j0 z7&ru?SMneTbwezw1zzAsmr4!d&GyE%J!38XnM-5Wy8%I3L zz7-a>-tIlFwiMzsQBUP9k>CD*2z%?Vr~0i=8Ap+g#! zZiWFQ6_}yBQ;cXa*ucAFC+le-QV`JEM?^pwDI|}|TIVt#@|ByM0$%P42w8O+S zfnm)}FVgoRDppHP+2o3#HMDn~+$71{G;r;&jN*CrdKIVETonz zE6RQ9+9O=o$gzrx>Z;^6)(WL)n56g0+B(}if?s-+@TIa^nZJ~7?tQ3ZsjB1DA59TC zq{uPwI-iN3xdcwe9I{*Zl2*XC;PNV8LO(yZ*jb1`ap*5QDu0tpL54$?h2${Bs2FEZ zQ&w}in2$MkGbzLe0vneOO5DAt29Kd&JqOoqd3{d~$$M+#ZwC;We*F8dM&ZjR_XXvy}xHW>0 z^7@EKxBXik^EjUbM7{t45_v4tK{4o}C^6yQQBQeeEj##h+=Kenu63v`iVVY2>}1@K zu8gNM{TmtHAWd27Nh&Ggi&!t81l2eOHvvVzoJ^Q+;KIG-;A((% zl_;kTX~wYT!guxmkqgO$qO>_d?qVXmT6wS61~UPIO@iUj6wz>Kz4#siEtF@72Rjz1 zJ|ViqAVMF9IGMUk zmf_@RmYx(%pRg&l@gCc7VzHt@+FElw?;5$!Fa(C624k&+hn$5EG`L5@4bGq#etHRo zJ#!Nd@agovp=-eow}IWsca>y4>*+kc2(di2;uf715pvd8g%0h(BPC+shzg#UY??Lv zE3Lc^nBv962~NnHZdnU(Vr!3UFrwEEh}?qDK7C81%i2Hk#mym5);y5{ZYGjL=T)}aRQ0OlsyQjr; z!%E}+`FHjV+Hy80-ff0{C}JmVPZCwQB*+WZ|4k6$D|MfY&7y}fO}{7E?LZu{sP8&M zY~K6ENtKe0>%tIh$*B&zfK0^c3i zz%RuIe4{5Y?j;u~UU%K1zZ+s)#bAhBN2b)&r}S{x%~EWr^engh++G)XQS zmI57L=VUaMAL?A&XwUfD*Zip@Dy0|K(7a8!tni01SYH~}NYSMbg=5ikBOduXB_8p4 zqm{l;ElxR!jDP0Y*E>#P?;;Tm)In*4$C5C*W_Y*ls3p*qA&0w#^I}n(c~j4RRYLGS z-qRG3JWh=8$(cv^DaohH6~JxW$RyCa_wt3-Y?oI%CV3;ncDZW%YTUT?#T1R7as{ET zX+TA571y}@3TYW+3+Hdg>KT#!xn=GmU6#pDF4p1bl^E|ry?+4Tue1l)it0Teq1v73 z)S4?xSoi5hsD?U}a94s*Rc@ozoj|R3py_Gj;m>dWV!y4R;Gls&NLd0$5Crz86mvJopmp-3 zKJO2MwNo?JD-)r@Urn%_qmMKX9E%FNbA{esih4hLT*tw<48 zq%0eh!jfFkx%pkbK``c!Gcm=`OygySCPvS=?Is#wLNfdn`D3kU7!`|dl+(3E6|ppI zfCDk+X`-RuAHT8Ji6rzz0dgz|yCaILnZ_TN!)1w{r1snl{pwWj}Jnb_#;} z^gb9oq>alH4V4_9@DM$3Z93<)Uku$OqBv8y|Vp_K` z&x-wWB;5JfEh>pzZEy3TfTmk3He!!Je}vmZ37kt2An8eR@=PaWfm#TwCP-0dWw!Fo zX&P%;^AU$SNT~=@g<<}Br|uh$_%fN;FAdSgQU2>O|Cuk5Yb4>7>$e8}XL>27%BL-w zKbX@er$1!aykCjmwWRu1$Qzi_Fyt44X8n0V%NqC}8kXs$rg!=o_vYEB=@zw_(pdAa zkBs)K9IkeRw5JG^t6O?I5|qI}Q8Ijx*wajZmLiNadFJC0G2yW>M3~t!TLu@q3b10A z3iXs${XB%em~QKVwPSbH<~Mz4bUlbs-VTDKuraPdhRtVlhDG3@RtqWVATNUb94nq5 zUJ66*RMDdC_zI+$&#hq-VP|3lO9P6o0_pM?{+H_Y8MA9oCozW++Nbf8_`V@+J^4R`d>lcG$RRP++_h6q7@y}xLSsuSUtH_~u?^PKSCtr#y5%*DX z6!}uY9X6n2<29(2<#f@z;>wm&ByT9ulrQ8sM5X9Wg@WMXL{Ui(=liwwe%p?UPpM1O z)y9y0A<#{YY?(AvhM>7-Qh)~32=%4|x$eXZwZxP|r{}d=-jw%Y{Ine)Js=+9xVowT zjG3R0DfC%+5-oUP{)nq(aAH;4E2{?iITTQBmB|X84%KHGqMR?6jsZ!3EVc9A%^#Z#5$9KNw|Qx3DRr)KxdiOLYkpL_ijN>8@#)7IXULi{fv)-yZPKxhCHZx%wtS~j9 zjOG-`P;v8J@v(!qn!`7%;Y)jrxH3Iq`s|v`x`{cCgi-Z_jT&8WO9fxYf)Vl7sgz9|XQb=uYaq&ohsLi(hn*(F{d%{ttJPC zArGm&kDPO`T62m{BEwlU!+RW__LX<^sklIKJ-P~oGx7}N#8n;K&P`#%ZjJPkxMm+; zfKqPa*r#lub6$4t9V}|a=YX!T#>?+Ue1wM`{2s44o>yDG=ntp_rg<6evAhQ{3AYg{ z7i-zKVBMUmPbRxp-gK3my5wdyxK7sq0Gxr>l8 zNUKj!7K~OkMHh(oZPc|6qGjoR;5@fB_Fyx`fGiaEJ>Gvv^_dL4Yev{foE=~5lHTU( zb{)s#`S&LWf8RX7WpQ5md42ac902-;(>H`2-8n@X{=Ub~?vCo~40Hc8ssE3!o`hJK zSb4aNgzQP>u$ct3)EEzclFI>s_15?De#&|h^51Ac9P?29*UuGy)*oOb<^HS9NKOb7 z|8IUB*FeXGYZM@YmPu|#|%Jf>6hsf6x7!MU{$Vf!kSx{e~4E-P>MLXHG7y0 zp!Eg+#Ucc+PnNEJ2IJ!aR)0E4nQj39A?<&ze+4%GdGkPza^4Il;h7if#ou`VQ0S!j zyz%NKz?SAXh!rROG6F73wig zPh_KEEt6HUgHc~gGqb~~)1jy{|5GXM`}f3Hi?xjt=WbhS-tzb$Rm3}3PMhbNupzIl zN4R;;ji?-`Cq#PJr$(IR}5}KD!Zttir2qkgVENX$~3<=or)f=;%x>UU2%m;S)B z+@J>juM!EQ!KTq)uqiZKxj_O*x(}D|WGwtKZ5+GYF%uu1Aw-T1D3~uSZJxUn?UzlpqC;O039|6;>$9&?=RW-lo`Mr&qeozJATGRf9Iip%sinK|@XJl0|);96dP3qCW1q!l&I+l8V?LFY{e zzV?(XM6(LM@by_!Oi`YHQk*&CeRU5z*`R4=;EEdqED`+UK-P5#6p;S=pEBN!+ zr&?6hV^2yw(X>)uT!cSDp5y(QN(N5P@F*yt)Ga#ng2Q4j? zRWT@8;Q|Y^e#CB&o4+)$hDHq!l7QvV)^KKD_WaTPCMLN#(E-m+OlF*teoMb5`*f}g zkymRCOD0>{qqCdqUaUiwnudR;KJQ0+EZ;xw z{ReOoqR)5cc}4N$6~J&?`D6{zu3WV=wogG)n&6$5HXcCuL5xW8FSa>Y$-L+G(3Rp3 zYinCr7k{e{gIL&#Bz5m~$&vdL(b?W&Rrz{iqZj zQ!IAx13C(B*@9z#xWF88G90te4vGR)-Vj1Gm`>h!>$Fb@HaYbtG&f%4 zpYZT~z@e9{`bkUz8S{F6D=ggd_OB8n@>6HD@QbI}9j8Y;h^-*T%<1->=TTkSss`Li z5M;7E4LWwyl`k|^>7{sP?%rz+`J?>yw&bZY4TUl!b9q~nu4Ua6XQ{MI{N(+p8G(fy zaaG#7_~2OR0lyh{KBK_S*8(O%J_R8Rqu`_7o3}bb3(iivE1sFSuM?W<>huNqyUL}T zo>%KB1_>yg<6aWw^t=SK`=0=$RC#Giy%?2p2Xy8=&uBQW~HUAH9CW+qjh`?g-6u*oe! zXnAT6)#9}gdr>>RjtrSSRn+r+LCW>1VGa?RjRw9yJhO6?3%|fdl(UG9+jjDQ0Ks1! z$4OlZzHd#+Bu4M!DluO(e)hj`3AJG_3)xKn9c}e3w}}#0{TbDOoqSPU+62fBT%FCm zG3*#01+`UF7V&J%{0Hztsk8;U{6!T*+e>ZA$F|!k{iHA;wV!t1!tU?P_t=X41jbjH zwF*7uI3=^A>ux9PtJRvtB96@P#pd03?Qf%}{bs878wfKm#eK1P70&d^vuQ%M1Dx=} zip~k8*n5V(n+`!baBLx+PBa6 zYQO0&aEaY9Kb&#gl!CY+GZbhKJu|x3U4XP?HV;E|eStd6VrqNrUOr6mN#?}0l_L7l zWNs=^xZG0Ne1E*jHgcA~2pcS=%lEBFiQX7lZDC_oDaPzG-Sp&Mi_Wt z=Y_&yg>Yrq6L2;kq-wV>kGU)bXskP(LGB99&fuS5hk;+#q~GWIuwJ`)TAONK%qn^d zc8}=m%lLvhMQ4I?uWlt`B4T?d9S_dy7vyPL$FBcHrP~c>x@#~ zu+6~iBSi(_H8Os_6r|V|UV4o82-g*l$saiUl2WMiY*6E_B6ZxX7 zJn1xIY94>pq0=zsQLv4Iz#uB!VT{~Ux*yJs`MXqU_O-wxqBxzO(`L5#SN@xA)iLG# zNz0DZTKS0mD+8AKIxJ^Uq~gbIM{mXf?{Z?A?vUv8K|By8yi0S@T#&ceNw*W$NHVrv zS>4xFP~@HFSkn)YrHdU?yz3lD|LPL`Y{n-V4|B(2XkW8i+?u-4lm zm)~2bmkq%YZcF^%5(k&lb&}~-T%R*u8%1a(Q_QW`Y-*0EGxL66o;8~)@mL-r6P>-o zPgZ$fwG*RgC*+tw_!U@kQlP??DR?Kd_W`f{yMaqp0)#DPwmaJ-a!@JXvzCcID{$Bz zM$)oj3d6Sp{R61=Jm8c#A@!%|06LaD+`;oCzvAfOlu3QZopYK@F_ZdHMvUuOMnzrW zdH%*fz+~~6+OB_=?9pVg`h3em3*&(68|G#5M47hTIjwJQ+c9woUW!ftxl|APa1_UYL6c*r+Jy}Tgn zXVdiT?FaH!(K}b;XHB=#V0;c1m}W~Ay3CrZI{i1x87bnYB`^( zeQ!oZxl>A24wU@sHwZ1?>@052htSg-WWIVmR~LSm>%7CpYXnIwk4_Av2!^7I$lk~) zaGZvf;En}dit--{XIQCoQJT}P6Q+Z{jSb$18o|n(hEW$EWb;ms0DhZ|NGNhkem|~Ut z#+{3;MO#2ANm5)NLIh+S;sqgE7SlZQCYZVnLyLLRh|hG=JP=CU6eig^2A$yKntbyV zs34VmuARCNm8!e1eee*w_-tVYeX<<+@FK6fA@&m;ts&aO^K9l}dNRDIKN`rC>Xkpa zSxopibzZtzmbwX~c7xAGY0L2ExPhM5L zL{j(Y^VlD7P;4$3^U4-#sFm29(fF*@YTOoXwS(EY=VRe#qZ;7wX51Sg%>bY7K zGy{djsBw?=q2&sd>DeYG{Psl^%cGEkK&bl7eB1zi;R}MIOqHm#$f;FBP+kmOa+Kld zN`7v+c4C+PQ&A{Rc$G}m3$bG)fbL4qq-*0VS4N$jWX9;B*O>m&@MGN=cbIiH9+sV` zTXByet<*2&L*JGQ<(qk82)_V1>Z6H!4>ZKJ(TyH8F(0TO0UOCK=HK?Aixq2|21%}_ zbvtc9-JXk&!=d1$igS@{3|>k%oSDCF@VuA$XU0J}1=t{st|6?hM6Mb&aXVS~P-_Sp zgML-BW!^h(Je!UqU)Mx=Tz7gpf_;}(@I3Dw-DFOwqO%WovCFV<+{B)HP)Rs*Al;OV zk130B#vG@sK4URv$g@6AR9g)#nM>EFnIaGdtb22iyz-vmS^w;o>f>?EC0vjp&z7N} znw?w&t-7RcC+>sRmKGfwzWCC}=-0;9L+YeVv3Pkl_y)u}g2n99$UGFqlW5D}es=@# z1MlcMQ2(F@6M-2Y#KhZhk66S${`%2ab6@uo^os)jwzF)54L zT=5QmH!-N_;>X1AA-LNa*XVvP^cS>IGG}E&u3^7AK zz{lzGSk_xnynd^5WQOC_E6BdzGVk-d+rL;9O4U~|yG8H9^~3mMTk;F#g;&Hswh%oD z8S2X>OGIy9CiYdoKV7O!KWXW$5IY_bR$n3$bChK4r+HX9?K~e6<2A@$lkOHst2iNF zC_Xtk=PZt(9+`f&97!nlM@>|=g3M9utiv*3q@$tb&sFBp%+uDuwD_M$Dp6+$9JHy| zd$)tQIk~tf2~d9*!ARH-`dase(eCkYo#yr%%D)aQ=wqTTYWJzhOb$E)|Fx)dQ^+{^ z(--756@#pC8O=#5(m07v> zUq+r71R4Gjj$3#=!r7wc&h1DiP*|9sFeJKFbnQZa*bWVwNU&L?)FXN=A>&akRvCMG|Qsw+56<_XT3W%ywe3~I9-!QulDD@=355s2yK%j ziV+60Y6#52j1b$m#|-9ma;!?$swA4jqF5xidH5!6GqHdwwwYQmQvH!CW-LZC8Vq_c zpOCwan=K6@Ff`}JF!j{4c`r>AU@5(y_1aq+?!wPJLWT?1;eiXoe(rK*@|BYy&!)$x z_LsNyY$fxbaSXZILQ9q$6l)g0HABYHnQWxdi03+*4R2~~{D-Kz{ikn_Cq{N!Yn|J@ zePGonMu9BmqAySph06=VylE2;p;vUYO{6}JoF$}o7aB!0R2q4YxzyBr` z4tVwDnmO*|=k*qVMAOeXm>}3Vc*%LD>E34@C!3u$pmJadOYbOuICm`7fW3g%FXz3z zgJ3*OK(Kh@w?_btmjLplv7ni%2mkTbloOo)!76%qPcoMrynbDD>;&UOo%@(W8UVso z|66kTU>Mf0QXzfljm2!0X2~C}q&O(esGjVE3=8 zqSAoJJaHW<>{BwVEIZ~yzF@;upH?%&W|+@V>yT# z*AHlYdBb>U3&0iPI>U&@l7an|Xl2RrvgM5oj#6NbgKMtzK2axB#PDfXXu7L(D_53s z&cOEIW1A;#Y|8m;GyE;Z>4TPM2h)q^CBHgbIiM)o5;aYBEiFL5$0J+{>_W^zHc$|k z6%0Y*Ju&nzD=2?AeDAMx==m63LxNhwc|F->dxWA*EXJV z6vD@tSGZtvHv8(Ado-X+#^`842GSl}K&XQOZ-9!}U9SfJ&CTeGls~*PfvtyYY*?I) z1W3Bh;U0*rH05CURlZyG_j2F&QvVs%6m<#t1y?h=j4k)Ap1C=MS{bpz4aDM(iyP@- z>fz>~o%eVycb%?|lNEF=SH6Q2wNm`zBOa!WbU#PB-q+L^B)=4$)UVMEW19G2874gR znw=KG?qko7LoJ6VyQ7`hIlb1#7zRTf+eQ15>ZhEGsY90n&VGtSVQy9*Xp-7e7!zRQ-&s6TPSur;1P8k&EN=eF2rW z{u*ai$P_N|+O=(~!^*jC(}qgjTA;D9H}W-Fa~DMje8|KclGIu3|5`~qN3F%|u%f_) z@srz%=sx%v@PtU(vl|osgUk6b`u!*`UZHJT6Re3WDlyF%-rv@LrO-(J)Zj6y(L#QCr zI`tW6D6-l0&DoS9f6Ij|NaDV>SkZ!C?{J~M9H0*9<9t+}z0&-KK_o?GS$a>H^zV_^ z@x!yVdNqY1kF9N60eSA9rnZ{2yed5K6D@hwCDu<)tfBAu+Q-^ z#lJLN72A?uHdGnv2<_7b-S@Kpht9~n>yRk4mK-34aBXU$Y`!y@ zZD8W2D!G^s447DB41_n3Z1uA@UgV0sD#B2I2xbGZ98A^=Ow{P!d^pIVdDYZHP7Egv?&fbB(N<^?TG-+h!$W0cB`)&Ly~wHaVU$3jVqKmbGWTn} zz>22OiiA_u@)Nxf()}1@+6yY$6`$kS_ond=4H3#9kbUf7ylHDtSSayMpuMlQNlu*} z(RVDbYy;!EWLy3vf<{&8!*n1yIKDvEv#pCuP{&*g+tZ4!$KO-e%rDon77=JIDsW#^ z)XZ+#Vc>XXD2OR#XR~zyL+n1}DQbec5y<|S$C0GPW>>((Qg*8zl@nIVN*oXEflgY^Ey)%DF3wh$x9oZlI!`hAH?dH3-F=`zh z7`WuQw%TiJh?DiXFA(Pa-f)AoEVE4nrR;wU-)Zy^pIjpR6f(}#&V|65CUVhjOW&#v zMysyCMS(zet7hJOg*YO>$JZ(~{=*=AHWU%?Ip0@5Ck{pr0@jz^dyj;)ib=K7C9 zkX)Qw0X^1BW+!80@bJx4j$Yy? z*^a_p&|;_15gLorDCLzPomHWIo8~b#L}l{Yxmc~-=0fSMMO>29h@66mgs#_w=zzkU zg0LY!-?nD$hCl-hq+ZM189hl7uo7tjrL&;jnKH2#P#8(jv5 zc*45+Wkj`Cg|q7eTFFTcjt3Jcsx`W?g2bULbnKipKTQ#&DU~T8XC4)llvvl(ZEKzd zcZxS?{l&sc1{Z!+rc7A=<9&zh_Se6OP(Sz^$&^Lwcs>S z{&Vuk{xegfLaQbj9`Z6!_%nZ=-&*_{zwM;ITBGlg{{TC7@ACRg zeo<`M1ld!7D5T(novCqwTbbvmaYsDGj0CKl`MUGu(XzAWgJ$8pg=`z4qHx|C%@Fr( zP+$xX*!#LYx#Q}%uC)F<*A!;MH#>5w?~vpRDXPU9 z7|Cv?JbEW>b~%Ue4WJHtG6a&$s>B3+ZG*!kj2iWVC=q%q7g$T~4`t|=Fz~anWm`fl z3iC8%G_@%s6p1R?g}?6*`1wFv6rL!vQ!3R5F!rYh6&~ox6-&qsnmh=T#BhJ!5P`yR zzf+>|gD;}-@dd74KU^>UwXCdFgI>A7_P@CCu@*Wo$p_fxS!;#RMhhZBS#1aXfI%wn zTKtorTa|F_#*WNz53*pU_tB``Q}scxf?~!Gd-gu_^d!5_M|{_+1=gb<6vs3ed#0C) zzGE110S!zN)UtDCP#mypcv`@K7+dmJF3@8;SnKPB${yQ^qqMiNit4afw~P$m{;Mu* z7Gi4d!r0)~#iA77>L4}_?N_`KxS!(8yfX`WlX$7mpRennReH{>U^rvzvG2fHpO)e4 zUQbnP(igMo<2EDRGadrPC!q3tdy>50#<`;JbJuw@T56 z`)(@4Q*&?LF~Lf-A?30uOeuzGN@;XFuqoXrYDZB*WB!_UvKSj3kv7{Q`~9^NJ0s_- zL*6<+j(**AO>GksVZLE4l#Uo31y+?lfJ}Ip4(_QJ=I#5=ar^_kqaSTX@CC#d=CZZq z_a<(I1}jVc)X?uZ5Ao2;dHX5{w^kLFm1nTFAo1Qv?5m~}&F8w&J;Sh7&Kiv}zG{O&B+5IjFd1lR5Aphe}>~FwsogJ_?}<-PXfo zTXVMh%-YcOnsU;UWti`GEPtOnZPC7bXL$1Y0``Zam{_C3#sjEyplIcB!07Q3&1Gws zS7VgDS>F4}V8Kn+2rQrR@ex1vZ7d{rwkqpoz=eb_#7L6aHwl||7jzzkLssH zv7Zz+*;+0@xOSb179{Qspvq)<>Jk8GSc#u)ri*I+wWZ3?#9iEHe)9`uAy$Hhi>T&= z68{LlZxBySyAmVf>gl;#=fk%#EHEgq88F_$R^WyxT1!grx{-~ywS8hI!bv)77UO@X zthMB(xu`I5`*p6@ZbLBJwjt=+uozXZ{zqLjyd$|1V*4vqEqd%ScW;(2HTFQ^?0Dh9 z99Iw6u(=@WB)O?;tH_yOU5To&x~pz4;JXr(-GU;NpDXvu}RkUeciV!`M1 zzo5fkZAVKqTp`LvN0$jKYBD#m?W!&$PC68&C0ifsBVVlf#g#V;V;QsWdPi>;m4f(c z3>hm01~asxzeE(~&1yhvyj=QR_6z((FB|f-t>E?++FtDT9Qb-o)zu(YSHH|-C$8$( z1o>G8{^}Lo1}pwY@f{DmPL7BTmKiJzv04HXtTSKHbG=!>#Rv$PCiv!{m`E|UUetawYSh}=SOrOt%#RnOGWoD{$ zSTLEPYEjthO_OId$jM{N(19m__eNVZp2sSj$?m0EV=!lnbwRX?`dP) z>?3Ru!?CNjy$IWPAK9|9HAmEFktwEVEe&>#xYs=AJBs28uZyXQYBIo(8LA6z=FQ$Q zB_B1u>*N9W?UCqY#oF0;;zdfFl0Bwq)UI=rS;A|yGlc5Eas2RfIclc#dv#bPuFghC z*0bSB>q0lS`3rpooT%1T-IA;q*$%-k0Ftj{Sn5)WK~MlIbWqyBF)>dU^Hk3Xl~OEh z(=aF}thjJ9=Gj-F;OIN5I#D%3nrb7do9-Fe|3%TY<0*?)ES~+b&2SkpDRB0K$?R<1 zDR6-(Xc0ZJ{E&-peRz#LXOo3RgEQO*v z+=3vAorjgWTeDVykBuIc$NE8#ESoNSgqQRAr_wpkpDLq_J?wIl>{(9}J%T8jeluYv zp^(m1yV=jJgPQ&|!%=uI;exwM)}@Twh8FKvCTNY2;W{kx??_20#>Fi(QH=P?!+Cl+ zMmO2-IkaXs4cbw(NN6}pyWFZcGN~)(46%U*)ZOwFUtblBXhGvIPJgyp@6Twcug;JD zwIrP;2!>{=a09&`@2QmN%@&fAp%YC+Ap;)z&HSkaOU7Gi%#4fHU(@8`RrA@7nj zKC627uf6%qN12&!Padkzt(igWQ6fJc?yweOgd7(F=G_8mT~J@X~7M%92t7{a8VP4$E?mMEb?dB3P<<1FTWTe7Vm z{HaecLNxTpHXgxeSXoAWqo@_q5d<<2a!!Ad&Vk3Nebh36mo z9dqNJ9YRxB+8md-%^$RN2x(iE&Si?Rh}jF|rA_AhnFjQf2QeW9aoL>rI?7Te!t@oF z4wq;ZS&$8>aym;sya(Pj70&l}Q#WipEkX=_8#I*^I1(63CULXt$K> z%bS*_{qNRs{(nU%(*NZC0RP)^i-T>{x&HQu{K(8vygbOn*&pDFT0%-O9IXM!p4Xqu z=d83l_uT!}17y?wLO9JeGTj+Yjhz7~_ULVbMH;$xocjReD8llplTiQ=G=;UDRptQL z|8K;f|JvXFpPJZ%NAu6O-#qxHb|YLAy6V^V=WB>=T`tX0gi!4F@k2Dfx^H2HoWWil zQ^xdZuGehSkWy2xPVWcPB6hJD*}pF=dEYSXFu!mrbFWk|afCoYT~SHP3OFIsZj}z{ z@*eb%MlXw5;<}Cd)FZ3x*u3itGB!D-{Yj~)y~P)C9eyPO0}}FuR`z~J<-OYVqXzFR zw;GUj^{30(EPzer7ad=|&W?N(6sl$wv3JNQ7W7(8}6q5X&FM zx!v1Q?yP_rKUyF&Xny3nX*d6rJjPzLnvwNa<@b8)Puv}a!uvVQ)@r)(aV{Yn@xR_D zUL%_m4G$P<<);;u+TOVxO0A(%7#}#jAAbp}s6&ze0sFEgCZg;ihvQIfUA#rYHT8UZ z)yzLrWmkr-E3G>D+kK8@c83Q`%rRS*ugMYYLGa0NmtXH{!&Cm`uSQ>03@FERGfgd6 zzF`C6l431QLgNjosS(;KeJ@%(6$1z$kmyMPenuI^ev?8hm+$yCq0a8{vwr}~8?N6| z<@d`;$27T9{yfuVFaXO>pF>z?CpK*|Y%`U>M1&?_pufDyujaddgZWc}l1S!$M#DORX?gTj`{cvhNMN+I&@8E^zi zfi-q*IS)7gmQUWv%GSLJzc(+FoS9~cZQ7HCPBAu;qr;`=(G8B`Z*eF!ADRc@s3A$v zEKkirvsQ%$*8xKB<3|o9QKM3VmUoETQPv0Q{xMW*Y^iYc z%#3lrjw~6!_8uD!2*&8zj7mOVajE84 zjPeJ`$Z*qPX)TtvHtaGDMD%;w6ygYq$WQT)jo=VX-+&{mz*bYKAHCk-1TQJ9E5DC8q zf_|hzH(z?DS~?(rJOc?#*crzmd|3E)ByVcLYxg>i@mF4y)Rh#cwbdtN>;RaFjaqfm zi;60ViZa@%??p}9r$V5rh-bvSCq0S&P0E{sbQwAL_)ztxXeUtkkX)1m1xWyU6z!*r zSXefKWbMC2D{6_H8x+!*y?K3}XRG)zogMg+G(!Vz$A*l>JgH@ClONa6(19AT-OXs= zX@@;Vk{Q8Do)S|?ysAmu$|0M@@B=+ghg?+#lGhopm^ej{ECq7U7Nh6Avb+}79IcoJ zPE!jsZQYVOJNdNGQ)I$4)e=411ERd*)M`7{&^Q&0FjlX9b<4wu?t_?cGR{ik70Mc$ zM>4KQ;UY?OL=9yw-V7MQS?Os6b(tHr;dHKIKfDguDC$PeNJO$=dIvhp+Rq}#gfx8x zb)tzOC3V?Q=805AVKwINS6s|pT%ZpJ6^UU=hDY3bDj87bdS33jbijMw!uKr%wNLw2 zje)$=dRs3ctV|Jhicd&2*Px$f`C*3U`2BN}EPH}+>OFW=v+uo%lATU}5|aP^MUDm` zKsyaf8;D}3rhNls+9zwdG@Di9BtC(})Y+m@;pJM57}#lwQX!>z6EH)@xT-(;fSZ}? z&fKgdZNGmv3AI1N$IlETl~33K>!Q@K_n~+~LvS^^nXZ*Rj>?uJX+v^R&UJ+vT@xWO zQ?RD}!oXMUHG77k_5zV~-axTjIz&ysCDva1$@C&G1>YVjpU_MCq}GUlmydGa-(Wzs zKW;?_^2ntg%9JF)M%tbN*D;I8rj%f#8L%|iA%BzdrVF0Jx7|p2$v1=-aI>iqZ=LJu z>)`t6N?DMq`amZ`fu7#c!mm!R)?Sc+|M^n5^Qs|NZLKw}qQpiqJBN^I(8KEeC(p%x zDCJ3S$~&u9Br(6uixSPb3>=pIyks=5?!eC_%e!xljf* zqG9|~4yPgxz_Z0er|D;m%YKS&<(;u)A}1n(^sHk~;KQ8?v!M+<9~zJG-{7cbl6ugx zR!5EZ`LHjCrUs&)mbOh3jM&g!N6JEwNQGZo_-mf5SyXwoLPb|tS)j}0o@SBpbNw)~ z@>gM@;l{(f)h{-Uqq+%FzQhVLxszGSuAm#QHXz^Da6jBh-W(z?AW9D-qWlMlJO@=^ znxokH>;|c=vNB89t9GSx%lb#Vj%xa;QhFu7U1$CJgf3A05W2vjy7ehljm=(QAOjINW%Y zc5CT@sjn>TR-;-n{sGLc_jNS_-#Un_3Ne^#{F?jiX?j#d4TWEratMp3rza^M5DfGh zM>O1%Hnt~QHf-5K-)ZM@$;V_iT_95QkW~;P%_cE>u@gs2dZe8zi@Q~V)i2%(KSp1n z)miGin}~On=N*Qd;*C@GgQf z_M__!&5Ka;$$VM0Caw!l<1-L4X+e8?;|Ng=U0IrI#OZ<<*3C4KkyX?}BQ&?X=aD%Z zgULicRb z59 zXx$Z!w{;O@;n~`X$r=2v<3AFY#uRV7-vJj@ekXH#X9D8{E+;UBd4U`^?+k-D_^_g|FPmt)ZT>W|nee)&JHKdXXsDJ&!xxViB<-GW z^gJ-X*j1`EQ#U31Z{AO=m^*fU#0@Q=5qnsoCZP6e>ZZ^X&n1oa$@ew1gH@e#GG51T z&HBHTyY#I1VvaPKcXD*6?Qy@)OXqo*2hHY9{$4O~It2L3avki9a@Ca7IB*!7^od=+ z9N}uO@ROAJ2e{W#V7NsxAM?l6tV%J|DS8$?aQ<6oBxW>kn!~d?*KPkF&LhvnDgD^rO{%{v2p_R8yP${P`vvB$ zf_^EdH}c7QzV7~`>{>%O(o=4}WcFT3C%5X3YnBFEQK3|4OT{-Ol+{@I4{((t;eA|Z zisww9V+ApciXf~DPW{Mb4{(xJZeDUjoUhK(`|-y}#&Bh}*?2Y(r=4dc+T{4!-OU0^ zwVvH*#OupKi82l50H2%RKKlJ}o9ipTqYJ8l}BBoh|9 zp*!(QLY#bEKG|va@CSCZ^%Bz@E2WJX`=(8+a&s9*X+j|y zlp{HK#!cqca-!0!`3LGk2X&@(`AHcp{lobZP4m$3`~3W2syK)_Sf|{<5g=yzv8g_) zy2Fj4pX7b9tA1SDWOJm+~PmPWNLjr+NDQz-RIxHN*TndVs= z_B^}TvAw^E-46j?-(^BR2C4qxcggSOhSGbsAU(UWBNP1$yLhqk232Sw4J5UR2U=DK zq4V5Dl5PX=c*9d?Z1q?k+VuH16ji@N%b#~|lt19qH2e11E()p&&5Q-I7^!dCNb#$e z6&_A8K%GUxs-aa!02eU`A4gc2kX$Y-BDG|RlNry(3THBWm5U9|(+5COjKigvL9<&e z;fvN#UT)W~JV);)@Y#;+e}?}69N8b#P_OoU)zwMfa)Z?-h{jG&mk`b>02P_I)5aSc zACh%oK4Ew3o&zRsFdsa5qHfxEY~qXYD;Z=ueJ~q(2w00|nJ0p!(XnwVIl`s|JW8x6 zkkfn|&$>VyGO3r&BYlKP1(K_&CSY4NBx^)zE@3sk7J~l*LB&TP(>{m9po-C?Phlxi z^#eE@Ljm%q7I1J@mJq{bk{l_}(rMeAW}GxCcU34j)QP9qNtXSC2%60QQWNJEH;X>) z=u`WK7?Cp^8o+%TL29vggkchfZ&lA8q6I$Baw%mKlQLdM9RTe&*(&z=d80jEP zo^wr0zcQ+kQg&ux8+1_N%_HFm!y>QWcH~|#VR~|f9qT~RdnMv@Wa}oCrtbf&DWS15 zb0;%3Rf8w}AU5?zC; zyN_w*-%$=8Xw*L+PR+oV{ivEy0(jH7>S&DCH<~&{qe3hKJR z)HR2bLDPJEeQXDjdeeENti<=H_Js#X!<|L6U7B*=nW2=daYveUguBS+w&~PqL2H+m zAkS!aN_&NDSd5T1H26(oK0RuEL9S3z#(ntXi5ja8_F3#4`Sl2eDpF%gNp+Svvh zrv5+#8r|2O&Pac=IUD;r9Fz63)f_LU-J52DXd7GlZ*$!2r?;A*&Yq^`R;DM*)OKn* zNiMYQQYqV1Lv&=pQVF>ho`@fV3<4X&AxzG7#QDXncTl4{o^6Nc4 zhARfBdHM15&!y`VCuEKiuCh3@$;6EeO`ZYtV-Y-t8XC4&MU`g@Lu^odPr%0YXqeJmAKqo`c>X@>jgFgrD4%{ zmugy+qu{-^n37;9Pflte?tIXq0O(uTt>Mp&lYS|aIa0PHS+-L$4hp;?XHvls6|1}@ z|J)cF3m(ce9=k941hmn4d^|F|{TxU-+D2Fvyx2j#7!lMV^vbyv*~UL-i#!$`I2KDg z7*p8GF#`(sR!NZtybd8%LA;^Ke%*F?-x&nf`x(gw=gwN&*K9?bz7`fm*SiDbadcG1Um;eKYGQmNK4&QD6aT#oGAn9T9!u6z9CZ3L#q?E~S@5Jbv>Az4g^`1)--9axa~fM!`|PnbBzX5NPfic9507wMU}IdWCOp8tog4G~wC3XsjMh zJEaV7gtA@M%S|R8&Uz+1DCb7C{0l#=RrT{YgZ#q&&Pm>_j4OJ=!*h%K57`~J4jVsF zC2Q&VzpH_d+US_R)aaIS8)JuirHs(N*?0<*r*`YdJv$%Q{ZOUTbrNNMPP_$bvu*|+Amb|Z#-Tg(&O5Ocq|4)VJ|m!RJO@YeCLR#h(|(>r-EeMwRk4?8QO#wA zvgD?H&!qQNkD^d~wf*Agx0*#RlD)I zSkvf0L{{dw_Jb4Re$Hv9iuQdlO)+_@TDM!a{US|lL*ykNb{iH1#$g}3B>DCkGqibA zD3pgs5CAHDZThU>c3URM2z+sPNw5lg%>GUm${TB5G2|GQD5n06mWQcGHiJH16PQFF z4-yQa&4<(DjV@3u>I!{(%^D>$v~NzMt6TRuwyko?G;(NoY)!r+fl`*r4tDI) zYp+KOhD~0~zc;<9vjayR$mVjHm zQnD$4b{4I)Ppe7ZyYUzOeyylF`c{~0yQ0b)Ic!kQZ z2`xGH_E-SFaqjE|cVvbUUf)Jh>^rnWQY`y>rCgxX1Zd!9M1=^bCRiZ|&LpLItt-*1 znLwEqDsHK&jiq|Z65C4gZIpV8@)e!%3s#VmbU&0~ zEHG8AhZ;W_SlJ>jMsU`MgXpWjl=pURAvcR2p7{LuQ)%U}$%2sZtDLJWO-N1hAV@d( z4nsW6(&T44=EU=~PN8}&qvAL5mO*r~E{{w?mFvzM3rl9Cy>tbskgteZungs1kw5z6 zkPup7a7tijjU&0GNT%~{BSjR#Dk#=2Ap_GCtomv78f^v27WxOH=%aitt@ftp6Lx?& z82%52`|EE|1$afjh+d)&Staa{nRcp5@y&aY@4rg*7hh(bHF(w6KRki{;a6i}{|BS+ z`tQLHFMUPc&blb-#Ly`RKc}FwoUt8m`)qqdz_& zrvE^0Z?v?{MuQGD(t~u5vs~+ZiB4|t54g1&xH+W^i=>}gln+F_%8k!|Ju7aECu9)Y z^5;Dwk!x`;^k>m5r=xTEyW#sitV#HE$!^{Cf^T$$twMuCDpapOHE|HU{QRO;8Oa*( zhm&mjbTr#b{=FVazG-gT^bhgJ>FMd}T92D2hyZjw*3a+9C*XbA?P3yVYwFJIo;aJZ z=D4bmeKf+BfWEDt1#UQct>*yeRPu_RwIDLNr^ry#)wI?L9ggL#))FGPQL0T5=ohyr zEJvGCx(RE>fX5^9AO27p3i%8@7P*KT#JOS zMab048tAar30!1KiwOxUdKK@Q#&;}vlmx+Lb7e60;ah#-myTl;gG-1fe82fT?3F7o z6Flo^XHTvhS%JEmzc}r)zh|kIQL*;YRPyT=5NH!b!Z_cUXCCcRLiW`nbwa;gcF0$X zk1pEW?_JH8xhw{BtO8Al0*^m!x`>*cYx8iRVPf;7M`?Bb-bbmnZtB$UAL4V9!3Z%A z5l%G}t-#z)cp(OU9O)f3Czr#WFGnP*4=Dx0#;p_{yM1gVf~X*>^<^-HmQRFY8N<;!d}$hvJM{dFO$48~1RRuMgYA(;Lu7 zpf{2Gb%0Gt?xIv4%iF7~+ET0)`JV=qRp|A?XU-67P+rZr!xDHCAja@xI5*Pg!=|uD>OH-tT+m# zqYEuAnWxJMi`H43i)mgVpRWJJdH3nAu-#|!de6gs&J?ZHa=A5;`Uj(vqnhgBx`Ml- z{w9UZ8qP`K50C+rp!*QPoey;G1M5!C@ijLF^_%hIxK7^BV?P#VR$R=RTygCl3N{l8 z$isw{oaoFxd(H4QJ%)AEu0xyqSLOR!?un#?l8rbYP-}q?5;oGVpMPLKN?iJ%1vH=b z|50XVED+5-f6JHO3ZCjVxaz_(hHsYiZn6hfr9gkJo7bN&oW~UZ8@mirY5>2=cuzJDbE0_s&$aQD&e#(JvIR&YY~dufIBI_#Br{N;1=bqbv< zmHRn)$^K|V^Db{Cw6%?kw#m#|t(s7uv#Awp*=@z|Ea+jG@uC6|m@9Y+^CYT>DV%`| zm^zTdFWdf3g(@L!|G{`#-20W9xLBfd2CGJ7wisbXAglHb)HDQ^5uLSv?&AZdKTiuj zF%U_uVrenj=Cu-Pe?6A+R6Dj6IT=Crxf*DV9;-<_bTdysDcvBPNgUCn+DYwI;dwW= z#3fRsFv2L|#9$sH0?n9jj0yN){LJ7ZCzf_GtCmstvCVI11R;WxfEVx&2BmYsvWj-i z+j8Qz9GGmjY+SZ?>eywQ`*nZb#ghN9flf{AT`>|@zXZJ$Pe%3HyFSfw>@yTD&KHOC z4S30awBpH-9#gKG$mwH2=!O=NA_G|*P1K)Aa~fQl5C_*ChkZ+&u4+jmtLD5g0K$@$ zV^6|FftuTV>!nS|$-}$Y#GLu_b3s~D9n@HBCy#Xi@u`=bduXvMx`c;&+O2#j2f$Cn z{Z^7(SBNvL)e1?)ieAw|hegS9H7VtT0m{@rUNn@Ys0Ru7Q6lBYU^^t2cRv3~5z= zav$D4Fn=u%?1TmC*%)e9R$Z6Y{(^)I>I`f@c->$UHGX_#7vjgkY(z~tP1`~7Ww=!8 zFvL((S$_67#%ojcsE%E4|qW|o1$2^Rn+Xx0N2 zy9EKLtH=T(TW61>P~=A86MNMlUcMLbCTUHeEjuW}Fho1@sOT+~f&vhS`VUXiD9M`ifENu}uP{GP`?_3R{SLNBe~N)^W*Lmr+>NNiz*_$p zwD$}BlZlCX41{1vHIbC!53m<2 zOV)5tjtN@*;p_V&a+@rqXA_Gm?Ue2K35`WW+VA)Bp3g*P^l~$3B0R$NRe=e_ET2IH z(DX8>Y=a{;i8QAF`dblI5d)YJ2Nggp7YB`RV}+*8{~`U!w9Fuw<`Ki^cl)!{P-rch z#7tq0(OfV-V064Z$=EH5)OY{HGh=Q-CnK>GM(7Z$ZA}IzWvW=xX$(uQlLm0aIQ*h4 zV;YY*6mn=;l=_$_{=MPgIrPxxGKI$^GiV*$^I&OznK<&c(O-I;zS#NOZD839XX0=CMJ2RO(Y(Uqyl_azfaJ)K(;y9$vB;n626Kt)2f+@8qyk_P5k8qp zXBcdh(Bgsm?!3Yao}*OvR1&%oXkaOs-KG6uudM^$Hy`;$#juMNJ8}j+6%SJbDv1aq z>*B4HZ1gi#3LOM9bXf!i>*7hIVsCHrWyTrajQo)eU!r{feH>4)H_A`&DEzlx1dWa; zVdmQ-T4Pn}v*PbsXsE^a_+9%{Kl!XzLl1M*@Jx35?Am$X5y`=8i{j;-Gb2cb(9PeP zk`Z1MT@BovKC;>?b_3E_M8@~bBv7ZVF%``%N;TEAm+Spytx#9F@Cdd-CKQ;cFYx=#*t#l^vyO|&z|CH0g4q7*#&9X*I<+Y0328(iP65yGpB~^n# zcrqR^pq;5K&oek(UGIb$o3QAfp*&St*YMdz1nHTm%<-u%)4*I~3!wS|GHX!XxaC&G?1+B*?yx2)NE`T`29&aG#4XHf~GUe+n+!Up5X zAmh>*9ToAKt?m`|2fClnGn(qh-d{4FZN1DNJqBbLU_}6W)8nI_-7MXSi;4GnftoJGuxq0mptzoIUhw0eA&NNsAWc7re|bYctfwQXc;(Rvkqkn zkKnrwR;X)89JHIBB?-o@Y zWMDVP@_&VlQ?MT3EWtt-tAdwvxdk-9ZXqOzu3|RpTsW!k%_V# z9&au}sVOf0!6+kIDUqXiEY*yXc~!vcK8~EcWa{{aFkvn{VNFU}^kUPBL$ZdlzIYc~ z6zx>}ND!KDO`If?P>4_&QNM|MWpkijoMH?*|nzw}xkmUJ%ThoD2p{F%l>W2n?GQxwY~q@&{a zI+l3wlo0jr)=x4NPB^kzpA1UtS!JUzw52yIu6)m+{w8 z$*BgvmQ|-VL4mgpfD&dJH&JU#aHG)EUVAW|oRJ%MwS&3z6Q|cd6IqLTvnWA`VxLP> z>yZYJmPD+ICb6GuDfqwbKyyesr^UMZoXq|o9l8od}bS_58xX-~EQYUk((jpzi@eA*0}-s&gbWOCk4&Xo&(5LY$Z z5)ilf!twa!SYUKH=(ZrK`|hfS8aj)2-e^A|%TiEPsXjR@d#7R;r2=(OZWW;uH;f+f zWa3gnjbwpkpw2uhxHNqw#l$2mv2Mueu!eQdQ2E1J2cjZ%sZq4q(>3awV3J4dnl)DI zM!qJ8LiqFB-{xx9cbpvvqGz^`uzZ3Vq3kgXIJTKJyIQiGOsnYLHm16Vfq+5UR=!cf z#3WSuD=oS#N1*j$Qt0qFhw29Jg3AepyL{R<^d9NaP>i+Q>M6HONK^my-0|G}6HmS3 zLE+ZCjz&;tpX4oV#oEHbKNwqw>cq8yKY}l_ib+`B4SU#=mAmZ^7kd)@y8d#`ppu|v z@?OOHfP`hru88FUyc>Q?`vI-c>ZLAq*td|LdIoUg_{gMKL`m%IbWI**5M z{z|>&B6QI34>!ZqIV^*Aare(`|JVayEyuL6w3-_H!+zT@KIWEK$6jlyo8Wjh$ZoBQ z{L~Qy>xjU8Jw_8YFUVjY-K)(tVFOIs?=|8t(B9uIU#wjsU$Uuw%N0m$Ttt+wcIsC;I_n0V@WuzJl}WDCV9~KRIUMSR zOB5HQp8Y}yj_Ss)#!emti_qs2P?_|`6{nY>nGJ7~PO9!Xd)}`uMluH0B{Ep*I3cvvJw?s~#j)I_WuQrbWQV(fl$ij_WV-Lv)MuDmA5X-yh zHSGptxg1X-ae64tE~TMx<=eH=S{;$)$j6)8pGCSnGBuyLzy^=nh#j*{+rohIxR5^x zRgPQXW0KA$ZH}SN_F91-DNFnQClbr?{I#7x1Jn@T$o*jMm{(qrrQ|*;J5<5K%t7)W z44AK!{$SVTz4PI!Fhma#%60%zoNrOl1kK2$PJ^_-I6tQ7`u6~=CP1KL4sO=hL)TGnhVPzG=vSAdwb#I<>DdjK@g8;?O+#+>9>pIUZWTj5y7PG|5 ztt)&eBwfVTiHf$Ws+^|ZIR^f09(ZVu`5P$Qgp^U)$Xzym@|7`bIfgLzDO-2soj1wM zDkE$Rk^0l;nLQ-B1eVVSJ>S7yj;nTFNq!A>bYgCQBMGpJkSYzFC**Ycyi^6ETJUq$ zpf^GXtk1aqRs4v&;&zme2p3B_U$QJSt}AqQ8LRok=ylIgjRx3*9+s~N#Bn2_eP$Pu9{WcNKGEwE2`{k0yR(Foz& z(drtP;U_IOI}DE@>n46am4n%=y8R06bJl)40=`Xn0pUVcjkG;K1RE_&guh@irh{Hup zmemG3Qn8nH=9IK%!^gL7LQbE(evDP^6>3~$^tbJC8s~uxa3D$cO>6aIF#&fL;@eh< z)yjG~qK46Nld2Aw7V~YjKx-?wkLB5fM@$6ZP*F{K@R_aPgwV1RM<$OLY$xlm$e+Fm& zsFl-cO0FzV2<9=&A{L_!-f>c;b_!Q@-k|yx}?jSY?0N~ z2iqYW{S33+a8A{L8^|LVMDI7u(8qz-(G|C_Kmc+VL&SZ0jYW|fTOp64vv?1yUnU*> zc+;k(B)Mt-9ghN}M^Q-%gFg}E&=ur!KyvFU7083fO9|DE^={eKU_}-(TBe0EY{r+C z0WMW)U}cRd`reNEmpS*~%NztF%&4eK^XI7`!`R37ZGFDxdeXE*BCT6H3xuRY^~-Tp zZUBxr2_LyN&pykt`9R=drih7?y4G*=z%EIDb5A0z-XyYLyl=LcOPb<1H1(g#rF?+< z$dF81C7Kx0Lv*IOpnf)*($_?I4I$f+5Bm#y8i1vE(W)7B9P-d{Z12Gr44L`1w#mYo z4agmd%QCoFA%eX6y;dtHz6h>IN|0{Q&Ne7{wk)QxtF%JCOZVWod1`3Ui?t4+7-(l5 zd0IX^`(Cq`2bFAmLA9lsI+fBK#j^^;lt^7h#;2VpaFMm)ouEYTHUhsm=hH8?Ass=I zH~wP1;O)T=f5`p0tT+Pk5BQUSJWc6bqzn~0IUUWom!RdPF0sO3bLgKU46y*XBSR>cvOxS5G4VXEWbZ@ zs9NQ0&#)CgK1)A@T}*nEQA0En=W^7sU|8_t4#;t|f#$CszndH#Z*f&I}mFTb16^hLW0sG(>mR0^=B-o?gDMEAivevhd4n--t@8!u!r zpa>S88(XHAHWS&depq;!3Do9w*)UGuC}}kId^NxSIjM)LX<#Gkv&IJRv$=|L!4;N0 zUDVKaOQDEasYe*k!VwX*UVHc?`9~~SPWmQegXM_w1j{*lCYl7)tnoHQ5PXH&JG|x{2%{PvE8DgO zRL=eQuy>EJ^}PIT8lZIiz_Itm@bEiKbd@j#?vS9Q(p3nA3si(!sJuG7>(#ZVs*`_* z%F=)Jcc9^wenxNZkE?$$nmGC7N;9OHvtLZr%*LJ`^Co%f zME-9L3I9>8+e|eZo%AICf8;-kVPDVQPTci8vD{HXJH{}u9vSD>C(w{gU?;j#klTXjp>d2Mo+7&WhuM zSq%NHBB*sqjE@$`6eD{ApsxN1|1%}zG_>W>-ggBrML1NGU zyN9!5YH?Q6$YBNpb)N&b%0LY2*Ans2eUo;V-$Ai85{=N70Q0ED9!Wna;g5>h92d z!>s(UzodJRtF}ZgCV5bkB*^h~_hhY!D%#e(&xiTceimc`KI*cm5x>+{SNAPDBorrQ zXae$}p7z+QfVop1%cq^y`S~nOfTn#A=_s$ zXkBr-IuD-jED=<7kS_Vtemi0Q9dq5^!ItVcBZ^zVi08F2)Lu1GAKd>}WaU}iR(~q3 zh>YU0igJ7~oaN@S#Lf4+zlaIFQeG0TotN8Xgx89B=#J!re16p#(QHeLNM{5Fp^=-T zw{$wv1kunpsNiV^_R4N@l!3ShKjJA+auSMx%>k;azxU!J55kf%D0up^Dacf!%9-oPra* zj;qFIU(~Y<5~#AMGEklPu3^7xvu_lXVBR*@N2^2xY%B5a>Ei+HX+BARDH_vkjZ92) zO&UufZxz}1lm$gKI1TKf31kTH>sUcDs9#Ma0eDkrSAtvSqbfyuTs@$48sg7C3ycXL z6aK1$=S#{A1sPGNy&AG7n;>7zX5`8k5UeO-L$E1nX;swIB$WVO1YAdxBJknkE@z%E zn%o1}LpzW*X7g(d-<$EUT_JNOz}FIY0Z&@c{Y8=Ck}SddV{1tPqsQHQaL6`BZ+(15s2tt;cl}m)J2-?C&s>g}8fN z`l;M8wt10nbY+mRwq^GY0;^2s2C<#Epf%t(jHrFbUkToB&pW3ozvn24Y5SWcT|J#& z+9+e_p)<1p(V3BAjxRxj@KGBlB6MqmDgLLD`U zB6L|~SR2N5Oj!t3DyM27D;TDIQEp}OvX5O3k|T?MDfbc|c%eh?%u%kQwxW@8R%iB_ zB{+3PQzx;Q5ZD7_<1rgFNh@aRShklhXc`FYPr}Fh+yp3NE7{sNh~^^{{5tc|{kWkXR-)RscOvy2LOD)^$Q!i}^NZ-mMtA&;P_O!?Z9?7owBN|vU? zF7MH%ys)Bo#mV-4LmNt(tfBGFwTcq&ilE6X8L^AXFpBWTY6&2r{ij|$=*H2P>gQg` zBQIjZY80iu(%do_AcT>b(Y)723ec zqAT{?z$pwIk&!uR#hs)bIS)88*anzDEkgtFu(dOvu~&?joS|kS);eE3<=r#emnrA5 z56jL&iVxx=>nvjOSOPHKH{bZoQjdnl)vvf-+D?Q@D(fQVwY|CZCPZhoh`qtR1BGq0Uxqx~1B)3cBkgE6ZV^fEKxpcc7f47q`(1MyKWoUq6-PD`w_yz~ zK3V)zH;$p$4MhS3mY6Q}qJdVXj++=Z z!ahIHcCjSmOzL1_ZREtWY^6-+Shl@LLsvP?PXcwN5LOQH)1^7jKRtVj@DSkCf|vKo zEvqJFU;+r`{ywEqtZF7Klk>=@f^P-OzK=|<1}+#1f;e{9$9}9yC*G#dDyw&zxxB~* z0M%)QF9dU12ycUte~tDqtGAU;p#LeIZJkKP zKNuk3^M~+UiL=7B+CekSn+GQW`deLCjuGDjXw%0pOX6K@cP<-cJg|35mCjMqfvZN ziWKAkbPZ{04n5zjE!eJN4=SC?xjvWxQRUCk?aNcV+ z$Tmra3N24?O4JmD(<)>}8wI^n`H>S7UGdEOr&tmEJYWkRSjtLySMo2aOK1jpC<@_d>? z>{PDwx_yTV*KyV^R=+1S%~8KXJ#-B2Cp1%jQq>&dP*y-Ni?-M;Y1i%*aony}iUdsy zfBHr5X#!jk8#i)#Ov~c1m&Ed6%L)?fkzAPnWqC&ShXy?_cG-Xia5NY4FTdxCktKC37;h5QYi^giHSuizHy8oJ+8-sapet-xGmK&piEg1)Bho-~`moPk!0_QxdW@3Pz(2HV4( zrt*TkLtR!?8yQt+sdQvtY6A{)I}Q-&_S>COhcFmXz7c_8_{i@lhZ>#MDa!o22ltHY zpEL2BuC=$<`yjX&v|nuXRB6}oJOM4sZmg`x54CA+S-YlbuM2Q87k^AQ$C$^{{V4x&hq&Z8d!3u*?d>4) z`H15hC&v5J;FTX9g+Vpxt zHapiH%arKk-4-Q5hw=1ITx0Sq;1UX}s80}By`B+?uqw^@Yh>dS>!O{T1q?+UO(m{= zBcs=XVlcfV1$jy8>ElChUr8`LV^(qV*jpn_ji_$Oo6jSE%b%#q2=DN1e{uf8OyDc@ zhEtx2!$e+1E;$%;s#t^qQ9YR)LDyAwjcm-h_~ zPMqz8);#^?o%m2VKs+5|RRdSmku6--r^S*i#3svAE8879d>aLc6O2Vjr8$OBPD?$; zEE!TnGwgn3>DbZ^aT43cr;k2Jyfxw#OFeW`+M&`^Jg>u=)94fSW{Zm(#Lc-$=rMX1 z6rlYK)+T!-cs^+CbkSp&BU{;o6(=BH0VZ2k)hk+7eg#75ox-VZHr7fe-nDP zUuXMMI>Bg`^Jq9j=_!QEPf3v{DJu8HXprZh(uS<2>_siWY&pK0ol0{!WJ%8>QH0qM z*JKFXn21MRwa)TJq&Jw_49X zo*?DznN{)MQirEpqW=X9gC`fT!&(~I=yK{S(fdXbwvtL~N4;PltM9qvqfT^bCk@Hu zIcP|K*$fR!f?NvCzD{x3;N7=kQ7kS+6*pe-CB;Q^o|c~@FYxVrBDF>)dHu9!Jx`+g zq8TFMdPz?cfZ8b;jEiHlHRhMUdB?+Ova0JiG7LX9qQ1S7iajdvaY#$)!yB*QYnSDE zhTx&Dg6H9US5qYt^2A0b0>8cu? zunZtnuxj?+#49ejltMI5QoCWq`1GY)O9Q*Dh>FtTaz(%y?RNlJJ8i)kgc~S9BvL0H zSL>K}9wjJVj$BiD!1>}ZV&7}Se?zTg7PZwCViCkvGRJhLlvqiEC4;?iRnlXet!)T5 zTyW~nVh^u}yIT`BIfi%6nK5^-vJM~VzG*7|TxtTAR0Fjdi0h161;=n3<5EnKyM&4- zmKt58w|IKn)%vs?|K8*?8mNU+G*EjASU`QnZfZ^Z@yt3wBZoH0Xs(8P4`ikiYXO_` z>6E9zHxQ>5xY3wT`-<9o2EpJSh}~=Y=zMnZ!YR3g;@8Ejsv{SF2(<_`etJVE++D6r z>fI+k71Q*^+-K>`05iODd3pE?buL$es=k&u3{1!o72N_CZwQKPj4n=Eono#DiAKC9EM)I_mTqqXB#D6tS_@7=yv+86MVuM-G^J zR20!=%Z@q>AZ}J(#@qFOC2Rk8z~=+7`aQb%&iA$qeYXAuA5#wqti?!gIlj?6ajks& zsE#H4ub-I&&D9PZ$NcvUX4a5(%v1NCmiuU@n*A@kLx1z$VNX^3fH6JYj=N`{a1U&` z=~2-xd`NZ<7`*Pj*+yY-VZ^KbgR#9Ij46WZyU)3O>4YJQn(Ll$X`%az#%XV0x(q#U z`Uj&61K4!#@4nkzy6pDzXAkyrx8%-sS%TSb-i zi*R6cNugSEI2QbWHz`Tb%a68>{qA=0^BK-RPxi^aoj{`k9V`Nhv!X-nR#j`zdTLNiY`hhBlTq%9rs zv5!9R=76;@9L?co?v*su;nCi%Hd#h0#a{HpRHh)-HSG@fJPRStKLrn##Xk!g1Y8uX zPIOMm1iXccho|BMmU=;c0!iemJSQ$SXXqKrT_-+C}c!Pde5L_O*ZsMZ7;JCkbQ96_3;9QsQY_IU3-QrDNCSksxdP3f4XcTb~&yrC3*))>dh+M=~Ht9T+6{{}>Kf3^bC_MF# za#rtm;Vf3Cg{dPG>#?y@^}aM+j>xFp$puc{nqzOa);}4F?RLp*U3%;j;%%3t2f}_S z@ya&jkwFx@icuOLbBNzY&bLZ_C!KjX#8+{asHrBd2J|z?63B(WRtvgEo&Pmo0Ab%J zKkGb8y48Diymm}&q^`XoPinn%0G^c8V0Da_Wi%qvJHL-R%kfhtQZQmZc=P>Ak-Ze% z*sVC1xSLp5J11su1diMU2F+24f-*<(?^^lUtF+*Tag^?fv|)m!gBnN z{%ShmV}w+UNW?5zKd}aUO_uaDw<;ohU=xoxn3bsAV#ho1ZI20d`#frBTR1R@?V$`(JwM>6eyo87@h;oOr@kr00_SjVzSepGf?4CD(PCTeS^tu|H*_E5pJQhY;Y;s4OyDGota2FcAXXmKISDZF{eOFfiHX zR%<=e7OlUXL=v17A7VdUlC7KLhLzWC;=RO<{A%|PM&o%05V=GGlh~4Yr%mldLV@5@ zk&Et7EaSlklcn!8Z;9=t(SeunJcC4A8)!(^@XVq4Pb&(0mp9~Ao0WB-ZB&D8{sZ75 z0fdht(0j{kgxcbeOlrYSMJ0ad<$yg@9{Mr4Ns)YKx)<76I#W^qjnVe4md*U8)Sm&t zR|bZfLqA;C`!>7|X*;*&mp((_YQCs&OEkFp*!7I=hVkYLB&y6axe_6B{Up&cuVail`O> z)17CMQOrs)IQX22&F=$Zo|c2W*7522vRMg#v!Qj8UC1hBs;Hb$D+d-WN(%v&kc{L) zKcHj>$J&}D2Wf1Gx^K_2A}2l<6b(|Jnl8X3l!qcC48SZx)7&_{m$??KvG0t+U&NQ% z?9fzmDlJ>GEYdhgu+l7(R!a$R-aISR(klJkXKOyrE>;zJ{n5`Cl>Fg_JZx2i=QuVr z2f$@_Bxi)Fr^6+_jLAVj+o$-sg;yLB{tciJJC$W~C6?TB))<48&7%DlD868)x94O2ZP20=&8~RQ4y|gmUTw<CdEJPP7%c-Xl~b|x-~mZB_paAsedAMHEJ{i_Y(uedmP31GTi0G| zLUilk(ooeI1D5cjJQxEU@M^9RST=l-k+fZodQ?Uz^Kg_LX1?1|{$FIhWmwZ;AND^| zT50Kyk^=@{K9AWAo5fTYyuj!}Y2OEbD9q*D-)^4Wdg&-3Dc{Exlau~&=V z@7i^p-}C&OBErH;gsq!F%bw5YM|?N)I6awJ^#2V(p?l^RizOxJlyFcRFs9lGR5j1qN=iCc5`7*)W&=1B4hM zP?0vOJcFvxG~Zw)&)_&xdc4VE|7a_Psor@wMJ>)h05NLQRk$9fFp49L*-Y?7?2Fuw zzYr}S3s{*Io|V2x{bY(z^6Ax!qbRiKN#%Z4)xYJ%t-$3=BFiSy$?o8t$wjHcdUY+P zsClO_nMnSlmIgW(yV-(_hS`5(eC~?vMEs5n)3yPXDu+u8glO@%+T~9*J^qT}RD2im z7Gj{Zb4;?VyLVr2b;Y&Z`nu=+Z@vU``vv($ff4=K1F&m-B&?Xfg5tt6aHu6XMawT~ zCM6t;erWN}KR{wl@;|_rd>P!@E1|~6)bdWzBX2%$wV9eyReLFTAtqH*X#yPB=hZaz zo;)d{xERHdrSR)w*LL3nN>B@bs%GxBL@t}l2UJiIJ6gHhJ_yDA!Sq<$R0xi`X6?e4 zkMgi?==de4v~4ybm|@qoC&TiSr85LemyM^O{8a<=>=FJ;chl-l76~IgE@zQe1(r8lAmg9W9b1-cm=;PZn zs?`=>Oei(}sO0U(mPGXjl=gPvCxwwxVu^J@(bcO1o1pjC-shU@GbLU2aRW1a-XUxS zTUj+(!jI`A)DjC46+8r7Mn;+fO`y{kaSV)WoBdaZ(w}+`Orm+kmdC~OA;r4_6%IVk zKVFZgMLLZMRxgdp?p6swpN62k?|H5gD`_b{8oaIOXd#%T{q5lU-pBw(oLT&#E>TJ8 z%9GVEII&wO|JC$M_gJaWbk<-^Tfes<5>;9Wr3JG$zPeCnDabotpW^Yw;n<`Fr8|T9 zr&{n*#(XQY+&f7!qvM5-Q^mwnzf=}B{)7~7H&Ow`rQHJ?2Y!)(HK`ffl>r8Zt@#|d`5Rh+qVD~Q9xnq1*0Dt+g5x$xg*~;sP2*it*H+dWHEU( zNzHHe1=)R5?2+P$=&)Al2r)uSJV^`Ym{V$DctUg;P|^7hKn_)(;514pY!bW3omYV* z&u&wS7wxyHb_-r`UMlC-!LHHvSpWSdw5q>rkHT{w;{IR$E9qrOR-BP~m+2jc;fd}5G{RLR} ze$Dh7^zFsq-Qwv#VIl-?0ZohQZT%|#;3>FYA)Q-hv@_Glp`w-G8?25!Qn(-t{evH74By7KGT_hP1{xl9(F~>OP8}ojcNHx^fSF-T+ zpr2sfxCqoKD099_9FAA)K1q6{q~aOZA3K5T6L%i(v-H5Z<@UyDpe(0kktW@-O)WPd zFbxJy^AtsFP8XH-JSm6h_Ok1I{NgxEr^9-{+B+{zvNh!LOZx1m3%LcTB_-cZY(c^> zqdqO@Iuuv9Mv{ZFTyr2(Kzg-$c74U;0>i~q^Po~Z+vD=2?a(%;BrO}{gFk7ICLlpd z#F7J?s^%*$itNGZ5 zC!jHm1%P%gZ}G)#f~zPNnMr)DM|TF=B&Tt(J!C)s#`=4oPQPCpJV|xp57S>`(vA1w z+qv78uEn)4DRYm{>5g(-L^?07cnNjQxlx3iHl2U_g1^RdZjpdYK!dr z5UD#p>}zv12cs~Bt=t;FHdVc~i8jGgcuA6c_E1Ps9hgRnFU;`fYm2F^X+L}5NF(6GF`wg+o>~$!xDv# zCu;5Lov~ow_Toa=mUgdE-8T+30)g*PCjn|ybT5(s!|x4Cc*JSXi!IyC5KUh_FwQ}M z!t*pVQ0`YZF<7PPMRD&UaSv`xW3qa z?!57IGFW-87>Zjl@0l?p%6!e)RopUhwWQoMzCH8cE$Qz{H(p+uO^oPjGD`M6DdA0&6YlN1n{fEoO*|&o~ zZ5pK%^7Tf%73+X=xs?O8!EqfOjpD7d{*5~r$*xO&ITtdnFVEsq+Ok>j6ggc-s7XnU zmF@5lf|{AKq+2`1Bko3awO>>9@fPl}pW!@8K-&CQpN}W~UYx%9tr*(5#X3@};s#>s&%4j6H)9EldC%jGoMqAbyt!|2i#mkOVU};nu>$6v2>Hs6D)ND^i`${iHzZk9qQuaszBHozE+x@p&Vi(?4cdNA= zg9BS7At@i;XA&nR@H1_e7$w5jKo=^Yqe+QgetNdz#8I@@L9#_yvq1twsRahqEMBUml$eLt z^RPP=1CsP5gPl>cvKgGyQt?*1O;-yifQ&jRN5t3PdC60z-`?pO3lStO)>#Du>8v5Q6R{tNNu)e;{on>d7KZfe~#A`>utN}ZcaxtV!E@`}FDvQWnS?sM5@WWWfz zuRcU9wC4Ofq05?IW3wgs*Pu4WMlN2*ct3ZRsR!X&s>1Q)cW+WS+H^jI=`=+~ajD_( zFz0?KS64_%SAfz<|Ma_73UIcbep>fbtCrc7kwH|Et?3CHAJ^DWv+u!l6?|Vz61;$( zCW|SseR@F2?9#zT9{iA7`fuYJ2T&-eBz?f;clo#eOR)SEq@Nq6^XYlHJ;fuJPd4eg zvVqQV6IR1O=Of$Nssaz%iz!5;q=uHHbpPLq^c*w8%o-Py<$obOj{iw=O8-bct4glRuVht?aGU4&j$&R{|1($74=~@~UwHBl;D~P*sBKsU{#_ng zAiYVCeEU^5p4H$#>K+lc(&F;9_~psF4EH39xBMruiRf3*d?w|d{BsO;(!KD5$qm#Z zAQ4?V$HEn^6;2OO0dTIY{G3=#_<>nkc6WWbalRVHcDslPCcc&qzWi&2sQ~!>5Qao& z52eVz)NlU!iY5R1@iC^g0l>Vzwf?3wV10ox^Y0H#*oW>K5BK@D(OYy)-&47NfS9bm zUjb$J{@Xskcj!W2VBkH{k4b4WT zT~Ix4Qz`ZkmaynDxHRx})Q$%$lWfyt1I2_mRkIo(BH|^g;21<5pF_D1U(t>F3q| z27WH5Qfa4gi@?DL*F{OKBjk2>7t9eo`L9{PZl&gFWwVtkrPHm=iP~?vw78xpZ{G6l z6m!#mum|y{n2k`^w-ng;aH*w9Wn{|3Jd+yZ)DnYUwD1p9`~oentXPLU)ofe6=kjSB znRQ;kad7GRh*nw$ns$7?AdQG1s^UKe4Zw$j0DMI1~DR>m#cC`+i zYHA$U%zw=JebjCeR9sQ%tiv;_kWuv$OpWHe`Q!H9#rY6w@Vi zP7{kJzma=IRmr);7pnNJI(4Dn69s{c3$kH>43miQi5sIwo6pG8rxCT$h2m0I1mFnT8 zN#&6w_*GSquaY@vIoN5dTB9tTyU3TMWDmeW@!FcfM+7e`wc}@qRW6Fvtew>P6-+7W z88nJ^4QR39kwDkl4~As}6_SG=?gpEVY!K*62uIiT+>a&)U`~TwVTH}`BrEY@l^VF)xp9^SxSgG_s;O|RRY-wDy14Zd8}If8RXMx zX^6E0o_NhUP9>6h@As+AJ5}%A^lkFeD3L>8H|pmm&Ey zy6`>*o_WHDn0;sU>vrx^)Qwq1Nf}5wwm1K_%;#nHAW~x|p8;00zBGNDQcv%2T<*gk z?d30PDzw2(N=o&~Jpatg=Tf!wS0>VdS&w_1+P-2FqXh zaJ5Teu-4+r_QZvzCQo&)Hs0Z`!uAJUs`GEmxtw1YyGOEY3Kd97S*-ipwu%O7uO!GW#itXov| zxRX6BpXzE_a1Y*?R5%dOdr-Cvdr8Y=bDPfG`&Kqss-KFMGuR9FPGz5sS$7Di!xg>k zpC2P$LlmXfUaD|a-MxG_gmi*}im0{dk5^p9c7v5v=>s!ZHkW4iSuh4T@cb3XBHwLo zkG}0`9~5zI#pGd1jzr4thW;|iuA;V0$?VZ^o*b_-v-6{1vkv!LaLdt;)q3dtaFF}f(=u%(z^=V z3(NA10Z;t)T@ieo>oi{bec&k?kACKM)^3K6$9?wNEtge1of=QeARBfPG$W)*scfGI zS$-057jZf#7W#;=N^)}Hjt|cXSC@9tRC>soPr;})xZWo)(hlEHz`5&ooUEWyd3u{X zZcs>X=q{U@`{PUErWvkzhmUvd7FgL@&ybGz`O!-LhkPQVfWOS_yBz7GkAZ4` z8Ai~9xv&zZ`{Vrw`~{O}X4|%+0*Mi9ieQu3MQCBSM@L~TFHuA4?Q$ZWnTyxZ9-e|i z368=RtK3A|;4j`?NBsUV3%Q{Swd?d}Zk1_hLG`c<6A@-f+XeSA--%0-vZIH3GJnE|^eLUs zZ56HMf8@9FiRy>BdFC@pqEo|e^V;9OBBjelrWBNukY2Kug z8}Rh%0cZwvW^Z=}<0uaRoiXYO`~2*x^h`Oq+0{_u4sqa$U^Ev=+WN0zbgha-dfVm9 zN^jLq2C4zCzor^nhF=S|JeQ3})@a#PL^e)174oiC55X$^9U1!K9S!5N(h0_A)}eO+ z*tV8|)rN03-EUmpecGR@>RE1LyQ$KdG`b=NVli@E5#*7QcHjUi*!kIEM&3=uTjmPO z;5-u!rOL&`iYFyy&d-`q$E!;jVKENR)OPE{nY9{Y02^l*i&3w_^ym~Yh*dOrRwkt{x^*mrgacd;i>KeX2nqJdGQv)E;CLf~ zT9HcruaBUqJ@)E$YcHAF_GcUxcYQOcU?otzoNC%6l2Uy)9tR7itm0Uhk`jbLQ>3mX ztBh8~E`m+a5a(NRxmivO1RsPBm#7=SRcGHTHdnq_0hqn{QN;vBBxsCY%JhS|$Uaok zhc`I_m+fFL)y~FvmyY@Z3?QQ92N1XycN~xF*R?km!!s`P&@M*_1A1mPu0e-%Njk!j zYwv1M?U3N>TrwYaiY%L+nUlkYSyAR>kCzGFV zPhgSj8xxq`EU{x)+X%W{Gh;Qf4)%p64GPtgGw7ZPsK+jTY&kILYMVdCm8cU}(;*3%HPYxm-UKFms0Od8@>ktiy|Muw>`y6w z!R^@Z!5LqhNMs={ZY(D8=@Z3i)`Epg;f}}GIaIC)2|gH#N!^-2FP4^}Qy-D(S}j@& zUIR;6_AB8)b#bx8 z^0wLGcyXq{8;&!s9I>+HPL{M>3r>5&<2X1QU9zF7E;USUG5?Lh>pf^1do8~df>{`b zZxO)*`GZ}Gi3!kIO?aoZoX_{S0LIzp{%a|y&wcv$pJvw_dwE^g98qow>9=hfo02Jf zE_$%HsuYC)LN8^dbRTaCi8kE3lS|2YRaJ%A)&S@8!Be*G10vrAE>+n1?C2#!=2oUo zrXGQ?>++k>0o>w-eMO+x?<1hFx|@)&dfH}>6}-hrOQ`2Dz}jD1ls_2zs7 z+%xj-Np**JMXcy~?~Y_t4a%f4Yb^V9o4$!2u_zi#GqbH!zV}jexPi=wF)589 z@gtea{D8x^mn~wEV1Mog4TvMAKFCs#0qvEFjmMSb0IH^su6u=Ya!WLWJDVu=Q5kY% zv@)NaPLGqDpLYxKXu(93XkSeanSAV()t)4?EXU&?&B>f2?Y3VX&st60XF+}k$M$f`7daPuvIQqj zIE2@P0{fwwHjoY2R-6g138a;~)-<+*9SWd}BCp^+Czw~`q92z9nz_$3_q!3-$|6?f zczkW{qfun82B9&S*IHZibSrABX&v2-tThO+CfaWIu60m z1g;+ULy(kL2IVbxL$l*4)lF|sdMvUI8Efxe?&3JerTpZYZ)iKH87`L5oJiAvwP(`Z56UfBrKJh@Q`{2eX-rzg7S;4;RuSSAbC=;l{$g%wu>o zI^+2vwDwEEh6nPo9Y9g)uCiec@LK-U!1a555ny~_`{g;HqA+6ehxX2^G647gbF`s- zZa69drx$3qTRG#Co&R{+_Wu4a&}sXL5}q@a$ISe^9Hmup{DQRf&Gm-ub-w{B(IYD7 zow#hzkCaIOnew$HpcJbj*X&RhomA!7j<`kwfSemA64WO{lUG^*W-_(K1j}R9?A@Ud zv(ZrinSATJFNl-yg*r4&%y@ZdFMRUW3_yvt)S&@nVgJ!9$tr3jvp(Xo{(pF4R+tn5 zcz6GoDz*uv4cIwW4G9asj1#HO<8K;Va!0NO#IgVIq8TlcVxDi7vk8)LkAhculc_zL z5C*d-Cn0(5v!T@36dC487qF6(T-+*-Pg|jTHX?~X$(DVB z6A`{>1#fbwWgQwI>Zd0RHtP{9mx}BR&qnFHZ++DTR0bza$;&L76WLf&nsga#lc%_^ zKG_d@t(#a;lT&0qrKEJae}u{H$}Jy1+R%jkqE!kbh((#jG`2z7Y%;XI`zwGv^!mho zT!L}t*J<+KB$TtP!`o99s6)Ge5|0#E`OJ@f5r$9oBs5l{`#3zOKVhxQ)Qoanh+?$L zRQl^ffW~jD7p*3v?G0M>vG4J=_L60g5>!md!ca6F;bpZ-5HfYQZ|838t58fe z-SaG(#F8urCp~Ky`PV=kbzNd$OD)d~bJ1Yts`fLM?@|R$5>?~kt;1gPZu-ga(>BY6 zc000S6Lr{LlQ+$;EJH9`YxxEwa*#-TGm5}8eY3q|tYOSo+1tmucm;(R_`sPLf&bLN zax@OY5TpNF2&5n(e`iBU2A=mYf*MXDS1sPu`vp(qS-Cez(QBCCt2qc`c(pXyvXVXu zzB`Cr_09pCVnpZuSr%r7I1>#zbzpQ5zO|Wa8w|m7r;%IAW&A)-aMvLq9aEsU`l8y$ zl<8@d*iNLG)%!#EdhaSyb<*}5_P03$I;y|NDeNz}F-d-?;W(v`8W$;Nr8vS87}SgdGR*p~17o#qyf-#r07@d;posH!0<~6}6MM zDzzpG)GLgxG$>sj@uOJr25o@(O2_)wGj|N>^$jh4%6qP^&yniehLbF7ic?e>+$$q1 zmkm@Ko}Uy#R?Thr1vY*NEM4(0JP$O9APExS07_{sur6TD2Cx4hPH)&9Vq_7U>yiC@ zjvHTnadiA8S5>9MPKkXoYNpA?-L>FIL@&x@T z@D*7sHXVl&Mb%eIuD=-->=x|K>>tjnH5-*%T{btLtDEyDU+CsKuTH}HVn>!7ObMpF zQ1Y8C)#dIteF$bdza$Ah2SJ ze5j!0Ar{zPx%Uu7uVCWTd0=+675AM;A#|35pyr!_T20Sd`L2*h^&iN}uK97|pl|UE zS*D27LY4R4joY1{_otSGofEuQC+ZnJNrGsy<#n zLSTfgvD{9q`HeyS;a5j0LfTXcg`7W*6P&UQlX_ z>j!qG?uNupFt^HLf9$N1gM;dPY=QHYo}pvlw?MOP&vb`JE`oeD2)fd){aH&4x|GWb z>lDJs!()p2t~<~tR=O>1?ZFj#5pnqOTaWM@p*5a-v4?5w2nUw%kun=FkIieq3p=VF zOkqe-PN7(2`dZ=M1W(G2%1r$|csZX_$#Jx{x4914vdW<&l9YCvXB(mQM5Mvx1cl(_ zJ5qV}K8$^AoL$M2wI!hx#>UEX!5VS>9mfjsnsClea!CrZl{&Y>p>aezWu{o6rxxS1 zAM~9-(WA0O$afsPxm5k;ueIlpXjKNqSZyS;cQz?awhL(G`EcS$OK@k2lehG!E{U^h zuR*2J%D@HXrX)T7+m@{w#E^cm85NPzi=wgMZ>pwGM6wT~23UES4Jqm+$4+0_?f~Vc zTCXUo6-{&kp*&z1JiA(41*Yfmeot${a;QOoG|{COT!Uc3zexQNwx01sCQCWr86G`O ziBj}`s58gkr`F|jtIDEmY}_1yi!urO^GJLA9*~l&`#6ev)+r-FOt`qLUk?qU)7iVu zvwu2rt7JtIU&O`%!CMSTCukj}Yf1YDn5LskP2k+y3x%f{pwCWbBjaX)$=fG52-o89 z_Nz+wh$GsSlha7vQa47zpmIKbwh*$ME(|lXq_Z~WcV2b&lm?Qmd*MN6ANLEsy}Cj# z=;QkT0RI5s5GK_&QtoRlm>3yZUPG>1Pj;FB;EN}Jh~=+Asr0{88+}qw!&QH>cQ>^& zo#`WH(yM0)12}Dmzt|5A_!w7O4!mSF@$Y?RJyR znd8|~UxYls_KJ1!zkiik+E0h~c1t<#wQ zX*8X?o(icc1{D|O`dl8l8&HXw7vW^iw-LY@|(7hs&uMrd86LhnOi?nkWk$DC|FCq^+o*m z!PDb;oW~CioGqU=>$q1fJ=1=&9&X#hhoom_Fa3kD)m7-OcQhou^WrYoHvnuOWT`vfH zuIg2A@b12%<4w_&a1p|pB%<$_R?u?O!W!KIr)}BWC968iWc{Rm?(J%1RAh$|;xk3L zV&`bwM?Z3J^*Vq#tQ0ntT25n^nWG|}na@Mjr9Ccnbk8^35rs^d)ON4bwdbIwlN&3& ze!FQ|pQ|$RI4!P}V$XlGkzbl?362TGu>f38cs`{+1;qx5V&TDE7l^=xoR&CUNas> zHjmPwAd+b>+v$U)_IQ!AAGTMv%lz#5z;p_6IG^-c?}|O$gdy>#1O>nIgL639&)H3U zT7CWKfqeA{tM+2w!&EN*ugkR1SzOezNR3G)sE6!_S5%CBdSYFJS6hxI$&urkSCepB z8VwDtdo`sw{Ax-hy8tKz`+6<13kBXEbs3c0jrJG%CZVqh(|iR~0GcEsNq z$?(wkV$chx`{bw3>XK|-aR6)o2e+tbVbFiJVk^FtJYW1)v#iZT=qC|`${99zY!P2q zllyN88OASIH1nNNd-(xMRz1a6_1up0>XcO)CM29c%QI1a!o@JMEeZ@hcjkMLA1tcK z3@xR!xelyt=Btb~pZC{o)HekrZE$(N&n(Op4lJ=;&oi?u&PULV413T*bETMO-)8gc z1-s4C9;+;OI1_`;Hl|?k4Rzq3InLnc9Sw?NxGR9u_vE%8q-0B)j@o5qB~O#OY;?c+ z$4d*ES4P_n0rzQz;gd|J!Ca1x5*2xFAF5|6;S(Iio!fY+Ln(eP^?!AMd1t9P;}rAX z)LZjfw$fO*(J63O|9ob$sRjtVY12)1jfL6f+g5s6{;XP%&Hnu$Ra)L2G`qh9-vocj zbH(ZK$7xrSnH2*I zy*y>u7X3LZP}))-ONOC_y=qQqZn|=;FCgAAfDJPROa1-iIVc{eUfk&xxB|6#yUaiK zEd#;Q?EhU2qQBg7XspA$B%u&5ka-0Z`pnf*8Ce)Z=hORJ16&{a8STXq%!6{ewFWVy z3Szu>nSCi_?&S>beeLyz6dauZgBf_|6X@W8`Kc+3RP`p0kC1MVVp-JJZlXWuG}oGB z)9fwu9Y2I>3S0PXw_1k{hS+%q6mH{|m@5uuOG34vosF(hb7$NHRgPps5m_>++g63J zl{^Y|_CqQhD)Ej+Cg!4$ry2%~?1u%_63%3+S(U%Cvq!#0VMLX zX=OhchWkzC9jf*tPI|=6xJN%5X&U%a_ht7N@u_2U&7MIo*ve~EQvXDW6NUSS$u&f5 z6%pVB%VQTgF&E-MS-|*eEXrnVv?iJin(>Q1$XQ;W-4NQ#PSE5G&+38 zYxo&1+h>01XQ5UGI?rSAfZAjGRL*fUE*h^XZ1A5T`DRPlkDGg?*%o;_%4M#;({mB- zeK{0=RYWX-@q$7i4$mH(tixay=SQ>zlDFP(B2y&v1up(78gXmdPSex#V(R3Y_cM4S zTh1gPk#Oi$7bHfoJ&1$1jg`lSd88{z+C?#jd=z^fB|=IZ!Ghe?D=A;IB`7sM<>&iM z6JL>SXiCoD4D@UF{kSB|H;6*51h+eUzi0-!_iTObN!m$ zXCTw*#VhLC$yU5rwsS>mLa)6IsX0ZGT~QO0waBBB=a?32K}r-AxgV@u&{6YPU}JY} z8(YVTa!ys*@X**mpVGWRLrfYG0=4>ySIj{w^kvz6J(c1c7B%(*0&IkgyWcax_BF-7 z?gGyn^;9xQqi#nFzG+@uvz%~<4uyl(eZ4n zR%&4(UlbWUvZex{Mu;vi1A?T+>%EKRkzzS+(6KAQ&AP|Sb0+l=U_in*e%IF9Rijn! zCZoWO0#3wo-?I24@26*M{{XCEWH9CN?~A8${NHE0F)TQ{K0BvWyrIaauOG6=^!HqN z1$s!Zm;3|fLN@7VUM6Yxbzox)q$@T&$;Dhoa{giegA^tvH^>#v$ns#~x4%0IUF|8+$8vR+#zh#9yVayC-GtKgaxO$6TOpt>F_R|N6zc8!bQ0 zVv4%0|9Qn z*T&x7Z%VvvVIsZ%mi?)h7~Pp&95-N*Vu;_@-=g1*kR%)Vo@bKp_SgFlc>j57q$Tro zSm>uxo!+_`pKv7RfB0OlQ>+rT^S@>oF29b&#{~c^t>X*Ma=!8a*^ZEl|FNI}P_@Xt zS)2c6NHJ)&)9!5iMzJ<`F<;afGd&fe0 zte6VK$APEeZRoH0wL(n&D1g*Spg^C<^}S|1HrTP_$5MFu4JP}jlwyaQ>gLpY_Bn(qto%)CMp#H zO#YhuZ;5xz#M`jwvwi=#7!O{hbFh5jcRKBps&V5%{~{B zVZP%qeIM^G;NgiA``j>J;M0p;#j6-D$fqWAo;Ul4j@4D?F(Rp>F9msM5=Gx){`EqW zJpo>-kg43<(wa4U_UO6FV8amw!;-QtD!)#uCApc{nwy*PHL>{xp?c)Q?T*w$?bX3e zD!NOa{(RKhjo+;j=+(ug3y#Gz%rH+?(0^8_gJuQhMs@C=ygJm{kp3InR(xP6T0&Jf zPCJ04fAKlRaI;wKnxgNNJxNNxH1#z*MYU#)Kj8Mw^_RCsFv^P2h8{$nqIMaB;ptKp zTd&NY!4*|Y9Qm{m>#@2x3u zyLL-v?bI`ZCWg*dV(iD8KoX<)O-rl$gZ2d8aA;jX_SDC^Kf8FAMV*bxRtE*+o5OwSf z`7o;XO?rKAy>oqeV*PC;Pp=|R^3%tf%q*%!8rQtf1Kv!XPiK8r6Y`O*9g8QEnF7z@ z2J~{>=M<*me?r&EWp?&X3+s#&WvMGkD?c0j zeGk6%=6{pWT6lCZou>kLcp!K8sFv!FFMd;#)K(qKX?qYx8;=V6wl9Ngs}ATQ6YAw5 zaD3pSCD|$6jkTc@hfAjukR6;{hgw)7QL~4YIDe!Vg&~bk5><9~C~q#ZJ%NT^+)!(v z05nMJ#TEq>-akN?U4zVYg_L+&kKh zog|~^do$lU4U~5wXIPtc^LiY9u?kC=EbB&VC28=g;k%X=GGlX?GT4Lk-AwV-%+y`g z9Qz=Vue?+6iI`jH{7QEqBX!jT5+kUFB78++Puc{>eKb(22&*L#<(I50O`(daQ8jJs zo}XT4Rou)b>?5hfQ_ydU5>$6-yU?dgrn2yJ!gI499v5SdMx!nlINH_6umnF>btbqIVlwDD~amMQjmWd6dlu`rf&O_*Ku`+T38IO zR?vx7h<})SxBM(qf4Nk8Q`s#$c|?)GT`egr%BbOtHEF7$Hk!(eI}f+%x_|0ZpkAak z;e?~T%fx>AtXjNK{#b7`ys4T?{YSE6ODM6ha zgm7g1^FG**7vqK{!r3mLx9n5`vMCYPd4!;$5$_{8fkaq)RyQm){o%#@gbs zN!`f52hYhiTMrTZ$R!CC(n_gD2=5vVC0Ym0g~l{Py32ODAE5i&DOYB*;{HL2f++40 zM~>+_*oRZnM@+Z!8*XM}I;;+=ITz!19>_6EI*YX(DWrKzEnGk5LPrLS>NKRxZzndn zzIHTHODJUeApEoC^8Cy=8TV^0nRo}&Gi-0cBwL`Ia^1#1fSRASmGeJ9rbeni0}VL* zYN+CJNAc)-(}{NpV*Ki1fj@H55?zp&iW0|ie_P*HCp23=+^Spv`@#K~GGw`Y8K-iq zQxmiK#ukZfzAR&ZZC9pM4XXNDre_1L29MLc)URp`=T)fs8TX4v>{f%XPpM+}FJT#^ zO0@iv&d0b~xo|glO>CDoTj@iVvxe#+BdC9>Qcqb{@CMSHzi`OFcD3U)jd*fgInID{{I zFFn&A9-m@by#Hl9jQW;G+dT6h;BjoGx_BwA5T9eeZGD5lDQ&rp{{3=TfQ;!Li^{AX zRnxZn_c4<&*nkl)cY8*1NadkBV=b7@LbSpZM~o7|5KUYbFN!%zu|B5XZah{%eROnLY4nZwd!+9lHF@b{xj zex&%_N-s^u@xW|!)GSdj4G;uu*Rn0)P6FM)@UUI5DZ<6CkuRV%B`#jqYTY5xudafe z;CD8|=rL^$LP8}1@nT`eFh#nPE@SA{A5Z;tOHnC+Y7v8_v6QstSxMp z$knsP%Yk2}21-FmG5nqxAJm<46QDr>=O5)RFpMNxuVsuck0$0fHSTEt-vl;W!EU=XMAXH||$7E5JQKyb(61l8bTB!`wX@?}C^jj&)!S9pc zrtIPk5$A9&+Vi0jIT^kMI`js#TBoHcmm?$0llN`xS^K1(h%C;qpcI(jKDFc#^}1a_ z9<1EC+v{_d@y)Rd+Kl*qaflZkQsr6dZ?Tk$&!Sc&6U#( zQg@6f4#A&UU&X#d*DMJ@)FcR|WX`OJPkfxNU{ybl?EbPPe=`8rJifqeEaL6(Hax7tl) zbR1O91w^oWzgp)w%GV;IuUNRvZWEzV9&_u7JB92#GTcE9Sl)zl%39|8el)fAkP^|8 zwjm_LhbM+;N?;DZo`0g|(M@@EIAJp1I0*+cJCwa;J=GM{WhH5XE=y+R$ka!r9&tl& zBO$0|g*PlMm4-=Fs^_X*{udtuYU<;4n|JGn=(HHAZIRFCZ)lZ!>y&G~-Er@7(bg03 z+qia_<8~wpU$DnD6pFQ)?$^p~|gs$TIk*UI`ab?Ttm2~Xf==vcLfYOrA z9&@X{$CB{pM1@Dp#N(>)QU;ysL-OuCfA0OpTmOJHyC?N9dzI3e6>C~Ew-)SoPq{#} zNw5HRa^JE$&p>G#2%L+aFtmNenmSKT$S=V4@t4a>q%^pfwc#`W=M(M(;~L5Sm~Xsu zm0!%vFm!24SILL87Oj&P>kzNVg-7fYM*G*&lSZxk@HrBle6sr=44t_c$UBqLSBXoa| zH(ns#*!+079A%Lu2@5LV*x&|~+$O%i7vAFbVV;bVHKOI)2yuQNa;JQk_WaP$5M#Xm733iy4!=9{i(hqJ~_kIhW`d^W0x1R?=6M~qSASs+e=zk8s|KF?&Y75mA9H|tw z!LVEZ4CMU!%QX>V@s4dduWCa;L=~vAUm!UTUcQV*iczP+nWi48qVh|LMGZcn@$IvE zW)HJDVXvThy&i>$v9;lpg}!)h0lBgF`yBnKQPuEYNl+8p&zVt2|G>%3 zm%WBIQ5MqG=EXS z7yZolvTZE$nxuW^k1p6q!n*|>WSqzHw1c)Nu7gF5T!JiwTQq&`5Q1%fa-5LdPK3>3 zF$e+M(31@`+1DMfA%v=j^&dp8Nh4=aa&8!66OC62awWLPGN^EP`u}Rna%JVb+vG7C zt@8_8>zBHAv9w`eW=Iwars+=pfCg|W`J+w&QHjO(5wIE`ib>^>zH{}6vA%aFvSD_j z3IYyMofI_l*UU*AqJ0fUGf!f=8=)+n=~Zc<+I`m3?&wtM)jx^x2z`awC&E7 z3IIV7jR*nIN@R3UJX8)Ghi%JMau%TgOvim+A!w7l*ABKc2dSKw(h9_CdhApy_e_)h z-l7MG)lzN;aBU^H5v(5Qz1gy>ka3OSh$1qw_(wD6wnJ3hEQiQ$J0<96GUanIsbOUj zFi+>hyB6BTr*Iht8U_IMdsUS?W=4Qob!WL3ZXqda2PmjoF--P1r6X8RgjT zZdCACv0S({a=^u=X(TnXH9_3ow_CBOm>!>W+3(5=EKeUAu?xQ{>jcd1S?SL=3ao&p zyMnw%R3upVaDvYC#jiN8zNW5bL0>KmR&L)6x#~~!MFc&BP=u#qLw}N?TZfWhAqR8A z#(D}|cnY&wD85s8etbD-UU#99KOb|}r7FO1cjBZ-F96Fz$W3I-c`>o31TrePw&~<5 zT9#EByMi7H1HSHu+F4x>5%xC-l+|s@zerog0iL0~pMEpA$Dq@gZ{VEDF6U_?McNtv zu=TjaT55XT&(>K`atSWEh}kKaRcsw>9B%xR${C%45lEpi3vi)S<7VD;(Yj2uIhUbb z?4mr%B2%o(wDPy!AEbE(xJ@{0tNw17YB1+pK&n72cQQUY+I;A;Wpcl?^HAY)5M@`$ zT;j+L^;i`wqA>{E7koZ<<-O^6?IGQjpq^;7H6JnTcZ&iFy4JktUQJvLE^hr_z)2tr z`ER!+?T>-eljQn2jE9gN`?xkblwHmVQuY2|z&0<*yqV$EF8Coy-)pnM*k-4fXNyj?l#Rhz374(aK<$!goB=1ThzKnl$BNHup|z$wdbZnG_ws ze)G3vY5=(|3oRS=d?=TELTZy6`k*>N!(|V4)Q5c-X|ubVpAT=82Y(XFqR3WKbJIN<%9{cU;;{hFk)fN6^b?TEwhmxTAK! zU6}ngFE^*jgim?`+eI?MiSdpJ)MSrw*MWm%t+f4p}TLWx>}|#$b#}8iHU%6{SNID2A&(yD2cy3hB7$_gAHn z70;XX7?g6BiMk@CY5;{+PPLBCcD77EKs(&!(fYl*V2`3LEce^AgKOD1@7xMa?P-NrR}rdP=mmqCo-QHuZIL*XY1$&P{rZa#e*zoJQgu2P!Uv7cvf#(h~B= zrV(!8sH&7)DW?uPT0TWq4NoyIbeK{bhPY4%QlN3#h(@pJ47cQPzDZ(u_xL1=nHeY| z7F3b%K}FD2d-he|X=<>$G^S%8AN4_70kMTvZ#>(Kfj7@|{n0x?(*t?ptyyA)G<-Awa((j@8T0riRY$u!gh1 zQ%OGd$IVgA_l_W3AE@$lZ(_YV{{0?Bw1CO96`hK5TznDIcQIso;BV#l?_A3lR%;sF zA5oT<)iTOip_{yCQHKI3np~5n$n@p!Hyigv{`UDTe72Lq|HDnS^A8-Xgv-D~vaV4O zsne}@254HW{j#1IZNMO+Z7?A4NO+dhJ)e8mleG6npyUg4HPb)$hb~++cGVR&#&;Z} zE7AUek<_P1;AD~XvbBc5zLZ|-+swj#@#SmCgt42YqBqrEio@~x6#I1b$QUfc`D#|` ze|SN~#I}y3EYg+tPHaPCckfe1A?X?161?2#u$pi7O#PnoA|17Q9rdWRgMv+}QNJfC zVM1ZEO3%y(+5t@2{Ph(McTayvwe6)z;@<|@Kp)sk_kGkHRh#kg)vQqrz$kr$fKo$C zh`Yc7;Bg;J5I-+W0c3tYI5~}=(*)?m6+}-s<9z~=5fqTo&m3TqeYwwX7f!5bx3i(2 zjLGDoMz{8G0JyZ_KFOPJrrl$;wPoY9#}z16n*+KOZZ`ObiQ3wNq_{1~-6ynPc)|VB zX|C<$Np04W2b<+wlog7FD;wJd`5lS-&f4i6={TPAN|n^Rg-JN(ltH4>=J<+tMcT{} zsqbXwMX(thlO~nhR|m)6@xc860V5XOeY4G}2~vMpzXO@4PZ! zIIBEmGp1wz&fk5vF}?;Q1i-D6U?3<2H{TB|;HTAFVJL3yUJ+}ah$PjFMj*kK*Bo^Q z+#4&K#)x+-7_iFwJyFW4gB^cP{V1^n1n44Fb^&DUWywWb3sDnZ=$(l#@B4sq&)}?= zo}kP9*V2>(Jjzl9!4M9!kg$AU`Vd%v=(JH`obGcVw!Qy5FdHhG*U9I*0pw} zDPadVfRm?8F0*r}^qkhdU1vFtd?DkPDO2%p+Ed5MR%!gJmS66BvpLv%zs3pDX<_zl zc7ckHw%w9g*GYI4fFT9nQa`j5*YO&>p5><}nLd?sud)c$h#nUdX8K|ZePniIVqL## z#oynWBAowMCp4&&{&Kk19%WGSSLJ4Ru~`sn6BdwOM2x5(OcoI5y_(jKdxP=Cm3Uk4 z5#yVya~APy_u#ehbEa_c6-;oFNwnakMh+jhhF^!$JJG>_m4@p!)j#sqYA(>`e&h5K zUg}&$l>XJfZd5iG9j&Ch8z2tFF(Ilt;Xpe#hz42a_ix{LfV!7#YVA7Ro{s#yFOPp) z0%mkub~()Ec$anAn0*2pidDu=-}QANA=AFD-&*-b*P0eAY6@Knp0h{%fZ~idJ`8ef zgdmx)rp;Hw=hp+`L#2LN0LWp+?TOfrczc( zEthmf5c%bS2)c5req@Zr+-`XM`VZNhS>Z#K^i=iybq!gXuaeT>aPf~+^z@aK{0!8} zZuG=zcG%Q8mFD3Nu`zyeWMXo1EF=PcK{O~An83(N2z4Ga*Ntu5jgCeN3ek?T2Hl>w zt3-n{FBp<}GTv@;$;7_MJzDUE;vh+jI&r|S27og7xE1G0{rRh<^_;_nD9iCxTFGje z`na)SY&s9dWS}Ja z3*-JLE0P3j%uoUm6A$TXP!;SM(-)Tf?=E;qm0CGvu})mkyc{++$L0${;&lmGtRoBF z)*SA9*=dQ}&Kl);&JL}$0s~F8!^%dPuKP-i0?w_EaONp?x=Sys4?t%PLa>!h1EZ#7 zx8|n}6=T}RwA(e9ktM3Cew%ImTwv>SU>u_;`nQ6WV?$i<2rDu@Hr;^)X@?Lh* ze0_*lFzAp@x|3nqsAP_~BFy;8EuSi6eJLzq`g4Ll>Zb_^B0$v!gweG-qP}oCEdR3zz`Lp`CeP! zUDmxau(XjGR`N($)(p!(gNO3d*pznIoZ&#bXmygpp+~6{N9@M1CHLi1R6W;_npb4v zHb<<4fTm~(e_J+D$-uG=$kY*iO((-?4-~f+u=iw&xJTh8VGI;V$ugmT)Z4SX?wU$l zmw1<~Eq3))g;gm!kA((G<;W{N|@Bly?E24NuuUD>05g{?KiK%XEDIz+W6dt0{0 zk79%9Au3?pHEIsiM>a8y`F$Ut)-n!<#9(|VjUo+5uDq3+ zJ~I(mhwy-_pf@L1r&O8t2BKW6%Vg{$>?=zx!V4evb zs#t;bt}$?%??9)Nm{Nq-6ll|EDM-z{MxU%@0fGnU%=3tA19E%-YTRtg7QQ&Xn^3Qh7WR@R?JX(cckI|6gZ z)#~Ek7Sy}{`(tcywB5NB-bnwIPDwqJ-hEc}Eq;CN%u_1NR$2az3)Qn3sy^$4=O*vm zy*ZrGB7yU+^eap7m+%PhcJB&$bc_2=CVCF*gayLm0V-$+LlD0Hk1F;5rc?;H)~pZx z|6!BJi6npW-Ro)DINiW`zj-}hokWCDm6IKnU15D^DD2WZPfw!5uKzpvtfERfPLv`c zSu<|19%qXR+85m!$=FpBp)M1okC;`%jO}PkuuX>dJo@Yj6G`buU34|rf3|)WOOeYf zp=JKQCi|jtilj-LotWIajwG^t`U znnh`9BxJr3sm6NE=%7jC6;_lCtUwV00)B{(c4c(n1RbBpPlO7(jPgu6ve%l<>Tp<| z--O{-3(C(Y(51)!Dv3VZ*%N^~ns8>zx8^&b8jft!asn@);}RDM znMGLfV1()8Rf7ouDg*MnE1xH@Z?VjbYhTDl_vd1496k1~vNyh;xy-o;PMK|Na?S4v zqFW~XRLL^|YzJV5QT<@!yidO(UKZ&vq**xVk&(2LD-p7} zk=-KEIq9rbb;g9mvq!J(utexcHNmr*a1GxY)hdFB*>dzNFT5cmnl_W!{uELMr3fW& z1*>2G5og;~W}NERbgeQ~UhdH#)i2=afr zmVEtcc)0E|G`p+tw*Tw&v*zHmxUAUE*+y-r$@Xm?UTJ+=xhCxFzxCI3Z&ati=_YF0 zZAukTOP%tWB02mQOo%nc@_OX6G4a~Y5e|_||G-AV4l{HzFG`iK2^D>FX!33N#pY?3 zX?0<3Wd&mJeQA{wl;9Xd$6|A2Vw%yrqp=~Kk5~o@>s()J3@0F2HagiTO;nKCXr8ue zs3+YM_;a+ivQ{NwT8gK9`ceGKKF&=<%*N}}NmR`O7+P=OFouD3#v}rI4rls^QqTpL z%Xx|%inQ*r&GBAWJ!o%lgWo3RhJcHpu>{MO)#_r3hs4@#cR$BWK9+Qap`vOBs8m z`lVqpSvZzu8XYJE)`xZG-PWuUn4})}?8IEZ*hQSxG(8cv8M)c85?zxR&kBS|4)4d+ z9d&=i4N86rW8xc&Wd$Wrb107{t)<%Zvv9f20CYd_noOwcTJp&eX>T%$sHrwuWWTi) z82;|}44=#Xz>PQU4+YvOtQrg+#=ig3oY&=UBkRLiF!<_Ivy6o#z847xt$*>%N7y_B~C09tanx;M~z-@a?FIs zu_s_l7f{Dueu}yqA7+A_dThM%ZETV>iPmA>u+5Eo?*RGh>lzR5K(Re9&kTlI;18 zC>~b{fgZ$@YC>m?GCl(n>skRSgnI@{al@6AEVH&juG|7llV=z;~_t2J^jR%=xiETWxdmhwKr9iM)Y8hJGW-}zsxm3UTy;)rDD80wR)k!`wWo*NvY~W8AdA23MYU*7RE3Ho{QMvfv|~mz%o(%7GL^!)Tne} zJhn~&y-p?G@)W>}tm%W6XH*R+(8D#{2%i*)C0~is%LEU?P=MNye8x`B>uqGVMEtlS zj9(Lvmsxpr@mcuAQ`9E`SpR0Bi&6G;)~*E`n6TD#z5`bU_AQ)OtiZuxmb0(VDz4#i z!S61!|5s(dQ%brrRitX1B2UTsyW@0gfAmVg6LW7mp3?036hH zJ!^&6nBHx>iUS-g{@iQZcAKRp#OfE6;{)f57eEfb?}nX>UD(mX5>T@)sz&huSIVhq z8)T9B&;%pVW3LgesxZ9JRMg~bF$RmyJpKbWM1%dRy7VlQcd%7Q#*@d(ytr9-koA^s z`gP)?DFujYoPr|?(09JjI^MAnp}qwq`e z8yzAY_dW(sgpUeIR}YA;EIlFnXK`vE-iynuipytzM7_Q)o;=Lst$ey*J;K~pNp8(wEt9P5qKzjdDXXh$Ys%2%Xze%y zEVrTrKr=(bbI%nhfU9YQS0Y^%JqY4ApNxN{XpeYRn zkfuI!CMWDT6~V>96~J(^blxmwG2*dmuoC(DE&0gA?SmvWc2^$|9VxJL^_gUKZMBb} z!Ltl{d6F|`H_t6P+s1U0Nr&+W-1g<}Bef-zqi7g)id9Xw=Mp>ptc+Od@L||W!XuWw zX)6{JL)N~YjFZM|%vMa^j7p$k>@gs1a%7PIu;so=&V7Y}dU3=$Ra@x~N@VyyaDX$! z?EU-v*J_>E3E{T36A90baD90@3!QO&Ndk^+`j6=-5}wP6S}~W|mf$%z5^zdT$S%cf z(}}D)1@1Ekk96*-X6|mbkW63ay7qlD@&>wE7xPmU&nf5N^@&@iwi))4@Pd-1-3~n`bZObp_xmb;vai&&NBpLs1I@j!)IAs~WXCv;3 z%R{M~;DgJvnAqGwtDR1)0nMwocBCLA*rLMA0>?|~w5u`Q++&WCV%1AS?|` zwJ~@9?x=pnPrw$88TR>eKw{?-5aH>*S@tGU(ior&3ylw(3)ZNjJSb+SXP7vQa%?Ea z$>oLm%+#i(qI_|@6kkPtRW-CK1?@_Y#mqi`z7+d%eP(;u`!+Jjozbzz9N#Og-oY1n zY8}n01Wcl|zu@@un_w1m&K9+!YkTeWVL$_hh9DlT6fWKrp__5|o#}CDH+U3hGoa@1|)fVPMAuU~F@ zOdi)_04l?lFaEw6ZtGfDHs7(M+CRfV!tLsdidWA~9~ zjfj?MPGjyUaf>9BeH5fpK<@93_qyfRYNgiD6a1ROARZk7tZiD_>WGeRYN7L~CFrMN z`a|wE-uy6bc;j=ekPBe0{JrRXYcZFTrxpk%1-8QLz=6UnPIYjT7=RcaBFyn*#5kXr zhsaHVs*o)ecYTdC9o3pWjvbF-AO3`@qJ>17KJD!I-r1m^=8$yY4&ldn>$x;CjHF!? zhMJbDQy-ebVRtwLjD$563;ih+quseVd}_MFP3xMzO@tadOo5L<)NcJdkR1l1ly)P5 zm%wbvI%!KYhjb)&eOg%qgZ z@OgdeyrwL0(l&qwq#oP<%kUq#ac>`>xvDzTz2)HShA$gomDj1uz~=rbH4bXCO=Zms zX86%}mh}22x;}rWpx{Jx4Ncy?s<*u&7D!8%vDcPzXTObA0#C|J-2C3XQ7*x&T zLfD(FGCryLv$6;eIwWR;a=0V6jC+pOOdSK4i_T?tswwSAs5_H{VBZsj;qZulA5XcPBb=qqt~r2q=CBqoe8IyA)czykc@|Y|^~O zmQXa^yq}RrR;_<)hPr6cyCY2NdXJQ+%F+tR3TXzCxthcgrGi^Rb?XRYra4T1ew;2w%xfqK1H9XK7Jp#4yhH%b6W@?Id|M&q6Yb2WDNN#zz_= z{QSGijSJEGRM}EtuCbD``E*lG44}>yCPfXZVtx{$QQ_GlLX&DxNfHc=;>kUcNv*j> z>TTpJCo_~Y=h$%zC}pk7qO>*v)35DtWKoOXPP|E}>v$A(yU-AzHO}IEbe*r%R0yv@ zrIJ-9pi^XB<)LX9FCfY)V5A~E6J&0a6i-pw_p~^o^!7G=Zycov4t^YDtD9rbY|~CA zWnU{(6Yplg0KuvlELzLVh9E|5cpmW$)iRJc5UdMQW=WXhQAa;)aXT*v%H- zByMR<8kKaHMrMp)OUFeYR^0G@=sA-Tr8#(Z9tKaRj~NasBXlz-{|x4{{W?2~rqbA> z9>-d#)i*aEpvB2+A|Z55nIlk{YHso;x8^+92FUwj`WAj{{XWUeM469>*eyLdj*1c{ zQxO$RKN0{`3aBmR&JR0mFoMtVWvRYW*eBcN56ce{`WjbyG{ z7KpAQg+c{#q5t=dY@l_xweZsCux;FE@@J{eX8RvTMVK24>au)E6hU%39DMP;C^BT( zo6G}0M5{E>^?Ri+e(VWG%g>`)3PY!8K*-6W({) zjJe9TDY$MR8}`3fo&?;9#@|U;GzQCvV5YqPf~J2zBCytW!@=Z_KUi$8B}-N>E;zhYCLn-Gd%cD}Db%ULPF2>!zg1;JfZaKZNA^j^33`eFj zd}u<@9y#6H3WJ(A+uQj?z`*NpTKM=v@V-lS3pMEf;-1hOF`JL>jl=)DNy_n;$Nt1a zTK-WP=HJlW?s1I?xqbcc&<*;x2`7e|SR?BHcNz5v)_TLkd=qg;_xJl>dk&rgU0Lal zF!|pAbl~3(USLpy!X$u(+nBFkx~+Emn?y~)Y|#?H9)f!)H&@1y8A(*G$1tQA*GP89 zu?+BdL}h@$_~3C#CA%)T`P7uIq^LVlpuY6%CtXH%wGR3ViJ2?sf-`Zsg2d7eEtSPf zNi+4FZwiv<=TaWKW}rJGTmc!-A8(w~?!d3be6P)S-m0`Lmz~#W=-*nM^eND4e`kP6 zkeTxYoI*yJ(%B_#!qZ3TtX+m1jqRUyY#4-^M`Qf4XSwEPXlQVl31cLsw1(@ks!sf| zvrMEQX|JJPRtpqMZByxtT>XV?Z&Hsa3HaO$KNtk2Ljq=T^W&ftG)AKWy@6FaLJZtx zZpKW8!Rzl%rNyQvO7wE`*Owm0$@C3vGOn^}TE69scy<*ViulQxg>Z2eNx6sE%rA^5{H ziBg@fE9ZA>^6C)wEFQ(>bx~pRIbo#oyCaUN4^ldYRjN6NFdlO1M^J>GMI>A(3B7}0 zsVo*`He0}O;^am#ZDxQIwK(^EN|U7mpJ}AJb4oc&6z(W6!MGqqwO* z6}PaVrVwSnm7Gu2!a)JI&jCylOM?Wr-cnaOH>#F4Qc<%Hov>UeD3|uFyU3}FH9wG^ zJ_8E;qV06sauCFLF4em`y#se!E&6z%gBJZlRyFcB6Bd$P=o6CIf77YZfUrb`*GiaI z5=d2C+#g0%x#rvZ#-hwkP&TAWnep_qgoUO|+%fr2NeoxL2e5SOM^?8!De{~~hkk(q z5MB|FoT8k702-38f_~F)iP{^L36$|G?maYnwsu~6TAsJQa(@YK2audUF#&(cUv$!P zC;tnl_eRq;b5cUEv&V`b2p0}}7sCR#mBX$L_|-aZv~ZH`Jm0j1#rY@dI;AYAVVTYf zbEG2-VO?j5_fEJnYOWku3P zYkjl(-}m?<;6x9Z^bN#i4rXkr-pn}ny@Q+0p?M5{pO71~3F&HvC(ki>o+&)l1D03* zg$J|n@K7cD1bED{7Yut~$BRdf5?{qCgY2%fo)Oc=e9=VumtPS#M`>0<>8FX@X4|jk zx!KRRu#q}ZAuCG@OUQVh2zjcKyPJ`$@#LhjEh6;+Mlw`p_``t4`s&fR&Te;(gZqV> z=pw!y@9mpP_DPklR2ZVI&02QzYQHPruaCxt)*{!n!mPRZPN?W$&D}30us1MoDfS-F z2WRR!vY1ek$KlFK(LR?#CY^=H^br|fvq@fz!_umOGX9v3zQvpvkbDc3w(&p9iZ~XCCz9sDRRWO` zFt3!-E|`Dfn0+6WP_8HywQD;C;^3#b!8$;g2{}?;MJ`eUO8YH$1)Y9e3S&wB2aeDe zt!FP^!a~*Ij%p|h-gRcr+S*ekC~Dh_xi2`)`uQ~8qGAVcu8F7nI=F^cS3ZWl&U|g2 z;vH2fJP%1hFB&Yb%cI`#Oh^^t3c+rV!C#Fz5267R%T+E~W~{xhii#>;Y2pZ9RUv#d zyC_x;MNO(uauS;^w3+**W`7I$h!V`a-?FO5JOV*8EG*64{fsY& zo^%ul0Pjo2>c*_*huf;&uYQdPi?L^UPL-TgshKq8WTtPMA3N3y_MGAr|DH_5QkxMK zHtV96sOpwR`hG+*)hI$=7DJuTlAEMXvuvzfr2l3Wjd+s+^QG=SG}9&;prco7Iq4PX zBzc@PYK3Fmm1@kfAQCk{YF0NrkttipnQ9uOA0*{6-4HfpYH}J}TZ&XTy!U&sJ}*m6 z29=0tz@9HX5z)oMB_}88cST~#*?VLPg%qxq6lVro0AoU+m}6(ac)btLf12?nLF{B?{&bMOI37PA~$Ll$saF? z49C=7at(sCoxs5GH5}>U$MT0}CU++MK7eDBqB2gAc%lU=bt8Idql*NX zUd)#gr{t^sS%eB4Hh*$vARK5)EL9UsVDd!8! z#)Tu%?jE*T$vS@zCoZ<0OmIY)lbep>tzg?Xbw9dZHmQt@lXX%(yDEQ88Rq8lp8NyX z%#(cQFH04*k|=fY`tz{N66>L*MMc5?WsP`K**REI%vOuwX@O@f=ve~;PVND z0b%aw>TXDL)$|;Xz8lYX1XA~5EF&|Bb%$M`ZRSfo%cr`GGg5Eu%0slif3Y{vAe`Pe~7YY$oIEn&>ZCozi{(x3X>Ug_5WCEPODsv4Ds-Dl)8UMhk|7AEH-18z#E?70w(yH+VNbA5y z%uAxoiR6d$YR4d%@FYWL@KVrc&_A@P<|t>PgHfh^Pm8m+Ui)^pY>q1i7D~OYGFUTa zy8!pRncMbFx_0-W)hcsT1wK+Hh4hR}N-Cs!TuB{IxO%VA%}v~YM4>9^e`I4;<%+ghp*ex&VN1OnDR)ydJYUUs&8HD*#XY5i>511)ii_5elpM zYVe?pL`v7)W_ME6&oM`<)xUYT6eliOU$*-q;%^#%1$sXG1@o*vkUj`rG?O?OMvSxk z17~loCWLFAGpdogs>Jb&ILmz#HMs@65hgfYa3^(j@WfWwTYWZ*3F!# z2QmWCXNzMyxqsji(sjaC#M(L}bgNXSh3oF4-&kZ=h%sqEyvqLAw66(7VzhnHkGHM7 z1SC7!Z)2i6`Y(gFS?ZQw|BeU~T~mpi*bq@<&!Unfd?nqo6K>Dkx{6HF@e%@BPEeO* z75@XbY6NX8A;B>Cu|ySsITVldCbs!lM!zJ45u0{wUI#|6aHl5U|IKRe6@SCi?$g$py*uT;p{sM`Z+v7gMokh2w>6x38`=Nw2u8<-5_^xUXpA}u8 zmj92^xCvP&%jPa&k@XJorwDXJ2}_=No@IE4W9#x9q1Ox5*&x>Fw&0yo8W!a7cOT@c zULumJkCI*zFrUd8gN+{dkt`G*X?tIG>(sS&K5*uiRGemMQp!Mw1O>ed z4C2A5N&ZqQMFmpc)mY6k=ntlmm!B(5UBHIeSsi&MLrq++nqDX>w7cjYLc3dJnj|b7 zFOWwAORp>b=rN$nbb)N3l+rZWZ})8^u`KBofG==#9DP8$fuc8M!ysyIJo)MC7r|Xw@V-b< zfVj5@&S??%9c?*vTj-};duOu&>c&k;NiVw%i~(yjSle^7^)bmu0j5xHq@@FErDkJr)WIC#z3(_jvOk^7 zXUvD3HKr^<2cW;nJN=E&SrXtnn&(PrNHzrg35QiKuj6w&mz;TWE_<`yi9BSdFlDFv zBuaC~;|1x*F+Sk#34Z4c)oD9P_T*18DLrwBXa2ZI)ljSpK|>W!(04A$Md$F{JoQHg zNm!tJ9A@hG?VY{jF|$mx(4eGB@86eH4kvAsLBl31GLcAs5q_`&nKm*)g;LC7j0)Vn zy|}V)Bvm*KP0a(t)j)~>&Zq=Auqe;B>CS09mO_nFb?w`1DYmxV#7ae;(#kxYMAuYP z_K#&_5}>+uVxJu#nxy;d^+mfzbVd{8abkE|%K#xO!ZE4+-MS)6gDHlzz)XV|j-hlb zk>mj$eOLq?0wp)d90O`U?fKUsTQ?=en8wPwjU*Ja?yFBP5S02_iWwty9+#Ai14oCO zRv(auY8hp}Rvxx^A~QHOS`yf+$99p!^ZsIXyyOnIbV<#iWJ=d|GEUX2Wj;kq z_V*h8Q%o}u-9gNNAs?)Ap7e{q+oXi}11DY;(n=Q%qtKYPq@9LF0Y45EHSH)Gx}e?i za)52-a;CMEA3@Vo(*z^^&Q$5yns@Uc%X7ng!{ewFXDrr9bneLqRp`EDS}e2T@Xsk1 z1SkeLsYp#stW?5!b5>g%oT{84Uvh$33ece{&7ilk;z^Owqr!0Q`|-jp-!$CpT~sh3 zkHHDzBT(~51s0HD9`EX_UGm*?lNzhd%N(nr%>dY`CW0wSRcPu$cSj&!ZG^^3(Iuiw ztPAN$3eSN}-Pj&x3uRIy=X_%4-NQqpu~-uuBn__aX4bkj+%!7@VdXSZxY>m=2ps;| zp;5HHuJqMA!W;VxF(#R@DpPl_>|g$+cB8<-hS2_ZFp@17SgZW~QO~gKA2=)@2|NHD z7TqDme=5QOZrTbu!f+Z8=TH#;CSh0!A?m`3#=_v_u){|nX}sTmkSHMU`gh8lBr8RH zecR#hglPYLiH4Z)ZVK+d=r20i!Iwp3&l_1Jpi#!dX;}>S!#jBDQfe)Sn43EUvHfjo zsCrNVw$L`-TrA=Zg3X`T*U%~ialz+EpRn6|-m3#Y?Mxz2^8^f^K9!7_Z9>w$y~HguN>ir%v7be)U5&xFuB+w-es{ zVIW4@rdr!Pc7JrcMZU!JAv6tLEi+DLX>N)9zh^d5Dev=NUm@vA+ayDh$~v7Ic(Jdv z{i|E4X}7P;VZGIZGG_S~S+($5K!uU5#dV`E)FcT)l~3Ek+{AfiCGc<;b_Tai!jzmf z3IJJrQsZA8NY@di^ted4E-^e<*sTP#o5e4?5Wpb!Xc6@r?2(GMkoGviMS& z;QYQ;*|xY9PJK9f8c+z30>seb5l>EM7{7tk-;RZ%XG3OpXY^q85ZO1W4VDi}qbgS^ zG*RK?Jo}*f2Ik6aQ+yKA*6>Y_DO|`N#|u~0$lq}oLUS-X-F!fn0MbswZCR;>c@#bU+i&m(x>ZaZnRYJbw_!3T!@6YkF4rmxS@V} z7yb(y7oA$#k^ywOCV!oy5JG1%x)pG$c?l*12dF#xxgl$4w}5S3HK2uCI?)lyX+{+* zYTCmWYq5qqXy?Q|l}iBnddq+_tOJ{)G8$*=pyHLG#hl24&Bg<|*^+2ceg{2zRZ}z_ z*tSKIUk3oqfrYQJAwMn}q)dt9g8`)889)^o!y**WJFlr=bcd7 zQ?=o=@!?mPh>CVtKDdng@KlUKjycGJ^t3I(x^UU(7!(k6#uf-rr_rFBg=ejhenW?` zuiac@frNu{!Q6b-HM#GijW*-w&B{RJ0rA2@;L)zWJMIzLQxpLLRc#i7u8CGsKw)9A z;?LZ$#KK5qoLF%*0-hG5uC@v0`IV6vRm=t;04}6?)PcMdWz%|jQ<3k)x@ka)$z8=Bjo58H93;#8br#EQHR{?K z9W~ICIu#R=@!EF5;@+A#N%2*koM3fKbD~u9CHMsnpSq+WH{LTAkY`_pHV#qJOh=YY z|7T9;a-ZkM?sqkU4pE?gZLJb!U@K0`#EpidO>+$C#)O~Z>UI7M%4?=yUNzygv?4K^ zw+-;`6&rQ;#%mYC7Ly?tBGV$;jS8!)$@}C!J3{g~inNkh&?wG_4Yt8HOzFKV{ay%f7qJ~XX4uq+&y^^@#u-wz%k0Q_K_s91y=V$wd_;F#@KFX? zWw8}wQT8dE3PxgVk2UzVX-s^^k`V2^tpC^86N1#g<3P&y{v;V2O3$+%Z*}{fziaaa z3-da~g(a%VA+GNi{Tb+A`@s7FC40EYXKL4NlD`tQO=}MW)7R0#H7|oYBlU>tsAt;9 z4EoIquw4-65^urCQY$otSEWzAO!%o#_hy0$SlbID(_5w@8>N~-B|o{tF^KJ(j}p(W zo;S4U2VEw_TA-1SP$CAmEvc)?rP*|Jw#c1re0)X23`p-AFS+7~Rs+ zASI=MbjN6r+USnaA>BxKBc0OeZy%rUAJ3mVj@7a4c<;Vn_j#RHdI8JYQVEE2krMG` z^c^?zV#DxfyOumtWzpGHS?9{gGLS#%n0%rseL4~lTYe$>dT2Cf?hML?SNn}M>6QG< zzQJ==H4H@sNnLD1)Y{PDrWqKc2(tA)F~om<5gCZ#nA4|D1}0gK>{ZMEMbY`q%}nt+ zGP+p)Kta|?W8?#24t~K2Nvj{e z4RH(%uC@i~rYdcc9wH8+OUvV;Ob+bdzOoy5rU-UUrJ`JRi!w)G6}`uV{CIVWOa!%c z=JkitPh`$?2M!c$WX<<5HjYef%RNg?rH&gU>KvaFO`8b*5UFY|_p&y%hbt@&F2jm^ zYYBN25HH0=m{Ej`_xUrBhUf|$053DKW8bo+Mf~crcF=X~vsc9K(a_RQJFsryYh`qO(AasO<6YV?t}HXPt;5H2K-ayA5f?Wo<4@WJ_Tp(0iED~h zQ)(j9*A@%E2#iYu=M7FJ@QTEx%1EE*#wS6A%2dij^$$u2T3bb%`Z(l%P{#nC_Y^7I zP*_j#rA9713(BVstt0LZo&E6;Zt6S2WjaA8mrM$uazXFjNnxor!2(bwBR>|5Qf9_3 z?$8}hJA$Ac($q9f{Gj??Syg6Hq4cU|(jHmI7|stY0R|J^l!x-Q?Neg#CS)cH0#s)2 zX1_d8cY(*})`!;#)FE3)^#Pb?cma|VwTwMA^6-i?9T(Rxw$#k`9@-E2xk$XPGW*@* z-oD56oSC4v6?1fw`Xc$w?IwFY3~MP!(fDO%tWVJBa+SyXpC7z)dgV4)y;0>J5MBnq zB_`6>R0`Tgy0!=RCf^tMy>HQ~U!vZs_Lxh@B6ml@@tm7IqdMLoEjD|!yrv!@nG|=3 zRi9v@P*^4{U@GrwiyTp6SPY^LVpz2wxah5izKSTJ`?)t;Q=BX;XMIJgsjKlxAnSrq z^x0pO^XE+adJ#HDqV=kaKL*`!D{Bb+om+pV^b=K)>&&Ge;w&zbfK9~5%Tx{4yMror zZwcej6ki(-#PQ~Fy>z3)nv|&7MKF^A+(o_jeQ%dV% zH$_l>YC7nVpRcU1$e|<0{4SA|e~RVYqM0ZtX-DXAJEKDtD5Y&#nhs|*fn_Mj=F-Ov zFaMxiq5P3!{eCIe0(0)0^4BEj;*3)LfDkweM!jU5RVZXXm-egNp6KcHKt@bW)}aWt zl$BBR>iPImk(9&T)sehfFcc`8@I@5Yy0Nu|WeRD*4rhPHW6{!G8kce&4N0c*r>c_u z#;IPI8tkt;)M)$8d|}immx;lWtrbsXsa-4EA~@Up&m(}4%t3po&ib;RCH1PPB=vg! zMm`!@`mn5&(Ucu>Tc_9ZE7HSpM zge@~SsqG!@O@h7HIo{GC!9Ue?ZAI>nEUgQ6I+TaOloNcye^EsJ_?8Tu*!3boeD)D) zW7ZTjycH8isza|xshh5)QeJz+p;Aij{{m~k>5C~Puz42MrnX_*3yaD?cA-SM*vX0Q zu8;Kqg(yCCyGbdaLG7qR8Rhku@yNG7k&i5Se^I1Cg1W1&uFpxdI=S4Cg?{{a}tt)KEY;@Q@B|}v=9b}XO-`&#{h}yTz#V@K9CFAWA4TnrH zPAxiV?T5-h3Iwootb<6dEn*Ch)9OfWP8I|706&ww1do8bJEhBJlw=%;;hb}0(809A zl=sUSBE?6xv9h_hrA7J+v-C=sFJrpXDIe{ld`C2mDZ$j?%%{xmf6M~C#jR6^eBzA= zE{WVbXA7*I`gCU4@{s^+fpu4X6Q4vQO?;qpPtQ-Zy;YaGs!AU5FVo&FF~ja&YBu*W zCpQJTVt-E71tWy{7-%^%Lqwb%Wcu6gsg|h+s*EBr9vJ8N4;atuJgK`B`16G=Q!paYc1& zVBlf*rdN4je)Aj$YBt_?U7)RG!Jti^#oI3h$R+6gOe(`XIy(J+3 zjcP3HIa7C2Z9GQhb1)_ocv2a@`0zq2#u5%pbXl;ndH(>wl?afSajtEyxFyG)}pH8+xnDl@(oxjWifc&IH zIGRkvjNbD`soxYeHP%=qnU^|9MnT;~NKaJ)V5=^#AEsBH*d%W~%@;7`R?a3{#kP=W zO_{W~AuUHTymLM!F;gTQ=u378u>oNRj{+Nu`>?%c~5w+TgCXv^J?vg@k z-IhrAHiIK3N+LkGFyXUCV#IR>HbQ~-r$?JHBQQk(CS9D;ix%nJls`1hU3%$N9n$D5*L zQ3F^n1u+|~6RE^!r=!QlCz$?XmV5<)7fsYLkG$35KMW~FPovbZVgAw)RJ(pt(O*}N zxf&~TEv8k;^ewh$!;7nodEPgjKD4)DLv*>DO=NA|^vA*491Rf6kJ#8+I#oWD8;{zfHW8Bk5X!0;&R}3~PHqr&^z}W3M=!53cY0$- zux>iY%1Q)DpW|7kcB`*gSG%=|E;CU6MTvyfxgKqoM4ZF2lyJ|8o;C?q#L!lkKvv3m zUzGbWmE$F4JL1tkbbaJws$5y|*ughDOX?yG(YR0f(ZxaK5%sVa3ArPsv0>cwRm6~o zh>2s@v8(JFX8Ix)P}%B=ZgJ<-6x!V+OV%CTsu;Kpw9)1`j)cTxAv9ruO> zX(nm&JR)VD1?jM=C9O0tx*9kAgsV3gm@ur&1qx405R!sUiYf9w^lqeDgE?r#o!*iw zdf^?{CcAyZ#iTVNBiG*pLqdk)=Gt67*7f8*4(EI2U?;~JlEfMDMm85LB=A8ki2Bn> zQ}Mu)+!`vVbkig__dAQxZi16>Uz?ldm)ss!J;TDYDnioFIbS)WB_mo|f zb*881_MeICp4d;I8|%yU60Rx+P=A{U_kkR&8XtJB6mS%z?zf%pXQ;>R9*JnbB1MN| zdCDoIv!g&mlA!gJVCROYcLTq8`x0xp%je5|s~klU3s%CR<;)ThO^qNIR~Ium)p8+h zl8#YJRZp)-^FP`|I0g8EX;64tk{7;yiRQWapGgI0hFEn$;&)f)f>+uo$#L7nbMg?LW$5|hapxsRNwI|(3X zy2(JWwye{PZc{RW&~;v>p7iP<2EW1480=-spw2Wwot)d7VEH9S<`YY|grYm@+8nX7VQuTG>e(yX;^>=~xaj z5u&syV8CaVy{!2-2`nuzNc2*g4b5-aXd|<`#i|d6v#h_h5@}_TsBzvFNcIC$$FCLk zz`uY5r3YU+o67mC2q7pGBfXXSJ7Fn&`MXz)t_f%n8{pY`2s2)2_-vm>|KKz(evxcR zu4eNdKk@y;=_gff?yYPww|_J_jx;&tjX-#Uwu#^G&fLlkBU9f0W`eKfQ#X7wZZLq( zaq<-eg>fD27hV5}_Uf6N0xa{I-5ulDO%9!r^r9ui{?|2Bb zEunh81)Od~!IL|9;#+;a(ngi5J7VCA!m>Rh)|OR~&j6iXFny6H?EPHsa_8dwfs1x+ z^OaVPZ#_Bgop=Qs%IA-0oWIR>+Nl~~u;Qwr^a5AjNApC@$UI#1!f(g~3;nk}9|_j~ zUPa(gLpkKDm{rVdTz4@m4q%&(rBR};4YBZK)Z*g&=lm)Mxr+^`)9oeJMUu{?2|ay8 zeTn~#CQ)^XU#uA|I=WofU-GZ+|0`2`Md-x-k#hif(L=qvTmL7_Lx&xH?RfH`E?C_; zlIm`_Md6J5?KpeU`QOv}fmLS3+=-9sZp9r7%IT1*?|&So%r}32if{J!Z$Ch`Db+S2$x1NZ8g2APk zeO49^N2s)X3)d@8`cGw7-4gF54wWCjZKB|=-#wAU7eAHX`$%XzR&LBvM_qSI$V6`r z>rjrgoVcvkwY8WQAPw@bq9wx$Qttz;ZblMIlN24+Zs;jPgGh#`4V2P_~{~w(`RvCtv9M zy^4pLAfz8>G^YE52fzoLFbNzfYvE<1}Hc4wF` z_<%+y%vntd`@9+>dKH#fG~k6>9{npk;p={_`70rwH>ItqL%*#sGJYt}MOH43)?Dgy zUZ~Ryl=K_OT88!#qGg)%O~#XdqJ`f8C-cz>K54`+*e#l{!=3{%1vA#6%5o$G4o#f` z^AzBB>}LMn@2-3m?qvp(2wWOP$-e{O8&EI0TWuZKi2v7%Q?>DU3zE3hQ~(S2LzTCr zn`O?_>s5`C(ZY5y=^_x(!I0jNUWT(G@FUQs**yJwk>u(tNZ|Z4qUqv=5&;kSI@~20 z0bE&haea`DU*4b3;ZAwX*ODK1w+`}zo2MDlrqNjTi)9E&8N&HH!uM$cwqUmjkFxBu|J5UtUKJ@M-I{)d={LMdV!?7?@c-Gi#e7~x6Tt3-dEQ#TivFKF zv0E3k8-LqmaJ9bst=bquouip6>3j*VgwX&Lt2N?%g!2z*&^Usy&>%xL3bZ)ZgX0xo z51WCs(buM0tC4P0%hhB?2C7O$O{7N&vW$+;CxFuZgMWZ0!Hinc-l~r*MBYP@U^9gz z(pTUB30(j`F<+IQhE_=tN_LP=(ocF%75f^YyDN(3X#6=bJpSv$$bz(*PljsEQ-|x; zq?B-BoEKcjM&+Y}gn)^C8&zx83EnwE2HYlxuXJef@eq}3VP+Uh311k;cd1lU3(zXi z#0TfB4Z?xrsk)xuh->xH*ONFOZ z^EhgJ!8@s1jA#h6v;U+vWKNLt2zQK?l9b!Wd|!E;sX&)}>0O4Q31^R_^L{m%F9ZgL zi5zNg#{bcP7Gd+PSE-A?RkxLiEd$7#D4O5^v9z0IAnBT=kcpQecw(MhH9UC=?5x=} z_vWbX%rIRQ=grwfg?81{7yuM}LmRT*$ML?j2RPBIzhOY8fC=OqjH z-1roYf7y!q*0@C&_(()c5Z589PCA}7wjz45zu_Jq(>Bm=us`WyA-Jr%ivzp zz_9+HzO;3~XgB-JS$F)>BqE=s-A8~rB(~;@nJ(YSMMk6W3%eoPGotxJG&lE^4*j}v z?+1T+(vYuEvtkinHnJ#&G2Y?d?P_b>iF~$pT&8vs=W{5WPL4gJe^HE+K!;z^7R*%a zDYn+lkD^XjLuaCkx4FdHmU)wT0d+nf-%2pV?X)6Z?CkaO4(e{yC=K!+w;LUr^Dm{k zs1q9kKKJ!yBH^=Sof`Xi3}Mv8x>7qk$8SHDt1O2rmSjfPlk7a~I7yvIC)xSom`s6Z z=Zjgf?V@9=Y(2c{IG$&roroLcLRmG~4)6U%``qg2%*o;;VUVo&8-bM8U69sr(oACT zMJ#wqQl1~3`t4x=l!migGJFCSgdQ89?l5~<{{?HC_~eu-@?Ojmz+;BR2FJ9=U<9>j zG9V;%R3dmHKyV;e92uBQ&+yfIWj)rErq2+UgeJ1FluqQZM@oILkDx_}cl*}N%HuR| zTI4(AY!d{{d`~n5NsO%wAsTX+_cD<8=u&Lsj{b673m4zm<%-Yj`7T||8eL>;O}dN~ zx%n(JscUPg*hQZ)ia^?eBujoM?{17)fD*1VA)NyIf#>WICNeYX)ZrW$FLPn6I>lbZ zTpf^b=HpJ?D+9&J3T&{uQpikyz;Iy-0OAu{uw=_1Xz1#ky9g64*bYBfjTim&~W zMQ0!=DiI}8tQhv}Me>o=Z`}~Xh-H7$L{YYUaQu7>aWLb&(D0x;!Er~9s|B_yxsfbH z-UX(s9c_-RhF&KdXGI?Jm&&i_F^>`n#9JEYT(8-_{4G3oQ6wY{ZAckVpGCU^-O}dVEW29Z+Ro`*&~*?g zib^FAia4Cmw?wfC0jT}%&%!H+{Qe?*R8GTU+u6Tk)JecFbZ2UrBq_{BEhywqA70>_ z-(XxObviFMm&lc94&xh=&gEp-w~VoSimOUyo={H|Q$AGqP&F$1Glw zAF!XF$e2!nB5X&;gKRC`?0;n_9WqOztEHyuV<~N^3(V7R#gr?4I>$GK#ue=@XwmbM zrvH4;a3!a)yi9kx>lg@U2xA;!aMdrFtmgZRGH19-EaLWlmY~ksUq(Vd{Nu8sBz2{d zZJ+JB8%4RG+UT)uw%=<(}=w(97PomDF0Rr0BxOm^qw=C+masL(;xQ zg>mG}htwXksYNB*;$Uxe9a|%;p`lu)Q5btLlq?!VKs{3OxbAW%R`8?)DbZC9R-6cV z0!~Gco^@#on%Pji_5VmufBGS(@&lE2-PKW5-OYj>5|a#8N&btXE2*n86!^Nvwtr1Xq#QC?Nm!`moKfEK9@bs`+cYnGD9yLThvPo#y88%0v9 zxvfX3wtuL&y;-f$f2X+_Q&fgF(VCCCij_TW#ZvUEiG|ES{yh|3#S$vWx1Q zcV;+0%7gCYwltFpe}cK9h3%avfZRat{PN6^7YRrit_|(dz3!Cx)E(W@HMhpn@^jKL z5$PD6^dIIsJ5g$yc0v5_YE;XZM6Hy{ZTHLMGu6r}3RFMyN&L9f1h3z$3^9s$a263f~=f@xA;dK?UJntR>|Uc+8)wz1-;7FdVcG z^U3b4(je+pCVuZ<{6*=G#Odw+*0dHYbL5@B)*G=0D6)&e7kk0puITy~WiXu7V;4?E zdA%u2!`AX+Vbo`T0&h-7Tf=2B)P6@Rw4A+VHsyP}=GK-gahfRpIdzpIp z#y<0>zLvb)AZd`nkx~r!*la-tcx#D4U?rNQp0l-lp8I*Cv>U*^US>`REa(4Jai!fJmSs6+M zN4%Dc5-Z;7+r)eg#M`nPP+*DljQmCbsXwi3xICq%xpz@w&Ii|&Nz_^?lavZ8?9V^c zYq^eR3KdDoP>s612aIM|L2;250&hpo$8t>sWe56Iy>G*;A;v|pqWB1fU{}A-QCu;j zu6{y<_?>&i(Zq7D&hloB=4T}4uspmP?bH+xb)*nf^%W4@ngnCv4k zMxFf=ViP&xqWuAS;13%d)7byS!uNbIQ>^HNJ%j#d_#h^_QTKO;x%@xH;d!U`YLot8 zG5UP+@YOJ4XNO5l zE$O^xp|Oxl9pS~@erB)+AuiJT9F^VvG&}HhtnyPE=y-uc;5abq_>FBq`XK?v4TO!K zWzLUHd33HYQO9C5Z$}sFMdZo?E0qFhF})8|kc_W}<&PAo#0TaWvD8s2u%hD7#-loLXE&VYw}tDK z5j!14&2|D@*SM#62&U5rq-~nq2N|p^-=@Y<2E(ATNtfIpp3Y?nmkqtBu8<8k85+c9 z+7IwW;By$FU}_;WcC#k-GTl)fMuo4(9nH7Dj@QeVy7W!PJdvPfk+H;|agUS~S0Xg~ zvtX|iT@Zwwj4=2`p0?;!1tpfAwk@)hZ%RWhFp#xN&AEpUFE^Q9frD?84x86$qC1%@ z`iLMuY1zuzsR@_z`C~X~6CWMa)Ckeb9r>fyOfV^tBu24G8Vrmy*pvDsGqIrvU@KTR z=_4qJ@CP7u+i-ze^ktZ+vYSHWm~`GmCe;v)TJ?dY2dSO~D6aDzXRL2FZ0c$P#LY{A zED-I;mR1YuKEMlf)$ADlbEzj6Mv`N}fko~ub z%n3LeZXA-acLeuUczTKz@1OuYDx8Y$$O#hmE}yVsWg#0QDGZLnlo4BvVtlP7clqPM z@<{nLt-+rcjy2P@mh)-x&i--MvKDw{AUF3^@~Nn`9}3I7nVKZKXQf|_cVhL^ZiKYv z&f<~GtO=BFF)CNe8EulHp-Uy$gw(OAONoF_&mTJ}U6#ixym4r~n0&K*h*2!g?Ai{V z(mndcOL9GAp9NC4-xQn+XI)3NhP-FUq?Mq)_XEgqBBON5Gb(gLmmW9aQG~~vo{w)@ z^oO*r8u?y2pEn||mK6=Dt5;l1e&9R)H?0}oMAuuiJ(@m(DL zqz&I@IN-TMYC@Q8`m|bSuHJ|l7b4sI=%B8(#HJ;0!W4-_V(s3#({+$9%+2i3 z!pjS+SzGQY+97e{g2d`0yO!NDz7uLDnv}G|7eL}AvsiXGk#OfDvo1)=n72l_fZu68TgwLmenr-ACT~pR^d3z90s^3KZ zR{fn7rbQoVEMlf4E6A8wxOhf{9KZFZZE>FGlW~Q(;aP)Sbgf>LxLdtue0}-`_p#ir zh4K!W4KMF>gB>wEu}GV-yAZ&l!}w(88{I*nkC(=CL;BfWcNYY9adB4mNG{MK_g)`n zH1iPhJX|{mq-wf<2a;i$DKBevl&HR3oAZ$0<5&JIibQtmzp*^0 z@uD{6zQB!&7;9|z&{oIDr9f>%)_PD^7&7<88?Ra4W#W`)%FjB71zlMyp;}fr3OxKV zu)G=g!7+!l_;s=O-jdI%xd&ACS-FaBk5SNbH2EgaHe?0+*X~7A6cKxuNPQXC+<v1Ct9>-N!mNP` z0A}VxD&Rk`Q?T&v?KNCZNQ*j#9Z^BRQoPARpf>UPyIT*?Qi#DSK#CoF&E(f7z< z#p98Jm63GdkB;3o4GOBa$I`z$qC{NRZ@;-ECV|?(>Isk?V(7?+)>Bs?zr6CBXk@}$ z0%?N$UxBFjB-Ov*!z(+x7v*pGmY&2X@xdkbG|$l_s2cuJrq5B)V>q*B>qXYbwHoYc zC^B?hb_Vzt2mhkPOuh>1LQX05#~hP)djFyrw(|7|g?EK{hW#55sL;`&pkDF|pzN&q zggpBy(sJhXEHFTUhTI^15d}A#MOjCM3#D7)e=&RIur>UPe}>2ZakDS~N6#_@2%;nF zME<)K_HS;18s5R+bvAJ8@gP1D)Q`cgAZc6t9BnJq@kC`;V>HS2a-q!gQxL5)q2Vwd zB~USv8I;KV%(<|bUWhISnh2N=hax^af*~&D#|Ng}2TR%+D(Bp7?^waOoff!f$LbRr z!NwN;TzT{4pB=6=3<(aUT|CM(+56A*n!36_s+^VJkN1!L6t_RI?rRE4+vmy~65GMj zVDaH;AP9AiSHM8U0&)uE6-!<>cjvhxxXf>+!5~YuHnm9mS&U9x!QNfmw~tI$?1$4s zG}YG2!>PFKB0u-zy-S{ph>+r@sW8ZCGCC=c5?})GN^Ph#r;1DBH` zOACvcXAZ)U_M*yY(Z2s&E1Br&%Es2BjS)<4O+`3zV-9=o!MmtsPwL< zY#cJt-9Ggq5#I-@7O8T(7S|-W`ITKr;2J4<6!!Iaut35>IAfnKD{umuiHY26l+`x8 z{PT>Z`|Q}N=9@0CPKOKl2LJ>An$uT&!s}N=cJeOU+dzJ; z>%}Ka1u1Y+vJ0F8vibHopyg>L_cypijgsmops@a2E3{kch1r_x0KST0opj5W_+=Pz z6e^lvRXXM#t^iZGrhnm8o{$BLid}xW>j!Mm#GF=b=3szb;`8TU{Px?1kU^3FqWJG; zJ5#l3_N#5nI+AjQP6iqItAA`_&3nwfNM&VpuAxWFah-Cgg>_0h#(OQZro-w*(Ubnf zD1m$l56UJsM3P#IDnhx5IVy-CM%JjNOhznEo1dxd=2{POZV6Bob;0gIX;rQZL-P0I zq;`iR%@{k21mS=Q0Sf!LUOt&kxm32{(wmY@jX}=|v^86ANHB2!=-)coG|3#QP!Q5;LS5;tw)W*jf8~cnma2dW0zxA zkhCc@H7!$_G50;0;R<0M%L_}m2vtmal7+o~QFG&`mpY$$Us=a-1cyorGn)A;0cde( z8s^g`%s^Wy8~AUG4KDJ{UOA8BuOzEVFTi}AF9Q@bQ#J)i72{J!Wz|z9nh}cP6F{px zNMwwtU}0pmYWh^z-rnwp>=~XXFR{Yd8SOo4HY&l!>>+i=Sd z;`bpzXnm3M4pI%;iG+H|1QnSpzG;2hdAi(?fln5rofp;5G|pdt3+wv$e(F8BLxN*O zVddEU9bxNIT$RhdeC3 zNt$_ZW@O!y9Tn-1l*zSavJ%tJ<(4&srX1cx)cLiRi!?9k-IJs6wdZq3lHun22(nwv z@sg=6EwJP_8j8?H{SHcs823!XwXWV~|7oeKrabaQ7axmCwDhC%8s&DAp&$?*{duyx zpm>OPxxuFCRhk&cOpriub%8g4p@21U-erkJ$6c~h!?8n^=;e?1G!MP^kv3qP{E;XR zBcJ_kbF(~d(_{}VC5VpA)@|ZhfuD;bX<)g2@%E27-`9gV123e@4^V%mN zXlH*&bNw*v$*G>#Us73lKYmRa0^MMFIKKw{mtO+0N!q1TBKVln=kDwmR#`W)%lp=M z@QGyvk2nAdl{A!$>kMysu~~%qYUkdTh4KO3)9S#Q6|KS|zY|=om)@6>JCECSni5@n)LJn$>onM!$2>FcbTm6QugpwN?ZGJ4^>5u;Be9TF;oto12LNGPw{(d zv5Zpa_O;&TMK3J^)dZmcAEf~qW<6O$)o_M=scp8#+`BF_OD&+5M6{Dy9IlAO3S|jt z;fys;*&b(jX;A$~zKIxGj#}&v{zmx;?Qdb$lJRT-)kOXCefma~u%-