Python





```python
from multiprocessing import Queue, Manager, Process
import time
def process1(queuex: Queue):
    while True:
        queuex.put("dasfdsa")
        time.sleep(1)
def process2(queuex: Queue):
    while True:
        if not queuex.empty():
            path = queuex.get()
            print(path)
if __name__ == '__main__':
    manager = Manager()
    dataq = manager.Queue()
    p1 = Process(target=process1, args=(dataq,))
    p2 = Process(target=process2, args=(dataq,))
    p1.start()
    p2.start()
    p1.join()
    p2.join()
```

Python 多进程通信

![在这里插入图片描述](/static/img/images/00179.png)





安装 
```shell
pip install kafka-python
```

生产者
```python
from kafka import KafkaProducer
producer = KafkaProducer(bootstrap_servers='101.112.143.254:59092')
future = producer.send('topic_wx_notice', key=b'my_key', value='my_valuefdsfsdf啊沙发舒服'.encode(), partition=0)
result = future.get(timeout=10)
print(result)
```

消费者

给group_id后，同一个group里面的消费者去消费消息只能消费一次。
```python
from kafka import KafkaConsumer
consumer = KafkaConsumer('topic_wx_notice', bootstrap_servers='101.112.143.254:59092',group_id="weiyiid",
                         value_deserializer=lambda m: m.decode())
for msg in consumer:
    print(msg)
```

python kafka 使用

![在这里插入图片描述](/static/img/images/00180.png)






```python
def get_wind_level_str(wind_speed_float):
    """
    :param wind_speed_float:  风速 m/s
    :return: 风级
    """
    if wind_speed_float < 0.2:
        return 0
    elif wind_speed_float < 1.5:
        return 1
    elif wind_speed_float < 3.3:
        return 2
    elif wind_speed_float < 5.4:
        return 3
    elif wind_speed_float < 7.9:
        return 4
    elif wind_speed_float < 10.7:
        return 5
    elif wind_speed_float < 13.8:
        return 6
    elif wind_speed_float < 17.1:
        return 7
    elif wind_speed_float < 20.7:
        return 8
    elif wind_speed_float < 24.4:
        return 8
    elif wind_speed_float < 28.4:
        return 10
    elif wind_speed_float < 32.6:
        return 11
    elif wind_speed_float < 36.9:
        return 12
    elif wind_speed_float < 41.4:
        return 13
    else:
        return 14
```

风速与风力等级关系 Python

Python





```python
# -*- coding: utf-8 -*-
import datetime
import json
import re
import time
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from pprint import pprint
import json
import requests
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import pandas as pd
import smtplib
from email.mime.text import MIMEText
from email.header import Header
# EmailReceivers = ['kevin.xie@chinaentropy.com']  # 邮件收件人
EmailReceivers = ["xxxx@chinaentropy.com", 'xxxx@chinaentropy.com', "xxxx@chinaentropy.com",
                  "xxxx@chinaentropy.com"]  # 邮件收件人
localFileName = "1234.xlsx"
def send_email_to_d(text, receivers):
    # 第三方 SMTP 服务
    mail_host = "smtp.exmail.qq.com"  # 设置服务器
    mail_user = "xxxx@chinaentropy.com"  # 用户名
    mail_pass = "qqqqqqqqqqq"  # 口令
    sender = 'xxxx@chinaentropy.com'
    message = MIMEMultipart('mixed')  # 邮件内容，mixed为混合形式可传附件
    message['From'] = Header("xxxx@chinaentropy.com", 'utf-8')
    message['To'] = Header(";".join(EmailReceivers), 'utf-8')
    message['Subject'] = Header('和风API调用费用统计', 'utf-8')
    # print(text)  # 邮件文本内容编辑
    text_plain = MIMEText(text, 'plain', 'utf-8')
    message.attach(text_plain)  # 实例化文本
    docxpart = MIMEApplication(open(localFileName, 'rb').read())
    docxpart.add_header('Content-Disposition', 'attachment', filename=localFileName)
    message.attach(docxpart)  # 要发送的exl实例化
    try:
        smtpObj = smtplib.SMTP_SSL(mail_host, 465)
        smtpObj.login(mail_user, mail_pass)
        smtpObj.sendmail(sender, receivers, message.as_string())
        smtpObj.quit()
        print("邮件发送成功")
    except smtplib.SMTPException:
        print("Error: 无法发送邮件")
class scrapyler:
    def __init__(self):
        pass
    def getCookie(self):
        userName = "xxxx"
        userPassword = "xxxxx"
        caps = {
            'browserName': 'chrome',
            'version': '',
            'platform': 'ANY',
            'goog:loggingPrefs': {'performance': 'ALL'},  # 记录性能日志
            'goog:chromeOptions': {'extensions': [], 'args': ['--headless']}  # 无界面模式
        }
        # driver = webdriver.Chrome(executable_path='chromedriver.exe', desired_capabilities=caps)
        driver = webdriver.Chrome(executable_path='/home/pythonPackage/chromedriver', desired_capabilities=caps)
        driver.get('https://id.qweather.com/#/login?redirect=https%3A%2F%2Fconsole.qweather.com%2F#/expenses')
        browser = driver
        print("网页响应中。。。")
        wait = WebDriverWait(browser, 30)
        browser.find_element(by="xpath",
                             value=r"""//*[@id="app"]/div/div/div[2]/div/div[3]/div/div[3]/form/div[1]/div/div/input""").clear()
        browser.find_element(by="xpath",
                             value=r"""//*[@id="app"]/div/div/div[2]/div/div[3]/div/div[3]/form/div[1]/div/div/input""").send_keys(
            userName)
        browser.find_element(by="xpath",
                             value=r"""//*[@id="app"]/div/div/div[2]/div/div[3]/div/div[3]/form/div[2]/div/div/input""").clear()
        browser.find_element(by="xpath",
                             value=r"""//*[@id="app"]/div/div/div[2]/div/div[3]/div/div[3]/form/div[2]/div/div/input""").send_keys(
            userPassword)
        browser.find_element(by="xpath",
                             value=r"""//*[@id="app"]/div/div/div[2]/div/div[3]/div/div[5]/button""").click()
        time.sleep(5)
        balance = float(browser.find_element(by="xpath",
                                             value=r"""//*[@id="page"]/section/main/div/div/div[2]/div[1]/div/div[3]/div[1]/div/span[2]""").text)
        logs = driver.get_log("performance")
        cookies = None
        for i in logs:
            if re.search(r'''"cookie":"[\s\S]{10,200}__HE_CORE__=[\s\S]{10,200}JSESSIONID=[0-9A-Za-z]{10,200}"''',
                         str(i)) is not None:
                cookies = re.search(
                    r'''"cookie":"[\s\S]{10,200}__HE_CORE__=[\s\S]{10,200}JSESSIONID=[0-9A-Za-z]{10,200}"''',
                    str(i)).group()
                break
        assert cookies is not None
        cookie = cookies[10:-1]
        browser.quit()
        # browser.close()
        return cookie, balance
    def getAppBinding(self, cookie):
        prog = requests.get(headers={"cookie": cookie}, url="https://console.qweather.com/v1/rest/app").json()["data"]
        idname = {lt["name"]: lt["id"] for lt in prog}
        print(idname)
        return idname
    def getPurchaseHistory(self, cookie, appId: str,
                           yearmonth: str):  # "https://console.qweather.com/v1/rest/fee/app/95491/type/2?date=2021-11"
        url = f"https://console.qweather.com/v1/rest/fee/app/{appId}/type/2?date={yearmonth}"
        prog = requests.get(headers={"cookie": cookie}, url=url).json()["data"]
        print(prog)
        return prog
    def run(self):
        year = datetime.datetime.now().year - 1 if datetime.datetime.now().month - 1 == 0 else datetime.datetime.now().year
        month = 12 if datetime.datetime.now().month - 1 == 0 else datetime.datetime.now().month - 1
        yearmonthstr = str(year) + "-" + str(month).zfill(2)
        cookie, balance = self.getCookie()
        time.sleep(0.5)
        idname = self.getAppBinding(cookie)
        time.sleep(0.5)
        resultdf = []
        need = ["森林消防", "二块板天气数据库专用", "环保生产"]
        for nd in need:
            prog = self.getPurchaseHistory(cookie, idname[nd], yearmonthstr)
            resultdf.append(pd.DataFrame(prog))
        df = pd.concat(resultdf)
        df["cost"] = df["cost"].apply(float)
        totalMonney = df["cost"].sum()
        df.to_excel(localFileName)
        return totalMonney, need, balance
if __name__ == '__main__':
    # 爬虫获取 总金额，行业名列表，本地文件“和风API调用费用明细.xlsx”
    scrapyler = scrapyler()
    totalMonney, need, balance = scrapyler.run()
    # 发邮件
    if datetime.datetime.now().day == 6 or balance <= 5:
        year = datetime.datetime.now().year - 1 if datetime.datetime.now().month - 1 == 0 else datetime.datetime.now().year
        month = 12 if datetime.datetime.now().month - 1 == 0 else datetime.datetime.now().month - 1
        yearmonthstr = str(year) + "-" + str(month).zfill(2)
        send_email_to_d(f"和风API调用当前余额还剩余  {balance}  元。\n"
                        f"\n"
                        f"\n"
                        f"和风API调用费用统计见附件，{yearmonthstr}月份总消费金额  {totalMonney}   元。\n"
                        f"此统计只包含这些项目行业{str(need)}。\n"
                        f"\n"
                        f"\n"
                        f"--------------------------------------------------------------------------"
                        f"\n"
                        f"此邮件是跑在    http://10.10.90.54:8888/dolphinscheduler/ui/#/resource/file/edit/280     的定时脚本。\n"
                        f"每月6号发一次邮件。每天检测一次余额，余额低于5元也会发本邮件。"
                        f"和风api登录地址: https://id.qweather.com/#/login?redirect=https%3A%2F%2Fconsole.qweather.com%2F#/expenses"
                        f"\n"
                        f"  "
                        ,
                        EmailReceivers)
```

Python selenium smtplib 爬虫获取信息后 发邮件 发附件

Python





```python
from bs4 import BeautifulSoup as bs
html = bs(open("test.html", "r").read(), 'html.parser')
trall = html.find_all(name="tr")
result = []
for tr in trall:
    want = tr.text.strip().replace("\n", " ").split()
    result.append(want)
print(result)
```

Python BeautifulSoup 处理html脚本

raise ValueError("check_hostname requires server_hostname")





ValueError: check_hostname requires server_hostname


关闭VPN即可！

ValueError: check_hostname requires server_hostname

Python





```python
import os
import re
def all_path(dirname):
    result = []
    for maindir, subdir, file_name_list in os.walk(dirname):
        for filename in file_name_list:
            apath = os.path.join(maindir, filename)
            result.append(apath)
    return result
result = []
for name in all_path("."):
    if name.endswith(".c") or name.endswith(".h"):
        singleTxt = open(name, "r").readlines()
        for k, line in enumerate(singleTxt):
            singleTxt[k] = re.sub("//[\s\S]*(?=[\n])", "", singleTxt[k])
            singleTxt[k] = re.sub("^[\s]*\n$", "", singleTxt[k])
        singleTxt_join = "".join(singleTxt)
        singleTxt_join = re.sub("/\*{1,2}[\s\S]*?\*/", "", singleTxt_join)
        open("temp.txt", "w").write(singleTxt_join)
        singleTxt = open("temp.txt", "r").readlines()
        result.append("/**\n")
        result.append("   * Copyright 2021-2021 Beijing Grant Technology Co., Ltd.\n")
        result.append("*/\n")
        result.append(f"/*{str(name)}*/\n")
        result.extend(singleTxt)
open("result.txt", "w").writelines(result)
```


打包：
```
pip install pyinstaller
```

打包方法：
```
pyinstaller -F setup.py 打包exe
pyinstaller -F -w setup.py 不带控制台的打包
pyinstaller -F -i xx.ico setup.py 打包指定exe图标打包
```

Python 删除 C语言中的注释 打上标 软 著

Python





```python
import requests
def save_picture_and_get_url(pic_path: str):
    pic_name = pic_path.split('/')[-1]
    url = "http://10.10.90.88:8113/upload"
    with open(pic_path, 'rb') as f:
        files = {'file': f}
        data = {"bucketName": "algorithmgroupfiles",
                'inputDir': 'chengguan/',
                'persistLink': 'true'}
        res = requests.request(method="POST", url=url, data=data, files=files)
    return res.json()['data'][pic_name]  # 返回url
if __name__ == '__main__':
    name = "./testpicture/img1coj3.jpg"
    print(save_picture_and_get_url(name))
```

Python minIO

执行：




```
mlflow ui --backend-store-uri sqlite:////mlflow/mlflow.db --default-artifact-root /mlflow/artifacts --host 0.0.0.0 --port 50004
```

报错：
```
2021/12/15 11:23:01 WARNING mlflow.store.db.utils: SQLAlchemy engine could not be created. The following exception is caught.
(sqlite3.OperationalError) unable to open database file
```

解决：

```shell
mkdir -p /mlflow/artifacts
```

mlflow (sqlite3.OperationalError) unable to open database file

官网文档：[https://docs.python.org/zh-cn/3/library/concurrent.futures.html](https://docs.python.org/zh-cn/3/library/concurrent.futures.html)






Python有GIL，故多线程最好只用于io密集型程序，比如需要网络下载很多个文件，下面截取部分代码来展示如何使用：
```python
from concurrent.futures import ThreadPoolExecutor, as_completed # 导入需要的包
threadPool = ThreadPoolExecutor(max_workers=140) # 线程池的池子大小，最多同时执行多少个任务，submit的任务数超过这个数字后后续任务会自行等待进入线程池
all_task = [threadPool.submit(self.download_01, url, path, delay * 1.5)
            for delay, url in enumerate(urls)] # 使用线程池ThreadPoolExecutor的submit方法发起任务，列表生成式里面装了所有任务的返回句柄。self.download_01是我的函数名称，url, path, delay * 1.5 全是我的self.download_01函数的参数。
for task in as_completed(all_task): # as_completed 可以判断列表里的线程是否完成
    data = task.result()
    print("任务{} down load success".format(data)) # 打印结果
threadPool.shutdown(wait=True) #　释放线程池资源
```