How to convert the log to JSON and put it in the Postgresql database?

Question

Python

Please tell me how to convert such a log to JSON in Python and add it to the Postgresql database? in order to be able to parse "key" : "value", and ignore other lines. from "fio version" to "filename" for example.

I'll attach the entire log just in case / https://drive.google.com/drive/folders/1t0tIVOMI36...

oltp_read_uniform: (g=0): rw=randread, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=1
...
oltp_write_uniform: (g=1): rw=randwrite, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=1
...
oltp_rw70_30_uniform: (g=2): rw=randrw, bs=(R) 8192B-8192B, (W) 8192B-8192B, (T) 8192B-8192B, ioengine=libaio, iodepth=1
...
fio-3.27-12-gd7a2
Starting 24 processes
3;fio-3.27-12-gd7a2;oltp_read_uniform;0;0;66362840;553019;69127;120001;3;662;7.105567;0.883065;17;23249;107.165068;60.988828;1.000000%=62;5.000000%=86;10.000000%=89;20.000000%=92;30.000000%=95;40.000000%=97;50.000000%=99;60.000000%=101;70.000000%=104;80.000000%=109;90.000000%=119;95.000000%=164;99.000000%=197;99.500000%=242;99.900000%=954;99.950000%=970;99.990000%=1613;0%=0;0%=0;0%=0;50;23256;114.506388;60.999852;489672;621288;100.000000%;553040.327731;1646.634228;0;0;0;0;0;0;0.000000;0.000000;0;0;0.000000;0.000000;1.000000%=0;5.000000%=0;10.000000%=0;20.000000%=0;30.000000%=0;40.000000%=0;50.000000%=0;60.000000%=0;70.000000%=0;80.000000%=0;90.000000%=0;95.000000%=0;99.000000%=0;99.500000%=0;99.900000%=0;99.950000%=0;99.990000%=0;0%=0;0%=0;0%=0;0;0;0.000000;0.000000;0;0;0.000000%;0.000000;0.000000;2.976875%;9.288333%;8295390;0;381;100.0%;0.0%;0.0%;0.0%;0.0%;0.0%;0.0%;0.00%;0.00%;0.00%;0.01%;0.01%;50.52%;49.00%;0.24%;0.01%;0.19%;0.03%;0.01%;0.01%;0.01%;0.01%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%
3;fio-3.27-12-gd7a2;oltp_write_uniform;1;0;0;0;0;0;0;0;0.000000;0.000000;0;0;0.000000;0.000000;1.000000%=0;5.000000%=0;10.000000%=0;20.000000%=0;30.000000%=0;40.000000%=0;50.000000%=0;60.000000%=0;70.000000%=0;80.000000%=0;90.000000%=0;95.000000%=0;99.000000%=0;99.500000%=0;99.900000%=0;99.950000%=0;99.990000%=0;0%=0;0%=0;0%=0;0;0;0.000000;0.000000;0;0;0.000000%;0.000000;0.000000;49500400;412499;51562;120001;3;316;7.214137;0.761373;58;26708;146.412453;77.802837;1.000000%=116;5.000000%=121;10.000000%=124;20.000000%=128;30.000000%=132;40.000000%=134;50.000000%=138;60.000000%=140;70.000000%=144;80.000000%=150;90.000000%=162;95.000000%=177;99.000000%=321;99.500000%=419;99.900000%=1003;99.950000%=1019;99.990000%=1581;0%=0;0%=0;0%=0;105;26715;153.876009;77.804117;321144;428728;100.000000%;412564.327731;1607.516327;2.312917%;6.936979%;6187579;0;364;100.0%;0.0%;0.0%;0.0%;0.0%;0.0%;0.0%;0.00%;0.00%;0.00%;0.00%;0.00%;0.01%;98.14%;1.48%;0.05%;0.23%;0.10%;0.01%;0.01%;0.01%;0.01%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%
3;fio-3.27-12-gd7a2;oltp_rw70_30_uniform;2;0;40471408;337258;42157;120001;3;277;7.143838;0.760130;32;20032;121.650831;72.261888;1.000000%=59;5.000000%=87;10.000000%=90;20.000000%=94;30.000000%=97;40.000000%=100;50.000000%=104;60.000000%=110;70.000000%=117;80.000000%=138;90.000000%=171;95.000000%=189;99.000000%=342;99.500000%=428;99.900000%=970;99.950000%=1028;99.990000%=1433;0%=0;0%=0;0%=0;54;20039;129.040904;72.264304;266669;352664;100.000000%;337357.924370;1312.378702;17335960;144465;18058;120001;3;344;7.430028;0.817761;60;18527;129.774764;79.488992;1.000000%=107;5.000000%=111;10.000000%=114;20.000000%=116;30.000000%=119;40.000000%=121;50.000000%=123;60.000000%=125;70.000000%=128;80.000000%=132;90.000000%=138;95.000000%=144;99.000000%=252;99.500000%=366;99.900000%=987;99.950000%=1011;99.990000%=2506;0%=0;0%=0;0%=0;100;18535;137.451811;79.487011;113567;152731;100.000000%;144509.420168;619.188392;2.693750%;8.134583%;7225950;0;585;100.0%;0.0%;0.0%;0.0%;0.0%;0.0%;0.0%;0.00%;0.00%;0.00%;0.00%;0.01%;25.80%;72.50%;1.34%;0.06%;0.24%;0.06%;0.01%;0.01%;0.01%;0.01%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%;0.00%
{
"fio version" : "fio-3.27-12-gd7a2",
"timestamp" : 1629816393,
"timestamp_ms" : 1629816393883,
"time" : "Tue Aug 24 14:46:33 2021",
"global options" : {
    "ioengine" : "libaio",
    "direct" : "1",
    "group_reporting" : "1",
    "size" : "100%",
    "iodepth" : "1",
    "numjobs" : "8",
    "runtime" : "120",
    "file_service_type" : "random",
    "log_hist_msec" : "1000",
    "log_hist_coarseness" : "4",
    "log_avg_msec" : "1000",
    "filename" : "/dev/mapper/3614529002e4c00140400800000000014"
},

The code is still divided into two separate files (1.connect to the database, 2.parser)

import psycopg2 
from psycopg2 import Error

try:
    # Подключиться к существующей базе данных
    connection = psycopg2.connect(user="postgres",
                                # пароль, который указали при установке PostgreSQL
                                password="*****",
                                host="127.0.0.1",
                                port="5432",
                                database="result_test_parsing")

    # Создайте курсор для выполнения операций с базой данных
    cursor = connection.cursor()
    # SQL-запрос для создания новой таблицы
    create_table_query = '''CREATE TABLE fio
                        (FIO VERSION    TEXT    NOT NULL,
                        JOBNAME           TEXT    NOT NULL,
                        TERSE_VERSION_3         INT); '''
    # Выполнение команды: это создает новую таблицу
    cursor.execute(create_table_query)
    connection.commit()
    print("Таблица успешно создана в PostgreSQL")

except (Exception, Error) as error:
    print("Ошибка при работе с PostgreSQL", error)
finally:
    if connection:
        cursor.close()
        connection.close()
        print("Соединение с PostgreSQL закрыто")

Well, here is the beginning of the parsing code.

import re
def reader(filename):

    with open(filename) as f:
        log = f.read()
    print(log)


if __name__ == '__main__':
    reader('test.log')

Answer 1

https://regex101.com/r/ZboRJD/1