Re: [RFC PATCH] python: add qmp-send program to send raw qmp commands to qemu

[Damien Hedde <damien.hedde@greensocs.com>]

On 5/25/22 18:06, Daniel P. Berrangé wrote:
> On Wed, Mar 16, 2022 at 10:54:55AM +0100, Damien Hedde wrote:
> 
> 
>> +def raw_load(file: TextIO) -> List[QMPMessage]:
>> +    """parse a raw qmp command file.
>> +
>> +    JSON formatted commands can expand on several lines but must
>> +    be separated by an end-of-line (two commands can not share the
>> +    same line).
>> +    File must not end with empty lines.
>> +    """
>> +    cmds: List[QMPMessage] = []
>> +    linecnt = 0
>> +    while True:
>> +        buf = file.readline()
>> +        if not buf:
>> +            return cmds
> 
> If you change this to 'break'...
> 
>> +        prev_err_pos = None
>> +        buf_linecnt = 1
>> +        while True:
>> +            try:
>> +                cmds.append(json.loads(buf))
> 
> ...and this to
> 
>    yield json.loads(buf)
> 
> then....
> 
>> +                break
>> +            except json.JSONDecodeError as err:
>> +                if prev_err_pos == err.pos:
>> +                    # adding a line made no progress so
>> +                    #  + either we're at EOF and json data is truncated
>> +                    #  + or the parsing error is before
>> +                    raise QmpRawDecodeError(err.msg, linecnt + err.lineno,
>> +                                            err.colno) from err
>> +                prev_err_pos = err.pos
>> +            buf += file.readline()
>> +            buf_linecnt += 1
>> +        linecnt += buf_linecnt
>> +
>> +
>> +def report_error(msg: str) -> None:
>> +    """Write an error to stderr."""
>> +    sys.stderr.write('ERROR: %s\n' % msg)
>> +
>> +
>> +def main() -> None:
>> +    """
>> +    qmp-send entry point: parse command line arguments and start the REPL.
>> +    """
>> +    parser = argparse.ArgumentParser(
>> +            description="""
>> +            Send raw qmp commands to qemu as long as they succeed. It either
>> +            connects to a remote qmp server using the provided socket or wrap
>> +            the qemu process. It stops sending the provided commands when a
>> +            command fails (disconnection or error response).
>> +            """,
>> +            epilog="""
>> +            When qemu wrap option is used, this script waits for qemu
>> +            to terminate but never send any quit or kill command. This
>> +            needs to be done manually.
>> +            """)
>> +
>> +    parser.add_argument('-f', '--file', action='store',
>> +                        help='Input file containing the commands')
>> +    parser.add_argument('-s', '--socket', action='store',
>> +                        help='< UNIX socket path | TCP address:port >')
>> +    parser.add_argument('-v', '--verbose', action='store_true',
>> +                        help='Verbose (echo commands sent and received)')
>> +    parser.add_argument('-p', '--pretty', action='store_true',
>> +                        help='Pretty-print JSON')
>> +
>> +    parser.add_argument('--wrap', nargs=argparse.REMAINDER,
>> +                        help='QEMU command line to invoke')
>> +
>> +    args = parser.parse_args()
>> +
>> +    socket = args.socket
>> +    wrap_qemu = args.wrap is not None
>> +
>> +    if wrap_qemu:
>> +        if len(args.wrap) != 0:
>> +            qemu_cmdline = args.wrap
>> +        else:
>> +            qemu_cmdline = ["qemu-system-x86_64"]
>> +        if socket is None:
>> +            socket = "qmp-send-wrap-%d" % os.getpid()
>> +        qemu_cmdline += ["-qmp", "unix:%s" % socket]
>> +
>> +    try:
>> +        address = QMPSend.parse_address(socket)
>> +    except QMPBadPortError:
>> +        parser.error(f"Bad port number: {socket}")
>> +        return  # pycharm doesn't know error() is noreturn
>> +
>> +    try:
>> +        with open(args.file, mode='rt', encoding='utf8') as file:
>> +            qmp_cmds = raw_load(file)
>> +    except QmpRawDecodeError as err:
>> +        report_error(str(err))
>> +        sys.exit(1)
> 
> ...change this to
> 
>      fh = sys.stdin
>      if args.file is not None and args.file != '-':
>        fh = open(args.file, mode='rt', encoding='utf8')
> 
> ....
> 
>> +
>> +    try:
>> +        with QMPSend(address, args.pretty, args.verbose,
>> +                     server=wrap_qemu) as qmp:
>> +            # starting with python 3.7 we could use contextlib.nullcontext
>> +            qemu = Popen(qemu_cmdline) if wrap_qemu else contextlib.suppress()
>> +            with qemu:
>> +                try:
>> +                    qmp.setup_connection()
>> +                except ConnectError as err:
>> +                    if isinstance(err.exc, OSError):
>> +                        report_error(f"Couldn't connect to {socket}: {err!s}")
>> +                    else:
>> +                        report_error(str(err))
>> +                    sys.exit(1)
>> +                try:
>> +                    for cmd in qmp_cmds:
> 
> ...finally this to
> 
>      for cmd in raw_load(fh)
> 
> 
> This means we can use qmp-send in a pipeline with commands
> sent to QEMU on the fly as they arrive, rather than having
> to read all the commands upfront before QEMU is started.

Yes. I was not sure which way was "better" between reading on the fly or 
buffering everything before. In we want pipelining, we don't have much 
choice.

> 
> BTW, as an example usage I was trying your impl here in the following
> way to extract information about CPUs that are deprecated
> 
>     echo -e '{ "execute": "query-cpu-definitions"}\n{"execute": "quit"}' | \
>       qmp-send -v -p --wrap ./build/qemu-system-x86_64 -nodefaults  -vnc :1 | \
>       jq -r  --slurp '.[1].return[] | [.name, .deprecated] | @csv'
> 
> 
>> +                        qmp.execute_cmd(cmd)
>> +                except QMPError as err:
>> +                    report_error(str(err))
>> +                    sys.exit(1)
>> +    finally:
>> +        if wrap_qemu:
>> +            os.unlink(socket)
>> +
>> +
>> +if __name__ == '__main__':
>> +    main()
> 
> 
> With regards,
> Daniel