Re: [RFC 7/8] fpga-region: add sysfs interface

[Alan Tull <delicious.quinoa@gmail.com>]

On Wed, Feb 15, 2017 at 1:49 PM, Jason Gunthorpe
<jgunthorpe@obsidianresearch.com> wrote:
> On Wed, Feb 15, 2017 at 12:23:28PM -0600, Alan Tull wrote:
>> > This is usually the sort of stuff I'd punt to userspace, but since the
>> > kernel is doing request_firmware it is hard to see how that is an
>> > option in this case...
>>
>> I like how extensible (and readable!) this is.  It wouldn't take much
>> kernel code to add this.  I'd like to see the python script.
>
> Sure, attached
>
> Jason

Hi Jason,

Thanks for sharing this.

So this script takes the bitfile and its build logs as input, parses
the build logs for image information, does some manipulations on bit
order as needed, and adds the header.  So it's really doing (at least)
two things: adding header info and doing bitorder changes where needed
so that the kernel won't need to do it.

Alan

>
> #!/usr/bin/env python
> # COPYRIGHT (c) 2016 Obsidian Research Corporation.
> # Permission is hereby granted, free of charge, to any person obtaining a copy
> # of this software and associated documentation files (the "Software"), to deal
> # in the Software without restriction, including without limitation the rights
> # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> # copies of the Software, and to permit persons to whom the Software is
> # furnished to do so, subject to the following conditions:
>
> # The above copyright notice and this permission notice shall be included in
> # all copies or substantial portions of the Software.
>
> # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
> # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
> # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
> # THE SOFTWARE.
>
> import subprocess,re,string,os,stat,mmap,sys,base64;
> import argparse
> import time;
>
> def timeRFC1123():
>     # Python doesn't have this as a built in
>     return subprocess.check_output(["date","-u",'+%a, %02d %b %Y %T GMT']).strip();
>
> def getTOT():
>     """Return the top of tree commit hash from git"""
>     try:
>         HEAD = subprocess.check_output(["git","rev-parse","--verify","HEAD"]).strip();
>     except subprocess.CalledProcessError:
>         return "??? %s"%(os.getcwd());
>
>     dirty = subprocess.check_output(["git","diff","--stat"]).strip();
>     if dirty:
>         return "%s-dirty"%(HEAD);
>     return HEAD;
>
> def parseISEPar(f,fmts):
>     """Read Xilinx ISE par log files to get relevant design information"""
>     f = string.join(f.readlines());
>     g = re.search('"([^"]+)" is an NCD, version [0-9.]+, device (\S+), package (\S+), speed -(\d+)',f).groups();
>     fmts.update({"Design": g[0],
>                  "Device": g[1],
>                  "Package": g[2],
>                  "Speed": g[3]});
>     fmts["PAR-Ver"] = re.search("(Release \S+ par \S+(?: \(\S+\))?)\n",f).groups()[0]
>     fmts["Speed-File"] = re.search('Device speed data version:\s+"([^"]+)"',f).groups()[0];
>
> def parseVivadoTwr(f,fmts):
>     """Read Vivado 'report_timing' log files to get relevant design information"""
>     f = string.join(f.readlines(1024));
>     g = re.search('Design\s+:\s+(\S+)',f).groups();
>     fmts["Design"] = g[0];
>
>     g = re.search('Speed File\s+:\s+-(\d+)\s+(.+)',f);
>     if g is not None:
>         g = g.groups();
>         fmts["Speed"] = g[0];
>         fmts["Speed-File"] = g[1];
>         g = re.search('Device\s+:\s+(\S+)-(\S+)',f).groups();
>         fmts["Device"] = g[0];
>         fmts["Package"] = g[1];
>     else:
>         g = re.search('Part\s+:\s+Device=(\S+)\s+Package=(\S+)\s+Speed=-(\d+)\s+\((.+)\)',f).groups();
>         fmts.update({"Device": g[0],
>                      "Package": g[1],
>                      "Speed": g[2],
>                      "Speed-File": g[3]});
>
>     g = re.search('Version\s+:\s+(.+)',f).groups();
>     fmts["Xilinx-Ver"] = g[0];
>
> def parseSrr(f,fmts):
>     """Read Synplify log files to get relevent design information"""
>     l = f.readline().strip();
>     fmts["Synplify-Ver"] = re.match("#Build: Synplify (.*)",l).groups()[0];
>
> def find_start(bitm):
>     """Locate the start of the actual bitsream in a Xilinx .bit file.  Xilinx
>        tools drop an Impact header in front of the sync word. The FPGA ignores
>        everything prior to the sync word."""
>     for I in range(len(bitm)):
>         if (bitm[I] == '\xaa' and bitm[I+1] == '\x99' and
>             bitm[I+2] == '\x55' and bitm[I+3] == '\x66'):
>             return I;
>     return 0;
>
> def align_bitstream(fmts,alignment=8):
>     """Adjust the header content so that the bitstream starts aligned. This is
>     so we can mmap this file with the header and still DMA from it."""
>     while True:
>         hdr = ("YYBIT/1.0\n" +
>                "\n".join("%s: %s"%(k,v) for k,v in sorted(fmts.iteritems())) +
>                "\n\n");
>         if len(hdr) % alignment == 0:
>             return hdr;
>         fmts["Pad"] = "x"*(alignment - ((len(hdr) + 6) % alignment));
>
> def makeHeader(out,args):
>     fmts = {
>         "Builder": os.getenv("USERNAME",os.getenv("USER","???")),
>         "Date": timeRFC1123(),
>         "GIT-TOT": getTOT(),
>         "Bit-Order": args.order,
>     };
>     for fn in args.logs:
>         with open(fn) as F:
>             if fn.endswith(".par"):
>                 parseISEPar(F,fmts);
>             if fn.endswith(".srr"):
>                 parseSrr(F,fmts);
>             if fn.endswith(".twr"):
>                 parseVivadoTwr(F,fmts);
>             if fn.endswith(".tsr"):
>                 parseVivadoTwr(F,fmts);
>
>     with open(args.bit) as bitf:
>         bitlen = os.fstat(bitf.fileno())[stat.ST_SIZE];
>
>         bitm = mmap.mmap(bitf.fileno(),bitlen,access=mmap.ACCESS_COPY);
>         start = 0;
>
>         # This is the format for our bit bang schemes. The pin labeled D0 is
>         # taken from bit 7.
>         if args.order == "reversed":
>             for i in range(0,bitlen):
>                 v = ord(bitm[i]);
>                 bitm[i] = chr(((v & (1<<0)) << 7) |
>                               ((v & (1<<1)) << 5) |
>                               ((v & (1<<2)) << 3) |
>                               ((v & (1<<3)) << 1) |
>                               ((v & (1<<4)) >> 1) |
>                               ((v & (1<<5)) >> 3) |
>                               ((v & (1<<6)) >> 5) |
>                               ((v & (1<<7)) >> 7));
>
>         # This is the format DMA to devcfg on the Zynq wants, impact header
>         # stripped, sync word in little endian and aligned.
>         if args.order == "byte-reversed":
>             start = find_start(bitm);
>             for i in range(start,bitlen//4*4,4):
>                 bitm[i],bitm[i+1],bitm[i+2],bitm[i+3] = bitm[i+3],bitm[i+2],bitm[i+1],bitm[i];
>
>         if start != 0:
>             fmts["Impact-Header"] = base64.b64encode(bitm[:start]);
>
>         fmts["Content-Length"] = bitlen - start;
>         out.write(align_bitstream(fmts));
>
>         out.write(bitm[start:]);
>
> parser = argparse.ArgumentParser(description="Format a Xilinx .bit file into a ybf with the necessary headers")
> parser.add_argument("--ybf",required=True,
>                     help="Output filename");
> parser.add_argument("--bit",required=True,
>                     help="Input bit filename");
> parser.add_argument("--archive",
>                     help="Optional directory to place a timestamped hardlink");
> parser.add_argument("--deps",
>                     help="File to write makefile dependencies list to");
> parser.add_argument("--order",default="reversed",
>                     help="Byte or bit order to use for the raw data");
> parser.add_argument('logs',nargs="+",
>                     help="Log files to pull meta data out of")
> args = parser.parse_args();
>
> with open(args.ybf,"wt") as F:
>     makeHeader(F,args);
>
> if args.archive:
>     os.link(args.ybf,os.path.join(args.archive,"%s-%s"%(os.path.basename(args.ybf),int(time.time()))));