fq(1) Manual Page

fq 'grep_by(.type=="data" and .value[0:4] == "book") | .value | apple_bookmark |
torepr' <sfl2 file>

$ fq -d bytes 'from_pem | asn1_ber | d' cert.pem

$ fq -d asn1_ber '.constructed[1].value | asn1_ber' file.ber

$ fq -d asn1_ber 'torepr as $r | ["version", "modulus", "private_exponent", "prime1", "prime2", "exponent1", "exponent2", "coefficient"] | with_entries({key: .value, value: $r[.key]})' pkcs1.der

$ fq '.streams[1].samples[] | tobytes' file.avi > stream01.mp3

$ fq -o decode_samples=false '[.chunks[0] | grep_by(.id=="LIST" and .type=="strl") | grep_by(.id=="strh") as {$type} | grep_by(.id=="strf") as {$format_tag, $compression} | {$type,$format_tag,$compression}]' *.avi

$ fq -o decode_samples=false -o decode_extended_chunks=false d file.avi

$ fq -d bencode torepr file.torrent

$ echo 'some {"a":1} json' | fq -d bits '.[40:-48] | fromjson'
{
  "a": 1
}

$ echo 'hello' | fq -d bits '.[4]'
1
$ echo 'hello' | fq -c -d bits '[.[range(8)]]'
[0,1,1,0,1,0,0,0]

$ fq d Info.plist

$ fq 'torepr.SomeTimeStamp | tovalue' Info.plist
685135328

$ fq 'torepr.SomeTimeStamp | todescription' Info.plist
"2022-09-17T19:22:08Z"

$ fq torepr com.apple.UIAutomation.plist
{
  "UIAutomationEnabled": true
}

# Assume $top.root is present
bplist> from_ns_keyed_archiver

# Specify optional root
bplist> from_ns_keyed_archiver(1)

$ fq -d bson torepr file.bson

$ fq -d bson 'torepr | select(.name=="bob")' file.bson

$ echo -n 'hello' | fq -d bytes '.[-3:]' > last_3_bytes
$ echo -n 'hello' | fq -d bytes '[.[-2:], .[0:2]] | tobytes' > first_last_2_bytes_swapped

$ echo 'some {"a":1} json' | fq -d bytes '.[5:-6] | fromjson'
{
  "a": 1
}

$ echo 'hello' | fq -d bytes '.[1]'
101

$ fq -d cbor torepr file.cbor

$ fq -d csv -o comma="\t" to_csv file.tsv

$ fq -d csv '.[0] as $t | .[1:] | map(with_entries(.key = $t[.key]))' file.csv

$ fq '[.data_records[] | select(.record_header.message_type == "data").data_message]' file.fit

# decode as object is the default
$ echo '<a href="url">text</a>' | fq -d html
{
  "html": {
    "body": {
      "a": {
        "#text": "text",
        "@href": "url"
      }
    },
    "head": ""
  }
}

$ echo '<a href="url">text</a>' | fq -d html -o array=true
[
  "html",
  null,
  [
    [
      "head",
      null,
      []
    ],
    [
      "body",
      null,
      [
        [
          "a",
          {
            "#text": "text",
            "href": "url"
          },
          []
        ]
      ]
    ]
  ]
]

# decode html files to a {file: "title", ...} object
$ fq -n -d html '[inputs | {key: input_filename, value: .html.head.title?}] | from_entries' *.html

# <a> href:s in file
$ fq -r -o array=true -d html '.. | select(.[0] == "a" and .[1].href)?.[1].href' file.html

$ fq '.load_commands[] | select(.cmd=="segment_64")' file

$ fq -d markdown '[.. | select(.type=="heading" and .level<=2)?.children[0]]' file.md

$ fq 'matroska_path(".Segment.Tracks[0]")' file.mkv

$ fq 'grep_by(.id == "Tracks") | matroska_path' file.mkv

fq -d midi '.. | select(.event=="track_name")? | "\(.track_name)"' midi/twinkle.mid

fq -d midi '.. | select(.event=="tempo")?.tempo' midi/twinkle.mid

fq -d midi '.. | select(.event=="key_signature")?.key_signature' midi/twinkle.mid

fq -d midi 'grep_by(.event=="note_on") | [.time.tick, .note_on.note] | join(" ")' midi/twinkle.mid

# manually decode first sample as an aac_frame
$ fq -o decode_samples=false '.tracks[0].samples[0] | aac_frame | d' file.mp4

$ fq 'first(grep_by(.type=="elst").entries) | tovalue' file.mp4

$ fq 'del(.tracks) | grep_by(.type=="mdat").data = "<excluded>" | tovalue' file.mp4

$ fq -n '"AAAAHGVsc3QAAAAAAAAAAQAAADIAAAQAAAEAAA==" | from_base64 | mp4({force:true}) | d'

# <decode value box> | mp4_path($path) -> <decode value box>
$ fq 'mp4_path(".moov.trak[1]")' file.mp4

# <decode value box> | mp4_path -> string
$ fq 'grep_by(.type == "trak") | mp4_path' file.mp4

$ fq -d msgpack torepr file.msgpack

$ fq -d negentropy dd file

$ echo '6186b7abb47c0001108e4206828ee3bf34258465809a337c6c00019a68e37b177a50b3ae7164ccc628b962020114019c1381281c9e3849d5fbd514b7bb65ad0101e601fbf7451f5d22e7fa36ae3e910e9f5215020157014a1b26853e06e9c32eb41b1df4f9ab300201e6011840e273c84bb1344f1d4e15d9aa67920200016f12ee2340888653f10b0ec2d438ac9f0101840156d2d796f4dff004ab369b9bcfa4d81e020187013f1b3c8a019800d5764e2de6bdfd2785020114017caaf0acb5dfe249aa0f7f742402168a01018301e7b8c4decb1eae455ca5714281e3245302017a01409c22636b097362df125ddffb6d944302015b01f332208bee82acf8ed922853ee54057f020001fc3e51fdb0b92966e38017f7959903850101cc01428ce0c96d49f15b50143e4fb228cb9300000131712d30e5296a7a45d07bba452d61cd' | fq -R 'from_hex | negentropy | dd'

$ fq -d negentropy '.bounds | length as $total | map(select(.mode == "fingerprint")) | length | {$total, fingerprint: .}' message

$ fq -d negentropy '.bounds | map(select(.mode == "idlist") | .idlist | .ids) | flatten' message

$ fq -r '.prg_rom[] | nes_toasm' file.nes

$ printf $(fq -r -d nes '.chr_rom[0] | nes_tokitty(5)' file.nes)

$ for line in $(fq -r '.chr_rom[] | nes_tokitty(5)' file.nes);do printf "%b%s" "$line";done

$ fq dd file.ots

$ fq '.operations | map(select(.attestation_type == "calendar") | .url)' file.ots

$ fq '.operations | map(select(.attestation_type == "bitcoin")) | length > 0' file.ots

# for a pcapng file you would use .[0].tcp_connections for first section
$ fq '.tcp_connections | group_by(.client.ip) | map({key: .[0].client.ip, value: map(.client.stream, .server.stream | tobytes.size) | add}) | from_entries'
{
  "10.1.0.22": 15116,
  "10.99.12.136": 234,
  "10.99.12.150": 218
}

$ fq -d pg_btree -o flavour=postgres14 ".[0] | d" 16404

$ fq -d pg_btree -o flavour=postgres14 ".[1]" 16404

$ fq -d pg_control -o flavour=postgres14 d pg_control

$ fq -d pg_control -o flavour=postgres14 ".state, .check_point_copy.redo, .wal_level" pg_control

$ fq -d pg_heap -o flavour=postgres14 ".[0]" 16994

$ fq -d pg_heap -o flavour=postgres14 ".[0].page_header" 16994

$ fq -d pg_heap -o flavour=postgres14 ".[0].pd_linp[0, -1]" 16994

$ fq -d pg_heap -o flavour=postgres14 ".[0].tuples[0, -1]" 16994

$ fq -d protobuf '.fields[6].wire_value | protobuf | d' file

fq '.tcp_connections[] | select(.server.port=="rtmp") | d' file.cap

fq -o bits_format=byte_array -d tap -V d /path/to/file.tap

$ tcpdump -i <iface> -w traffic.pcap

$ SSLKEYLOGFILE=traffic.keylog curl --tls-max 1.2 https://host/path

# decode and show whole tree
$ fq -o keylog=@traffic.keylog d traffic.pcap

# write unencrypted server response to a file.
# first .stream is the TCP stream, second .stream is TLS application data stream
#
# first TCP connections:
$ fq -o keylog=@traffic.keylog '.tcp_connections[0].server.stream.stream | tobytes' traffic.pcap > data
# first TLS connection:
$ fq -o keylog=@traffic.keylog  'first(grep_by(.server.stream | format == "tls")).server.stream.stream | tobytes' > data

fq '.v2plusdatablock.transition_times[-1] | tovalue' tziffile

fq '.v2plusdatablock.leap_second_records | length' tziffile

fq -o bits_format=byte_array -d tzx -V d /path/to/file.tzx

$ fq '.sections[] | select(.id == "code_section") | [.. | .opcode? // empty] | count | map({key: .[0], value: .[1]}) | from_entries' file.wasm

$ fq '.sections | {import: map(select(.id == "import_section").content.im.x[].nm.b), export: map(select(.id == "export_section").content.ex.x[].nm.b)}' file.wasm

# decode as object is the default
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o seq=true
{
  "a": {
    "b": [
      {
        "#seq": 0
      },
      {
        "#seq": 1,
        "#text": "bbb"
      }
    ],
    "c": {
      "#seq": 2,
      "#text": "ccc",
      "@attr": "value"
    }
  }
}

# access text of the <c> element
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq '.a.c["#text"]'
"ccc"

# decode to object and encode to xml
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o seq=true 'to_xml({indent:2})'
<a>
  <b></b>
  <b>bbb</b>
  <c attr="value">ccc</c>
</a>

# decode as array
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -d xml -o array=true
[
  "a",
  null,
  [
    [
      "b",
      null,
      []
    ],
    [
      "b",
      {
        "#text": "bbb"
      },
      []
    ],
    [
      "c",
      {
        "#text": "ccc",
        "attr": "value"
      },
      []
    ]
  ]
]

# decode to array and encode to xml
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -r -d xml -o array=true -o seq=true 'to_xml({indent:2})'
<a>
  <b></b>
  <b>bbb</b>
  <c attr="value">ccc</c>
</a>

# access text of the <c> element, the object variant above is probably easier to use
$ echo '<a><b/><b>bbb</b><c attr="value">ccc</c></a>' | fq -o array=true '.[2][2][1]["#text"]'
"ccc"