GENERAL

Codepoints

Work with explode/implode

Smart quotes

Wrap string with quotes when a space, new line, or tab is present

# single quote
def smart_squotes($s):
  $s | if (test("[\\s\\n\\t]";"x")) then "\([39]|implode)\($s)\([39]|implode)" else $s end;

# double quote
def smart_dquotes($s):
  $s | if (test("[\\s\\n\\t]";"x")) then "\($s|@json)" else $s end;

    $ jq -n -r '["hello","hello world","hello\nworld","hello\tworld"] | map("var="+smart_squotes(.))[]'
var=hello
var='hello world'
var='hello
world'
var='hello      world'

$ jq -n -r '["hello","hello world","hello\nworld","hello\tworld"] | map("var="+smart_dquotes(.))[]'
var=hello
var="hello world"
var="hello\nworld"
var="hello\tworld"

  

Chart

    $ jq -n -r '[range(9622;9632)]|map("\(.) \([.]|implode)")[]'
▖
▗
▘
▙
▚
▛
▜
▝
▞
▟

  

Github

Inputs

tmp.tsv

$ jq -R 'inputs' data/tmp.tsv

"1   2   3"
"4   5   6"

$  jq -c -R 'split("[\\s\\t]+";"x")' data/tmp.tsv

["foo","bar","baz"]
["1","2","3"]
["4","5","6"]

$ jq -R 'split("[\\s\\t]+";"x") as $h | [$h,inputs]' data/tmp.tsv

[
  [
    "foo",
    "bar",
    "baz"
  ],
  "1   2   3",
  "4   5   6"
]

Github

kv

split("\n")
| map(
    select(. != "") |
    split("=") |
    {"key": .[0], "value": (.[1:] | join("="))}
  )
| from_entries

$ env | jq -sR 'split("\n") | map(select(. != "") | split("=") | {"key": .[0], "value": (.[1:] | join("="))}) | from_entries'

{
  "XDG_SESSION_ID": "c18",
  "SHELL": "/bin/bash",
  "TERM": "xterm-256color",
  "HISTSIZE": "100000",
  "HISTFILESIZE": "10000000",
  "USER": "root",
  "MAIL": "/var/spool/mail/root",
  "PATH": "/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/puppetlabs/bin:/usr/local/bin:/root/bin:/misc/software/database/engineering/bin",
  "PWD": "/root",
  "LANG": "en_US.UTF-8",
  "PS1": "\\[\\e]0;\\w\\a\\]\\n\\[\\e[32m\\]\\u@\\h \\[\\e[33m\\]\\w\\[\\e[0m\\]\\n$ ",
  "HISTIGNORE": "exit:^history*:clear",
  "HISTCONTROL": "ignoredups",
  "SHLVL": "1",
  "HOME": "/root",
  "LOGNAME": "root",
  "LESSOPEN": "||/usr/bin/lesspipe.sh %s",
  "_": "/bin/env"
}

Github

Range

$ jq -c -n '{x: range(2), y: range(2;5) }'
{"x":0,"y":2}
{"x":0,"y":3}
{"x":0,"y":4}
{"x":1,"y":2}
{"x":1,"y":3}
{"x":1,"y":4}

Github

Reduce

empty

$ echo 1 2 3 | jq -cn 'reduce inputs as $i ([]; if $i==0 then empty else .+[$i] end)'
[1,2,3]

$ echo 1 2 3 | jq -cn 'reduce inputs as $i ([]; if $i==1 then empty else .+[$i] end)'
[2,3]

sum

$ seq 5 | jq -n 'reduce inputs as $i (0;.+($i|tonumber))'
15

flatten and match string

$ jq --arg match_text "nntrn" '. as $data 
| [path(..| select(scalars and (tostring | test($match_text)))) ] | map({ (.|join(".")): (. as $path | .=$data | getpath($path)) }) 
| reduce .[] as $item ({}; . * $item)' data/sample-github-events.json

{
  "0.actor.login": "nntrn",
  "0.actor.display_login": "nntrn",
  "0.actor.url": "https://api.github.com/users/nntrn",
  "0.repo.name": "nntrn/jq-recipes",
  "0.repo.url": "https://api.github.com/repos/nntrn/jq-recipes"
}

Learn more:

Reduce in jq on Exercism
https://stackoverflow.com/a/74687036/7460613
https://github.com/stedolan/jq/issues/873#issuecomment-125393055

Github

regex

jq uses the Oniguruma regular expression library

Match
Positive lookahead	?=
Negative lookahead	?!
Positive lookbehind	?<=
Negative lookbehind	?<!

Extended groups

  i: ignore case
  m: multi-line (dot (.) also matches newline)
  x: extended form
  W: ASCII only word (\w, \p{Word}, [[:word:]])
     ASCII only word bound (\b)
  D: ASCII only digit (\d, \p{Digit}, [[:digit:]])
  S: ASCII only space (\s, \p{Space}, [[:space:]])
  P: ASCII only POSIX properties (includes W,D,S)
      (alnum, alpha, blank, cntrl, digit, graph,
      lower, print, punct, space, upper, xdigit, word)

  y{?}: Text Segment mode
    This option changes the meaning of \X, \y, \Y.
    Currently, this option is supported in Unicode only.

    y{g}: Extended Grapheme Cluster mode (default)
    y{w}: Word mode

Lookahead

Remove repeating characters

$ jq -n '"aaaabbbbcccc" | gsub("(.)(?=.*\\1)";"")'
"abc"

$ jq -n '"aaaabbbbccccaaaa" | gsub("(.)(?=.*\\1)";"")'
"bca"

Only remove first repeating 'a'

$ jq -n '"aaaabbbbcccc126186" | gsub("\\b(.)(?=.*\\1)";"")'
"abbbbcccc126186"

$ jq -n '"aaaa bbbb cccc 126 126" | gsub("(\\b.)(?=.*\\1)";"")'
"abc 126"

Resources:

Github

Scalars

Scalars are variables that hold an individual value (strings, integers, and booleans). If it's not an object or array — it's most likely a scalar

sample-github-events.json

Extract

map(with_entries(select(.value|scalars)))

Flatten

. as $data
| [path(..|select(scalars))]
| map({ (.|join(".")): (. as $path | .=$data | getpath($path)) })
| add

$ jq '. as $data | [path(..| select(scalars))] | map({ (.|join(".")): (. as $path | .=$data | getpath($path)) }) | add' sample-github-events.json

{
  "0.id": "30572272710",
  "0.type": "CreateEvent",
  "0.actor.id": 17685332,
  "0.actor.login": "nntrn",
  "0.actor.display_login": "nntrn",
  "0.actor.gravatar_id": "",
  "0.actor.url": "https://api.github.com/users/nntrn",
  "0.actor.avatar_url": "https://avatars.githubusercontent.com/u/17685332?",
  "0.repo.id": 582752600,
  "0.repo.name": "nntrn/jq-recipes",
  "0.repo.url": "https://api.github.com/repos/nntrn/jq-recipes",
  "0.payload.ref": "master",
  "0.payload.ref_type": "branch",
  "0.payload.master_branch": "main",
  "0.payload.pusher_type": "user",
  "0.public": true,
  "0.created_at": "2023-07-21T00:01:11Z"
}

Target URLs

Select paths that begin with https://

. as $data
| [path(..| select(scalars and (tostring | test("^https://";"x"))))]
| map({ (.|join(".")): (. as $path | .=$data | getpath($path)) })
| add

$ jq '. as $data | [path(..| select(scalars and (tostring | test("^https://";"x"))))] | map({ (.|join(".")): (. as $path | .=$data | getpath($path)) })|add' sample-github-events.json

{
  "0.actor.url": "https://api.github.com/users/nntrn",
  "0.actor.avatar_url": "https://avatars.githubusercontent.com/u/17685332?",
  "0.repo.url": "https://api.github.com/repos/nntrn/jq-recipes"
}

Adapted from 5 Useful jq Commands to Parse JSON on the CLI:

path

now with `..`, traverse with path

$ jq -c '. as $data | [path(..)][]' sample-github-events.json

[]
[0]
[0,"id"]
[0,"type"]
[0,"actor"]
[0,"actor","id"]
[0,"actor","login"]
[0,"actor","display_login"]
[0,"actor","gravatar_id"]
[0,"actor","url"]
[0,"actor","avatar_url"]
[0,"repo"]
[0,"repo","id"]
[0,"repo","name"]
[0,"repo","url"]
[0,"payload"]
[0,"payload","ref"]
[0,"payload","ref_type"]
[0,"payload","master_branch"]
[0,"payload","description"]
[0,"payload","pusher_type"]
[0,"public"]
[0,"created_at"]

select

Select only paths that contain nntrn (github username) in the value

$ jq -c '. as $data | [path(..| select(scalars and (tostring | test( "nntrn")))) ][]' sample-github-events.json

[0,"actor","login"]
[0,"actor","display_login"]
[0,"actor","url"]
[0,"repo","name"]
[0,"repo","url"]

getpath

$ jq '. as $data | [path(..| select(scalars and (tostring | test("nntrn")))) ] | map({ (.|join(".")): (. as $path | .=$data | getpath($path)) })' data/sample-github-events.json

[
  {
    "0.actor.login": "nntrn"
  },
  {
    "0.actor.display_login": "nntrn"
  },
  {
    "0.actor.url": "https://api.github.com/users/nntrn"
  },
  {
    "0.repo.name": "nntrn/jq-recipes"
  },
  {
    "0.repo.url": "https://api.github.com/repos/nntrn/jq-recipes"
  }
]

reduce

reduce all key-value matches to a single object

$ jq '. as $data | [path(..| select(scalars and (tostring | test("nntrn")))) ] | map({ (.|join(".")): (. as $path | .=$data | getpath($path)) }) | reduce .[] as $item ({}; . * $item)' data/sample-github-events.json

{
  "0.actor.login": "nntrn",
  "0.actor.display_login": "nntrn",
  "0.actor.url": "https://api.github.com/users/nntrn",
  "0.repo.name": "nntrn/jq-recipes",
  "0.repo.url": "https://api.github.com/repos/nntrn/jq-recipes"
}

add

add can also be used in this case

$ jq '. as $data | [path(..| select(scalars and (tostring | test("nntrn")))) ] | map({ (.|join(".")): (. as $path | .=$data | getpath($path)) })|add' data/sample-github-events.json

{
  "0.actor.login": "nntrn",
  "0.actor.display_login": "nntrn",
  "0.actor.url": "https://api.github.com/users/nntrn",
  "0.repo.name": "nntrn/jq-recipes",
  "0.repo.url": "https://api.github.com/repos/nntrn/jq-recipes"
}

Github

Slurp

Slurpfile

$ jq --slurpfile cars data/cars.json '{titanic: .[0:1], cars: $cars[][0:1]}' data/titanic.json
{
  "titanic": [
    {
      "Survived": 0,
      "Pclass": 3,
      "Name": "Mr. Owen Harris Braund",
      "Sex": "male",
      "Age": 22,
      "Siblings_Spouses_Aboard": 1,
      "Parents_Children_Aboard": 0,
      "Fare": 7.25
    }
  ],
  "cars": [
    {
      "name": "AMC Ambassador Brougham",
      "brand": "AMC",
      "economy_mpg_": 13,
      "cylinders": 8,
      "displacement_cc_": 360,
      "power_hp_": 175,
      "weight_lb_": 3821,
      "0_60_mph_s_": 11,
      "year": 1973
    }
  ]
}

Github

Streams

Data used: nested.json

{
  "outer1": {
    "outer2": {
      "outer3": {
        "key1": "value1",
        "key2": "value2"
      },
      "outer4": {
        "key1": "value1",
        "key2": "value2"
      }
    }
  }
}

$ jq -c --stream '' data/outer.json

[["outer1","outer2","outer3","key1"],"value1"]
[["outer1","outer2","outer3","key2"],"value2"]
[["outer1","outer2","outer3","key2"]]
[["outer1","outer2","outer4","key1"],"value1"]
[["outer1","outer2","outer4","key2"],"value2"]
[["outer1","outer2","outer4","key2"]]
[["outer1","outer2","outer4"]]
[["outer1","outer2"]]
[["outer1"]]

Github

Update

|=

{
  "key1": {
    "attr1": "foo"
  },
  "key2": {
    "attr1": "foo",
    "attr2": "bar"
  }
}

$ jq '.[] |= if .attr2 then (.attr2 = "bax") else . end'
{
  "key1": {
    "attr1": "foo"
  },
  "key2": {
    "attr1": "foo",
    "attr2": "bax"
  }
}

Github