back to index | | home .

git trend json

---

tested on debian distro.
requirment: bash, jq.
optional: soimort/translate-shell, ffmpeg, libttspico-utils and its dependencies (from synaptic).

Remove trailing spaces on each line when copy code from here, by sed 's/[[:space:]]*$//'

#!/usr/bin/env bash  
[ ! -d /tmp ] && exit;  
cd /tmp/;  
# file path variables  
jsonOutput=filename  
sortedOutput=  
prjsInfo=  
prjsInfo2=  
outxt=  
trans= 'path/to/excutable/trans'  
# for tts  
tmpOut=  
audio=No_extension_filename  

# download data from someone else  
curl https://raw.githubusercontent.com/Unpublished/GithubTrending/trends/trending_monthly-all.json > $jsonOutput  

curl https://raw.githubusercontent.com/Unpublished/GithubTrending/trends/trending_monthly-unknown.json >> $jsonOutput  

# sort by stars  
sed -i 's/\]\[/,/' $jsonOutput  
jq -s '.[]|sort_by(.stars)|reverse' $jsonOutput \  
> $sortedOutput && \  
mv $sortedOutput $jsonOutput  

# extract prj lang, name by owner,stars into string  
# remove " [] blank_lines leading_spaces ,$  
# rounding stars number  
# remove leading comma, extra comma space, .0K to K,  _ to ' '. add like at end  
#  
cat $jsonOutput |jq '.|map([.language,.repo+" by "+.owner,(.stars|tostring)]|join(","))' \  
| tr -d '"'|sed 's/\[//; s/\]//;/^$/d; s/^[[:space:]]*//; s/,$//;' \  
| numfmt  --to=si --round=down --invalid=ignore --delimiter=, --field 3  \  
| sed 's/, */, /g; s/\.0K/K/g; s/$/ like/; s/^, //g' \  
| tr _ ' ' > $prjsInfo  

# extract description  
# remove (and stuff inside) :1 perline: ^null$  
# replace @ to at  
#  
cat $jsonOutput | jq '.[].description'|tr -d '"'| \  
sed "s/[(][^)]*[)]//g; s/[:][^:]*[:]//g; s/^null$//g; s/@/ at /; " \  
> $prjsInfo2  

# translate non-english line  
## french/german wont be translate, unless there's special char  
# remove punctuation, grep non-english line and line number.  
# --> Extract line number and map to array  
# using ./trans line by line from match  
#  
if [[ -f "$trans" ]]; then  
mapfile -t lineNm < <(cat $prjsInfo2 |sed 's/[[:punct:]]//g'|\  
 grep -Pn '[^\x00-\x7F]'|\  
 cut -f1 -d':')  
if [ ${#lineNm[@]} != 0 ]; then  
 echo translating;  
 for i in "${lineNm[@]}";do n=$(echo "$i");  
 ext="$(bash $trans -t en -b "$(sed -n $n\p $prjsInfo2)")" && sleep 1.5;  
 sed -i $n"s@.*@$ext@" $prjsInfo2; done  
fi  
fi  


# join $prjsInfo/2 files  
# cut off after 258 chars per line, and add . if not exist at end  
#  
echo "fetch on `date '+%Y %B %d.' `" > $outxt  
paste -d "\n" $prjsInfo $prjsInfo2 | sed n\;G|cut -c -258 \  
| sed "s/[[:space:]]*$//; s/,$/./"|awk '!/[[:punct:]]$/ && NF{$NF=$NF"."}1' \  
| cat -s >> $outxt  
find $outxt -exec sed -i '' 's/, */, /g' {} + 2> /dev/null  

#exit # after this line for audio  
if ! which pico2wave >/dev/null;then exit;fi  

# small correction for $audio's transcript, WIP  
if grep -q 'JavaScript' $outxt; then  
 find $outxt -exec sed '' -e 's/JavaScript/Java-Script/g;  
 s/.js/ dot JS;/g; s/macos/mac OS/gI; s/OS/ OS/g;' {} + \  
 > $tmpOut 2> /dev/null  
 outxt=$tmpOut;  
fi;  

# tts, output audio. apply treble and speed up filter.  
# test filter w/ $ffplay $audio.wav -af treble=g=25:f=2500,atempo=...  
if [[ -f "$outxt" ]];then  
pico2wave --wave="$audio.wav" "$(cat $outxt)" && \  
ffmpeg -loglevel panic -hide_banner -y -i "$audio.wav" \  
 -af "treble=g=25,atempo=1.35" \  
 -ac 1 -c:a libmp3lame -q:a 2 \  
 "$audio.mp3";  

# ffmpeg -y -i "$audio.wav" -map_metadata -1 -c:a libvorbis -af "treble=g=25,atempo=1.35" -ac 1 -q:a 2 "$audio.ogg";  

fi  

exit  

---

jq for git search api. example

# sort by stars base on prjects create 1 month ago or newer  
at='-1 month'  
curl -G https://api.github.com/search/repositories \  
    --data-urlencode "q=created:>`date --date="$at" '+%Y-%m-%d'`" \  
    --data-urlencode "sort=stars" \  
    --data-urlencode "order=desc" \  
    -H "Accept: application/vnd.github.preview" \  
    > $jsonOutput  

# prj name, stars, lang, license  
cat $jsonOutput \  
| jq -r '.items[]| [.name,.watchers,.language,.license?.key]|@csv' \  
| tr -d '"'|sed 's/,other$//; s/\.0$//; s/,,/,/' \  
| numfmt  --to=si --round=down --invalid=ignore --delimiter=, --field 2  \  
| sed 's/, */, /g; s/[^,]*$/\U&/; s/\.0K/K/g'| tr _ ' ' \  
> $prjsInfo  

# prj description  
cat $jsonOutput | jq .items[].description \  
| tr -d '"'|sed "s/[(][^)]*[)]//g; s/[:][^:]*[:]//g" \  
> $prjsInfo2  

paste -d "\n" $prjsInfo $prjsInfo2 # | sed .... rest like above  

other stuff I've tried w/ data-urlencode

    --data-urlencode "q=p5js mirror:false" \  
    --data-urlencode "sort=updated" \  
##  
    "q=json OR 3d mirror:false"  
##  
    "q=topic:app"  
##  

there's no date on commits history ie.https://api.github.com/repos/Unpublished/GithubTrending/commits/trends