Extracting certain patterns by searching all columns for each line, and writing them in specified columns in output file

Viewed 100

I am a beginner programming and was given a task, where certain strings from "INFO" column in a text file should be extracted using awk. Here is the code:

awk -F '\t'  '/^[^#]/ {n=split($8,a,/[;]/); for(i=1;i<=n;i++) {if(a[i] ~ /^CLNDN=/) printf("%s\t",a[i]); else if(a[i] ~ /^CLNREVSTAT=/) printf("%s\t",a[i]); else if(a[i] ~ /^CLNSIG=/) printf("%s\t",a[i]);else if(a[i] ~ /^CLNSIGCONF=/) printf("%s\t",a[i]); else if(a[i] ~ /^ORIGIN=/) printf("%s\t",a[i]); } printf("\n");}' test.vcf > trial.vcf

The input file here:

1   879375  950448  C   T   .   .   ALLELEID=929884;CLNDISDB=MedGen:CN517202;CLNDN=not_provided;CLNHGVS=NC_000001.10:g.879375C>T;CLNREVSTAT=criteria_provided,_single_submitter;CLNSIG=Pathogenic;CLNVC=single_nucleotide_variant;CLNVCSO=SO:0001483;GENEINFO=SAMD11:148398;MC=SO:0001587|nonsense;ORIGIN=1

1   955619  210112  G   C   .   .   AF_EXAC=0.03475;AF_TGP=0.00879;ALLELEID=206690;CLNDISDB=MONDO:MONDO:0014052,MedGen:C3808739,OMIM:615120|MedGen:CN169374|MedGen:CN517202;CLNDN=Myasthenic_syndrome,_congenital,_8|not_specified|not_provided;CLNHGVS=NC_000001.10:g.955619G>C;CLNREVSTAT=criteria_provided,_conflicting_interpretations;CLNSIG=Conflicting_interpretations_of_pathogenicity;CLNSIGCONF=Benign(1),Likely_benign(2),Uncertain_significance(1);CLNVC=single_nucleotide_variant;CLNVCSO=SO:0001483;GENEINFO=AGRN:375790;MC=SO:0001583|missense_variant;ORIGIN=1;RS=201073369

Below, you can see the example output what I want to get.

CLNDN=not_provided      CLNREVSTAT=criteria_provided,_single_submitter  CLNSIG=Pathogenic            ORIGIN=1           
CLNDN=Myasthenic_syndrome,_congenital,_8|not_specified|not_provided     CLNREVSTAT=criteria_provided,_conflicting_interpretations       CLNSIG=Conflicting_interpretations_of_pathogenicity         CLNSIGCONF=Benign(1),Likely_benign(2),Uncertain_significance(1) ORIGIN=1

There is a gap between CLNSIG and ORIGIN in the first line because this line does not contain CLNSIGCONF= information. Here, I would like to extract strings starting with CLDN=, CLNREVSTAT=, CLNSIG=, CLNSIGCONF= and ORIGIN=, and print them into column1-5 in output file, respectively. The code is able to extract the interests but I am stuck on printing them into specified columns.

I would be very appreciated if you could help me out (And very open to any suggestion).

Thank you very much in advance.

2 Answers

EDIT: In case your any of the element could be missing in line then try following. This will also print a statement in case NONE of the matches found in any line(in case you want to remove this then remove if(cldn=="" &&....) block from this solution.

awk '
BEGIN{
  OFS="\t"
}
match($0,/CLNDN=[^;]*/){
  cldn=substr($0,RSTART,RLENGTH)
}
match($0,/CLNREVSTAT=[^;]*/){
  clnrevstat=substr($0,RSTART,RLENGTH)
}
match($0,/CLNSIG=[^;]*/){
  clnsig=substr($0,RSTART,RLENGTH)
}
match($0,/CLNSIGCONF=[^;]*/){
  clnsisconf=substr($0,RSTART,RLENGTH)
}
match($0,/ORIGIN=[^;]*/){
  origin=substr($0,RSTART,RLENGTH)
}
NF{
  if(cldn=="" && clnrevstat=="" && clnsig=="" && clnsisconf=="" && origin==""){
    print "NO matched value found in this line."
    next
  }
  print cldn,clnrevstat,clnsig,clnsisconf,origin
  cldn=clnrevstat=clnsig=clnsisconf=origin=""
  next
}
1
'  Input_file


Could you please try following, written based on your shown samples in GNU awk.

awk '
BEGIN{
  OFS="\t"
}
{ cldn=clnrevstat=clnsig=clnsisconf="" }
match($0,/CLNDN=[^;]*/){
  cldn=substr($0,RSTART,RLENGTH)
}
match($0,/CLNREVSTAT=[^;]*/){
  clnrevstat=substr($0,RSTART,RLENGTH)
}
match($0,/CLNSIG=[^;]*/){
  clnsig=substr($0,RSTART,RLENGTH)
}
match($0,/CLNSIGCONF=[^;]*/){
  clnsisconf=substr($0,RSTART,RLENGTH)
}
match($0,/ORIGIN=[^;]*/){
  print cldn,clnrevstat,clnsig,clnsisconf,substr($0,RSTART,RLENGTH)
}
' Input_file

Explanation: Adding detailed explanation for above.

awk '                                    ##Starting awk program from here.
BEGIN{                                   ##Starting BEGIN section of this program from here.
  OFS="\t"                               ##Setting OFS as tab here.
}
{ cldn=clnrevstat=clnsig=clnsisconf="" }
match($0,/CLNDN=[^;]*/){                 ##Using match function to match from string CLNDN= till semi colon here.
  cldn=substr($0,RSTART,RLENGTH)         ##Creating cldn which has matched regex sub string.
}
match($0,/CLNREVSTAT=[^;]*/){            ##Using match function to match from string CLNREVSTAT= till semi colon here.
  clnrevstat=substr($0,RSTART,RLENGTH)   ##Creating clnrevstat which has matched regex sub string here.
}
match($0,/CLNSIG=[^;]*/){                ##Using match function to match from string CLNSIG= till semi colon here.
  clnsig=substr($0,RSTART,RLENGTH)       ##Creating clnsig which has matched regex sub string here.
}
match($0,/CLNSIGCONF=[^;]*/){            ##Using match function to match from string CLNSIGCONF= till semi colon here.
  clnsisconf=substr($0,RSTART,RLENGTH)   ##Creating clnsisconf which has matched regex sub string here.
}
match($0,/ORIGIN=[^;]*/){                ##Using match function to match from string ORIGIN= till semi colon here.
  print cldn,clnrevstat,clnsig,clnsisconf,substr($0,RSTART,RLENGTH)
                                         ##Printing all variables value and sub string of matched regex.
}
' Input_file                             ##Mentioning Input_file name here. 

Whenever you have tag=value pairs in your data it's best to first build an array of that mapping (f[] below) and then just print the values you want by their tags (names):

$ cat tst.awk
BEGIN { OFS="\t" }
NF {
    delete f
    split($NF,tagVals,/;/)
    for (i in tagVals) {
        tag = tagVals[i]
        sub(/=.*/,"",tag)
        f[tag] = tagVals[i]
    }
    print f["CLNDN"], f["CLNREVSTAT"], f["CLNSIG"], f["CLNSIGCONF"], f["ORIGIN"]
}

$ awk -f tst.awk file
CLNDN=not_provided      CLNREVSTAT=criteria_provided,_single_submitter  CLNSIG=Pathogenic               ORIGIN=1
CLNDN=Myasthenic_syndrome,_congenital,_8|not_specified|not_provided     CLNREVSTAT=criteria_provided,_conflicting_interpretations     CLNSIG=Conflicting_interpretations_of_pathogenicity     CLNSIGCONF=Benign(1),Likely_benign(2),Uncertain_significance(1)       ORIGIN=1

FWIW here's how I think you should be reallly doing this instead of having every field in every line containing both the tag and the value:

$ cat tst.awk
BEGIN {
    OFS = "\t"
    n = split("CLNDN CLNREVSTAT CLNSIG CLNSIGCONF ORIGIN",tags)
    for (i=1; i<=n; i++) {
        tag = tags[i]
        printf "%s%s", tag, (i<n ? OFS : ORS)
    }
}
{
    delete tag2val

    split($NF,tagVals,/;/)
    for (i in tagVals) {
        tag = val = tagVals[i]
        sub(/=.*/,"",tag)
        sub(/[^=]+=/,"",val)
        tag2val[tag] = val
    }

    for (i=1; i<=n; i++) {
        tag = tags[i]
        val = tag2val[tag]
        printf "%s%s", val, (i<n ? OFS : ORS)
    }
}

$ awk -f tst.awk file
CLNDN   CLNREVSTAT      CLNSIG  CLNSIGCONF      ORIGIN
not_provided    criteria_provided,_single_submitter     Pathogenic              1
Myasthenic_syndrome,_congenital,_8|not_specified|not_provided   criteria_provided,_conflicting_interpretationsConflicting_interpretations_of_pathogenicity    Benign(1),Likely_benign(2),Uncertain_significance(1)    1
Related