ggml : fix another case of quants nans

[slaren]

Ref: #7311 (comment)

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 556 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=8400.47ms p(95)=20778.83ms fails=, finish reason: stop=504 truncated=52
• Prompt processing (pp): avg=91.99tk/s p(95)=365.55tk/s
• Token generation (tg): avg=62.52tk/s p(95)=47.91tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=sl/fix-quant-near-zero-2 commit=a84d76b40ec990be066a384508b79b1038e7555b

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 556 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1716151175 --> 1716151803
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 417.99, 417.99, 417.99, 417.99, 417.99, 808.68, 808.68, 808.68, 808.68, 808.68, 754.35, 754.35, 754.35, 754.35, 754.35, 780.99, 780.99, 780.99, 780.99, 780.99, 854.53, 854.53, 854.53, 854.53, 854.53, 868.36, 868.36, 868.36, 868.36, 868.36, 862.16, 862.16, 862.16, 862.16, 862.16, 873.25, 873.25, 873.25, 873.25, 873.25, 869.68, 869.68, 869.68, 869.68, 869.68, 879.16, 879.16, 879.16, 879.16, 879.16, 902.55, 902.55, 902.55, 902.55, 902.55, 894.8, 894.8, 894.8, 894.8, 894.8, 869.51, 869.51, 869.51, 869.51, 869.51, 881.18, 881.18, 881.18, 881.18, 881.18, 890.83, 890.83, 890.83, 890.83, 890.83, 891.73, 891.73, 891.73, 891.73, 891.73, 893.07, 893.07, 893.07, 893.07, 893.07, 890.05, 890.05, 890.05, 890.05, 890.05, 893.09, 893.09, 893.09, 893.09, 893.09, 893.32, 893.32, 893.32, 893.32, 893.32, 890.48, 890.48, 890.48, 890.48, 890.48, 887.42, 887.42, 887.42, 887.42, 887.42, 888.91, 888.91, 888.91, 888.91, 888.91, 887.51, 887.51, 887.51, 887.51, 887.51, 885.66, 885.66, 885.66, 885.66, 885.66, 886.68, 886.68, 886.68, 886.68, 886.68, 887.76, 887.76, 887.76, 887.76, 887.76, 821.69, 821.69, 821.69, 821.69, 821.69, 822.76, 822.76, 822.76, 822.76, 822.76, 826.27, 826.27, 826.27, 826.27, 826.27, 830.18, 830.18, 830.18, 830.18, 830.18, 828.41, 828.41, 828.41, 828.41, 828.41, 832.47, 832.47, 832.47, 832.47, 832.47, 843.34, 843.34, 843.34, 843.34, 843.34, 843.16, 843.16, 843.16, 843.16, 843.16, 826.02, 826.02, 826.02, 826.02, 826.02, 825.56, 825.56, 825.56, 825.56, 825.56, 823.35, 823.35, 823.35, 823.35, 823.35, 826.51, 826.51, 826.51, 826.51, 826.51, 828.16, 828.16, 828.16, 828.16, 828.16, 828.22, 828.22, 828.22, 828.22, 828.22, 819.05, 819.05, 819.05, 819.05, 819.05, 793.03, 793.03, 793.03, 793.03, 793.03, 792.95, 792.95, 792.95, 792.95, 792.95, 791.96, 791.96, 791.96, 791.96, 791.96, 790.11, 790.11, 790.11, 790.11, 790.11, 788.43, 788.43, 788.43, 788.43, 788.43, 787.67, 787.67, 787.67, 787.67, 787.67, 784.07, 784.07, 784.07, 784.07, 784.07, 787.72, 787.72, 787.72, 787.72, 787.72, 791.24, 791.24, 791.24, 791.24, 791.24, 788.41, 788.41, 788.41, 788.41, 788.41, 787.95, 787.95, 787.95, 787.95, 787.95, 795.86, 795.86, 795.86, 795.86, 795.86, 797.34, 797.34, 797.34, 797.34, 797.34, 797.5, 797.5, 797.5, 797.5, 797.5, 798.34, 798.34, 798.34, 798.34, 798.34, 798.72, 798.72, 798.72, 798.72, 798.72, 799.54, 799.54, 799.54, 799.54, 799.54, 801.56, 801.56, 801.56, 801.56, 801.56, 802.54, 802.54, 802.54, 802.54, 802.54]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 556 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1716151175 --> 1716151803
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 38.63, 38.63, 38.63, 38.63, 38.63, 26.38, 26.38, 26.38, 26.38, 26.38, 28.75, 28.75, 28.75, 28.75, 28.75, 31.97, 31.97, 31.97, 31.97, 31.97, 31.95, 31.95, 31.95, 31.95, 31.95, 33.14, 33.14, 33.14, 33.14, 33.14, 33.96, 33.96, 33.96, 33.96, 33.96, 34.3, 34.3, 34.3, 34.3, 34.3, 34.61, 34.61, 34.61, 34.61, 34.61, 35.0, 35.0, 35.0, 35.0, 35.0, 35.0, 35.0, 35.0, 35.0, 35.0, 34.79, 34.79, 34.79, 34.79, 34.79, 34.36, 34.36, 34.36, 34.36, 34.36, 32.01, 32.01, 32.01, 32.01, 32.01, 31.89, 31.89, 31.89, 31.89, 31.89, 30.27, 30.27, 30.27, 30.27, 30.27, 30.57, 30.57, 30.57, 30.57, 30.57, 30.7, 30.7, 30.7, 30.7, 30.7, 30.49, 30.49, 30.49, 30.49, 30.49, 30.46, 30.46, 30.46, 30.46, 30.46, 30.55, 30.55, 30.55, 30.55, 30.55, 30.62, 30.62, 30.62, 30.62, 30.62, 30.63, 30.63, 30.63, 30.63, 30.63, 30.67, 30.67, 30.67, 30.67, 30.67, 30.82, 30.82, 30.82, 30.82, 30.82, 30.85, 30.85, 30.85, 30.85, 30.85, 30.93, 30.93, 30.93, 30.93, 30.93, 31.17, 31.17, 31.17, 31.17, 31.17, 31.45, 31.45, 31.45, 31.45, 31.45, 31.62, 31.62, 31.62, 31.62, 31.62, 31.78, 31.78, 31.78, 31.78, 31.78, 31.89, 31.89, 31.89, 31.89, 31.89, 31.96, 31.96, 31.96, 31.96, 31.96, 31.76, 31.76, 31.76, 31.76, 31.76, 31.62, 31.62, 31.62, 31.62, 31.62, 31.35, 31.35, 31.35, 31.35, 31.35, 31.19, 31.19, 31.19, 31.19, 31.19, 31.37, 31.37, 31.37, 31.37, 31.37, 31.45, 31.45, 31.45, 31.45, 31.45, 31.58, 31.58, 31.58, 31.58, 31.58, 31.72, 31.72, 31.72, 31.72, 31.72, 31.38, 31.38, 31.38, 31.38, 31.38, 31.35, 31.35, 31.35, 31.35, 31.35, 31.06, 31.06, 31.06, 31.06, 31.06, 29.9, 29.9, 29.9, 29.9, 29.9, 29.78, 29.78, 29.78, 29.78, 29.78, 29.77, 29.77, 29.77, 29.77, 29.77, 29.72, 29.72, 29.72, 29.72, 29.72, 29.73, 29.73, 29.73, 29.73, 29.73, 29.84, 29.84, 29.84, 29.84, 29.84, 29.89, 29.89, 29.89, 29.89, 29.89, 29.75, 29.75, 29.75, 29.75, 29.75, 29.71, 29.71, 29.71, 29.71, 29.71, 29.79, 29.79, 29.79, 29.79, 29.79, 29.86, 29.86, 29.86, 29.86, 29.86, 29.95, 29.95, 29.95, 29.95, 29.95, 30.05, 30.05, 30.05, 30.05, 30.05, 30.08, 30.08, 30.08, 30.08, 30.08, 30.17, 30.17, 30.17, 30.17, 30.17, 30.24, 30.24, 30.24, 30.24, 30.24]
                    

Loading

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 556 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1716151175 --> 1716151803
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.01, 0.01, 0.01, 0.01, 0.01, 0.28, 0.28, 0.28, 0.28, 0.28, 0.24, 0.24, 0.24, 0.24, 0.24, 0.14, 0.14, 0.14, 0.14, 0.14, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.13, 0.13, 0.13, 0.13, 0.13, 0.14, 0.14, 0.14, 0.14, 0.14, 0.11, 0.11, 0.11, 0.11, 0.11, 0.21, 0.21, 0.21, 0.21, 0.21, 0.22, 0.22, 0.22, 0.22, 0.22, 0.12, 0.12, 0.12, 0.12, 0.12, 0.28, 0.28, 0.28, 0.28, 0.28, 0.33, 0.33, 0.33, 0.33, 0.33, 0.42, 0.42, 0.42, 0.42, 0.42, 0.32, 0.32, 0.32, 0.32, 0.32, 0.24, 0.24, 0.24, 0.24, 0.24, 0.14, 0.14, 0.14, 0.14, 0.14, 0.28, 0.28, 0.28, 0.28, 0.28, 0.23, 0.23, 0.23, 0.23, 0.23, 0.15, 0.15, 0.15, 0.15, 0.15, 0.18, 0.18, 0.18, 0.18, 0.18, 0.13, 0.13, 0.13, 0.13, 0.13, 0.3, 0.3, 0.3, 0.3, 0.3, 0.14, 0.14, 0.14, 0.14, 0.14, 0.11, 0.11, 0.11, 0.11, 0.11, 0.2, 0.2, 0.2, 0.2, 0.2, 0.08, 0.08, 0.08, 0.08, 0.08, 0.11, 0.11, 0.11, 0.11, 0.11, 0.11, 0.11, 0.11, 0.11, 0.11, 0.14, 0.14, 0.14, 0.14, 0.14, 0.12, 0.12, 0.12, 0.12, 0.12, 0.14, 0.14, 0.14, 0.14, 0.14, 0.33, 0.33, 0.33, 0.33, 0.33, 0.27, 0.27, 0.27, 0.27, 0.27, 0.32, 0.32, 0.32, 0.32, 0.32, 0.2, 0.2, 0.2, 0.2, 0.2, 0.11, 0.11, 0.11, 0.11, 0.11, 0.14, 0.14, 0.14, 0.14, 0.14, 0.11, 0.11, 0.11, 0.11, 0.11, 0.16, 0.16, 0.16, 0.16, 0.16, 0.42, 0.42, 0.42, 0.42, 0.42, 0.65, 0.65, 0.65, 0.65, 0.65, 0.51, 0.51, 0.51, 0.51, 0.51, 0.48, 0.48, 0.48, 0.48, 0.48, 0.14, 0.14, 0.14, 0.14, 0.14, 0.26, 0.26, 0.26, 0.26, 0.26, 0.25, 0.25, 0.25, 0.25, 0.25, 0.12, 0.12, 0.12, 0.12, 0.12, 0.13, 0.13, 0.13, 0.13, 0.13, 0.21, 0.21, 0.21, 0.21, 0.21, 0.31, 0.31, 0.31, 0.31, 0.31, 0.08, 0.08, 0.08, 0.08, 0.08, 0.22, 0.22, 0.22, 0.22, 0.22, 0.13, 0.13, 0.13, 0.13, 0.13, 0.19, 0.19, 0.19, 0.19, 0.19, 0.13, 0.13, 0.13, 0.13, 0.13, 0.11, 0.11, 0.11, 0.11, 0.11, 0.15, 0.15, 0.15, 0.15, 0.15, 0.19, 0.19, 0.19, 0.19, 0.19, 0.15, 0.15, 0.15, 0.15, 0.15]
                    

Loading

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 556 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1716151175 --> 1716151803
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 2.0, 2.0, 2.0, 2.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 1.0, 1.0, 1.0, 1.0, 1.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 3.0, 3.0, 3.0, 3.0, 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 8.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0]
                    

Loading