Repository navigation
Expand file tree
/
Copy pathNumSharp.Interop.OpenBLAS.targets
More file actions
1193 lines (1074 loc) · 56.2 KB
/
Copy pathNumSharp.Interop.OpenBLAS.targets
File metadata and controls
1193 lines (1074 loc) · 56.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<?xml version="1.0" encoding="utf-8"?>
<!-- ==================================================================================
NumSharp.Interop.OpenBLAS — BUILD-phase override delivery (delivery design §5–§7).
Imported (via buildTransitive/) into every project referencing the package directly
or transitively. Decides which OpenBLAS binary lands in the consumer's output:
no override -> nothing runs; the bundled runtime assets ship as-is
OpenBlasPath -> a source marker records the read-in-place directory
OpenBlasVersion -> the pinned scipy-openblas wheel is downloaded from PyPI,
double-hash verified, the native library AND the vendored
GCC runtime it loads (libgfortran/libquadmath/libgcc_s on
Linux and macOS; Windows is self-contained) extracted
into a global temp cache (the wheel is then DELETED),
staged over the bundle in the output, and a source
marker flags the folder as a REQUIRED override
Knobs are metadata on ANY PackageReference (so a package that depends on this one
exposes the identical experience to its consumers — design §6.1) or environment
variables, which win. A version override that cannot be satisfied FAILS THE BUILD:
the pin is a contract, and silently shipping a different binary is the one failure
a pin exists to prevent (design Goal 3, §10.5).
The wheel→library extraction reproduces tools/fetch_openblas.py exactly (same RID →
distribution/platform-tag map read from the same manifest, same /lib/ member pick,
same file name, same runtimes/<rid>/native/ layout, same co-staging of the vendored
runtime — beside the main on Linux, under native/.dylibs/ on macOS, where the
package's loader materializes the ../.dylibs sibling the dylib's load commands name)
— that is the design's "same binary" invariant (§5.4), and it is what lets an
override transparently overwrite the bundle in the same folder.
================================================================================== -->
<Project>
<!-- ============================ the resolve gate ================================
Cheap detection of whether ANY override is configured. This is the only thing a
default build evaluates: an item scan and a few property reads. The inline task
below is not even COMPILED unless a stage target finds work to do. -->
<Target Name="_NumSharpOpenBlasResolve" Condition="'$(NumSharpOpenBlasEnabled)' == 'true'">
<ItemGroup>
<!-- Every reference carrying OpenBLAS metadata, unless that reference opted out. -->
<_NumSharpOpenBlasRef Include="@(PackageReference)"
Condition="('%(PackageReference.OpenBlasVersion)' != '' or
'%(PackageReference.OpenBlasPath)' != '' or
'%(PackageReference.OpenBlasDistribution)' != '' or
'%(PackageReference.OpenBlasFeed)' != '' or
'%(PackageReference.OpenBlasSha256)' != '' or
'%(PackageReference.OpenBlasDelivery)' != '') and
'%(PackageReference.OpenBlasDelivery)' != 'none'" />
</ItemGroup>
<PropertyGroup>
<_NumSharpOpenBlasHasOverride>false</_NumSharpOpenBlasHasOverride>
<_NumSharpOpenBlasHasOverride Condition="'@(_NumSharpOpenBlasRef)' != '' or
'$(NUMSHARP_OPENBLAS_VERSION)' != '' or
'$(NUMSHARP_OPENBLAS_SEARCH_PATH)' != ''">true</_NumSharpOpenBlasHasOverride>
<!-- The RID whose asset is staged: an explicit RuntimeIdentifier, else the host. -->
<_NumSharpOpenBlasRid Condition="'$(RuntimeIdentifier)' != ''">$(RuntimeIdentifier)</_NumSharpOpenBlasRid>
<_NumSharpOpenBlasRid Condition="'$(_NumSharpOpenBlasRid)' == ''">$(NETCoreSdkPortableRuntimeIdentifier)</_NumSharpOpenBlasRid>
<_NumSharpOpenBlasRid Condition="'$(_NumSharpOpenBlasRid)' == ''">$(NETCoreSdkRuntimeIdentifier)</_NumSharpOpenBlasRid>
</PropertyGroup>
</Target>
<!-- ============================ build staging ===================================
Runs AFTER the ordinary output copy so a staged override lands ON TOP of the
bundled runtime asset already copied there.
The has-override gate sits on the TASK, not the target: a target's Condition is
evaluated BEFORE its DependsOnTargets run (the classic MSBuild gotcha), so a
condition reading $(_NumSharpOpenBlasHasOverride) up there would test the property
before _NumSharpOpenBlasResolve had a chance to set it — and the override would
silently never stage. At the task level the condition evaluates at execution time,
after the dependency ran; a default build still does nothing but the (cheap) item
scan, and the task is never even compiled. -->
<Target Name="_NumSharpOpenBlasStageBuild"
AfterTargets="CopyFilesToOutputDirectory"
DependsOnTargets="_NumSharpOpenBlasResolve"
Condition="'$(NumSharpOpenBlasEnabled)' == 'true' and '$(DesignTimeBuild)' != 'true'">
<!-- An override REMOVED between builds must not leave last build's marker behind: a
stale mode=version marker would keep declaring the folder a hard-required
override the consumer no longer asked for. -->
<Delete Files="$(OutDir)openblas.source.json;$(OutDir)runtimes/$(_NumSharpOpenBlasRid)/native/openblas.source.json"
Condition="'$(_NumSharpOpenBlasHasOverride)' != 'true'"
TreatErrorsAsWarnings="true" />
<NumSharpOpenBlasFetchTask
Condition="'$(_NumSharpOpenBlasHasOverride)' == 'true'"
References="@(_NumSharpOpenBlasRef)"
EnvVersion="$(NUMSHARP_OPENBLAS_VERSION)"
EnvPath="$(NUMSHARP_OPENBLAS_SEARCH_PATH)"
EnvDistribution="$(NUMSHARP_OPENBLAS_DISTRIBUTION)"
EnvFeed="$(NUMSHARP_OPENBLAS_PYPI_FEED_URL)"
EnvSha256="$(NUMSHARP_OPENBLAS_SHA256)"
EnvDelivery="$(NUMSHARP_OPENBLAS_DELIVERY)"
ManifestPath="$(NumSharpOpenBlasManifest)"
ProjectDirectory="$(MSBuildProjectDirectory)"
OutputDirectory="$(OutDir)"
Rid="$(_NumSharpOpenBlasRid)"
FlattenNative="false"
CacheDirectory="$(NumSharpOpenBlasCacheDir)">
<Output TaskParameter="ResolvedMode" PropertyName="_NumSharpOpenBlasMode" />
<Output TaskParameter="ResolvedDelivery" PropertyName="_NumSharpOpenBlasDelivery" />
<Output TaskParameter="StagedPackageFiles" ItemName="_NumSharpOpenBlasPackFile" />
</NumSharpOpenBlasFetchTask>
</Target>
<!-- ============================ publish staging =================================
The publish layout differs: a RID-specific publish FLATTENS native assets next to
the app, a portable publish keeps the runtimes/ tree. Stage accordingly, after
publish has copied its file set. -->
<Target Name="_NumSharpOpenBlasStagePublish"
AfterTargets="CopyFilesToPublishDirectory"
DependsOnTargets="_NumSharpOpenBlasResolve"
Condition="'$(NumSharpOpenBlasEnabled)' == 'true' and '$(DesignTimeBuild)' != 'true'">
<PropertyGroup>
<_NumSharpOpenBlasFlatten>false</_NumSharpOpenBlasFlatten>
<_NumSharpOpenBlasFlatten Condition="'$(RuntimeIdentifier)' != ''">true</_NumSharpOpenBlasFlatten>
</PropertyGroup>
<Delete Files="$(PublishDir)openblas.source.json;$(PublishDir)runtimes/$(_NumSharpOpenBlasRid)/native/openblas.source.json"
Condition="'$(_NumSharpOpenBlasHasOverride)' != 'true'"
TreatErrorsAsWarnings="true" />
<NumSharpOpenBlasFetchTask
Condition="'$(_NumSharpOpenBlasHasOverride)' == 'true'"
References="@(_NumSharpOpenBlasRef)"
EnvVersion="$(NUMSHARP_OPENBLAS_VERSION)"
EnvPath="$(NUMSHARP_OPENBLAS_SEARCH_PATH)"
EnvDistribution="$(NUMSHARP_OPENBLAS_DISTRIBUTION)"
EnvFeed="$(NUMSHARP_OPENBLAS_PYPI_FEED_URL)"
EnvSha256="$(NUMSHARP_OPENBLAS_SHA256)"
EnvDelivery="$(NUMSHARP_OPENBLAS_DELIVERY)"
ManifestPath="$(NumSharpOpenBlasManifest)"
ProjectDirectory="$(MSBuildProjectDirectory)"
OutputDirectory="$(PublishDir)"
Rid="$(_NumSharpOpenBlasRid)"
FlattenNative="$(_NumSharpOpenBlasFlatten)"
CacheDirectory="$(NumSharpOpenBlasCacheDir)" />
</Target>
<!-- ============================ pack hook (§6.2) ================================
A dependent package author setting OpenBlasVersion + OpenBlasDelivery=package gets
the staged binaries (ALL manifest RIDs) and their markers included in THEIR OWN
nupkg as runtimes/<rid>/native/ assets — their consumers then get the pinned
version with no build-time download, and the marker keeps it a required override
at runtime. NuGet's nearest-wins asset resolution makes the dependent's copy beat
this package's transitive bundle. -->
<PropertyGroup>
<TargetsForTfmSpecificContentInPackage>$(TargetsForTfmSpecificContentInPackage);_NumSharpOpenBlasPackContent</TargetsForTfmSpecificContentInPackage>
</PropertyGroup>
<Target Name="_NumSharpOpenBlasPackContent"
DependsOnTargets="_NumSharpOpenBlasResolve;_NumSharpOpenBlasStageBuild"
Condition="'$(NumSharpOpenBlasEnabled)' == 'true'">
<!-- Same dependency-before-condition rule as above: the has-override / delivery gate
must sit below the target level. -->
<ItemGroup Condition="'$(_NumSharpOpenBlasHasOverride)' == 'true' and '$(_NumSharpOpenBlasDelivery)' == 'package'">
<TfmSpecificPackageFile Include="@(_NumSharpOpenBlasPackFile)" PackagePath="%(_NumSharpOpenBlasPackFile.PackagePath)" />
</ItemGroup>
</Target>
<!-- ============================ the inline task =================================
Compiled by RoslynCodeTaskFactory ONLY when a stage target actually runs with an
override configured. The factory compiles against netstandard2.0's DEFAULT
references (simple-name <Reference> items do not resolve on the dotnet-sdk MSBuild
— MSB3755), which is why the task uses only netstandard-2.0 surface: HttpClient,
ZipArchive, SHA256 — and a small self-contained JSON reader/writer instead of
System.Text.Json (which netstandard2.0 lacks). That also keeps the task working
under Visual Studio's .NET Framework MSBuild. No Python, no extra dependency. -->
<UsingTask TaskName="NumSharpOpenBlasFetchTask"
TaskFactory="RoslynCodeTaskFactory"
AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll">
<Task>
<Code Type="Class" Language="cs">
<![CDATA[
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
/// <summary>
/// Resolves the OpenBLAS override knobs (env beats PackageReference metadata), downloads and
/// verifies the pinned scipy-openblas wheel when a version is set, extracts the one native
/// library into the global cache (deleting the wheel), stages it into the output, and writes
/// the openblas.source.json marker the runtime uses to tell an override from the bundle.
/// Reproduces tools/fetch_openblas.py's extraction exactly (the "same binary" invariant).
/// </summary>
public class NumSharpOpenBlasFetchTask : Task
{
public ITaskItem[] References { get; set; }
public string EnvVersion { get; set; }
public string EnvPath { get; set; }
public string EnvDistribution { get; set; }
public string EnvFeed { get; set; }
public string EnvSha256 { get; set; }
public string EnvDelivery { get; set; }
[Required] public string ManifestPath { get; set; }
[Required] public string ProjectDirectory { get; set; }
[Required] public string OutputDirectory { get; set; }
[Required] public string Rid { get; set; }
public bool FlattenNative { get; set; }
[Required] public string CacheDirectory { get; set; }
[Output] public string ResolvedMode { get; set; }
[Output] public string ResolvedDelivery { get; set; }
[Output] public ITaskItem[] StagedPackageFiles { get; set; }
private const string MarkerFileName = "openblas.source.json";
private const string DefaultFeed = "https://pypi.org";
public override bool Execute()
{
try
{
StagedPackageFiles = new ITaskItem[0];
string version, path, distribution, feed, sha256, delivery;
if (!ResolveKnobs(out version, out path, out distribution, out feed, out sha256, out delivery))
return false;
ResolvedDelivery = delivery;
if (string.IsNullOrEmpty(version) && string.IsNullOrEmpty(path))
{
ResolvedMode = "none";
return true;
}
// ---- path-only mode: record the read-in-place directory, nothing downloaded ----
if (string.IsNullOrEmpty(version))
{
ResolvedMode = "path";
string absPath = ResolvePath(path);
WriteMarker(Path.Combine(OutputDirectory, MarkerFileName),
"path", null, null, null, false, absPath);
// A mode SWITCH (version -> path between builds) must not leave the old
// version marker declaring the staged folder a hard-required override.
TryDelete(Path.Combine(OutputDirectory, "runtimes", Rid, "native", MarkerFileName));
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: recorded OpenBlasPath override -> {0} (read in place, non-binding)",
absPath);
return true;
}
// ---- version mode: a hard requirement ----
ResolvedMode = "version";
Dictionary<string, object> manifest = LoadManifest();
Dictionary<string, object> runtimes = (Dictionary<string, object>)manifest["runtimes"];
if (delivery == "package" && !string.IsNullOrEmpty(path))
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: OpenBlasDelivery=package stages every RID into the " +
"build output for packing and cannot combine with OpenBlasPath (which names one " +
"read/download directory). Drop one of the two.");
return false;
}
string[] rids = delivery == "package" ? runtimes.Keys.ToArray() : new[] { Rid };
var packFiles = new List<ITaskItem>();
foreach (string rid in rids)
{
string staged;
List<KeyValuePair<string, string>> stagedVendored;
if (!StageRid(manifest, runtimes, rid, version, path, distribution, feed, sha256,
out staged, out stagedVendored))
return false;
if (delivery == "package")
{
var lib = new TaskItem(staged);
lib.SetMetadata("PackagePath", "runtimes/" + rid + "/native/" + Path.GetFileName(staged));
packFiles.Add(lib);
// The vendored runtime travels with the main, in the same relative layout
// (Linux: beside it; macOS: native/.dylibs/ — NuGet delivers nested paths).
foreach (var dep in stagedVendored)
{
var depItem = new TaskItem(dep.Value);
depItem.SetMetadata("PackagePath", "runtimes/" + rid + "/native/" + dep.Key);
packFiles.Add(depItem);
}
var marker = new TaskItem(Path.Combine(Path.GetDirectoryName(staged), MarkerFileName));
marker.SetMetadata("PackagePath", "runtimes/" + rid + "/native/" + MarkerFileName);
packFiles.Add(marker);
}
}
StagedPackageFiles = packFiles.ToArray();
return true;
}
catch (Exception e)
{
// A version override is a contract (design Goal 3): any failure to satisfy it is a
// BUILD error, never a silent fallback to a different binary.
Log.LogError("NumSharp.Interop.OpenBLAS: the OpenBLAS override could not be satisfied: {0}",
e.Message);
Log.LogMessage(MessageImportance.Low, e.ToString());
return false;
}
}
// ---------------------------------------------------------------- knob resolution
/// <summary>
/// Env beats metadata; among references, the highest OpenBlasVersion wins (design §10.2,
/// logged); a version on one reference conflicting with a path-only override on another is
/// a hard error (the two name different contracts and neither can silently lose).
/// </summary>
private bool ResolveKnobs(out string version, out string path, out string distribution,
out string feed, out string sha256, out string delivery)
{
version = path = distribution = feed = sha256 = null;
delivery = "build";
var refs = References ?? new ITaskItem[0];
var versions = new List<KeyValuePair<string, string>>(); // (refName, version)
string pathRef = null, versionRef = null;
foreach (var r in refs)
{
string v = r.GetMetadata("OpenBlasVersion");
string p = r.GetMetadata("OpenBlasPath");
if (!string.IsNullOrEmpty(v)) { versions.Add(new KeyValuePair<string, string>(r.ItemSpec, v)); versionRef = r.ItemSpec; }
if (!string.IsNullOrEmpty(p))
{
if (path != null && path != p)
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: two PackageReferences disagree on OpenBlasPath " +
"('{0}' vs '{1}'). Resolve the conflict explicitly.", path, p);
return false;
}
path = p;
if (string.IsNullOrEmpty(v))
pathRef = r.ItemSpec;
}
if (distribution == null) distribution = NullIfEmpty(r.GetMetadata("OpenBlasDistribution"));
if (feed == null) feed = NullIfEmpty(r.GetMetadata("OpenBlasFeed"));
if (sha256 == null) sha256 = NullIfEmpty(r.GetMetadata("OpenBlasSha256"));
string d = r.GetMetadata("OpenBlasDelivery");
if (!string.IsNullOrEmpty(d) && d != "build") delivery = d;
}
if (versions.Count > 0)
{
versions.Sort((a, b) => CompareVersions(b.Value, a.Value));
version = versions[0].Value;
if (versions.Select(kv => kv.Value).Distinct().Count() > 1)
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: multiple OpenBlasVersion values ({0}); the highest, {1} (from '{2}'), wins.",
string.Join(", ", versions.Select(kv => kv.Key + "=" + kv.Value)), version, versions[0].Key);
}
// A reference that pins a VERSION and a different reference that pins a bare PATH name
// different contracts (enforced download vs read-in-place) — hard error (§10.2).
if (version != null && pathRef != null && versionRef != null && pathRef != versionRef)
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: reference '{0}' pins OpenBlasVersion={1} while reference '{2}' " +
"sets a bare OpenBlasPath — an enforced download and a read-in-place directory cannot both " +
"win. Put both knobs on one reference (download INTO the path) or drop one.",
versionRef, version, pathRef);
return false;
}
// Environment wins over everything the references said.
if (!string.IsNullOrEmpty(EnvVersion)) version = EnvVersion;
if (!string.IsNullOrEmpty(EnvPath)) path = EnvPath;
if (!string.IsNullOrEmpty(EnvDistribution)) distribution = EnvDistribution;
if (!string.IsNullOrEmpty(EnvFeed)) feed = EnvFeed;
if (!string.IsNullOrEmpty(EnvSha256)) sha256 = EnvSha256;
if (!string.IsNullOrEmpty(EnvDelivery)) delivery = EnvDelivery;
if (delivery != "build" && delivery != "package" && delivery != "none")
{
Log.LogError("NumSharp.Interop.OpenBLAS: OpenBlasDelivery must be none|build|package, got '{0}'.", delivery);
return false;
}
if (distribution != null)
{
if (distribution == "64") distribution = "scipy-openblas64";
else if (distribution == "32") distribution = "scipy-openblas32";
if (distribution != "scipy-openblas64" && distribution != "scipy-openblas32")
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: OpenBlasDistribution must be scipy-openblas64|scipy-openblas32 " +
"(or the 64|32 aliases), got '{0}'.", distribution);
return false;
}
}
return true;
}
private static string NullIfEmpty(string s)
{
return string.IsNullOrEmpty(s) ? null : s;
}
/// <summary>
/// scipy-openblas versions have five numeric parts; System.Version stops at four. Missing
/// parts compare as 0; a non-numeric PRESENT part drops the whole comparison to ordinal.
/// </summary>
private static int CompareVersions(string a, string b)
{
string[] pa = a.Split('.'), pb = b.Split('.');
for (int i = 0; i < Math.Max(pa.Length, pb.Length); i++)
{
int na = 0, nb = 0;
if (i < pa.Length && !int.TryParse(pa[i], out na))
return string.CompareOrdinal(a, b);
if (i < pb.Length && !int.TryParse(pb[i], out nb))
return string.CompareOrdinal(a, b);
if (na != nb)
return na.CompareTo(nb);
}
return 0;
}
private string ResolvePath(string p)
{
return Path.GetFullPath(Path.IsPathRooted(p) ? p : Path.Combine(ProjectDirectory, p));
}
// ---------------------------------------------------------------- staging one RID
private bool StageRid(Dictionary<string, object> manifest, Dictionary<string, object> runtimes,
string rid, string version, string customPath,
string distributionOverride, string feed, string expectedSha,
out string stagedFile, out List<KeyValuePair<string, string>> stagedVendored)
{
stagedFile = null;
stagedVendored = new List<KeyValuePair<string, string>>();
object entryObj;
if (string.IsNullOrEmpty(rid) || !runtimes.TryGetValue(rid, out entryObj))
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: no manifest entry for RID '{0}'. Supported RIDs: {1}. " +
"(The manifest is the shared RID -> wheel map; an unsupported RID cannot be staged.)",
rid, string.Join(", ", runtimes.Keys));
return false;
}
var entry = (Dictionary<string, object>)entryObj;
string manifestDistribution = (string)entry["distribution"];
string distribution = distributionOverride ?? manifestDistribution;
string tag = PlatformTagAnchor((string)entry["wheel"]);
string pinnedVersion = (string)manifest["distribution_version"];
// The manifest hash pins only the DEFAULT version+distribution; any other override must
// bring its own OpenBlasSha256 (or none, in which case the wheel hash from the index is
// the verification).
string sha = expectedSha;
if (sha == null && version == pinnedVersion && distribution == manifestDistribution)
sha = (string)entry["sha256"];
string feedBase = string.IsNullOrEmpty(feed) ? DefaultFeed : feed.TrimEnd('/');
if (feedBase != DefaultFeed && expectedSha == null)
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: a non-pypi.org OpenBlasFeed ({0}) requires an explicit " +
"OpenBlasSha256 — a mirror's own index cannot attest its own artifacts (design §10.6).",
feedBase);
return false;
}
// ---- the global cache: extracted libraries only, keyed (distribution, version, rid, sha) ----
string cacheRoot = Path.Combine(CacheDirectory, distribution, version, rid);
List<KeyValuePair<string, string>> vendored;
string cached = FindInCache(cacheRoot, sha, out vendored);
if (cached == null)
{
if (!DownloadIntoCache(feedBase, distribution, version, rid, tag, sha, cacheRoot, out cached, out vendored))
return false;
}
else
{
Log.LogMessage(MessageImportance.Normal,
"NumSharp.Interop.OpenBLAS: cache hit {0}", cached);
}
string stagedSha = Path.GetFileName(Path.GetDirectoryName(cached));
// ---- stage into the destination (over the bundle) ----
string destDir = !string.IsNullOrEmpty(customPath)
? ResolvePath(customPath)
: FlattenNative
? Path.GetFullPath(OutputDirectory)
: Path.GetFullPath(Path.Combine(OutputDirectory, "runtimes", rid, "native"));
Directory.CreateDirectory(destDir);
string dest = Path.Combine(destDir, Path.GetFileName(cached));
if (StageOne(cached, dest, stagedSha))
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: staged {0} {1} ({2}) -> {3}",
distribution, version, rid, dest);
else
Log.LogMessage(MessageImportance.Normal,
"NumSharp.Interop.OpenBLAS: {0} already staged and verified", dest);
// The vendored runtime goes wherever the main went, in the wheel's relative layout —
// except that a flattened (RID-specific publish) layout drops the sub-folder, exactly
// as the SDK flattens the bundle's own native/.dylibs/ files beside the main; the
// package's loader looks in both places.
foreach (var dep in vendored)
{
string depDest = FlattenNative
? Path.Combine(destDir, Path.GetFileName(dep.Value))
: Path.Combine(destDir, dep.Key.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(Path.GetDirectoryName(depDest));
if (StageOne(dep.Value, depDest, null))
Log.LogMessage(MessageImportance.Normal,
"NumSharp.Interop.OpenBLAS: staged vendored {0} -> {1}", dep.Key, depDest);
stagedVendored.Add(new KeyValuePair<string, string>(dep.Key, depDest));
}
// ---- the source marker: next to the staged binary (its own directory is the read
// location), plus a root-level marker pointing at a custom directory the runtime's
// app-relative probe could not otherwise find. The unused location is CLEARED so a
// mode switch between builds cannot leave a stale marker shadowing the current one —
// but ONLY when it is a distinct file: in the flattened publish layout destDir IS the
// output root, and the cleanup would delete the marker just written. ----
WriteMarker(Path.Combine(destDir, MarkerFileName),
"version", distribution, version, stagedSha, true, null);
if (!string.IsNullOrEmpty(customPath))
{
WriteMarker(Path.Combine(OutputDirectory, MarkerFileName),
"version", distribution, version, stagedSha, true, destDir);
// Staging went to the custom directory — a rid-dir marker from an earlier
// default-staged build would still declare the OLD location an override.
// (Guarded: a custom path pointed AT the rid dir must not lose its fresh marker.)
string ridNative = Path.Combine(OutputDirectory, "runtimes", Rid, "native");
if (!SamePath(destDir, ridNative))
TryDelete(Path.Combine(ridNative, MarkerFileName));
}
else if (!SamePath(destDir, OutputDirectory))
{
TryDelete(Path.Combine(OutputDirectory, MarkerFileName));
}
stagedFile = dest;
return true;
}
/// <summary>
/// The anchor component of the wheel's platform tag, reproduced from the manifest's wheel
/// FILE NAME the way fetch_openblas.py's RID_MAP spells it: the piece after "py3-none-",
/// first dot-separated component (compound manylinux tags list the anchor first, e.g.
/// "manylinux2014_x86_64.manylinux_2_17_x86_64").
/// </summary>
private static string PlatformTagAnchor(string wheelFileName)
{
const string sep = "-py3-none-";
int i = wheelFileName.IndexOf(sep, StringComparison.Ordinal);
if (i < 0)
throw new InvalidDataException("unexpected wheel name in manifest: " + wheelFileName);
string tags = wheelFileName.Substring(i + sep.Length);
if (tags.EndsWith(".whl", StringComparison.Ordinal))
tags = tags.Substring(0, tags.Length - 4);
int dot = tags.IndexOf('.');
return dot < 0 ? tags : tags.Substring(0, dot);
}
/// <summary>Copies <paramref name="source"/> to <paramref name="dest"/> unless an identical file is already there; true when it copied.</summary>
private static bool StageOne(string source, string dest, string expectedSha)
{
if (File.Exists(dest) && new FileInfo(dest).Length == new FileInfo(source).Length)
{
string have = HashFile(dest);
if (expectedSha != null ? have.Equals(expectedSha, StringComparison.OrdinalIgnoreCase)
: have.Equals(HashFile(source), StringComparison.OrdinalIgnoreCase))
return false;
}
File.Copy(source, dest, true);
return true;
}
/// <summary>
/// The sidecar a cache entry carries: { "main": "<file name>", "files": { "<relative path>": "<sha256>" } }
/// — the vendored runtime the wheel shipped beside the main, in the wheel's relative layout.
/// </summary>
private const string EntrySidecar = ".entry.json";
/// <summary>
/// A cache entry is a directory named by the MAIN library's sha256, holding the main plus its
/// vendored dependencies (Linux: beside it; macOS: under .dylibs/) as recorded in its sidecar.
/// A hit is only a hit when the main still hashes to the directory name AND every vendored
/// file to what the sidecar recorded; anything else — a truncated write (killed build), a
/// tampered file, or an entry written by an older build that carried the main alone (which
/// cannot load on Linux/macOS) — is discarded and re-downloaded (§10.6 — never silently
/// substitute; fetch_openblas.py treats its poisoned cache entries the same way).
/// </summary>
private string FindInCache(string cacheRoot, string expectedSha, out List<KeyValuePair<string, string>> vendored)
{
vendored = null;
if (!Directory.Exists(cacheRoot))
return null;
// With a pinned sha only that entry qualifies; without one, any entry for this
// (distribution, version, rid) was wheel-hash verified when it was downloaded.
string[] dirs = expectedSha != null
? new[] { Path.Combine(cacheRoot, expectedSha.ToLowerInvariant()) }
: Directory.GetDirectories(cacheRoot);
foreach (string dir in dirs)
{
if (!Directory.Exists(dir) || dir.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase))
continue;
string main;
List<KeyValuePair<string, string>> deps;
string problem = ValidateEntry(dir, out main, out deps);
if (problem == null)
{
vendored = deps;
return main;
}
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: DISCARDING poisoned cache entry {0} ({1})", dir, problem);
try
{
Directory.Delete(dir, true);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
return null;
}
/// <summary>Null when the entry is intact; otherwise what is wrong with it.</summary>
private static string ValidateEntry(string dir, out string main, out List<KeyValuePair<string, string>> deps)
{
main = null;
deps = new List<KeyValuePair<string, string>>();
string sidecar = Path.Combine(dir, EntrySidecar);
if (!File.Exists(sidecar))
return "no " + EntrySidecar + " sidecar — written by an older build that cached the main library alone";
Dictionary<string, object> doc;
try
{
doc = MiniJson.Parse(File.ReadAllText(sidecar)) as Dictionary<string, object>;
}
catch (Exception e)
{
return "unreadable sidecar: " + e.Message;
}
object mainObj;
if (doc == null || !doc.TryGetValue("main", out mainObj) || !(mainObj is string))
return "sidecar names no main library";
string mainPath = Path.Combine(dir, (string)mainObj);
if (!File.Exists(mainPath))
return "main library missing";
string actual = HashFile(mainPath);
if (!actual.Equals(Path.GetFileName(dir), StringComparison.OrdinalIgnoreCase))
return "main library hashes " + actual + ", entry claims " + Path.GetFileName(dir);
object filesObj;
if (doc.TryGetValue("files", out filesObj) && filesObj is Dictionary<string, object>)
{
foreach (var kv in (Dictionary<string, object>)filesObj)
{
string rel = kv.Key;
string p = Path.Combine(dir, rel.Replace('/', Path.DirectorySeparatorChar));
if (!File.Exists(p))
return "vendored file missing: " + rel;
string sha = kv.Value as string;
if (sha != null && !HashFile(p).Equals(sha, StringComparison.OrdinalIgnoreCase))
return "vendored file " + rel + " does not hash to what the sidecar recorded";
deps.Add(new KeyValuePair<string, string>(rel, p));
}
}
main = mainPath;
return null;
}
// ---------------------------------------------------------------- download & extract
private bool DownloadIntoCache(string feedBase, string distribution, string version, string rid,
string tag, string expectedSha, string cacheRoot, out string cached,
out List<KeyValuePair<string, string>> vendored)
{
cached = null;
vendored = new List<KeyValuePair<string, string>>();
string indexUrl = feedBase + "/pypi/" + distribution + "/" + version + "/json";
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: resolving {0} {1} for {2} from {3}",
distribution, version, rid, indexUrl);
using (var http = new HttpClient())
{
http.Timeout = TimeSpan.FromMinutes(5);
http.DefaultRequestHeaders.UserAgent.ParseAdd("NumSharp.Interop.OpenBLAS-build");
string indexJson;
try
{
indexJson = http.GetStringAsync(indexUrl).GetAwaiter().GetResult();
}
catch (Exception e)
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: could not resolve {0}=={1} from {2}: {3}. A version " +
"override is a hard requirement — the build fails rather than shipping a different " +
"binary. (Offline? Prime the cache once, or point OpenBlasFeed at a mirror; see " +
"docs/stale-docs/OPENBLAS_DELIVERY_DESIGN.md §10.5.)",
distribution, version, indexUrl, FirstLine(e));
return false;
}
string wheelUrl = null, wheelSha = null, wheelName = null;
object indexRoot = MiniJson.Parse(indexJson);
var urls = (List<object>)((Dictionary<string, object>)indexRoot)["urls"];
var matches = new List<Dictionary<string, object>>();
foreach (object u in urls)
{
var file = (Dictionary<string, object>)u;
string name = (string)file["filename"];
// Compound platform tags ("…-manylinux2014_x86_64.manylinux_2_17_x86_64.whl")
// anchor on "-<tag>." exactly like fetch_openblas.py.
if (name.Contains("-" + tag + "."))
matches.Add(file);
}
if (matches.Count != 1)
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: expected exactly one {0} {1} wheel for platform tag " +
"'{2}', found {3}.", distribution, version, tag, matches.Count);
return false;
}
wheelUrl = (string)matches[0]["url"];
wheelSha = (string)((Dictionary<string, object>)matches[0]["digests"])["sha256"];
wheelName = (string)matches[0]["filename"];
// The wheel is TRANSIENT (design Goal 7): downloaded to a temp file, verified,
// extracted, deleted. Nothing wheel-shaped survives the build.
string wheelTemp = Path.GetTempFileName();
string libTemp = null;
try
{
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: downloading {0}", wheelName);
using (var response = http.GetAsync(wheelUrl, HttpCompletionOption.ResponseHeadersRead)
.GetAwaiter().GetResult())
{
response.EnsureSuccessStatusCode();
using (var src = response.Content.ReadAsStreamAsync().GetAwaiter().GetResult())
using (var dst = File.Create(wheelTemp))
src.CopyTo(dst);
}
string actualWheelSha = HashFile(wheelTemp);
if (!actualWheelSha.Equals(wheelSha, StringComparison.OrdinalIgnoreCase))
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: CHECKSUM MISMATCH for {0}: expected {1}, got {2}. " +
"The artifact changed under its URL — hard failure, never a warning.",
wheelName, wheelSha, actualWheelSha);
return false;
}
// The one shared library in a scipy-openblas wheel: prefer members under /lib/,
// take the LARGEST native file (the real library, not a stub) — exactly
// fetch_openblas.py's pick_native_member. Every OTHER native member is the
// vendored GCC runtime the main loads (its DT_NEEDED / LC_LOAD_DYLIB name the
// content-hashed vendored file, so a system libgfortran can never satisfy it) —
// fetch_openblas.py's vendored_members, co-extracted here for the same reason.
libTemp = Path.GetTempFileName();
string memberName;
var depTemps = new List<KeyValuePair<string, string>>(); // (relative path, temp file)
// ZipArchive over a FileStream, NOT ZipFile.OpenRead: ZipFile lives in a different
// assembly per runtime and is not part of the factory's default references.
using (var wheelStream = File.OpenRead(wheelTemp))
using (var zip = new ZipArchive(wheelStream, ZipArchiveMode.Read))
{
ZipArchiveEntry member = PickNativeMember(zip);
if (member == null)
{
Log.LogError("NumSharp.Interop.OpenBLAS: no shared library found in {0}.", wheelName);
return false;
}
memberName = member.FullName;
using (var src = member.Open())
using (var dst = File.Create(libTemp))
src.CopyTo(dst);
foreach (ZipArchiveEntry dep in VendoredMembers(zip, member))
{
string depTemp = Path.GetTempFileName();
using (var src = dep.Open())
using (var dst = File.Create(depTemp))
src.CopyTo(dst);
depTemps.Add(new KeyValuePair<string, string>(VendoredRelativePath(dep.FullName), depTemp));
}
}
try
{
string libSha = HashFile(libTemp);
if (expectedSha != null && !libSha.Equals(expectedSha, StringComparison.OrdinalIgnoreCase))
{
Log.LogError(
"NumSharp.Interop.OpenBLAS: CHECKSUM MISMATCH for the library extracted from {0} " +
"(member {1}): expected {2}, got {3}.",
wheelName, memberName, expectedSha, libSha);
return false;
}
string cacheDir = Path.Combine(cacheRoot, libSha.ToLowerInvariant());
string mainName = Path.GetFileName(memberName.Replace('\\', '/'));
// Build the WHOLE entry (main + vendored + sidecar) in a temp sibling and rename
// it into place, never file by file into the final directory: a killed build
// cannot leave a half-built entry under the content-hash name (which a later
// hit would otherwise have to detect and discard), and two builds racing the
// same entry cannot clobber each other mid-write — the loser keeps the winner's
// entry, which carries the identical verified content by construction.
string tmpDir = cacheDir + "." + Guid.NewGuid().ToString("N") + ".tmp";
Directory.CreateDirectory(tmpDir);
File.Copy(libTemp, Path.Combine(tmpDir, mainName), true);
var sidecar = new StringBuilder();
sidecar.Append("{\n \"main\": ").Append(MiniJson.Quote(mainName)).Append(",\n \"files\": {");
bool first = true;
foreach (var dep in depTemps)
{
string target = Path.Combine(tmpDir, dep.Key.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(Path.GetDirectoryName(target));
File.Copy(dep.Value, target, true);
sidecar.Append(first ? "\n " : ",\n ")
.Append(MiniJson.Quote(dep.Key)).Append(": ").Append(MiniJson.Quote(HashFile(target)));
first = false;
}
sidecar.Append(first ? "}\n}\n" : "\n }\n}\n");
File.WriteAllText(Path.Combine(tmpDir, EntrySidecar), sidecar.ToString());
Directory.CreateDirectory(cacheRoot);
try
{
Directory.Move(tmpDir, cacheDir);
}
catch (IOException)
{
TryDeleteDirectory(tmpDir);
if (!Directory.Exists(cacheDir))
throw; // a genuine IO failure, not a lost race
}
List<KeyValuePair<string, string>> deps;
string main;
string problem = ValidateEntry(cacheDir, out main, out deps);
if (problem != null)
{
Log.LogError("NumSharp.Interop.OpenBLAS: the cache entry {0} could not be verified after writing it: {1}",
cacheDir, problem);
return false;
}
cached = main;
vendored = deps;
Log.LogMessage(MessageImportance.High,
"NumSharp.Interop.OpenBLAS: cached {0} (sha256 {1}, {2} vendored file(s))", cached, libSha, deps.Count);
return true;
}
finally
{
foreach (var dep in depTemps)
TryDelete(dep.Value);
}
}
finally
{
TryDelete(wheelTemp);
if (libTemp != null)
TryDelete(libTemp);
}
}
}
/// <summary>Every native member of the wheel other than the main library — the vendored GCC runtime.</summary>
private static List<ZipArchiveEntry> VendoredMembers(ZipArchive zip, ZipArchiveEntry main)
{
return zip.Entries.Where(e => e != main && IsNativeMember(e)).ToList();
}
/// <summary>
/// Where a vendored member sits relative to the main library once staged: the wheel keeps the
/// Linux deps beside the main (RUNPATH $ORIGIN) and the macOS deps in .dylibs/ — kept as a
/// .dylibs/ sub-folder UNDER native/ (NuGet delivers nested paths, never a sibling folder),
/// exactly like fetch_openblas.py's vendored_dest; the package's loader materializes the
/// ../.dylibs sibling the dylib's load commands name.
/// </summary>
private static string VendoredRelativePath(string memberFullName)
{
string norm = memberFullName.Replace('\\', '/');
string baseName = Path.GetFileName(norm);
return norm.Contains("/.dylibs/") ? ".dylibs/" + baseName : baseName;
}
private static bool IsNativeMember(ZipArchiveEntry e)
{
string baseName = Path.GetFileName(e.FullName.Replace('\\', '/'));
if (string.IsNullOrEmpty(baseName) || baseName.StartsWith("."))
return false;
return baseName.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) ||
baseName.EndsWith(".so", StringComparison.OrdinalIgnoreCase) ||
baseName.EndsWith(".dylib", StringComparison.OrdinalIgnoreCase) ||
baseName.Contains(".so.");
}
private static void TryDeleteDirectory(string dir)
{
try
{
if (Directory.Exists(dir))
Directory.Delete(dir, true);
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
}
private static ZipArchiveEntry PickNativeMember(ZipArchive zip)
{
List<ZipArchiveEntry> candidates = zip.Entries
.Where(e => IsNativeMember(e) && e.FullName.Replace('\\', '/').Contains("/lib/"))
.ToList();
if (candidates.Count == 0)
candidates = zip.Entries.Where(IsNativeMember).ToList();
return candidates.OrderByDescending(e => e.Length).FirstOrDefault();
}
// ---------------------------------------------------------------- helpers
private void WriteMarker(string file, string mode, string distribution, string version,
string sha256, bool required, string path)
{
Directory.CreateDirectory(Path.GetDirectoryName(file));
var sb = new StringBuilder();
sb.Append("{\n");
sb.Append(" \"mode\": ").Append(MiniJson.Quote(mode));
if (distribution != null) sb.Append(",\n \"distribution\": ").Append(MiniJson.Quote(distribution));
if (version != null) sb.Append(",\n \"version\": ").Append(MiniJson.Quote(version));
if (sha256 != null) sb.Append(",\n \"sha256\": ").Append(MiniJson.Quote(sha256.ToLowerInvariant()));
sb.Append(",\n \"required\": ").Append(required ? "true" : "false");
if (path != null) sb.Append(",\n \"path\": ").Append(MiniJson.Quote(path));
sb.Append("\n}\n");
File.WriteAllText(file, sb.ToString());
Log.LogMessage(MessageImportance.Normal,
"NumSharp.Interop.OpenBLAS: wrote source marker {0} (mode={1})", file, mode);
}
private Dictionary<string, object> LoadManifest()
{
if (string.IsNullOrEmpty(ManifestPath) || !File.Exists(ManifestPath))
throw new FileNotFoundException(
"openblas-manifest.json not found (looked at '" + ManifestPath + "') — it ships in the " +
"package's buildTransitive/ folder and is the shared RID -> wheel map.");
return (Dictionary<string, object>)MiniJson.Parse(File.ReadAllText(ManifestPath));
}
private static bool SamePath(string a, string b)
{
char[] seps = { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar };
return string.Equals(Path.GetFullPath(a).TrimEnd(seps), Path.GetFullPath(b).TrimEnd(seps),
StringComparison.OrdinalIgnoreCase);
}
private static string HashFile(string file)