1// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
2// for details. All rights reserved. Use of this source code is governed by a
3// BSD-style license that can be found in the LICENSE file.
4
5#include <bin/elf_loader.h>
6#include <bin/file.h>
7#include <platform/elf.h>
8#include <platform/globals.h>
9#include <vm/bss_relocs.h>
10#include <vm/cpu.h>
11#include <vm/virtual_memory.h>
12
13#if defined(HOST_OS_FUCHSIA)
14#include <sys/mman.h>
15#endif
16
17#include <memory>
18#include <utility>
19
20namespace dart {
21namespace bin {
22
23namespace elf {
24
25class Mappable {
26 public:
27 static Mappable* FromPath(const char* path);
28#if defined(HOST_OS_FUCHSIA) || defined(HOST_OS_LINUX)
29 static Mappable* FromFD(int fd);
30#endif
31 static Mappable* FromMemory(const uint8_t* memory, size_t size);
32
33 virtual MappedMemory* Map(File::MapType type,
34 uint64_t position,
35 uint64_t length,
36 void* start = nullptr) = 0;
37
38 virtual bool SetPosition(uint64_t position) = 0;
39 virtual bool ReadFully(void* dest, int64_t length) = 0;
40
41 virtual ~Mappable() {}
42
43 protected:
44 Mappable() {}
45
46 private:
47 DISALLOW_COPY_AND_ASSIGN(Mappable);
48};
49
50class FileMappable : public Mappable {
51 public:
52 explicit FileMappable(File* file) : Mappable(), file_(file) {}
53
54 ~FileMappable() override { file_->Release(); }
55
56 MappedMemory* Map(File::MapType type,
57 uint64_t position,
58 uint64_t length,
59 void* start = nullptr) override {
60 return file_->Map(type, position, length, start);
61 }
62
63 bool SetPosition(uint64_t position) override {
64 return file_->SetPosition(position);
65 }
66
67 bool ReadFully(void* dest, int64_t length) override {
68 return file_->ReadFully(dest, length);
69 }
70
71 private:
72 File* const file_;
73 DISALLOW_COPY_AND_ASSIGN(FileMappable);
74};
75
76class MemoryMappable : public Mappable {
77 public:
78 MemoryMappable(const uint8_t* memory, size_t size)
79 : Mappable(), memory_(memory), size_(size), position_(memory) {}
80
81 ~MemoryMappable() override {}
82
83 MappedMemory* Map(File::MapType type,
84 uint64_t position,
85 uint64_t length,
86 void* start = nullptr) override {
87 if (position > size_) return nullptr;
88 MappedMemory* result = nullptr;
89 const uword map_size = Utils::RoundUp(length, VirtualMemory::PageSize());
90 if (start == nullptr) {
91 auto* memory = VirtualMemory::Allocate(
92 map_size, type == File::kReadExecute, "dart-compiled-image");
93 if (memory == nullptr) return nullptr;
94 result = new MappedMemory(memory->address(), memory->size());
95 memory->release();
96 delete memory;
97 } else {
98 result = new MappedMemory(start, map_size,
99 /*should_unmap=*/false);
100 }
101
102 size_t remainder = 0;
103 if ((position + length) > size_) {
104 remainder = position + length - size_;
105 length = size_ - position;
106 }
107 memcpy(result->address(), memory_ + position, length); // NOLINT
108 memset(reinterpret_cast<uint8_t*>(result->address()) + length, 0,
109 remainder);
110
111 auto mode = VirtualMemory::kReadOnly;
112 switch (type) {
113 case File::kReadExecute:
114 mode = VirtualMemory::kReadExecute;
115 break;
116 case File::kReadWrite:
117 mode = VirtualMemory::kReadWrite;
118 break;
119 case File::kReadOnly:
120 mode = VirtualMemory::kReadOnly;
121 break;
122 default:
123 UNREACHABLE();
124 }
125
126 VirtualMemory::Protect(result->address(), result->size(), mode);
127
128 return result;
129 }
130
131 bool SetPosition(uint64_t position) override {
132 if (position > size_) return false;
133 position_ = memory_ + position;
134 return true;
135 }
136
137 bool ReadFully(void* dest, int64_t length) override {
138 if ((position_ + length) > (memory_ + size_)) return false;
139 memcpy(dest, position_, length);
140 return true;
141 }
142
143 private:
144 const uint8_t* const memory_;
145 const size_t size_;
146 const uint8_t* position_;
147 DISALLOW_COPY_AND_ASSIGN(MemoryMappable);
148};
149
150Mappable* Mappable::FromPath(const char* path) {
151 return new FileMappable(File::Open(/*namespc=*/nullptr, path, File::kRead));
152}
153
154#if defined(HOST_OS_FUCHSIA) || defined(HOST_OS_LINUX)
155Mappable* Mappable::FromFD(int fd) {
156 return new FileMappable(File::OpenFD(fd));
157}
158#endif
159
160Mappable* Mappable::FromMemory(const uint8_t* memory, size_t size) {
161 return new MemoryMappable(memory, size);
162}
163
164/// A loader for a subset of ELF which may be used to load objects produced by
165/// Dart_CreateAppAOTSnapshotAsElf.
166class LoadedElf {
167 public:
168 explicit LoadedElf(std::unique_ptr<Mappable> mappable,
169 uint64_t elf_data_offset)
170 : mappable_(std::move(mappable)), elf_data_offset_(elf_data_offset) {}
171
172 ~LoadedElf();
173
174 /// Loads the ELF object into memory. Returns whether the load was successful.
175 /// On failure, the error may be retrieved by 'error()'.
176 bool Load();
177
178 /// Reads Dart-specific symbols from the loaded ELF.
179 ///
180 /// Stores the address of the corresponding symbol in each non-null output
181 /// parameter.
182 ///
183 /// Fails if any output parameter is non-null but points to null and the
184 /// corresponding symbol was not found, or if the dynamic symbol table could
185 /// not be decoded.
186 ///
187 /// Has the side effect of initializing the relocated addresses for the text
188 /// sections corresponding to non-null output parameters in the BSS segment.
189 ///
190 /// On failure, the error may be retrieved by 'error()'.
191 bool ResolveSymbols(const uint8_t** vm_data,
192 const uint8_t** vm_instrs,
193 const uint8_t** isolate_data,
194 const uint8_t** isolate_instrs);
195
196 const char* error() { return error_; }
197
198 private:
199 bool ReadHeader();
200 bool ReadProgramTable();
201 bool LoadSegments();
202 bool ReadSectionTable();
203 bool ReadSectionStringTable();
204 bool ReadSections();
205
206 static uword PageSize() { return VirtualMemory::PageSize(); }
207
208 // Unlike File::Map, allows non-aligned 'start' and 'length'.
209 MappedMemory* MapFilePiece(uword start,
210 uword length,
211 const void** mapping_start);
212
213 // Initialized on a successful Load().
214 std::unique_ptr<Mappable> mappable_;
215 const uint64_t elf_data_offset_;
216
217 // Initialized on error.
218 const char* error_ = nullptr;
219
220 // Initialized by ReadHeader().
221 dart::elf::ElfHeader header_;
222
223 // Initialized by ReadProgramTable().
224 std::unique_ptr<MappedMemory> program_table_mapping_;
225 const dart::elf::ProgramHeader* program_table_ = nullptr;
226
227 // Initialized by LoadSegments().
228 std::unique_ptr<VirtualMemory> base_;
229
230 // Initialized by ReadSectionTable().
231 std::unique_ptr<MappedMemory> section_table_mapping_;
232 const dart::elf::SectionHeader* section_table_ = nullptr;
233
234 // Initialized by ReadSectionStringTable().
235 std::unique_ptr<MappedMemory> section_string_table_mapping_;
236 const char* section_string_table_ = nullptr;
237
238 // Initialized by ReadSections().
239 const char* dynamic_string_table_ = nullptr;
240 const dart::elf::Symbol* dynamic_symbol_table_ = nullptr;
241 uword dynamic_symbol_count_ = 0;
242 uword* vm_bss_ = nullptr;
243 uword* isolate_bss_ = nullptr;
244
245 DISALLOW_COPY_AND_ASSIGN(LoadedElf);
246};
247
248#define CHECK(value) \
249 if (!(value)) { \
250 ASSERT(error_ != nullptr); \
251 return false; \
252 }
253
254#define ERROR(message) \
255 { \
256 error_ = (message); \
257 return false; \
258 }
259
260#define CHECK_ERROR(value, message) \
261 if (!(value)) { \
262 error_ = (message); \
263 return false; \
264 }
265
266bool LoadedElf::Load() {
267 VirtualMemory::Init();
268
269 if (error_ != nullptr) {
270 return false;
271 }
272
273 CHECK_ERROR(Utils::IsAligned(elf_data_offset_, PageSize()),
274 "File offset must be page-aligned.");
275
276 ASSERT(mappable_ != nullptr);
277 CHECK_ERROR(mappable_->SetPosition(elf_data_offset_), "Invalid file offset.");
278
279 CHECK(ReadHeader());
280 CHECK(ReadProgramTable());
281 CHECK(LoadSegments());
282 CHECK(ReadSectionTable());
283 CHECK(ReadSectionStringTable());
284 CHECK(ReadSections());
285
286 return true;
287}
288
289LoadedElf::~LoadedElf() {
290 // Unmap the image.
291 base_.reset();
292
293 // Explicitly destroy all the mappings before closing the file.
294 program_table_mapping_.reset();
295 section_table_mapping_.reset();
296 section_string_table_mapping_.reset();
297}
298
299bool LoadedElf::ReadHeader() {
300 CHECK_ERROR(mappable_->ReadFully(&header_, sizeof(dart::elf::ElfHeader)),
301 "Could not read ELF file.");
302
303 CHECK_ERROR(header_.ident[dart::elf::EI_DATA] == dart::elf::ELFDATA2LSB,
304 "Expected little-endian ELF object.");
305
306 CHECK_ERROR(header_.type == dart::elf::ET_DYN,
307 "Can only load dynamic libraries.");
308
309#if defined(TARGET_ARCH_IA32)
310 CHECK_ERROR(header_.machine == dart::elf::EM_386, "Architecture mismatch.");
311#elif defined(TARGET_ARCH_X64)
312 CHECK_ERROR(header_.machine == dart::elf::EM_X86_64,
313 "Architecture mismatch.");
314#elif defined(TARGET_ARCH_ARM)
315 CHECK_ERROR(header_.machine == dart::elf::EM_ARM, "Architecture mismatch.");
316#elif defined(TARGET_ARCH_ARM64)
317 CHECK_ERROR(header_.machine == dart::elf::EM_AARCH64,
318 "Architecture mismatch.");
319#else
320#error Unsupported architecture architecture.
321#endif
322
323 CHECK_ERROR(header_.version == dart::elf::EV_CURRENT,
324 "Unexpected ELF version.");
325 CHECK_ERROR(header_.header_size == sizeof(dart::elf::ElfHeader),
326 "Unexpected header size.");
327 CHECK_ERROR(
328 header_.program_table_entry_size == sizeof(dart::elf::ProgramHeader),
329 "Unexpected program header size.");
330 CHECK_ERROR(
331 header_.section_table_entry_size == sizeof(dart::elf::SectionHeader),
332 "Unexpected section header size.");
333
334 return true;
335}
336
337bool LoadedElf::ReadProgramTable() {
338 const uword file_start = header_.program_table_offset;
339 const uword file_length =
340 header_.num_program_headers * sizeof(dart::elf::ProgramHeader);
341 program_table_mapping_.reset(
342 MapFilePiece(file_start, file_length,
343 reinterpret_cast<const void**>(&program_table_)));
344 CHECK_ERROR(program_table_mapping_ != nullptr,
345 "Could not mmap the program table.");
346 return true;
347}
348
349bool LoadedElf::ReadSectionTable() {
350 const uword file_start = header_.section_table_offset;
351 const uword file_length =
352 header_.num_section_headers * sizeof(dart::elf::SectionHeader);
353 section_table_mapping_.reset(
354 MapFilePiece(file_start, file_length,
355 reinterpret_cast<const void**>(&section_table_)));
356 CHECK_ERROR(section_table_mapping_ != nullptr,
357 "Could not mmap the section table.");
358 return true;
359}
360
361bool LoadedElf::ReadSectionStringTable() {
362 const dart::elf::SectionHeader header =
363 section_table_[header_.shstrtab_section_index];
364 section_string_table_mapping_.reset(
365 MapFilePiece(header.file_offset, header.file_size,
366 reinterpret_cast<const void**>(&section_string_table_)));
367 CHECK_ERROR(section_string_table_mapping_ != nullptr,
368 "Could not mmap the section string table.");
369 return true;
370}
371
372bool LoadedElf::LoadSegments() {
373 // Calculate the total amount of virtual memory needed.
374 uword total_memory = 0;
375 uword maximum_alignment = PageSize();
376 for (uword i = 0; i < header_.num_program_headers; ++i) {
377 const dart::elf::ProgramHeader header = program_table_[i];
378
379 // Only PT_LOAD segments need to be loaded.
380 if (header.type != dart::elf::ProgramHeaderType::PT_LOAD) continue;
381
382 total_memory = Utils::Maximum(
383 static_cast<uword>(header.memory_offset + header.memory_size),
384 total_memory);
385 CHECK_ERROR(Utils::IsPowerOfTwo(header.alignment),
386 "Alignment must be a power of two.");
387 maximum_alignment =
388 Utils::Maximum(maximum_alignment, static_cast<uword>(header.alignment));
389 }
390 total_memory = Utils::RoundUp(total_memory, PageSize());
391
392 base_.reset(VirtualMemory::AllocateAligned(
393 total_memory, /*alignment=*/maximum_alignment,
394 /*is_executable=*/false, "dart-compiled-image"));
395 CHECK_ERROR(base_ != nullptr, "Could not reserve virtual memory.");
396
397 for (uword i = 0; i < header_.num_program_headers; ++i) {
398 const dart::elf::ProgramHeader header = program_table_[i];
399
400 // Only PT_LOAD segments need to be loaded.
401 if (header.type != dart::elf::ProgramHeaderType::PT_LOAD) continue;
402
403 const uword memory_offset = header.memory_offset,
404 file_offset = header.file_offset;
405 CHECK_ERROR(
406 (memory_offset % PageSize()) == (file_offset % PageSize()),
407 "Difference between file and memory offset must be page-aligned.");
408
409 const intptr_t adjustment = header.memory_offset % PageSize();
410
411 void* const memory_start =
412 static_cast<char*>(base_->address()) + memory_offset - adjustment;
413 const uword file_start = elf_data_offset_ + file_offset - adjustment;
414 const uword length = header.memory_size + adjustment;
415
416 File::MapType map_type = File::kReadOnly;
417 if (header.flags == (dart::elf::PF_R | dart::elf::PF_W)) {
418 map_type = File::kReadWrite;
419 } else if (header.flags == (dart::elf::PF_R | dart::elf::PF_X)) {
420 map_type = File::kReadExecute;
421 } else if (header.flags == dart::elf::PF_R) {
422 map_type = File::kReadOnly;
423 } else {
424 ERROR("Unsupported segment flag set.");
425 }
426
427#if defined(HOST_OS_FUCHSIA)
428 // mmap is less flexible on Fuchsia than on Linux and Darwin, in (at least)
429 // two important ways:
430 //
431 // 1. We cannot map a file opened as RX into an RW mapping, even if the
432 // mode is MAP_PRIVATE (which implies copy-on-write).
433 // 2. We cannot atomically replace an existing anonymous mapping with a
434 // file mapping: we must first unmap the existing mapping.
435
436 if (map_type == File::kReadWrite) {
437 CHECK_ERROR(mappable_->SetPosition(file_start),
438 "Could not advance file position.");
439 CHECK_ERROR(mappable_->ReadFully(memory_start, length),
440 "Could not read file.");
441 continue;
442 }
443
444 CHECK_ERROR(munmap(memory_start, length) == 0,
445 "Could not unmap reservation.");
446#endif
447
448 std::unique_ptr<MappedMemory> memory(
449 mappable_->Map(map_type, file_start, length, memory_start));
450 CHECK_ERROR(memory != nullptr, "Could not map segment.");
451 CHECK_ERROR(memory->address() == memory_start,
452 "Mapping not at requested address.");
453 }
454
455 return true;
456}
457
458bool LoadedElf::ReadSections() {
459 for (uword i = 0; i < header_.num_section_headers; ++i) {
460 const dart::elf::SectionHeader header = section_table_[i];
461 const char* const name = section_string_table_ + header.name;
462 if (strcmp(name, ".dynstr") == 0) {
463 CHECK_ERROR(header.memory_offset != 0, ".dynstr must be loaded.");
464 dynamic_string_table_ =
465 static_cast<const char*>(base_->address()) + header.memory_offset;
466 } else if (strcmp(name, ".dynsym") == 0) {
467 CHECK_ERROR(header.memory_offset != 0, ".dynsym must be loaded.");
468 dynamic_symbol_table_ = reinterpret_cast<const dart::elf::Symbol*>(
469 base_->start() + header.memory_offset);
470 dynamic_symbol_count_ = header.file_size / sizeof(dart::elf::Symbol);
471 } else if (strcmp(name, ".bss") == 0) {
472 auto const bss_size =
473 (BSS::kVmEntryCount + BSS::kIsolateEntryCount) * kWordSize;
474 CHECK_ERROR(header.memory_offset != 0, ".bss must be loaded.");
475 CHECK_ERROR(header.file_size >= bss_size,
476 ".bss does not have enough space.");
477 vm_bss_ = reinterpret_cast<uword*>(base_->start() + header.memory_offset);
478 isolate_bss_ = vm_bss_ + BSS::kVmEntryCount;
479 // We set applicable BSS entries in ResolveSymbols().
480 }
481 }
482
483 CHECK_ERROR(dynamic_string_table_ != nullptr, "Couldn't find .dynstr.");
484 CHECK_ERROR(dynamic_symbol_table_ != nullptr, "Couldn't find .dynsym.");
485 CHECK_ERROR(vm_bss_ != nullptr, "Couldn't find .bss.");
486 return true;
487}
488
489bool LoadedElf::ResolveSymbols(const uint8_t** vm_data,
490 const uint8_t** vm_instrs,
491 const uint8_t** isolate_data,
492 const uint8_t** isolate_instrs) {
493 if (error_ != nullptr) {
494 return false;
495 }
496
497 // The first entry of the symbol table is reserved.
498 for (uword i = 1; i < dynamic_symbol_count_; ++i) {
499 const dart::elf::Symbol sym = dynamic_symbol_table_[i];
500 const char* name = dynamic_string_table_ + sym.name;
501 const uint8_t** output = nullptr;
502
503 if (strcmp(name, kVmSnapshotDataAsmSymbol) == 0) {
504 output = vm_data;
505 } else if (strcmp(name, kVmSnapshotInstructionsAsmSymbol) == 0) {
506 output = vm_instrs;
507 if (output != nullptr) {
508 // Store the value of the symbol in the VM BSS, as it contains the
509 // address of the VM instructions section relative to the DSO base.
510 BSS::InitializeBSSEntry(BSS::Relocation::InstructionsRelocatedAddress,
511 sym.value, vm_bss_);
512 }
513 } else if (strcmp(name, kIsolateSnapshotDataAsmSymbol) == 0) {
514 output = isolate_data;
515 } else if (strcmp(name, kIsolateSnapshotInstructionsAsmSymbol) == 0) {
516 output = isolate_instrs;
517 if (output != nullptr) {
518 // Store the value of the symbol in the isolate BSS, as it contains the
519 // address of the isolate instructions section relative to the DSO base.
520 BSS::InitializeBSSEntry(BSS::Relocation::InstructionsRelocatedAddress,
521 sym.value, isolate_bss_);
522 }
523 }
524
525 if (output != nullptr) {
526 *output = reinterpret_cast<const uint8_t*>(base_->start() + sym.value);
527 }
528 }
529
530 CHECK_ERROR(isolate_data == nullptr || *isolate_data != nullptr,
531 "Could not find isolate snapshot data.");
532 CHECK_ERROR(isolate_instrs == nullptr || *isolate_instrs != nullptr,
533 "Could not find isolate instructions.");
534 return true;
535}
536
537MappedMemory* LoadedElf::MapFilePiece(uword file_start,
538 uword file_length,
539 const void** mem_start) {
540 const uword adjustment = (elf_data_offset_ + file_start) % PageSize();
541 const uword mapping_offset = elf_data_offset_ + file_start - adjustment;
542 const uword mapping_length =
543 Utils::RoundUp(elf_data_offset_ + file_start + file_length, PageSize()) -
544 mapping_offset;
545 MappedMemory* const mapping =
546 mappable_->Map(bin::File::kReadOnly, mapping_offset, mapping_length);
547
548 if (mapping != nullptr) {
549 *mem_start = reinterpret_cast<uint8_t*>(mapping->start() +
550 (file_start % PageSize()));
551 }
552
553 return mapping;
554}
555
556} // namespace elf
557} // namespace bin
558} // namespace dart
559
560using namespace dart::bin::elf; // NOLINT
561
562#if defined(HOST_OS_FUCHSIA) || defined(HOST_OS_LINUX)
563DART_EXPORT Dart_LoadedElf* Dart_LoadELF_Fd(int fd,
564 uint64_t file_offset,
565 const char** error,
566 const uint8_t** vm_snapshot_data,
567 const uint8_t** vm_snapshot_instrs,
568 const uint8_t** vm_isolate_data,
569 const uint8_t** vm_isolate_instrs) {
570 std::unique_ptr<Mappable> mappable(Mappable::FromFD(fd));
571 std::unique_ptr<LoadedElf> elf(
572 new LoadedElf(std::move(mappable), file_offset));
573
574 if (!elf->Load() ||
575 !elf->ResolveSymbols(vm_snapshot_data, vm_snapshot_instrs,
576 vm_isolate_data, vm_isolate_instrs)) {
577 *error = elf->error();
578 return nullptr;
579 }
580
581 return reinterpret_cast<Dart_LoadedElf*>(elf.release());
582}
583#endif
584
585#if !defined(HOST_OS_FUCHSIA)
586DART_EXPORT Dart_LoadedElf* Dart_LoadELF(const char* filename,
587 uint64_t file_offset,
588 const char** error,
589 const uint8_t** vm_snapshot_data,
590 const uint8_t** vm_snapshot_instrs,
591 const uint8_t** vm_isolate_data,
592 const uint8_t** vm_isolate_instrs) {
593 std::unique_ptr<Mappable> mappable(Mappable::FromPath(filename));
594 if (mappable == nullptr) {
595 *error = "Couldn't open file.";
596 return nullptr;
597 }
598 std::unique_ptr<LoadedElf> elf(
599 new LoadedElf(std::move(mappable), file_offset));
600
601 if (!elf->Load() ||
602 !elf->ResolveSymbols(vm_snapshot_data, vm_snapshot_instrs,
603 vm_isolate_data, vm_isolate_instrs)) {
604 *error = elf->error();
605 return nullptr;
606 }
607
608 return reinterpret_cast<Dart_LoadedElf*>(elf.release());
609}
610#endif
611
612DART_EXPORT Dart_LoadedElf* Dart_LoadELF_Memory(
613 const uint8_t* snapshot,
614 uint64_t snapshot_size,
615 const char** error,
616 const uint8_t** vm_snapshot_data,
617 const uint8_t** vm_snapshot_instrs,
618 const uint8_t** vm_isolate_data,
619 const uint8_t** vm_isolate_instrs) {
620 std::unique_ptr<Mappable> mappable(
621 Mappable::FromMemory(snapshot, snapshot_size));
622 if (mappable == nullptr) {
623 *error = "Couldn't open file.";
624 return nullptr;
625 }
626 std::unique_ptr<LoadedElf> elf(
627 new LoadedElf(std::move(mappable), /*file_offset=*/0));
628
629 if (!elf->Load() ||
630 !elf->ResolveSymbols(vm_snapshot_data, vm_snapshot_instrs,
631 vm_isolate_data, vm_isolate_instrs)) {
632 *error = elf->error();
633 return nullptr;
634 }
635
636 return reinterpret_cast<Dart_LoadedElf*>(elf.release());
637}
638
639DART_EXPORT void Dart_UnloadELF(Dart_LoadedElf* loaded) {
640 delete reinterpret_cast<LoadedElf*>(loaded);
641}
642