1/* 2 * Copyright (C) 2009-2010 Julien BLACHE <jb@jblache.org> 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 17 */ 18 19#ifdef HAVE_CONFIG_H 20# include <config.h> 21#endif 22 23#include <stdio.h> 24#include <string.h> 25#include <unistd.h> 26#include <stdlib.h> 27#include <sys/types.h> 28#include <sys/stat.h> 29#include <fcntl.h> 30#include <errno.h> 31 32#include <sys/mman.h> 33 34#include <stdint.h> 35#include <inttypes.h> 36 37#include <avl.h> 38#include <plist/plist.h> 39 40#include "evhttp/evhttp.h" 41 42#include "logger.h" 43#include "db.h" 44#include "filescanner.h" 45#include "conffile.h" 46#include "misc.h" 47 48 49/* Mapping between iTunes library IDs and our DB IDs */ 50struct itml_to_db_map { 51 uint64_t itml_id; 52 uint32_t db_id; 53}; 54 55/* Mapping between iTunes library metadata keys and the offset 56 * of the equivalent metadata field in struct media_file_info */ 57struct metadata_map { 58 char *key; 59 plist_type type; 60 size_t offset; 61}; 62 63static struct metadata_map md_map[] = 64 { 65 { "Name", PLIST_STRING, mfi_offsetof(title) }, 66 { "Artist", PLIST_STRING, mfi_offsetof(artist) }, 67 { "Album Artist", PLIST_STRING, mfi_offsetof(album_artist) }, 68 { "Composer", PLIST_STRING, mfi_offsetof(composer) }, 69 { "Album", PLIST_STRING, mfi_offsetof(album) }, 70 { "Genre", PLIST_STRING, mfi_offsetof(genre) }, 71 { "Comments", PLIST_STRING, mfi_offsetof(comment) }, 72 { "Track Count", PLIST_UINT, mfi_offsetof(total_tracks) }, 73 { "Track Number", PLIST_UINT, mfi_offsetof(track) }, 74 { "Disc Count", PLIST_UINT, mfi_offsetof(total_discs) }, 75 { "Disc Number", PLIST_UINT, mfi_offsetof(disc) }, 76 { "Year", PLIST_UINT, mfi_offsetof(year) }, 77 { "Total Time", PLIST_UINT, mfi_offsetof(song_length) }, 78 { "Bit Rate", PLIST_UINT, mfi_offsetof(bitrate) }, 79 { "Sample Rate", PLIST_UINT, mfi_offsetof(samplerate) }, 80 { "BPM", PLIST_UINT, mfi_offsetof(bpm) }, 81 { "Rating", PLIST_UINT, mfi_offsetof(rating) }, 82 { "Compilation", PLIST_BOOLEAN, mfi_offsetof(compilation) }, 83 { "Date Added", PLIST_DATE, mfi_offsetof(time_added) }, 84 { NULL, 0, 0 } 85 }; 86 87static avl_tree_t *itml_to_db; 88 89 90static int 91itml_to_db_compare(const void *aa, const void *bb) 92{ 93 struct itml_to_db_map *a = (struct itml_to_db_map *)aa; 94 struct itml_to_db_map *b = (struct itml_to_db_map *)bb; 95 96 if (a->itml_id < b->itml_id) 97 return -1; 98 99 if (a->itml_id > b->itml_id) 100 return 1; 101 102 return 0; 103} 104 105 106/* plist helpers */ 107static int 108get_dictval_int_from_key(plist_t dict, const char *key, uint64_t *val) 109{ 110 plist_t node; 111 112 node = plist_dict_get_item(dict, key); 113 114 if (!node) 115 return -1; 116 117 if (plist_get_node_type(node) != PLIST_UINT) 118 return -1; 119 120 plist_get_uint_val(node, val); 121 122 return 0; 123} 124 125static int 126get_dictval_date_from_key(plist_t dict, const char *key, uint32_t *val) 127{ 128 plist_t node; 129 int32_t secs; 130 int32_t dummy; 131 132 node = plist_dict_get_item(dict, key); 133 134 if (!node) 135 return -1; 136 137 if (plist_get_node_type(node) != PLIST_DATE) 138 return -1; 139 140 plist_get_date_val(node, &secs, &dummy); 141 142 *val = (uint32_t) secs; 143 144 return 0; 145} 146 147static int 148get_dictval_bool_from_key(plist_t dict, const char *key, uint8_t *val) 149{ 150 plist_t node; 151 152 node = plist_dict_get_item(dict, key); 153 154 /* Not present means false */ 155 if (!node) 156 { 157 *val = 0; 158 159 return 0; 160 } 161 162 if (plist_get_node_type(node) != PLIST_BOOLEAN) 163 return -1; 164 165 plist_get_bool_val(node, val); 166 167 return 0; 168} 169 170static int 171get_dictval_string_from_key(plist_t dict, const char *key, char **val) 172{ 173 plist_t node; 174 175 node = plist_dict_get_item(dict, key); 176 177 if (!node) 178 return -1; 179 180 if (plist_get_node_type(node) != PLIST_STRING) 181 return -1; 182 183 plist_get_string_val(node, val); 184 185 return 0; 186} 187 188static int 189get_dictval_dict_from_key(plist_t dict, const char *key, plist_t *val) 190{ 191 plist_t node; 192 193 node = plist_dict_get_item(dict, key); 194 195 if (!node) 196 return -1; 197 198 if (plist_get_node_type(node) != PLIST_DICT) 199 return -1; 200 201 *val = node; 202 203 return 0; 204} 205 206static int 207get_dictval_array_from_key(plist_t dict, const char *key, plist_t *val) 208{ 209 plist_t node; 210 211 node = plist_dict_get_item(dict, key); 212 213 if (!node) 214 return -1; 215 216 if (plist_get_node_type(node) != PLIST_ARRAY) 217 return -1; 218 219 *val = node; 220 221 return 0; 222} 223 224 225/* We don't actually check anything (yet) despite the name */ 226static int 227check_meta(plist_t dict) 228{ 229 char *appver; 230 char *folder; 231 uint64_t major; 232 uint64_t minor; 233 int ret; 234 235 ret = get_dictval_int_from_key(dict, "Major Version", &major); 236 if (ret < 0) 237 return -1; 238 239 ret = get_dictval_int_from_key(dict, "Minor Version", &minor); 240 if (ret < 0) 241 return -1; 242 243 ret = get_dictval_string_from_key(dict, "Application Version", &appver); 244 if (ret < 0) 245 return -1; 246 247 ret = get_dictval_string_from_key(dict, "Music Folder", &folder); 248 if (ret < 0) 249 { 250 free(appver); 251 return -1; 252 } 253 254 DPRINTF(E_INFO, L_SCAN, "iTunes XML playlist Major:%" PRIu64 " Minor:%" PRIu64 255 " Application:%s Folder:%s\n", major, minor, appver, folder); 256 257 free(appver); 258 free(folder); 259 260 return 0; 261} 262 263 264/* Best-effort attempt at locating the file: 265 * - first exact location given in itml 266 * - somewhere under the location of itml 267 * - anywhere (filename only) 268 */ 269static int 270find_track_file(char *location, char *base) 271{ 272 char *filename; 273 int plen; 274 int mfi_id; 275 276 location = evhttp_decode_uri(location); 277 278 plen = strlen("file://localhost/"); 279 280 /* Not a local file ... */ 281 if (strncmp(location, "file://localhost/", plen) != 0) 282 return 0; 283 284 /* Windows pathspec, from iTunes Win32 */ 285 if (location[plen + 1] == ':') 286 plen += 2; 287 else 288 plen -= 1; 289 290 /* Try exact path first */ 291 filename = m_realpath(location + plen); 292 if (filename) 293 { 294 mfi_id = db_file_id_bypath(filename); 295 296 free(filename); 297 298 if (mfi_id > 0) 299 { 300 free(location); 301 302 return mfi_id; 303 } 304 } 305 306 filename = strrchr(location, '/'); 307 if (!filename) 308 { 309 DPRINTF(E_WARN, L_SCAN, "Could not extract filename from location\n"); 310 311 free(location); 312 return 0; 313 } 314 315 filename++; 316 317 /* Try to locate the file under the playlist location */ 318 mfi_id = db_file_id_byfilebase(filename, base); 319 if (mfi_id > 0) 320 { 321 free(location); 322 323 return mfi_id; 324 } 325 326 /* Last resort, filename only */ 327 mfi_id = db_file_id_byfile(filename); 328 329 free(location); 330 331 return mfi_id; 332} 333 334static int 335process_track_file(plist_t trk, char *base) 336{ 337 char *location; 338 struct media_file_info *mfi; 339 char *string; 340 uint64_t integer; 341 char **strval; 342 uint32_t *intval; 343 char *chrval; 344 uint8_t boolean; 345 int mfi_id; 346 int i; 347 int ret; 348 349 ret = get_dictval_string_from_key(trk, "Location", &location); 350 if (ret < 0) 351 { 352 DPRINTF(E_WARN, L_SCAN, "Track type File with no Location\n"); 353 354 return 0; 355 } 356 357 mfi_id = find_track_file(location, base); 358 359 if (mfi_id <= 0) 360 { 361 DPRINTF(E_INFO, L_SCAN, "Could not match location '%s' to any known file\n", location); 362 363 free(location); 364 return 0; 365 } 366 367 free(location); 368 369 if (!cfg_getbool(cfg_getsec(cfg, "library"), "itunes_overrides")) 370 return mfi_id; 371 372 /* Override our metadata with what's provided by iTunes */ 373 mfi = db_file_fetch_byid(mfi_id); 374 if (!mfi) 375 { 376 DPRINTF(E_LOG, L_SCAN, "Could not retrieve file info for file id %d\n", mfi_id); 377 378 return mfi_id; 379 } 380 381 for (i = 0; md_map[i].key != NULL; i++) 382 { 383 switch (md_map[i].type) 384 { 385 case PLIST_UINT: 386 ret = get_dictval_int_from_key(trk, md_map[i].key, &integer); 387 if (ret < 0) 388 break; 389 390 intval = (uint32_t *) ((char *) mfi + md_map[i].offset); 391 392 *intval = (uint32_t)integer; 393 break; 394 395 case PLIST_STRING: 396 ret = get_dictval_string_from_key(trk, md_map[i].key, &string); 397 if (ret < 0) 398 break; 399 400 strval = (char **) ((char *) mfi + md_map[i].offset); 401 402 if (*strval) 403 free(*strval); 404 405 *strval = string; 406 break; 407 408 case PLIST_BOOLEAN: 409 ret = get_dictval_bool_from_key(trk, md_map[i].key, &boolean); 410 if (ret < 0) 411 break; 412 413 chrval = (char *) mfi + md_map[i].offset; 414 415 *chrval = boolean; 416 break; 417 418 case PLIST_DATE: 419 intval = (uint32_t *) ((char *) mfi + md_map[i].offset); 420 421 get_dictval_date_from_key(trk, md_map[i].key, intval); 422 break; 423 424 default: 425 DPRINTF(E_WARN, L_SCAN, "Unhandled metadata type %d\n", md_map[i].type); 426 break; 427 } 428 } 429 430 /* Don't let album_artist set to "Unknown artist" if we've 431 * filled artist from the iTunes data in the meantime 432 */ 433 if (strcmp(mfi->album_artist, "Unknown artist") == 0) 434 { 435 free(mfi->album_artist); 436 mfi->album_artist = strdup(mfi->artist); 437 } 438 439 unicode_fixup_mfi(mfi); 440 db_file_update(mfi); 441 442 free_mfi(mfi, 0); 443 444 return mfi_id; 445} 446 447static int 448process_track_stream(plist_t trk) 449{ 450 char *url; 451 int ret; 452 453 ret = get_dictval_string_from_key(trk, "Location", &url); 454 if (ret < 0) 455 { 456 DPRINTF(E_WARN, L_SCAN, "Track type URL with no Location entry!\n"); 457 458 return 0; 459 } 460 461 ret = db_file_id_byurl(url); 462 463 free(url); 464 465 return ret; 466} 467 468static int 469process_tracks(plist_t tracks, char *base) 470{ 471 plist_t trk; 472 plist_dict_iter iter; 473 struct itml_to_db_map *map; 474 avl_node_t *mapnode; 475 char *str; 476 uint64_t trk_id; 477 uint8_t disabled; 478 int ntracks; 479 int mfi_id; 480 int ret; 481 482 if (plist_dict_get_size(tracks) == 0) 483 { 484 DPRINTF(E_WARN, L_SCAN, "No tracks in iTunes library\n"); 485 return 0; 486 } 487 488 ntracks = 0; 489 490 iter = NULL; 491 plist_dict_new_iter(tracks, &iter); 492 493 plist_dict_next_item(tracks, iter, NULL, &trk); 494 while (trk) 495 { 496 if (plist_get_node_type(trk) != PLIST_DICT) 497 { 498 plist_dict_next_item(tracks, iter, NULL, &trk); 499 continue; 500 } 501 502 ret = get_dictval_int_from_key(trk, "Track ID", &trk_id); 503 if (ret < 0) 504 { 505 DPRINTF(E_WARN, L_SCAN, "Track ID not found!\n"); 506 507 plist_dict_next_item(tracks, iter, NULL, &trk); 508 continue; 509 } 510 511 ret = get_dictval_bool_from_key(trk, "Disabled", &disabled); 512 if (ret < 0) 513 { 514 DPRINTF(E_WARN, L_SCAN, "Malformed track record (id %" PRIu64 ")\n", trk_id); 515 516 plist_dict_next_item(tracks, iter, NULL, &trk); 517 continue; 518 } 519 520 if (disabled) 521 { 522 DPRINTF(E_INFO, L_SCAN, "Track %" PRIu64 " disabled; skipping\n", trk_id); 523 524 plist_dict_next_item(tracks, iter, NULL, &trk); 525 continue; 526 } 527 528 ret = get_dictval_string_from_key(trk, "Track Type", &str); 529 if (ret < 0) 530 { 531 DPRINTF(E_WARN, L_SCAN, "Track %" PRIu64 " has no track type\n", trk_id); 532 533 plist_dict_next_item(tracks, iter, NULL, &trk); 534 continue; 535 } 536 537 if (strcmp(str, "URL") == 0) 538 mfi_id = process_track_stream(trk); 539 else if (strcmp(str, "File") == 0) 540 mfi_id = process_track_file(trk, base); 541 else 542 { 543 DPRINTF(E_LOG, L_SCAN, "Unknown track type: %s\n", str); 544 545 free(str); 546 plist_dict_next_item(tracks, iter, NULL, &trk); 547 continue; 548 } 549 550 free(str); 551 552 if (mfi_id <= 0) 553 { 554 plist_dict_next_item(tracks, iter, NULL, &trk); 555 continue; 556 } 557 558 ntracks++; 559 560 map = (struct itml_to_db_map *)malloc(sizeof(struct itml_to_db_map)); 561 if (!map) 562 { 563 DPRINTF(E_WARN, L_SCAN, "Out of memory for itml -> db mapping\n"); 564 565 plist_dict_next_item(tracks, iter, NULL, &trk); 566 continue; 567 } 568 569 map->itml_id = trk_id; 570 map->db_id = mfi_id; 571 572 mapnode = avl_insert(itml_to_db, map); 573 if (!mapnode) 574 { 575 if (errno == EEXIST) 576 DPRINTF(E_WARN, L_SCAN, "Track %" PRIu64 " already in itml -> db map?!\n", trk_id); 577 else 578 DPRINTF(E_WARN, L_SCAN, "Track %" PRIu64 ": AVL insert error: %s\n", trk_id, strerror(errno)); 579 580 free(map); 581 } 582 583 plist_dict_next_item(tracks, iter, NULL, &trk); 584 } 585 586 free(iter); 587 588 return ntracks; 589} 590 591 592static void 593process_pl_items(plist_t items, int pl_id) 594{ 595 struct itml_to_db_map needle; 596 struct itml_to_db_map *map; 597 plist_t trk; 598 avl_node_t *mapnode; 599 uint32_t alen; 600 uint32_t i; 601 int ret; 602 603 alen = plist_array_get_size(items); 604 for (i = 0; i < alen; i++) 605 { 606 trk = plist_array_get_item(items, i); 607 608 if (plist_get_node_type(trk) != PLIST_DICT) 609 continue; 610 611 ret = get_dictval_int_from_key(trk, "Track ID", &needle.itml_id); 612 if (ret < 0) 613 { 614 DPRINTF(E_WARN, L_SCAN, "No Track ID found for playlist item %u\n", i); 615 continue; 616 } 617 618 mapnode = avl_search(itml_to_db, &needle); 619 if (!mapnode) 620 { 621 DPRINTF(E_INFO, L_SCAN, "Track ID %" PRIu64 " dropped\n", needle.itml_id); 622 continue; 623 } 624 625 map = (struct itml_to_db_map *)mapnode->item; 626 627 ret = db_pl_add_item_byid(pl_id, map->db_id); 628 if (ret < 0) 629 DPRINTF(E_WARN, L_SCAN, "Could not add ID %d to playlist\n", map->db_id); 630 } 631} 632 633static int 634ignore_pl(plist_t pl, char *name) 635{ 636 uint64_t kind; 637 int smart; 638 uint8_t master; 639 uint8_t party; 640 641 kind = 0; 642 smart = 0; 643 master = 0; 644 party = 0; 645 646 /* Special (builtin) playlists */ 647 get_dictval_int_from_key(pl, "Distinguished Kind", &kind); 648 649 /* If only we could recover the smart playlists ... */ 650 if (plist_dict_get_item(pl, "Smart Info") 651 || plist_dict_get_item(pl, "Smart Criteria")) 652 smart = 1; 653 654 /* Not interested in the Master playlist */ 655 get_dictval_bool_from_key(pl, "Master", &master); 656 /* Not interested in Party Shuffle playlists */ 657 get_dictval_bool_from_key(pl, "Party Shuffle", &party); 658 659 if ((kind > 0) || smart || party || master) 660 { 661 DPRINTF(E_INFO, L_SCAN, "Ignoring playlist '%s' (k %" PRIu64 " s%d p%d m%d)\n", name, kind, smart, party, master); 662 663 return 1; 664 } 665 666 return 0; 667} 668 669static void 670process_pls(plist_t playlists, char *file) 671{ 672 plist_t pl; 673 plist_t items; 674 struct playlist_info *pli; 675 char *name; 676 uint64_t id; 677 int pl_id; 678 uint32_t alen; 679 uint32_t i; 680 int ret; 681 682 alen = plist_array_get_size(playlists); 683 for (i = 0; i < alen; i++) 684 { 685 pl = plist_array_get_item(playlists, i); 686 687 if (plist_get_node_type(pl) != PLIST_DICT) 688 continue; 689 690 ret = get_dictval_int_from_key(pl, "Playlist ID", &id); 691 if (ret < 0) 692 { 693 DPRINTF(E_DBG, L_SCAN, "Playlist ID not found!\n"); 694 continue; 695 } 696 697 ret = get_dictval_string_from_key(pl, "Name", &name); 698 if (ret < 0) 699 { 700 DPRINTF(E_DBG, L_SCAN, "Name not found!\n"); 701 continue; 702 } 703 704 if (ignore_pl(pl, name)) 705 { 706 free(name); 707 continue; 708 } 709 710 pli = db_pl_fetch_bytitlepath(name, file); 711 712 if (pli) 713 { 714 pl_id = pli->id; 715 716 free_pli(pli, 0); 717 718 db_pl_ping(pl_id); 719 db_pl_clear_items(pl_id); 720 } 721 else 722 pl_id = 0; 723 724 ret = get_dictval_array_from_key(pl, "Playlist Items", &items); 725 if (ret < 0) 726 { 727 DPRINTF(E_INFO, L_SCAN, "Playlist '%s' has no items\n", name); 728 729 free(name); 730 continue; 731 } 732 733 if (pl_id == 0) 734 { 735 ret = db_pl_add(name, file, &pl_id); 736 if (ret < 0) 737 { 738 DPRINTF(E_LOG, L_SCAN, "Error adding iTunes playlist '%s' (%s)\n", name, file); 739 740 free(name); 741 continue; 742 } 743 744 DPRINTF(E_INFO, L_SCAN, "Added playlist as id %d\n", pl_id); 745 } 746 747 free(name); 748 749 process_pl_items(items, pl_id); 750 } 751} 752 753 754void 755scan_itunes_itml(char *file) 756{ 757 struct stat sb; 758 char *itml_xml; 759 char *ptr; 760 plist_t itml; 761 plist_t node; 762 int fd; 763 int ret; 764 765 DPRINTF(E_INFO, L_SCAN, "Processing iTunes library: %s\n", file); 766 767 fd = open(file, O_RDONLY); 768 if (fd < 0) 769 { 770 DPRINTF(E_WARN, L_SCAN, "Could not open iTunes library '%s': %s\n", file, strerror(errno)); 771 772 return; 773 } 774 775 ret = fstat(fd, &sb); 776 if (ret < 0) 777 { 778 DPRINTF(E_WARN, L_SCAN, "Could not stat iTunes library '%s': %s\n", file, strerror(errno)); 779 780 close(fd); 781 return; 782 } 783 784 itml_xml = mmap(NULL, sb.st_size, PROT_READ, MAP_SHARED, fd, 0); 785 if (itml_xml == MAP_FAILED) 786 { 787 DPRINTF(E_WARN, L_SCAN, "Could not map iTunes library: %s\n", strerror(errno)); 788 789 close(fd); 790 return; 791 } 792 793 itml = NULL; 794 plist_from_xml(itml_xml, sb.st_size, &itml); 795 796 ret = munmap(itml_xml, sb.st_size); 797 if (ret < 0) 798 DPRINTF(E_LOG, L_SCAN, "Could not unmap iTunes library: %s\n", strerror(errno)); 799 800 close(fd); 801 802 if (!itml) 803 { 804 DPRINTF(E_WARN, L_SCAN, "iTunes XML playlist '%s' failed to parse\n", file); 805 806 return; 807 } 808 809 if (plist_get_node_type(itml) != PLIST_DICT) 810 { 811 DPRINTF(E_WARN, L_SCAN, "Malformed iTunes XML playlist '%s'\n", file); 812 813 plist_free(itml); 814 return; 815 } 816 817 /* Meta data */ 818 ret = check_meta(itml); 819 if (ret < 0) 820 { 821 plist_free(itml); 822 return; 823 } 824 825 /* Tracks */ 826 ret = get_dictval_dict_from_key(itml, "Tracks", &node); 827 if (ret < 0) 828 { 829 DPRINTF(E_WARN, L_SCAN, "Could not find Tracks dict\n"); 830 831 plist_free(itml); 832 return; 833 } 834 835 itml_to_db = avl_alloc_tree(itml_to_db_compare, free); 836 if (!itml_to_db) 837 { 838 DPRINTF(E_FATAL, L_SCAN, "iTunes library parser could not allocate AVL tree\n"); 839 840 plist_free(itml); 841 return; 842 } 843 844 ptr = strrchr(file, '/'); 845 if (!ptr) 846 { 847 DPRINTF(E_FATAL, L_SCAN, "Invalid filename\n"); 848 849 avl_free_tree(itml_to_db); 850 plist_free(itml); 851 return; 852 } 853 854 *ptr = '\0'; 855 856 ret = process_tracks(node, file); 857 if (ret <= 0) 858 { 859 DPRINTF(E_WARN, L_SCAN, "No tracks loaded\n"); 860 861 avl_free_tree(itml_to_db); 862 plist_free(itml); 863 return; 864 } 865 866 *ptr = '/'; 867 868 DPRINTF(E_INFO, L_SCAN, "Loaded %d tracks from iTunes library\n", ret); 869 870 /* Playlists */ 871 ret = get_dictval_array_from_key(itml, "Playlists", &node); 872 if (ret < 0) 873 { 874 DPRINTF(E_WARN, L_SCAN, "Could not find Playlists dict\n"); 875 876 avl_free_tree(itml_to_db); 877 plist_free(itml); 878 return; 879 } 880 881 process_pls(node, file); 882 883 avl_free_tree(itml_to_db); 884 plist_free(itml); 885} 886