EF6 Child entities not updating in many-to-many relationship

I have two entities in a many-to-many relationship: A Map which can have many Tags (and a Tag in turn can be used by many Maps).

I'm trying to update a parent Map entity, including removing items from its child Tags collection. While the Map entity is honoring changes in the database changes to the Tags collection are never honored (apart from the initial creation of them). Any ideas what I'm doing wrong?

In the database are 3 tables:

• Map
• Tag
• MapTags

The entity classes:

public class Map
{
    public Map()
    {
        Tags = new List<Tag>();
    }

    public string Id { get; set; }
    ...
    public ICollection<Tag> Tags { get; set; }
}

public class Tag
{
    public Tag()
    {
        Maps = new List<Map>();
    }

    public int Id { get; set; }
    public string Text { get; set; }
    public ICollection<Map> Maps { get; set; }
}

And the EF6 mappings:

public class MapMap : EntityTypeConfiguration<Map>
{
    public MapMap()
    {
        // Primary Key
        this.HasKey(t => new { t.Id });

        // Properties

        this.Property(t => t.Id)
            .IsRequired()
            .HasMaxLength(32);

        ...

        this.ToTable("Map");
        this.Property(t => t.Id).HasColumnName("Id");
        ...

        // Relationships

        this.HasMany(m => m.Tags)
            .WithMany(t => t.Maps)
            .Map(m =>
            {
                m.MapLeftKey("MapId");
                m.MapRightKey("MapTagId");
                m.ToTable("MapTags");
            });
    }
}

public class TagMap : EntityTypeConfiguration<Tag>
{
    public TagMap()
    {
        // Primary Key
        this.HasKey(t => new { t.Id });

        // Properties

        this.Property(t => t.Id)
            .HasDatabaseGeneratedOption(DatabaseGeneratedOption.Identity);

        this.Property(t => t.Text)
            .IsRequired()
            .HasMaxLength(256);

        // Table & Column Mappings

        this.ToTable("Tag");
        this.Property(t => t.Id).HasColumnName("Id");
        this.Property(t => t.Text).HasColumnName("Text");

        // Relationships

        this.HasMany(t => t.Maps)
            .WithMany(m => m.Tags)
            .Map(m =>
            {
                m.MapLeftKey("TagId");
                m.MapRightKey("MapId");
                m.ToTable("MapTags");
            });
    }
}

Code to update a Map's Tags:

map.Tags = new List<Tag>();
foreach (string item in data.tags)
{
    Tag tag = MapRepository.FindTagByText(item);
    if (tag == null)
    {
        try
        {
            tag = WebMapRepository.CreateTag(new Tag()
                {
                    Text = item
                });
        }
        catch (DbEntityValidationException ex)
        {
            DisplayValidationErrors(ex, "Tag [" + item + "] validation errors:");
            throw; // Abort
        }
    }
    map.Tags.Add(tag);
}

And the DAL code updating the Map:

public static Map UpdateMap(Map map)
{
    using (MapContext context = new MapContext())
    {
        context.Maps.Attach(map);
        context.Entry(map).State = EntityState.Modified;
        context.SaveChanges();
        return GetMap(map.Id);
    }
}

Workaround While I'd prefer a more elegant solution, for now I'm just running SQL directly to refresh my relationships manually.